feat: add webmentions
69adcc74
8 file(s) · +397 −13
| 20 | 20 | site: "https://stevedylan.dev", |
|
| 21 | 21 | outDir: "dist", |
|
| 22 | 22 | compressHTML: true, |
|
| 23 | + | // Webmention endpoint must accept cross-origin form POSTs; Astro's default |
|
| 24 | + | // CSRF check (checkOrigin) would reject them with a 403. |
|
| 25 | + | security: { |
|
| 26 | + | checkOrigin: false, |
|
| 27 | + | }, |
|
| 23 | 28 | image: { |
|
| 24 | 29 | domains: ["kagifeedback.org", "api.iconify.design", "files.stevedylan.dev"], |
|
| 25 | 30 | }, |
| 10 | 10 | "preview": "astro preview", |
|
| 11 | 11 | "format": "biome format --write src package.json", |
|
| 12 | 12 | "deploy": "bun run build && wrangler deploy -c dist/server/wrangler.json", |
|
| 13 | - | "parse:birds": "bun run scripts/parse-birds.ts" |
|
| 13 | + | "parse:birds": "bun run scripts/parse-birds.ts", |
|
| 14 | + | "send:webmentions": "bun run scripts/send-webmentions.ts" |
|
| 14 | 15 | }, |
|
| 15 | 16 | "devDependencies": { |
|
| 16 | 17 | "@astrojs/markdown-satteri": "0.3.3", |
| 1 | + | /** |
|
| 2 | + | * Webmention sender. |
|
| 3 | + | * |
|
| 4 | + | * Run after `bun run build`. Scans each built blog post for outbound links, |
|
| 5 | + | * discovers each target's webmention endpoint, and POSTs source+target. |
|
| 6 | + | * |
|
| 7 | + | * State is kept in scripts/.webmention-sent.json so a given source->target |
|
| 8 | + | * pair is only sent once (commit that file to persist across machines/CI). |
|
| 9 | + | * |
|
| 10 | + | * Usage: bun run scripts/send-webmentions.ts [--dry] |
|
| 11 | + | */ |
|
| 12 | + | ||
| 13 | + | import { Glob } from "bun"; |
|
| 14 | + | import { join } from "node:path"; |
|
| 15 | + | ||
| 16 | + | const SITE = "https://stevedylan.dev"; |
|
| 17 | + | const SITE_HOST = "stevedylan.dev"; |
|
| 18 | + | const POSTS_GLOB = "dist/client/posts/*/index.html"; |
|
| 19 | + | const STATE_PATH = join(import.meta.dir, ".webmention-sent.json"); |
|
| 20 | + | const DRY = process.argv.includes("--dry"); |
|
| 21 | + | ||
| 22 | + | type SentState = Record<string, string>; // "source|target" -> ISO timestamp |
|
| 23 | + | ||
| 24 | + | async function loadState(): Promise<SentState> { |
|
| 25 | + | const file = Bun.file(STATE_PATH); |
|
| 26 | + | if (await file.exists()) return (await file.json()) as SentState; |
|
| 27 | + | return {}; |
|
| 28 | + | } |
|
| 29 | + | ||
| 30 | + | async function saveState(state: SentState): Promise<void> { |
|
| 31 | + | await Bun.write(STATE_PATH, `${JSON.stringify(state, null, 2)}\n`); |
|
| 32 | + | } |
|
| 33 | + | ||
| 34 | + | // Post URL for the source, derived from the built path: |
|
| 35 | + | // dist/client/posts/<slug>/index.html -> https://stevedylan.dev/posts/<slug>/ |
|
| 36 | + | function sourceUrlFor(path: string): string { |
|
| 37 | + | const slug = path.split("/posts/")[1].replace(/\/index\.html$/, ""); |
|
| 38 | + | return `${SITE}/posts/${slug}/`; |
|
| 39 | + | } |
|
| 40 | + | ||
| 41 | + | // Grab only the article body (between the prose container and the |
|
| 42 | + | // Webmentions section) so nav/header/footer links are ignored. |
|
| 43 | + | function extractArticle(html: string): string { |
|
| 44 | + | const start = html.indexOf("prose-cactus"); |
|
| 45 | + | const end = html.indexOf('id="webmentions"'); |
|
| 46 | + | if (start === -1) return ""; |
|
| 47 | + | return html.slice(start, end === -1 ? undefined : end); |
|
| 48 | + | } |
|
| 49 | + | ||
| 50 | + | function extractTargets(articleHtml: string): string[] { |
|
| 51 | + | const hrefs = new Set<string>(); |
|
| 52 | + | const re = /href="(https?:\/\/[^"]+)"/g; |
|
| 53 | + | let m: RegExpExecArray | null; |
|
| 54 | + | while ((m = re.exec(articleHtml)) !== null) { |
|
| 55 | + | try { |
|
| 56 | + | const url = new URL(m[1]); |
|
| 57 | + | if (url.host === SITE_HOST) continue; // skip self-links |
|
| 58 | + | url.hash = ""; // normalize away fragments |
|
| 59 | + | hrefs.add(url.href); |
|
| 60 | + | } catch { |
|
| 61 | + | // ignore malformed URLs |
|
| 62 | + | } |
|
| 63 | + | } |
|
| 64 | + | return [...hrefs]; |
|
| 65 | + | } |
|
| 66 | + | ||
| 67 | + | // Discover a target's webmention endpoint via the Link header first, then a |
|
| 68 | + | // <link>/<a rel="webmention"> in the body. Returns absolute URL or null. |
|
| 69 | + | async function discoverEndpoint(target: string): Promise<string | null> { |
|
| 70 | + | let res: Response; |
|
| 71 | + | try { |
|
| 72 | + | res = await fetch(target, { |
|
| 73 | + | headers: { "user-agent": "stevedylan.dev-webmention-sender/1.0" }, |
|
| 74 | + | redirect: "follow", |
|
| 75 | + | }); |
|
| 76 | + | } catch { |
|
| 77 | + | return null; |
|
| 78 | + | } |
|
| 79 | + | if (!res.ok) return null; |
|
| 80 | + | ||
| 81 | + | // 1. Link header: Link: <url>; rel="webmention" |
|
| 82 | + | const linkHeader = res.headers.get("link"); |
|
| 83 | + | if (linkHeader) { |
|
| 84 | + | const found = parseLinkHeader(linkHeader); |
|
| 85 | + | if (found) return new URL(found, res.url).href; |
|
| 86 | + | } |
|
| 87 | + | ||
| 88 | + | // 2. HTML <link>/<a rel="webmention" href="..."> |
|
| 89 | + | const body = await res.text(); |
|
| 90 | + | const re = |
|
| 91 | + | /<(?:link|a)[^>]+rel=["'][^"']*\bwebmention\b[^"']*["'][^>]*>/gi; |
|
| 92 | + | const tag = re.exec(body); |
|
| 93 | + | if (tag) { |
|
| 94 | + | const href = /href=["']([^"']+)["']/i.exec(tag[0]); |
|
| 95 | + | if (href) return new URL(href[1], res.url).href; |
|
| 96 | + | } |
|
| 97 | + | return null; |
|
| 98 | + | } |
|
| 99 | + | ||
| 100 | + | function parseLinkHeader(header: string): string | null { |
|
| 101 | + | for (const part of header.split(",")) { |
|
| 102 | + | const seg = part.trim(); |
|
| 103 | + | const relMatch = /rel=["']?([^"';]+)["']?/i.exec(seg); |
|
| 104 | + | if (relMatch && /\bwebmention\b/i.test(relMatch[1])) { |
|
| 105 | + | const urlMatch = /^<([^>]+)>/.exec(seg); |
|
| 106 | + | if (urlMatch) return urlMatch[1]; |
|
| 107 | + | } |
|
| 108 | + | } |
|
| 109 | + | return null; |
|
| 110 | + | } |
|
| 111 | + | ||
| 112 | + | async function sendWebmention( |
|
| 113 | + | endpoint: string, |
|
| 114 | + | source: string, |
|
| 115 | + | target: string, |
|
| 116 | + | ): Promise<boolean> { |
|
| 117 | + | const res = await fetch(endpoint, { |
|
| 118 | + | method: "POST", |
|
| 119 | + | headers: { |
|
| 120 | + | "content-type": "application/x-www-form-urlencoded", |
|
| 121 | + | "user-agent": "stevedylan.dev-webmention-sender/1.0", |
|
| 122 | + | }, |
|
| 123 | + | body: new URLSearchParams({ source, target }).toString(), |
|
| 124 | + | }); |
|
| 125 | + | return res.ok; |
|
| 126 | + | } |
|
| 127 | + | ||
| 128 | + | async function main() { |
|
| 129 | + | const state = await loadState(); |
|
| 130 | + | let sent = 0; |
|
| 131 | + | let skipped = 0; |
|
| 132 | + | let noEndpoint = 0; |
|
| 133 | + | ||
| 134 | + | for await (const path of new Glob(POSTS_GLOB).scan(".")) { |
|
| 135 | + | const source = sourceUrlFor(path); |
|
| 136 | + | const html = await Bun.file(path).text(); |
|
| 137 | + | const targets = extractTargets(extractArticle(html)); |
|
| 138 | + | ||
| 139 | + | for (const target of targets) { |
|
| 140 | + | const key = `${source}|${target}`; |
|
| 141 | + | if (state[key]) { |
|
| 142 | + | skipped++; |
|
| 143 | + | continue; |
|
| 144 | + | } |
|
| 145 | + | ||
| 146 | + | const endpoint = await discoverEndpoint(target); |
|
| 147 | + | if (!endpoint) { |
|
| 148 | + | noEndpoint++; |
|
| 149 | + | continue; |
|
| 150 | + | } |
|
| 151 | + | ||
| 152 | + | if (DRY) { |
|
| 153 | + | console.log(`[dry] would send ${source} -> ${target} @ ${endpoint}`); |
|
| 154 | + | continue; |
|
| 155 | + | } |
|
| 156 | + | ||
| 157 | + | const ok = await sendWebmention(endpoint, source, target); |
|
| 158 | + | if (ok) { |
|
| 159 | + | state[key] = new Date().toISOString(); |
|
| 160 | + | sent++; |
|
| 161 | + | console.log(`sent ${source} -> ${target}`); |
|
| 162 | + | } else { |
|
| 163 | + | console.warn(`failed ${source} -> ${target} @ ${endpoint}`); |
|
| 164 | + | } |
|
| 165 | + | } |
|
| 166 | + | } |
|
| 167 | + | ||
| 168 | + | if (!DRY) await saveState(state); |
|
| 169 | + | console.log( |
|
| 170 | + | `\nDone. sent=${sent} skipped=${skipped} no-endpoint=${noEndpoint}`, |
|
| 171 | + | ); |
|
| 172 | + | } |
|
| 173 | + | ||
| 174 | + | main().catch((err) => { |
|
| 175 | + | console.error(err); |
|
| 176 | + | process.exit(1); |
|
| 177 | + | }); |
| 1 | + | --- |
|
| 2 | + | // Renders verified webmentions for the current page. |
|
| 3 | + | // Blog posts are statically prerendered, so mentions are fetched client-side |
|
| 4 | + | // from the SSR endpoint at /api/webmention?target=<canonical url>. |
|
| 5 | + | --- |
|
| 6 | + | ||
| 7 | + | <section |
|
| 8 | + | id="webmentions" |
|
| 9 | + | class="mt-12 border-t border-zinc-200 pt-6 dark:border-zinc-700" |
|
| 10 | + | data-show="false" |
|
| 11 | + | hidden |
|
| 12 | + | > |
|
| 13 | + | <h2 class="title text-lg">Webmentions</h2> |
|
| 14 | + | <ul id="webmention-list" class="mt-4 space-y-2 text-sm"></ul> |
|
| 15 | + | </section> |
|
| 16 | + | ||
| 17 | + | <script> |
|
| 18 | + | interface Mention { |
|
| 19 | + | source: string; |
|
| 20 | + | target: string; |
|
| 21 | + | verifiedAt: string; |
|
| 22 | + | } |
|
| 23 | + | ||
| 24 | + | async function loadWebmentions() { |
|
| 25 | + | const section = document.getElementById("webmentions"); |
|
| 26 | + | const list = document.getElementById("webmention-list"); |
|
| 27 | + | if (!section || !list) return; |
|
| 28 | + | ||
| 29 | + | const target = `${window.location.origin}${window.location.pathname}`; |
|
| 30 | + | try { |
|
| 31 | + | const res = await fetch( |
|
| 32 | + | `/api/webmention?target=${encodeURIComponent(target)}`, |
|
| 33 | + | ); |
|
| 34 | + | if (!res.ok) return; |
|
| 35 | + | const data = (await res.json()) as { mentions: Mention[] }; |
|
| 36 | + | if (!data.mentions?.length) return; |
|
| 37 | + | ||
| 38 | + | list.innerHTML = ""; |
|
| 39 | + | for (const m of data.mentions) { |
|
| 40 | + | const li = document.createElement("li"); |
|
| 41 | + | const a = document.createElement("a"); |
|
| 42 | + | a.href = m.source; |
|
| 43 | + | a.rel = "nofollow ugc noopener noreferrer"; |
|
| 44 | + | a.target = "_blank"; |
|
| 45 | + | a.className = "underline hover:text-link"; |
|
| 46 | + | try { |
|
| 47 | + | a.textContent = new URL(m.source).host; |
|
| 48 | + | } catch { |
|
| 49 | + | a.textContent = m.source; |
|
| 50 | + | } |
|
| 51 | + | const when = document.createElement("span"); |
|
| 52 | + | when.className = "ml-2 text-zinc-500"; |
|
| 53 | + | when.textContent = new Date(m.verifiedAt).toLocaleDateString(); |
|
| 54 | + | li.append(a, when); |
|
| 55 | + | list.append(li); |
|
| 56 | + | } |
|
| 57 | + | ||
| 58 | + | section.hidden = false; |
|
| 59 | + | section.dataset.show = "true"; |
|
| 60 | + | } catch { |
|
| 61 | + | // silently ignore — mentions are non-critical |
|
| 62 | + | } |
|
| 63 | + | } |
|
| 64 | + | ||
| 65 | + | loadWebmentions(); |
|
| 66 | + | </script> |
| 1 | 1 | /// <reference path="../.astro/types.d.ts" /> |
|
| 2 | - | // <reference path="../.astro/types.d.ts" /> |
|
| 3 | 2 | ||
| 4 | - | // type Runtime = import("@astrojs/cloudflare").Runtime<ENV>; |
|
| 5 | - | // declare namespace App { |
|
| 6 | - | // interface Locals extends Runtime {} |
|
| 7 | - | // } |
|
| 3 | + | // Typed bindings for `import { env } from "cloudflare:workers"` |
|
| 4 | + | declare namespace Cloudflare { |
|
| 5 | + | interface Env { |
|
| 6 | + | SESSION: KVNamespace; |
|
| 7 | + | WEBMENTIONS: KVNamespace; |
|
| 8 | + | } |
|
| 9 | + | } |
| 5 | 5 | ||
| 6 | 6 | import BaseLayout from "./Base.astro"; |
|
| 7 | 7 | import BlogHero from "@/components/blog/Hero.astro"; |
|
| 8 | + | import Webmentions from "@/components/blog/Webmentions.astro"; |
|
| 8 | 9 | ||
| 9 | 10 | interface Props { |
|
| 10 | 11 | post: CollectionEntry<"post">; |
|
| 80 | 81 | > |
|
| 81 | 82 | <slot /> |
|
| 82 | 83 | </div> |
|
| 84 | + | <Webmentions /> |
|
| 83 | 85 | </article> |
|
| 84 | 86 | </div> |
|
| 85 | 87 | <button |
|
| 1 | + | import type { APIRoute } from "astro"; |
|
| 2 | + | import { env } from "cloudflare:workers"; |
|
| 3 | + | ||
| 4 | + | export const prerender = false; |
|
| 5 | + | ||
| 6 | + | const ALLOWED_HOST = "stevedylan.dev"; |
|
| 7 | + | ||
| 8 | + | interface StoredMention { |
|
| 9 | + | source: string; |
|
| 10 | + | target: string; |
|
| 11 | + | verifiedAt: string; |
|
| 12 | + | } |
|
| 13 | + | ||
| 14 | + | function isValidHttpUrl(value: string): URL | null { |
|
| 15 | + | try { |
|
| 16 | + | const url = new URL(value); |
|
| 17 | + | if (url.protocol !== "http:" && url.protocol !== "https:") return null; |
|
| 18 | + | return url; |
|
| 19 | + | } catch { |
|
| 20 | + | return null; |
|
| 21 | + | } |
|
| 22 | + | } |
|
| 23 | + | ||
| 24 | + | // KV key groups mentions by target so they can be listed for a given page. |
|
| 25 | + | function mentionKey(target: string, source: string): string { |
|
| 26 | + | return `mention:${encodeURIComponent(target)}:${encodeURIComponent(source)}`; |
|
| 27 | + | } |
|
| 28 | + | ||
| 29 | + | // Fetch the source and confirm it actually links to the target, per the |
|
| 30 | + | // Webmention spec (https://www.w3.org/TR/webmention/#request-verification). |
|
| 31 | + | async function verifyAndStore( |
|
| 32 | + | kv: KVNamespace, |
|
| 33 | + | source: string, |
|
| 34 | + | target: string, |
|
| 35 | + | ): Promise<void> { |
|
| 36 | + | const key = mentionKey(target, source); |
|
| 37 | + | try { |
|
| 38 | + | const res = await fetch(source, { |
|
| 39 | + | headers: { "user-agent": "stevedylan.dev-webmention/1.0" }, |
|
| 40 | + | redirect: "follow", |
|
| 41 | + | }); |
|
| 42 | + | if (!res.ok) { |
|
| 43 | + | await kv.delete(key); |
|
| 44 | + | return; |
|
| 45 | + | } |
|
| 46 | + | const body = await res.text(); |
|
| 47 | + | if (body.includes(target)) { |
|
| 48 | + | const mention: StoredMention = { |
|
| 49 | + | source, |
|
| 50 | + | target, |
|
| 51 | + | verifiedAt: new Date().toISOString(), |
|
| 52 | + | }; |
|
| 53 | + | await kv.put(key, JSON.stringify(mention)); |
|
| 54 | + | } else { |
|
| 55 | + | // Source no longer mentions the target — remove any prior record. |
|
| 56 | + | await kv.delete(key); |
|
| 57 | + | } |
|
| 58 | + | } catch { |
|
| 59 | + | await kv.delete(key); |
|
| 60 | + | } |
|
| 61 | + | } |
|
| 62 | + | ||
| 63 | + | export const POST: APIRoute = async ({ request }) => { |
|
| 64 | + | const contentType = request.headers.get("content-type") ?? ""; |
|
| 65 | + | if (!contentType.includes("application/x-www-form-urlencoded")) { |
|
| 66 | + | return new Response("Content-Type must be application/x-www-form-urlencoded", { |
|
| 67 | + | status: 400, |
|
| 68 | + | }); |
|
| 69 | + | } |
|
| 70 | + | ||
| 71 | + | const form = await request.formData(); |
|
| 72 | + | const source = form.get("source"); |
|
| 73 | + | const target = form.get("target"); |
|
| 74 | + | ||
| 75 | + | if (typeof source !== "string" || typeof target !== "string") { |
|
| 76 | + | return new Response("Missing source or target", { status: 400 }); |
|
| 77 | + | } |
|
| 78 | + | ||
| 79 | + | const sourceUrl = isValidHttpUrl(source); |
|
| 80 | + | const targetUrl = isValidHttpUrl(target); |
|
| 81 | + | if (!sourceUrl || !targetUrl) { |
|
| 82 | + | return new Response("source and target must be valid http(s) URLs", { |
|
| 83 | + | status: 400, |
|
| 84 | + | }); |
|
| 85 | + | } |
|
| 86 | + | ||
| 87 | + | if (sourceUrl.href === targetUrl.href) { |
|
| 88 | + | return new Response("source and target must differ", { status: 400 }); |
|
| 89 | + | } |
|
| 90 | + | ||
| 91 | + | if (targetUrl.host !== ALLOWED_HOST) { |
|
| 92 | + | return new Response(`target must be on ${ALLOWED_HOST}`, { status: 400 }); |
|
| 93 | + | } |
|
| 94 | + | ||
| 95 | + | const kv = env.WEBMENTIONS; |
|
| 96 | + | ||
| 97 | + | // Verify the source links back, then store, before responding. |
|
| 98 | + | await verifyAndStore(kv, sourceUrl.href, targetUrl.href); |
|
| 99 | + | ||
| 100 | + | return new Response("Webmention accepted", { status: 201 }); |
|
| 101 | + | }; |
|
| 102 | + | ||
| 103 | + | // Read endpoint: list verified mentions for a given ?target= page. |
|
| 104 | + | export const GET: APIRoute = async ({ url }) => { |
|
| 105 | + | const target = url.searchParams.get("target"); |
|
| 106 | + | const kv = env.WEBMENTIONS; |
|
| 107 | + | ||
| 108 | + | const prefix = target |
|
| 109 | + | ? `mention:${encodeURIComponent(target)}:` |
|
| 110 | + | : "mention:"; |
|
| 111 | + | const list = await kv.list({ prefix }); |
|
| 112 | + | ||
| 113 | + | const mentions: StoredMention[] = []; |
|
| 114 | + | for (const entry of list.keys) { |
|
| 115 | + | const value = await kv.get(entry.name); |
|
| 116 | + | if (value) mentions.push(JSON.parse(value) as StoredMention); |
|
| 117 | + | } |
|
| 118 | + | ||
| 119 | + | return new Response(JSON.stringify({ mentions }), { |
|
| 120 | + | status: 200, |
|
| 121 | + | headers: { "content-type": "application/json" }, |
|
| 122 | + | }); |
|
| 123 | + | }; |
| 1 | 1 | { |
|
| 2 | - | "name": "stevedylandev", |
|
| 3 | - | "compatibility_date": "2024-09-23", |
|
| 4 | - | "compatibility_flags": ["nodejs_compat"], |
|
| 5 | - | "kv_namespaces": [ |
|
| 6 | - | { "binding": "SESSION" } |
|
| 7 | - | ] |
|
| 8 | - | } |
|
| 2 | + | "name": "stevedylandev", |
|
| 3 | + | "compatibility_date": "2024-09-23", |
|
| 4 | + | "compatibility_flags": [ |
|
| 5 | + | "nodejs_compat" |
|
| 6 | + | ], |
|
| 7 | + | "kv_namespaces": [ |
|
| 8 | + | { |
|
| 9 | + | "binding": "SESSION" |
|
| 10 | + | }, |
|
| 11 | + | { |
|
| 12 | + | "binding": "WEBMENTIONS", |
|
| 13 | + | "id": "f2418ef9b3f84441b25472c784145848" |
|
| 14 | + | } |
|
| 15 | + | ] |
|
| 16 | + | } |