Fix the SEO audit's findings (#142): sitemap coverage, preview images, honest dates, answer-led descriptions

@@ -55,4 +55,10 @@ jobs:
           SITE: ${{ steps.pages.outputs.origin }}
           BASE_PATH: ${{ steps.pages.outputs.base_path }}
+      # what search engines and link previews read: sitemap coverage, canonical addresses,
+      # descriptions, og:images that exist, JSON-LD and internal links (scripts/seo-audit.mjs)
+      - run: npm run seo:audit
+        env:
+          SITE: ${{ steps.pages.outputs.origin }}
+          BASE_PATH: ${{ steps.pages.outputs.base_path }}
       - uses: actions/upload-pages-artifact@v5
         with:

@@ -13,4 +13,5 @@
     "books:test": "node --test scripts/books-status.test.cjs",
     "history:test": "node --test scripts/history-scope.test.cjs",
+    "seo:audit": "node scripts/seo-audit.mjs",
     "answers:test": "node --test scripts/answers.test.mjs scripts/profile-claims.test.mjs",
     "articles:backlog": "node scripts/article-backlog.cjs",

@@ -0,0 +1,120 @@
+// A check of the built site (dist/) for what search engines and link previews read, run after the
+// build in the deploy workflow:
+//   - the sitemap lists every indexable page, and no page marked noindex or missing from the build
+//   - each page has a title, a description and a canonical address that is its own
+//   - every og:image a page advertises exists in the build as a PNG
+//   - every JSON-LD block parses, and every URL in it on this site resolves
+//   - internal links point at pages that exist
+//   - sitemap dates are real dates no later than today
+// It prints counts with a few examples, and exits with 1 when something is wrong.
+//   npm run build && npm run seo:audit
+//   node scripts/seo-audit.mjs dist --all      every example, not the first eight
+import fs from "node:fs";
+import path from "node:path";
+
+const ALL = process.argv.includes("--all");
+const DIST = path.resolve(process.argv.slice(2).find((a) => !a.startsWith("--")) ?? "dist");
+const SITE = (process.env.SITE ?? "https://cannabisfreedom.fyi").replace(/\/$/, "");
+const BASE = ("/" + (process.env.BASE_PATH ?? "").replace(/^\/+|\/+$/g, "") + "/").replace("//", "/");
+const ORIGIN_BASE = SITE + BASE.replace(/\/$/, "");
+
+const walk = (dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => e.isDirectory() ? walk(path.join(dir, e.name)) : [path.join(dir, e.name)]);
+const files = walk(DIST);
+const exists = new Set(files.map((f) => "/" + path.relative(DIST, f).split(path.sep).join("/")));
+// the address of a built page: dist/country/de/index.html → /country/de/
+const urlOf = (file) => "/" + path.relative(DIST, file).split(path.sep).join("/").replace(/index\.html$/, "").replace(/\.html$/, "/");
+// does an address on this site lead to a built file?
+const resolves = (p) => {
+  const clean = decodeURI(p.split(/[?#]/)[0]);
+  if (!clean.startsWith(BASE)) return true; // another site on the same domain: not ours to check
+  const rel = "/" + clean.slice(BASE.length);
+  return exists.has(rel) || exists.has(rel.replace(/\/?$/, "/index.html")) || exists.has(rel + ".html");
+};
+
+const problems = {};
+const report = (kind, example) => { (problems[kind] ??= []).push(example); };
+const attr = (tag, name) => tag.match(new RegExp(`\\b${name}=(?:"([^"]*)"|'([^']*)')`))?.slice(1).find((x) => x != null);
+const meta = (html, key, value) => { for (const m of html.matchAll(/<meta\b[^>]*>/g)) if (attr(m[0], key) === value) return attr(m[0], "content"); return null; };
+const unescape = (s) => s?.replace(/&amp;/g, "&").replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/&lt;/g, "<").replace(/&gt;/g, ">");
+
+// ── the sitemap ──
+const sitemap = fs.readFileSync(path.join(DIST, "sitemap.xml"), "utf8");
+const entries = [...sitemap.matchAll(/<url><loc>([^<]+)<\/loc>(?:<lastmod>([^<]+)<\/lastmod>)?/g)].map((m) => [m[1].replace(SITE, ""), m[2] ?? null]);
+const listed = new Map(entries);
+for (const [loc] of entries.filter(([loc], i) => entries.findIndex(([x]) => x === loc) !== i)) report("sitemap lists a page twice", loc);
+// a commit's date is its author's local day, which can be a day ahead of UTC
+const today = new Date(Date.now() + 864e5).toISOString().slice(0, 10);
+for (const [loc, mod] of listed) {
+  if (!resolves(loc)) report("sitemap lists a page that was not built", loc);
+  if (mod && (!/^\d{4}-\d{2}-\d{2}$/.test(mod) || mod > today)) report("sitemap date that is not a past date", `${loc} ${mod}`);
+}
+if (/<priority>/.test(sitemap)) report("sitemap carries <priority>, which search engines ignore", "sitemap.xml");
+
+// ── every page ──
+const pages = files.filter((f) => f.endsWith(".html"));
+const indexable = new Set();
+const descriptions = new Map();
+let checkedLinks = 0;
+for (const file of pages) {
+  const html = fs.readFileSync(file, "utf8");
+  if (!/<html\b/i.test(html)) continue; // a fragment loaded into another page
+  const url = BASE.replace(/\/$/, "") + urlOf(file);
+  const robots = meta(html, "name", "robots") ?? "";
+  const noindex = /noindex/.test(robots);
+  if (/<meta http-equiv="refresh"/i.test(html)) continue; // a redirect
+  if (!noindex) indexable.add(url);
+  if (noindex && listed.has(url)) report("sitemap lists a noindex page", url);
+  if (!noindex && !listed.has(url)) report("indexable page missing from the sitemap", url);
+
+  const title = html.match(/<title>([^<]*)<\/title>/)?.[1]?.trim();
+  if (!title) report("page without a title", url);
+  if (noindex) continue; // a page kept out of search needs a title, not the rest
+  const description = unescape(meta(html, "name", "description"));
+  if (!description) report("page without a description", url);
+  else (descriptions.get(description) ?? descriptions.set(description, []).get(description)).push(url);
+  const canonical = html.match(/<link rel="canonical" href="([^"]+)"/)?.[1];
+  if (!canonical) report("page without a canonical address", url);
+  else if (canonical.replace(SITE, "").replace(/^https?:\/\/[^/]+/, "") !== url) report("canonical address that is not the page's own", `${url} → ${canonical}`);
+
+  const og = meta(html, "property", "og:image");
+  if (!og) report("page without og:image", url);
+  else {
+    const p = og.replace(/^https?:\/\/[^/]+/, "");
+    const rel = "/" + decodeURI(p).slice(BASE.length);
+    if (!exists.has(rel)) report("og:image that was not built", `${url} → ${p}`);
+    else if (rel.endsWith(".png")) {
+      const head = Buffer.alloc(8);
+      const fd = fs.openSync(path.join(DIST, rel), "r"); fs.readSync(fd, head, 0, 8, 0); fs.closeSync(fd);
+      if (head.toString("hex") !== "89504e470d0a1a0a") report("og:image that is not a PNG", `${url} → ${p}`);
+    }
+  }
+
+  for (const m of html.matchAll(/<script type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/g)) {
+    let data;
+    try { data = JSON.parse(m[1]); } catch { report("JSON-LD that does not parse", url); continue; }
+    for (const u of JSON.stringify(data).match(/https?:\/\/[^"\s]+/g) ?? []) {
+      if (u.startsWith(ORIGIN_BASE) && !resolves(u.replace(/^https?:\/\/[^/]+/, ""))) report("JSON-LD link to a page that was not built", `${url} → ${u}`);
+    }
+  }
+
+  for (const m of html.matchAll(/<a\b[^>]*\bhref="([^"]+)"/g)) {
+    const href = unescape(m[1]);
+    if (/^(https?:|mailto:|tel:|#|javascript:|data:)/.test(href) || !href.startsWith("/")) continue;
+    checkedLinks++;
+    if (!resolves(href)) report("internal link to a page that was not built", `${url} → ${href}`);
+  }
+}
+for (const [description, urls] of descriptions) if (urls.length > 3) report("description shared by more than three pages", `${urls.length}× ${urls[0]} … "${description.slice(0, 60)}"`);
+
+// ── the report ──
+const kinds = Object.keys(problems);
+console.log(`${pages.length} HTML files, ${indexable.size} indexable pages, ${listed.size} sitemap entries, ${checkedLinks} internal links checked`);
+for (const kind of kinds) {
+  const list = [...new Set(problems[kind])];
+  console.log(`\n✗ ${kind}: ${list.length}`);
+  const shown = ALL ? list : list.slice(0, 8);
+  for (const x of shown) console.log(`    ${x}`);
+  if (list.length > shown.length) console.log(`    … and ${list.length - shown.length} more`);
+}
+if (!kinds.length) console.log("✓ no problems found");
+process.exit(kinds.length ? 1 : 0);

@@ -81,4 +81,6 @@ const countries = defineCollection({
       complete: z.coerce.date().optional(),
       summary: z.string().min(1),
+      // the search-result description, where the generated one (src/lib/seo.js) would say it wrong
+      seo_description: z.string().min(50).max(300).optional(),
       scores: perSegment(score),
       answers,
@@ -128,4 +130,5 @@ const units = defineCollection({
       updated: z.coerce.date(),
       summary: z.string().min(1),
+      seo_description: z.string().min(50).max(300).optional(),
       scores: perSegment(score),
       answers,

@@ -104,4 +104,5 @@ const everyPlace = (await getCollection("countries"))
       med: MED_CODE[data.medical],
       answers: data.answers ?? null, // quick answers the profile states itself (answers.js)
+      seoDescription: data.seo_description ?? null, // its search-result description, where written by hand (seo.js)
       note: data.summary,
       details: {
@@ -165,4 +166,5 @@ export function subunitsOf(id) {
         med: MED_CODE[data.medical],
         answers: data.answers ?? null,
+        seoDescription: data.seo_description ?? null,
         note: data.summary,
         reviewed: isoDate(data.updated),

@@ -0,0 +1,21 @@
+// When a file last changed in git: "YYYY-MM-DD", or null when git or the full history is not
+// available (a shallow checkout), so a caller can leave the date out instead of inventing one.
+// Read once per build, for the sitemap's <lastmod> and the dateModified of articles.
+import { execFileSync } from "node:child_process";
+
+const dates = new Map();
+try {
+  // newest first: the first date seen for a file is its last change
+  const log = execFileSync("git", ["log", "--format=@%cs", "--name-only", "--no-renames"], { cwd: process.cwd(), maxBuffer: 1 << 29, stdio: ["ignore", "pipe", "ignore"] }).toString();
+  const shallow = execFileSync("git", ["rev-parse", "--is-shallow-repository"], { stdio: ["ignore", "pipe", "ignore"] }).toString().trim() === "true";
+  if (!shallow) {
+    let date = null;
+    for (const line of log.split("\n")) {
+      if (line.startsWith("@")) date = line.slice(1);
+      else if (line && !dates.has(line)) dates.set(line, date);
+    }
+  }
+} catch {}
+
+// the last change of a file, or the latest of several (paths from the repository root)
+export const lastChanged = (...files) => files.map((f) => dates.get(f)).filter(Boolean).sort().at(-1) ?? null;

@@ -1,8 +1,9 @@
 // Wording and structured data for search engines: titles that match what people type
 // ("is weed legal in Germany"), and schema.org objects for the pages.
-import { REC_LABEL, REC_HELP, MED_LABEL, estimateNote } from "./countries.js";
+import { REC_LABEL, REC_HELP } from "./countries.js";
 import { SEGMENT_IDS } from "./segment-ids.js";
 import { answersFor } from "./answers.js";
 import { SITE } from "./exports.js";
+import { lastChanged } from "./lastmod.js";
 
 const THE = /^(United |Netherlands|Bahamas|Philippines|Maldives|Gambia|Dominican Republic|Central African Republic|Comoros|Seychelles|Solomon Islands|Marshall Islands|DR Congo)/;
@@ -26,9 +27,35 @@ const crumbs = (site, trail) => ({
 });
 
+// The search-result description of a country or state: the answer first. Its status in a
+// sentence, then two of the quick answers (answers.js, checked against each profile): whether you
+// can carry it, and whether you can buy it or, where there is no legal source, what happens if you
+// are caught. Then the medical status and when the profile was reviewed; the score only while the
+// whole stays short. Nothing is cut to fit: where that runs long or reads badly, the profile
+// writes its own `seo_description` (the review date is still added here, so it cannot go stale).
+const REC_SENTENCE = {
+  legal: "is legal for adults, within limits",
+  partial: "is partly legal or tolerated",
+  decrim: "is decriminalised but still prohibited",
+  illegal: "is illegal",
+  death: "is illegal, with the death penalty for trafficking",
+};
+const MED_SENTENCE = { yes: "Medical cannabis: legal.", limited: "Medical cannabis: limited.", no: "No medical cannabis." };
+const monthYear = (iso) => new Date(iso).toLocaleString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
+const lower = (s) => s[0].toLowerCase() + s.slice(1);
+export function placeDescription(x, name, reviewed) {
+  if (x.seoDescription) return `${x.seoDescription} Reviewed ${monthYear(reviewed)}.`;
+  const a = Object.fromEntries(answersFor(x).map((q) => [q.id, q]));
+  // the carry and buy texts stand alone ("legal up to a limit"); "if caught" keeps its label ("Prison")
+  const second = a.buy.answer === "no" ? `If caught: ${lower(a.caught.label)}; ${lower(a.caught.text)}.` : `Buying: ${lower(a.buy.text)}.`;
+  const parts = [`Recreational cannabis ${inPlace(name)} ${REC_SENTENCE[x.rec]}.`, `Carrying: ${lower(a.carry.text)}.`, second, MED_SENTENCE[x.med], `Reviewed ${monthYear(reviewed)}.`];
+  const text = parts.join(" ");
+  const score = ` Freedom score ${x.score}/100.`;
+  return text.length + score.length <= 230 ? text + score : text;
+}
+
 // A country page: title, description and structured data (the questions the page answers).
 export function countrySeo(c, site) {
   const where = inPlace(c.name);
   const year = c.details.reviewed.slice(0, 4);
-  const status = `Recreational cannabis: ${REC_LABEL[c.rec].toLowerCase()}. Medical: ${MED_LABEL[c.med].toLowerCase()}.`;
   const faq = {
     "@context": "https://schema.org",
@@ -42,5 +69,5 @@ export function countrySeo(c, site) {
   return {
     seoTitle: `Is weed legal ${where}? Cannabis laws ${year}`,
-    description: `${status} ${c.note} Cannabis freedom score: ${c.score} out of 100${estimateNote(c) ? " (national estimate)" : ""}${c.rank ? `, #${c.rank} in the world` : ""}.`,
+    description: placeDescription(c, c.name, c.details.reviewed),
     jsonld: [faq, crumbs(site, [["Cannabis Freedom Index", ""], [c.name, `country/${c.slug}/`]])],
   };
@@ -50,4 +77,5 @@ export function countrySeo(c, site) {
 export function subunitSeo(c, sub, site) {
   return {
+    description: placeDescription(sub, `${sub.name}, ${c.name}`, sub.reviewed),
     seoTitle: `Is weed legal in ${sub.name}? Cannabis laws in ${sub.name}, ${c.name}`,
     jsonld: [crumbs(site, [["Cannabis Freedom Index", ""], [c.name, `country/${c.slug}/`], [sub.name, `country/${c.slug}/${sub.id}/`]])],
@@ -55,4 +83,51 @@ export function subunitSeo(c, sub, site) {
 }
 
+// A page of the history or the cultural atlas as an Article: what it is about, the works it cites,
+// and dates only where they are known: a long read has the day it was written and its file's last
+// change; an entry without one has no dates at all (changes.yaml keeps none per entry).
+const isoDay = (d) => d && (d instanceof Date ? d.toISOString() : String(d)).slice(0, 10);
+const cites = (sources) => sources.filter(([, url]) => url).map(([name, url]) => ({ "@type": "CreativeWork", name, url }));
+function article(site, path, { headline, description, about, sources, written, modified }) {
+  const url = root(site) + path;
+  return {
+    "@context": "https://schema.org",
+    "@type": "Article",
+    headline,
+    description,
+    url,
+    mainEntityOfPage: url,
+    inLanguage: "en",
+    image: `${root(site)}og/${path.replace(/\/$/, "")}.png`,
+    isPartOf: { "@type": "WebSite", name: "Cannabis Freedom Index", url: root(site) },
+    publisher: { "@type": "Organization", name: "Cannabis Freedom Index", url: root(site) },
+    ...(about.length && { about: about.map((name) => ({ "@type": "Place", name })) }),
+    ...(written && { datePublished: written }),
+    ...(modified && { dateModified: modified }),
+    ...(sources.length && { citation: cites(sources) }),
+  };
+}
+
+// A history entry, with its long read when it has one (`long`, from articles/<id>.md).
+export function historySeo(ch, long, site) {
+  const written = isoDay(long?.data.written);
+  const modified = long ? [lastChanged(`articles/${ch.id}.md`), written].filter(Boolean).sort().at(-1) : null;
+  const path = `history/${ch.id}/`;
+  return [
+    article(site, path, { headline: ch.title, description: ch.description, about: ch.country ? [ch.unit ? `${ch.unit.name}, ${ch.country.name}` : ch.country.name] : [],
+      sources: [...(long?.data.sources ?? []).map((s) => [s.title, s.url]), ...ch.sources], written, modified }),
+    crumbs(site, [["Cannabis Freedom Index", ""], ["History", "history/"], [ch.title, path]]),
+  ];
+}
+
+// An entry of the cultural atlas.
+export function cultureSeo(e, section, site) {
+  const written = isoDay(e.written);
+  const modified = [lastChanged(`culture/${e.entry.id}.md`), written].filter(Boolean).sort().at(-1) ?? null;
+  return [
+    article(site, e.url, { headline: e.title, description: e.summary, about: e.countries.map((c) => c.name), sources: e.sources, written, modified }),
+    crumbs(site, [["Cannabis Freedom Index", ""], ["Culture", "culture/"], [section.name, `culture/${section.id}/`], [e.title, e.url]]),
+  ];
+}
+
 // The home page: the site itself and the dataset behind it.
 export function homeSeo(site, count) {

@@ -44,5 +44,5 @@ const moves = g.places.filter((p) => p.scores?.length || (p.status === "added" &
   .sort((a, b) => Math.abs((b.to ?? 0) - (b.from ?? 0)) - Math.abs((a.to ?? 0) - (a.from ?? 0)));
 ---
-<Base title={`${c.subject} · Changelog`} description={`${formatDate(c.date.slice(0, 10))}: ${summary.map(([n, w]) => `${n} ${w}`).join(", ") || "changes to the site"}.`}>
+<Base noindex={c.short === "uncommitted"} title={`${c.subject} · Changelog`} description={`${formatDate(c.date.slice(0, 10))}: ${summary.map(([n, w]) => `${n} ${w}`).join(", ") || "changes to the site"}.`}>
   <main class="changes-page gl gl-commit">
     <section class="hero hero-sm">

@@ -4,5 +4,5 @@ import Base from "../../../layouts/Base.astro";
 import CountryArticle from "../../../components/CountryArticle.astro";
 import { countries, subunitsOf } from "../../../lib/countries.js";
-import { tierOf, tierColor } from "../../../lib/tiers.js";
+import { tierColor } from "../../../lib/tiers.js";
 import { subunitSeo } from "../../../lib/seo.js";
 
@@ -15,7 +15,6 @@ export function getStaticPaths() {
 const { c, sub } = Astro.props;
 const base = import.meta.env.BASE_URL;
-const description = `${sub.name} (${c.name}) scores ${sub.score} out of 100 on the Cannabis Freedom Index (${tierOf(sub.raw).name}). ${sub.note}`;
 ---
-<Base title={`${sub.name}, ${c.name}`} description={description} {...subunitSeo(c, sub, Astro.site)} type="article">
+<Base title={`${sub.name}, ${c.name}`} {...subunitSeo(c, sub, Astro.site)} type="article">
   <a slot="here" class="topbar-here" href="#top" style={`--c:${tierColor(sub.raw)}`}><span aria-hidden="true">{c.flag}</span><span class="here-name">{sub.name}, {c.name}</span><b>{sub.score}</b></a>
   <main class="country-page">

@@ -14,4 +14,5 @@ import References from "../../../components/References.astro";
 import { exhibitionsOfAtlas } from "../../../lib/exhibitions.js";
 import ExhibitionBanner from "../../../components/ExhibitionBanner.astro";
+import { cultureSeo } from "../../../lib/seo.js";
 
 export function getStaticPaths() {
@@ -31,5 +32,5 @@ const written = e.written?.toISOString().slice(0, 10);
 const exhibits = exhibitionsOfAtlas(e);
 ---
-<Base type="article" title={`${e.title} · ${s.name}`} description={e.summary}>
+<Base type="article" title={`${e.title} · ${s.name}`} description={e.summary} jsonld={cultureSeo(e, s, Astro.site)}>
   <main class="change-page ev culture-entry">
     <div class="page-bar"><a class="back" href={`${base}culture/${s.id}/`}>← {s.name}</a></div>

@@ -16,4 +16,5 @@ import { relatedTo } from "../../lib/related.js";
 import { exhibitionsOf } from "../../lib/exhibitions.js";
 import ExhibitionBanner from "../../components/ExhibitionBanner.astro";
+import { historySeo } from "../../lib/seo.js";
 import "@fontsource/limelight";
 
@@ -59,5 +60,5 @@ const unit = unitGroup?.units.find((x) => x.id === ch.unit.id);
 const ledeInBody = !article;
 ---
-<Base type="article" title={ch.title} description={ch.score ? `${ch.title}: ${c.name} moved from ${ch.score.before} to ${ch.score.after} on the Cannabis Freedom Index. ${ch.description}` : `${ch.title}. ${ch.description}`}>
+<Base type="article" jsonld={historySeo(ch, article, Astro.site)} title={ch.title} description={ch.score ? `${ch.title}: ${c.name} moved from ${ch.score.before} to ${ch.score.after} on the Cannabis Freedom Index. ${ch.description}` : `${ch.title}. ${ch.description}`}>
   <main class="change-page ev">
     <div class="page-bar">

@@ -4,5 +4,5 @@
 // entries with a lead picture get it beside their name and summary; every other
 // page gets its name and a line about it. Drawn by src/og/render.js.
-import { SEGMENTS, DATA_SNAPSHOT, REC_LABEL, MED_LABEL, countries, places, changes, articles, subunitsOf, worldAvg, changeDate, historyPeriods, CHANGE_CATEGORIES } from "../../lib/countries.js";
+import { SEGMENTS, DATA_SNAPSHOT, REC_LABEL, MED_LABEL, countries, places, changes, articles, subunitsOf, worldAvg, changeDate, historyPeriods, CHANGE_CATEGORIES, CHANGE_TAGS } from "../../lib/countries.js";
 import { tierOf, tierIndex } from "../../lib/tiers.js";
 import { MAP_VIEWS } from "../../lib/map-views.js";
@@ -59,4 +59,6 @@ const cards = {
   "did-you-know": page("Did you know?", "Strange rules, forgotten laws and what the scores show.", [[String(getFacts().length), "CURIOUS CASES"]]),
   articles: page("Articles", "Moments from the history of cannabis law told in full.", [[num(articles.size), "ARTICLES"]]),
+  charts: page("Cannabis (un)freedom in charts", "How prohibition spread after 1860, the waves of law, the regions, use and the law, drawn from the index's scores and history.", indexFacts, `CHARTS · ${BRAND}`),
+  exhibitions: page("Exhibitions", "Self-contained experiences, each about one strange episode in humanity's relationship with cannabis.", [], `EXHIBITIONS · ${BRAND}`),
   culture: page("A cultural atlas of cannabis", "The words people have for the plant, the rituals and traditions it belongs to, and the people behind the laws.", [[String(cultureEntries.length), "ENTRIES"]]),
   "culture/stories": page("The people behind the laws", "A participatory anthropology project on how cannabis shapes, and is shaped by, society.", [], `CULTURE · ${BRAND}`),
@@ -85,4 +87,5 @@ for (const c of commits) cards[`changelog/${c.short}`] = page(c.subject, summary
 for (const v of MAP_VIEWS) cards[v.slug ? `map/${v.slug}` : "map"] = page(v.title, v.lede, [], `MAP · ${BRAND}`);
 for (const [kind, k] of Object.entries(CHANGE_CATEGORIES)) cards[`history/kind/${kind}`] = page(k.label, `${k.help}. Every such entry in the history of cannabis.`, [[num(dated.filter((ch) => ch.category === kind).length), "ENTRIES"]], "HISTORY OF CANNABIS");
+for (const [tag, t] of Object.entries(CHANGE_TAGS)) cards[`history/tag/${tag}`] = page(t.label, `${t.help}. Every entry with this tag in the history of cannabis.`, [[num(dated.filter((ch) => ch.tags.some((x) => x.key === tag)).length), "ENTRIES"]], "HISTORY OF CANNABIS");
 for (const p of historyPeriods) cards[`history/period/${p.slug}`] = page(`History, ${p.label}`, `Cannabis laws, court rulings and treaties from ${p.from} to ${p.to}.`, [], "HISTORY OF CANNABIS");
 for (const ch of changes) cards[`history/${ch.id}`] = historyCard(ch); // estimated first bans have a page too

@@ -1,41 +1,69 @@
-// Every page of the site, for search engines. Linked from robots.txt.
+// Every indexable page of the site, for search engines. Linked from robots.txt.
+// `lastmod` is given only where it is known what last changed the page, and is then that date:
+//   - a country or state: the date its profile was last reviewed
+//   - a page made from every profile (home, rankings, maps, comparisons, downloads): the latest review
+//   - a page whose content is one file (an article, a culture entry, the about page): that file's
+//     last commit (src/lib/lastmod.js); an article's history entry the same
+//   - a changelog page: its commit
+// Lists of history entries (decades, kinds, tags) and other hubs get no date: no single change
+// stands for them, and a date that is not the page's own would mislead. No <priority>: search
+// engines ignore it. Pages marked noindex (research, the survey scanner, "your submissions") are
+// left out. scripts/seo-audit.mjs checks after the build that this list and the site agree.
 import { openExhibitions, exhibitionPath } from "../lib/exhibitions.js";
-import { places, countries, changes, subunitsOf, historyPeriods, CHANGE_CATEGORIES } from "../lib/countries.js";
+import { places, countries, changes, articles, subunitsOf, historyPeriods, CHANGE_CATEGORIES, CHANGE_TAGS } from "../lib/countries.js";
 import { SITE } from "../lib/exports.js";
 import { MAP_VIEWS } from "../lib/map-views.js";
 import { CULTURE_SECTIONS, cultureEntries } from "../lib/culture.js";
+import { commits } from "../lib/gitlog.js";
+import { lastChanged } from "../lib/lastmod.js";
 
 export function GET({ site }) {
   const root = (site?.href ?? SITE + "/").replace(/\/$/, "") + import.meta.env.BASE_URL;
   const latest = places.map((c) => c.details.reviewed).sort().at(-1);
-  // [path, last change (YYYY-MM-DD, optional), priority]
+  const page = (file) => lastChanged(`src/pages/${file}`);
+  const later = (...dates) => dates.filter(Boolean).sort().at(-1) ?? null;
+  const committed = commits.filter((c) => c.short !== "uncommitted");
+  // [path, last change (YYYY-MM-DD) or null]
   const pages = [
-    ["", latest, 1],
-    ["can-i/", latest, 0.8],
-    ["history/", latest, 0.8],
-    ["charts/", latest, 0.7],
-    ["exhibitions/", latest, 0.6],
-    ...openExhibitions.map((e) => [exhibitionPath(e), latest, 0.7]),
-    ["compare/", latest, 0.6],
-    ["survey/", latest, 0.6],
-    ["survey/take/", latest, 0.5],
-    ["about/", latest, 0.6],
-    ["culture/", latest, 0.7],
-    ...CULTURE_SECTIONS.filter((x) => !x.soon).map((x) => [`culture/${x.id}/`, latest, 0.6]),
-    ...cultureEntries.map((e) => [e.url, e.written?.toISOString().slice(0, 10) ?? null, 0.6]),
-    ["did-you-know/", latest, 0.5],
-    ["score-changes/", latest, 0.5],
-    ["changelog/", latest, 0.4],
-    ["data/", latest, 0.5],
-    ...MAP_VIEWS.map((v) => [v.slug ? `map/${v.slug}/` : "map/", latest, 0.7]),
-    ...places.map((c) => [`country/${c.slug}/`, c.details.reviewed, 0.9]),
-    ...countries.flatMap((c) => (subunitsOf(c.id)?.units ?? []).map((sub) => [`country/${c.slug}/${sub.id}/`, c.details.reviewed, 0.7])),
-    ...historyPeriods.map((p) => [`history/period/${p.slug}/`, null, 0.4]),
-    ...Object.keys(CHANGE_CATEGORIES).map((kind) => [`history/kind/${kind}/`, null, 0.4]),
-    ...changes.map((ch) => [`history/${ch.id}/`, null, 0.5]),
+    // made from every profile
+    ["", latest],
+    ["can-i/", latest],
+    ["compare/", latest],
+    ["score-changes/", latest],
+    ["data/", latest],
+    ["map/poster/", latest],
+    ...MAP_VIEWS.map((v) => [v.slug ? `map/${v.slug}/` : "map/", latest]),
+    // profiles
+    ...places.map((c) => [`country/${c.slug}/`, c.details.reviewed]),
+    ...countries.flatMap((c) => (subunitsOf(c.id)?.units ?? []).map((sub) => [`country/${c.slug}/${sub.id}/`, sub.reviewed])),
+    // pages whose text is their own file
+    ["about/", page("about.astro")],
+    ["survey/", page("survey/index.astro")],
+    ["survey/take/", page("survey/take.astro")],
+    ["culture/stories/", page("culture/stories.astro")],
+    ...openExhibitions.map((e) => [exhibitionPath(e), null]),
+    // hubs and lists
+    ["history/", null],
+    ["charts/", null],
+    ["articles/", null],
+    ["sources/", null],
+    ["exhibitions/", null],
+    ["culture/", null],
+    ["did-you-know/", null],
+    // stories is a section too, listed above with its own file's date
+    ...CULTURE_SECTIONS.filter((x) => !x.soon && x.id !== "stories").map((x) => [`culture/${x.id}/`, null]),
+    ...historyPeriods.map((p) => [`history/period/${p.slug}/`, null]),
+    ...Object.keys(CHANGE_CATEGORIES).map((kind) => [`history/kind/${kind}/`, null]),
+    ...Object.keys(CHANGE_TAGS).map((tag) => [`history/tag/${tag}/`, null]),
+    // the atlas and the history: a culture entry is its file; an entry with a long read, its article
+    ...cultureEntries.map((e) => [e.url, later(lastChanged(`culture/${e.entry.id}.md`), e.written?.toISOString().slice(0, 10))]),
+    ...changes.map((ch) => [`history/${ch.id}/`, articles.has(ch.id) ? lastChanged(`articles/${ch.id}.md`) : null]),
+    // the changelog
+    ["changelog/", committed[0]?.date.slice(0, 10) ?? null],
+    ...committed.map((c) => [`changelog/${c.short}/`, c.date.slice(0, 10)]),
   ];
   const xml = `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-${pages.map(([path, mod, pri]) => `  <url><loc>${root}${path}</loc>${mod ? `<lastmod>${mod}</lastmod>` : ""}<priority>${pri}</priority></url>`).join("\n")}
+${pages.map(([path, mod]) => `  <url><loc>${root}${path}</loc>${mod ? `<lastmod>${mod}</lastmod>` : ""}</url>`).join("\n")}
 </urlset>
 `;

@@ -15,5 +15,5 @@ const names = Object.fromEntries(countries.map((c) => [c.id, `${c.flag} ${c.name
 const regions = Object.fromEntries(countries.filter((c) => subunitsOf(c.id)).map((c) => [c.id, Object.fromEntries(subunitsOf(c.id).units.map((u) => [u.id, u.name]))]));
 ---
-<Base title="Your submissions" description="The answers this browser has submitted to the State of Cannabis survey: read them, change them, delete them, or back up the key they are signed with.">
+<Base noindex title="Your submissions" description="The answers this browser has submitted to the State of Cannabis survey: read them, change them, delete them, or back up the key they are signed with.">
   <main class="changes-page survey sv-me" data-config={JSON.stringify(submitConfig)} data-questions={JSON.stringify(questions)} data-names={JSON.stringify(names)} data-regions={JSON.stringify(regions)}>
     <section class="hero hero-sm">