Research page: how the research grew, commit by commit

@@ -47,4 +47,5 @@ jobs:
             .cache/og
             .cache/changelog
+            .cache/progress
           key: og-${{ hashFiles('src/og/**', 'package-lock.json') }}-${{ github.run_id }}
           restore-keys: |

@@ -0,0 +1,127 @@
+---
+// How the research grew, commit by commit (src/lib/progress.js): four small charts on one time
+// axis, each on its own scale, because the counts differ a hundredfold and one axis would flatten
+// the small ones. Lines are steps: a count changes only when a commit lands. Hovering or touching
+// any chart reads all four at the same moment and names the commit; a click opens it in the
+// changelog. The table under it holds the same numbers for reading without the charts.
+import { progress } from "../lib/progress.js";
+
+const base = import.meta.env.BASE_URL;
+const num = (n) => n.toLocaleString("en-GB");
+const day = (t) => new Date(t).toLocaleDateString("en-GB", { day: "numeric", month: "short", timeZone: "UTC" });
+const SERIES = [
+  { i: 1, label: "History entries", note: (p) => `${Math.round((p[2] / p[1]) * 100)}% checked against a source` },
+  { i: 5, label: "Web pages cited", note: () => "in profiles, history, articles and the atlas" },
+  { i: 3, label: "Articles", note: () => "long reads beside history entries" },
+  { i: 4, label: "Culture entries", note: () => "words, traditions and people" },
+];
+const first = progress[0], last = progress.at(-1);
+const T0 = first?.[0], T1 = last?.[0];
+const x = (t) => ((t - T0) / Math.max(1, T1 - T0)) * 1000;
+// a round top for each scale: 3,178 → 3,500
+const niceTop = (v) => { const p = 10 ** Math.floor(Math.log10(Math.max(1, v))), m = v / p; return (m <= 1 ? 1 : m <= 1.5 ? 1.5 : m <= 2 ? 2 : m <= 2.5 ? 2.5 : m <= 3 ? 3 : m <= 3.5 ? 3.5 : m <= 4 ? 4 : m <= 5 ? 5 : m <= 6 ? 6 : m <= 8 ? 8 : 10) * p; };
+const week = T1 - 7 * 864e5;
+const at = (t) => { let k = 0; while (k + 1 < progress.length && progress[k + 1][0] <= t) k++; return progress[k]; };
+const panels = SERIES.map((s) => {
+  const top = niceTop(Math.max(...progress.map((p) => p[s.i])));
+  const y = (v) => 200 - (v / top) * 190;
+  // a step line: across at the old count, then up at each commit
+  const line = progress.map((p, k) => (k ? `H${x(p[0]).toFixed(1)}V${y(p[s.i]).toFixed(1)}` : `M0,${y(p[s.i]).toFixed(1)}`)).join("") + "H1000";
+  return { ...s, top, line, area: `${line}V200H0Z`, now: last[s.i], gained: last[s.i] - at(week)[s.i] };
+});
+// days on the axis: the first, the last, and two between
+const ticks = [0, 1 / 3, 2 / 3, 1].map((f) => ({ left: f * 100, label: day(T0 + f * (T1 - T0)) }));
+// what the browser needs to read a moment: [x 0–1000, …counts, short hash, subject]
+const data = progress.map((p) => [+x(p[0]).toFixed(1), p[1], p[2], p[3], p[4], p[5], p[6], p[7], day(p[0])]);
+---
+{progress.length > 1 && (
+  <section class="section rg" aria-labelledby="rgTitle">
+    <div class="section-head"><div>
+      <h2 id="rgTitle">How the research grew</h2>
+      <p class="sub">Every commit since {day(T0)}, counted from the files as they stood then. Each chart has its own scale. Point at a moment to read all four; click to open that change.</p>
+    </div></div>
+    <p class="rg-when" aria-live="polite"><span data-rg-when>Today</span> <a data-rg-commit hidden></a></p>
+    <div class="rg-grid">
+      {panels.map((p) => (
+        <figure class="rg-panel" data-i={p.i}>
+          <figcaption>
+            <span class="rg-label">{p.label}</span>
+            <b class="rg-now" data-rg-value>{num(p.now)}</b>
+            <span class="rg-delta" data-rg-delta>{p.gained > 0 ? `+${num(p.gained)} in the last 7 days` : "no change in the last 7 days"}</span>
+            <span class="rg-note" data-rg-note>{p.note(last)}</span>
+          </figcaption>
+          <div class="rg-plot">
+            <svg viewBox="0 0 1000 200" preserveAspectRatio="none" aria-hidden="true">
+              <line class="rg-grid-line" x1="0" x2="1000" y1="105" y2="105" />
+              <path class="rg-area" d={p.area} />
+              <path class="rg-line" d={p.line} />
+            </svg>
+            <span class="rg-top" aria-hidden="true">{num(p.top)}</span>
+            <span class="rg-mid" aria-hidden="true">{num(p.top / 2)}</span>
+            <i class="rg-rule" hidden></i><i class="rg-dot" hidden></i>
+          </div>
+          <div class="rg-axis" aria-hidden="true">{ticks.map((t) => <span style={`left:${t.left}%`}>{t.label}</span>)}</div>
+        </figure>
+      ))}
+    </div>
+    <details class="rg-table">
+      <summary>The numbers by day</summary>
+      <div class="rs-scroll"><table class="src-table">
+        <thead><tr><th>Day</th>{SERIES.map((s) => <th class="num">{s.label}</th>)}</tr></thead>
+        <tbody>{[...new Map(progress.map((p) => [day(p[0]), p])).entries()].reverse().map(([d, p]) => <tr><td>{d}</td>{SERIES.map((s) => <td class="num">{num(p[s.i])}</td>)}</tr>)}</tbody>
+      </table></div>
+    </details>
+    <script type="application/json" id="rgData" set:html={JSON.stringify({ data, base, series: SERIES.map((s) => s.i) })} />
+  </section>
+)}
+
+<script>
+  // Reading a moment: the commit nearest the pointer, the same in all four charts.
+  const box = document.querySelector(".rg");
+  if (box) {
+    const { data, base } = JSON.parse(document.getElementById("rgData").textContent);
+    const num = (n) => n.toLocaleString("en-GB");
+    const panels = [...box.querySelectorAll(".rg-panel")];
+    const when = box.querySelector("[data-rg-when]"), link = box.querySelector("[data-rg-commit]");
+    const resting = panels.map((p) => ({ value: p.querySelector("[data-rg-value]").textContent, delta: p.querySelector("[data-rg-delta]").textContent, note: p.querySelector("[data-rg-note]").textContent }));
+    let current = null;
+    const nearest = (fx) => { let k = 0; while (k + 1 < data.length && data[k + 1][0] <= fx) k++; return k; };
+    function show(k) {
+      current = k;
+      const d = data[k], left = `${d[0] / 10}%`;
+      when.textContent = d[8];
+      link.hidden = false;
+      link.href = `${base}changelog/${d[6]}/`;
+      link.textContent = d[7];
+      panels.forEach((p, n) => {
+        const i = +p.dataset.i, plot = p.querySelector(".rg-plot"), svgTop = +p.querySelector(".rg-top").textContent.replace(/,/g, "");
+        p.querySelector("[data-rg-value]").textContent = num(d[i]);
+        const before = k ? d[i] - data[k - 1][i] : 0;
+        p.querySelector("[data-rg-delta]").textContent = before ? `${before > 0 ? "+" : "−"}${num(Math.abs(before))} in this commit` : "unchanged in this commit";
+        if (i === 1) p.querySelector("[data-rg-note]").textContent = `${Math.round((d[2] / d[1]) * 100)}% checked against a source`;
+        const rule = plot.querySelector(".rg-rule"), dot = plot.querySelector(".rg-dot");
+        rule.hidden = dot.hidden = false;
+        rule.style.left = dot.style.left = left;
+        dot.style.top = `${100 - (d[i] / svgTop) * 95}%`;
+      });
+    }
+    function rest() {
+      current = null;
+      when.textContent = "Today";
+      link.hidden = true;
+      panels.forEach((p, n) => {
+        p.querySelector("[data-rg-value]").textContent = resting[n].value;
+        p.querySelector("[data-rg-delta]").textContent = resting[n].delta;
+        p.querySelector("[data-rg-note]").textContent = resting[n].note;
+        p.querySelectorAll(".rg-rule, .rg-dot").forEach((x) => (x.hidden = true));
+      });
+    }
+    for (const plot of box.querySelectorAll(".rg-plot")) {
+      const under = (e) => { const r = plot.getBoundingClientRect(); return nearest(((e.clientX - r.left) / r.width) * 1000); };
+      plot.addEventListener("pointermove", (e) => { if (e.pointerType === "mouse") show(under(e)); });
+      // a mouse has shown the moment already, so a click opens it; a tap shows it, a second tap opens it
+      plot.addEventListener("click", (e) => { const k = under(e); if (current === k) location.href = link.href; else show(k); });
+    }
+    box.querySelector(".rg-grid").addEventListener("pointerleave", rest);
+  }
+</script>

@@ -0,0 +1,52 @@
+// How the research grew, read from git: at every commit, how many history entries there were (and
+// how many of them checked against a source), long-read articles, culture entries, and distinct web
+// pages cited by the profiles, the history, the articles and the atlas. For the chart on
+// /research/. Counted straight from each commit's files without parsing them (a history entry is a
+// line "- id:", a cited page a `url:` field), so it stays quick; a commit never changes, so each is
+// counted once and kept in .cache/progress. Without git, or in a shallow copy, the chart shows the
+// commits there are.
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+const ROOT = process.cwd();
+const CACHE = path.join(ROOT, ".cache", "progress");
+const VERSION = "1"; // raise to count every commit again after changing what is counted
+const git = (...args) => {
+  try { return execFileSync("git", args, { cwd: ROOT, maxBuffer: 1 << 29, stdio: ["ignore", "pipe", "ignore"] }).toString(); } catch { return ""; }
+};
+const lines = (text) => text.split("\n").filter(Boolean);
+
+// the counts at one commit
+function countAt(hash) {
+  const file = path.join(CACHE, `${VERSION}-${hash}.json`);
+  if (existsSync(file)) return JSON.parse(readFileSync(file, "utf8"));
+  const entries = +git("grep", "-h", "-c", "^- id:", hash, "--", "changes.yaml").trim() || 0;
+  const checked = +git("grep", "-h", "-c", "^  checked: true", hash, "--", "changes.yaml").trim() || 0;
+  const md = (dir) => lines(git("ls-tree", "-r", "--name-only", hash, "--", dir)).filter((f) => f.endsWith(".md")).length;
+  // a `url:` field holds its address on the same line, or on the next when folded (url: >-)
+  const urls = new Set();
+  const found = lines(git("grep", "-h", "-E", "-A1", "^\\s*(- )?url:", hash, "--", "changes.yaml", "countries", "articles", "culture"));
+  found.forEach((line, i) => {
+    const m = line.match(/url:\s*(.*)$/);
+    if (!m) return;
+    const value = /^>-?$/.test(m[1].trim()) ? (found[i + 1] ?? "").trim() : m[1].trim();
+    const url = value.replace(/^["']|["']$/g, "");
+    if (/^https?:\/\//.test(url)) urls.add(url);
+  });
+  const counts = { entries, checked, articles: md("articles"), culture: md("culture"), sources: urls.size };
+  mkdirSync(CACHE, { recursive: true });
+  writeFileSync(file, JSON.stringify(counts));
+  return counts;
+}
+
+// every commit, oldest first: [time (ms), entries, checked, articles, culture, sources, short hash, subject]
+// (the short hash is the changelog's: its first seven characters, so each moment links to its page)
+export const progress = lines(git("log", "--reverse", "--format=%H %cI %s"))
+  .map((line) => {
+    const [hash, date, ...subject] = line.split(" ");
+    const c = countAt(hash);
+    return [Date.parse(date), c.entries, c.checked, c.articles, c.culture, c.sources, hash.slice(0, 7), subject.join(" ")];
+  })
+  // a commit before the history existed says nothing about it
+  .filter((p) => p[1] > 0);

@@ -5,4 +5,5 @@
 import Base from "../layouts/Base.astro";
 import { research, researchTotals as T } from "../lib/research.js";
+import ResearchGrowth from "../components/ResearchGrowth.astro";
 
 const base = import.meta.env.BASE_URL;
@@ -32,4 +33,6 @@ const stats = [
     </dl>
 
+    <ResearchGrowth />
+
     <section class="section">
       <div class="section-head"><div>

@@ -2166,4 +2166,35 @@ button.sv-row .sv-chip { font: 400 10.5px var(--mono); text-transform: uppercase
 /* ── Research progress (/research/, internal) ──────────────────────────────── */
 .research-page code { font: 400 .9em var(--mono); }
+/* how the research grew (ResearchGrowth.astro): four small step charts on one time axis */
+.rg-when { margin: 0 0 14px; min-height: 1.5em; font: 400 12px var(--mono); color: var(--muted); text-transform: uppercase; letter-spacing: .06em; }
+.rg-when a { margin-left: 10px; text-transform: none; letter-spacing: 0; font: 400 13px Inter, sans-serif; color: var(--text-2); text-decoration: underline; text-underline-offset: 3px; }
+.rg-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 28px 40px; }
+.rg-panel { margin: 0; padding-top: 14px; border-top: 1px solid var(--border); }
+.rg-panel figcaption { display: grid; grid-template-columns: 1fr auto; align-items: baseline; gap: 2px 12px; margin-bottom: 12px; }
+.rg-label { font: 400 11px var(--mono); text-transform: uppercase; letter-spacing: .06em; color: var(--text-2); }
+.rg-now { grid-row: span 2; font: 400 clamp(30px, 3.4vw, 44px)/1 var(--display); font-variant-numeric: tabular-nums; text-align: right; }
+.rg-delta { font-size: 12.5px; color: var(--text-2); }
+.rg-note { grid-column: 1 / -1; font-size: 12px; color: var(--muted); }
+.rg-plot { position: relative; height: 150px; cursor: crosshair; touch-action: pan-y; }
+.rg-plot svg { display: block; width: 100%; height: 100%; overflow: visible; }
+.rg-plot path, .rg-plot line { vector-effect: non-scaling-stroke; }
+.rg-area { fill: color-mix(in srgb, var(--u3) 16%, transparent); }
+.rg-line { fill: none; stroke: var(--u3); stroke-width: 2; stroke-linejoin: round; }
+.rg-grid-line { stroke: var(--border); stroke-width: 1; stroke-dasharray: 2 4; }
+.rg-plot::after { content: ""; position: absolute; left: 0; right: 0; bottom: 0; border-bottom: 1px solid var(--text-2); }
+/* the scale on the left, where every line starts low */
+.rg-top, .rg-mid { position: absolute; left: 0; font: 400 10.5px var(--mono); color: var(--muted); background: var(--bg); padding-right: 6px; transform: translateY(-50%); pointer-events: none; }
+.rg-top { top: 5%; }
+.rg-mid { top: 52.5%; }
+.rg-rule { position: absolute; top: 0; bottom: 0; width: 1px; background: var(--text-2); pointer-events: none; }
+.rg-dot { position: absolute; width: 10px; height: 10px; margin: -5px 0 0 -5px; border-radius: 50%; background: var(--u3); box-shadow: 0 0 0 2px var(--bg); pointer-events: none; }
+.rg-rule[hidden], .rg-dot[hidden] { display: none; }
+.rg-axis { position: relative; height: 18px; margin-top: 6px; font: 400 10.5px var(--mono); color: var(--muted); }
+.rg-axis span { position: absolute; transform: translateX(-50%); white-space: nowrap; }
+.rg-axis span:first-child { transform: none; }
+.rg-axis span:last-child { transform: translateX(-100%); }
+.rg-table { margin-top: 24px; }
+.rg-table summary { cursor: pointer; font: 400 11.5px var(--mono); text-transform: uppercase; letter-spacing: .06em; color: var(--text-2); }
+@media (max-width: 760px) { .rg-grid { grid-template-columns: minmax(0, 1fr); } .rg-plot { height: 120px; } }
 .rs-checks { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 40px; }
 #rsFind { align-self: end; min-width: 240px; }