add research progress; croatia and bulgaria laws

@@ -662,2 +662,21 @@ from. A relay may ignore deletion requests, which is why a deleted part is also
 an empty one. Someone who clears their browser without a backup of the key can no longer change
 their answers.
+
+## Research progress (/research/)
+
+A hidden page, not linked, not in the search or the sitemap and not indexed. It shows for each
+country how far the research has come: a percentage, the same split into profile and history,
+the number of entries, scored entries and articles, the year of the first ban, the decades with
+no entry and the longest gap between two entries. Clicking a row lists what is missing.
+
+The checks and their points are in `src/lib/research.js`. For the profile: every score cited,
+every score resting on more than Wikipedia or a weak site, the ten context sections, a measured
+figure for use, a review in the last year, sources archived. For the history: a dated first
+restriction, entries checked, a score timeline with no flag, no decade without an entry, an
+entry in the last three years, fifteen entries or more, entries resting on more than Wikipedia,
+an article for every score change.
+
+The checks see only what is in the files, so they stop at 99%. A country reaches 100% only when
+every check passes and its file carries `complete: YYYY-MM-DD`, set by hand by whoever went
+through the country and found nothing left to add. An empty decade is a prompt to look, not
+proof of a gap: some decades had no event.

Not shown: a picture, a generated file, or past the length shown per change.

@@ -75,4 +75,7 @@ const countries = defineCollection({
       ...status,
       updated: z.coerce.date(),
+      // set by hand, once someone has gone through the country and found nothing left to add:
+      // only then can the research page show 100% (see src/lib/research.js)
+      complete: z.coerce.date().optional(),
       summary: z.string().min(1),
       scores: perSegment(score),

@@ -0,0 +1,102 @@
+// How far the research on each country has come: a list of checks on its profile and on its
+// history, each worth some points, and the share of the points it has. The page /research/
+// shows the result. A check can only see what is in the files, so on its own it never reaches
+// 100%: that needs `complete: <date>` in the country file, set by hand once someone has gone
+// through the country and found nothing left to add.
+import { getCollection } from "astro:content";
+import { SEGMENTS, dateTime, articles } from "./countries.js";
+import { CONTEXT_IDS, STATUS_SECTIONS } from "./segment-ids.js";
+import { archivedUrl } from "./archive.js";
+
+// sites that proved unreliable, and the encyclopedia: fine as a lead, not enough on their own
+const WEAK = /cannabisregulations\.ai|tripbase\.com|sensiseeds\.com/;
+const WIKI = /wikipedia\.org/;
+const solid = (src) => src && (!src.url || !(WEAK.test(src.url) || WIKI.test(src.url))); // a book has no url
+const SECTIONS = [...CONTEXT_IDS, "planting", ...Object.keys(STATUS_SECTIONS), "black_market", "arrests"];
+const NOW = new Date();
+const THIS_YEAR = NOW.getFullYear();
+const total = (scores) => SEGMENTS.reduce((sum, seg) => sum + (scores[seg.id] * seg.weight) / 10, 0);
+const UP = new Set(["easing", "legalisation", "medical"]), DOWN = new Set(["ban", "tightening", "death"]);
+
+const entries = (await getCollection("changes")).map(({ data }) => data);
+const flagOf = (id) => String.fromCodePoint(...[...id.toUpperCase()].map((ch) => 0x1f1a5 + ch.charCodeAt(0)));
+
+export const research = (await getCollection("countries")).map(({ id: slug, data }) => {
+  const checks = []; // { part, label, got, of, missing: [..] }
+  const add = (part, label, got, of, missing = []) => checks.push({ part, label, got, of, missing });
+  const src = (keys) => keys.map((k) => data.sources[k]);
+
+  // ── the profile ──
+  const uncited = SEGMENTS.filter((seg) => !data.segments[seg.id].sources.length);
+  add("profile", "Every score is cited", SEGMENTS.length - uncited.length, SEGMENTS.length, uncited.map((s) => `${s.name}: no source`));
+  const soft = SEGMENTS.filter((seg) => data.segments[seg.id].sources.length && !src(data.segments[seg.id].sources).some(solid));
+  add("profile", "Every score rests on more than Wikipedia", SEGMENTS.length - uncited.length - soft.length, SEGMENTS.length, soft.map((s) => `${s.name}: only Wikipedia or a weak site`));
+  const empty = SECTIONS.filter((k) => !data.context[k]?.sources.length);
+  add("profile", "Context sections researched", SECTIONS.length - empty.length, SECTIONS.length, empty.map((k) => `${k.replace("_", " ")}: ${data.context[k] ? "no source" : "missing"}`));
+  const use = data.cannabis_use;
+  add("profile", "Use is a measured figure", use.year && !use.estimate ? 1 : 0, 1, use.year && !use.estimate ? [] : ["prevalence of use is an estimate"]);
+  const age = (NOW - data.updated) / 864e5;
+  add("profile", "Reviewed in the last year", age <= 365 ? 1 : 0, 1, age <= 365 ? [] : [`last reviewed ${data.updated.toISOString().slice(0, 10)}`]);
+  const urls = Object.values(data.sources).map((s) => s.url).filter(Boolean);
+  const saved = urls.filter((u) => archivedUrl(u)).length;
+  add("profile", "Sources archived", urls.length ? (2 * saved) / urls.length : 0, 2, saved < urls.length ? [`${urls.length - saved} of ${urls.length} sources have no archived copy`] : []);
+
+  // ── the history ──
+  const own = entries.filter((e) => e.country === slug).sort((a, b) => dateTime(a.date) - dateTime(b.date));
+  const dated = own.filter((e) => !e.estimate);
+  const scored = own.filter((e) => e.before);
+  const first = scored[0];
+  add("history", "The first restriction is dated", first && !first.estimate ? 3 : 0, 3, first && !first.estimate ? [] : [first ? "the first ban is an estimate" : "no first ban recorded"]);
+  const unchecked = dated.filter((e) => !e.checked);
+  add("history", "Entries checked against a source", dated.length ? (2 * (dated.length - unchecked.length)) / dated.length : 0, 2, unchecked.map((e) => `not checked: ${e.id}`));
+  const flags = [];
+  scored.forEach((e, i) => {
+    const b = total(e.before), a = total(scored[i + 1] ? scored[i + 1].before : data.scores);
+    if (UP.has(e.category) && a < b) flags.push(`${e.id}: ${e.category}, but the score falls`);
+    if (DOWN.has(e.category) && a > b) flags.push(`${e.id}: ${e.category}, but the score rises`);
+    if (i === 0 && b < 90 && !e.estimate) flags.push(`${e.id}: the timeline starts below the unregulated state`);
+  });
+  add("history", "The score timeline makes sense", scored.length && !flags.length ? 2 : 0, 2, scored.length ? flags : ["no scored entry"]);
+  // a decade with no entry at all, between the first dated entry (1900 at the earliest) and now
+  const years = dated.map((e) => +e.date.slice(0, 4));
+  const from = years.length ? Math.max(1900, Math.floor(Math.min(...years) / 10) * 10) : 1900;
+  const decades = [];
+  for (let d = from; d <= THIS_YEAR; d += 10) decades.push(d);
+  const silent = decades.filter((d) => !years.some((y) => y >= d && y < d + 10));
+  add("history", "No decade without an entry", years.length ? (6 * (decades.length - silent.length)) / decades.length : 0, 6, years.length ? silent.map((d) => `${d}s: nothing recorded`) : ["no dated entry"]);
+  const recent = years.some((y) => y >= THIS_YEAR - 3);
+  add("history", "Followed up to the present", recent ? 1 : 0, 1, recent ? [] : [`nothing since ${years.length ? Math.max(...years) : "ever"}`]);
+  add("history", "Depth (15 entries or more)", (4 * Math.min(dated.length, 15)) / 15, 4, dated.length < 15 ? [`${dated.length} dated entries`] : []);
+  const thin = dated.filter((e) => !e.sources.some(solid));
+  add("history", "Entries rest on more than Wikipedia", dated.length ? (3 * (dated.length - thin.length)) / dated.length : 0, 3, thin.length ? [`${thin.length} entries cite only Wikipedia or a weak site`] : []);
+  const told = scored.filter((e) => articles.has(e.id));
+  add("history", "An article for every score change", scored.length ? (3 * told.length) / scored.length : 0, 3, scored.filter((e) => !articles.has(e.id)).map((e) => `no article: ${e.id}`));
+
+  const share = (part) => { const cs = checks.filter((c) => !part || c.part === part); return cs.reduce((a, c) => a + c.got, 0) / cs.reduce((a, c) => a + c.of, 0); };
+  const signed = data.complete ? data.complete.toISOString().slice(0, 10) : null;
+  // every check passed is 99%; the last point is the signature
+  const pct = signed && share() === 1 ? 100 : Math.min(99, Math.floor(share() * 100));
+  const gaps = years.slice(1).map((y, i) => y - years[i]);
+  return {
+    slug, name: data.name, flag: data.flag ?? flagOf(slug), region: data.region, disputed: Boolean(data.disputed),
+    population: data.population[Object.keys(data.population).sort().at(-1)],
+    pct, profile: Math.floor(share("profile") * 100), history: Math.floor(share("history") * 100), signed,
+    entries: own.length, scored: scored.length, articles: own.filter((e) => articles.has(e.id)).length,
+    first: first ? (first.estimate ? "est." : first.date.slice(0, 4)) : "—",
+    gap: gaps.length ? Math.max(...gaps) : null, silent: silent.length,
+    reviewed: data.updated.toISOString().slice(0, 10),
+    checks,
+  };
+}).sort((a, b) => b.pct - a.pct || b.entries - a.entries || a.name.localeCompare(b.name));
+
+export const researchTotals = {
+  places: research.length,
+  average: Math.round(research.reduce((a, r) => a + r.pct, 0) / research.length),
+  // the same, counted by people: a well-researched large country weighs more
+  byPopulation: Math.round(research.reduce((a, r) => a + r.pct * r.population, 0) / research.reduce((a, r) => a + r.population, 0)),
+  complete: research.filter((r) => r.pct === 100).length,
+  good: research.filter((r) => r.pct >= 75).length,
+  poor: research.filter((r) => r.pct < 40).length,
+  entries: entries.length,
+  estimates: research.filter((r) => r.first === "est.").length,
+};

@@ -4,8 +4,16 @@ import Base from "../../layouts/Base.astro";
 import HistoryList from "../../components/HistoryList.astro";
 import { historyPeriods } from "../../lib/countries.js";
+import { researchTotals } from "../../lib/research.js";
+
+const base = import.meta.env.BASE_URL;
 ---
 <Base title="History" description="Laws, court rulings and treaties that shaped cannabis freedom, from the first bans to today, and how each one moved a country's score on the Cannabis Freedom Index.">
   <main class="changes-page">
-    <section class="hero hero-sm">
+    <section class="hero hero-sm hero-progress">
+      <a class="rs-pill" href={`${base}research/`} title={`Average research progress over ${researchTotals.places} countries and territories. Open the details.`}>
+        <span>Research progress</span>
+        <i style={`--p:${researchTotals.average}%`}></i>
+        <b>{researchTotals.average}%</b>
+      </a>
       <p class="eyebrow">History</p>
       <h1>A history of cannabis law</h1>

@@ -59,4 +59,5 @@ const cards = {
   "survey/take": page("Take the survey", "State of Cannabis: about ten minutes, no account, answers encrypted.", [], `STATE OF CANNABIS · ${survey.edition}`),
   "survey/me": page("Your submissions", "State of Cannabis: read, change or delete what you sent.", [], `STATE OF CANNABIS · ${survey.edition}`),
+  research: page("Research progress", "How much of each country's profile and history has been researched.", [], `INTERNAL · ${BRAND}`),
   "survey/scan": page("Submission scanner", "What the survey's public relays hold, as anyone can see it.", [], `STATE OF CANNABIS · ${survey.edition}`),
 };

@@ -0,0 +1,140 @@
+---
+// Research progress: how much of each country's profile and history has been filled in, and what
+// is still missing. An internal working page: not linked from the menu, the search or the
+// sitemap, and not indexed. The checks and their weights are in src/lib/research.js.
+import Base from "../layouts/Base.astro";
+import { research, researchTotals as T } from "../lib/research.js";
+
+const base = import.meta.env.BASE_URL;
+const num = (n) => n.toLocaleString("en-US");
+const example = research[0].checks;
+const points = (part) => example.filter((c) => c.part === part).reduce((a, c) => a + c.of, 0);
+const stats = [
+  [`${T.average}%`, "Average"],
+  [`${T.byPopulation}%`, "Weighted by population"],
+  [num(T.complete), "Complete (100%)"],
+  [num(T.good), "At 75% or more"],
+  [num(T.poor), "Below 40%"],
+  [num(T.estimates), "First ban only estimated"],
+];
+---
+<Base title="Research progress" description="How much of each country's profile and history has been researched, and what is still missing." noindex>
+  <main class="changes-page research-page">
+    <section class="hero hero-sm">
+      <p class="eyebrow">Internal</p>
+      <h1>Research progress</h1>
+      <p class="lede">For each of the {num(T.places)} countries and territories: how much of the profile is filled in and sourced, how many history entries there are, and how complete that history looks. The figure is the share of points a country has out of {points("profile")} for its profile and {points("history")} for its history.</p>
+      <p class="lede">The checks only see what is in the files. They cannot know that nothing is missing, so they stop at 99%. A country shows 100% only when every check passes and someone has signed it off by hand, with <code>complete: YYYY-MM-DD</code> in its file.</p>
+    </section>
+
+    <dl class="hist-stats">
+      {stats.map(([n, label]) => <div><dt>{label}</dt><dd>{n}</dd></div>)}
+    </dl>
+
+    <section class="section">
+      <div class="section-head"><div>
+        <h2>What is counted</h2>
+        <p class="sub">Each check gives points; a partly met check gives part of them.</p>
+      </div></div>
+      <div class="rs-checks">
+        {["profile", "history"].map((part) => (
+          <table class="src-table">
+            <thead><tr><th>{part === "profile" ? "Profile" : "History"}</th><th class="num">Points</th></tr></thead>
+            <tbody>{example.filter((c) => c.part === part).map((c) => <tr><td>{c.label}</td><td class="num">{c.of}</td></tr>)}</tbody>
+          </table>
+        ))}
+      </div>
+    </section>
+
+    <section class="section">
+      <div class="section-head"><div>
+        <h2>By country</h2>
+        <p class="sub">Click a column to sort by it, and a row to see what is missing.</p>
+      </div>
+        <input type="search" id="rsFind" placeholder="Filter by name or region" aria-label="Filter countries" />
+      </div>
+      <div class="rs-scroll"><table class="src-table rs-table" id="rsTable">
+        <thead><tr>
+          <th data-sort="name">Country</th>
+          <th class="num" data-sort="pct" aria-sort="descending">Research</th>
+          <th class="num" data-sort="profile">Profile</th>
+          <th class="num" data-sort="history">History</th>
+          <th class="num" data-sort="entries">Entries</th>
+          <th class="num" data-sort="scored" title="Entries that moved a score">Scored</th>
+          <th class="num" data-sort="articles">Articles</th>
+          <th class="num" data-sort="first" title="Year of the first recorded restriction">First ban</th>
+          <th class="num" data-sort="silent" title="Decades since the first entry (1900 at the earliest) with no entry">Empty decades</th>
+          <th class="num" data-sort="gap" title="Longest stretch of years between two entries">Longest gap</th>
+          <th class="num" data-sort="reviewed">Reviewed</th>
+        </tr></thead>
+        {research.map((r) => (
+          <tbody data-name={r.name.toLowerCase()} data-region={r.region.toLowerCase()} data-pct={r.pct} data-profile={r.profile} data-history={r.history} data-entries={r.entries} data-scored={r.scored} data-articles={r.articles} data-first={r.first === "est." ? 9998 : r.first === "—" ? 9999 : r.first} data-silent={r.silent} data-gap={r.gap ?? -1} data-reviewed={r.reviewed}>
+            <tr class="rs-row" tabindex="0" aria-expanded="false">
+              <th scope="row"><a href={`${base}country/${r.slug}/`}>{r.flag} {r.name}</a>{r.disputed && <small> territory</small>}</th>
+              <td class="num"><span class="rs-bar" style={`--p:${r.pct}%`}><b>{r.pct}%</b></span></td>
+              <td class="num">{r.profile}%</td>
+              <td class="num">{r.history}%</td>
+              <td class="num">{r.entries}</td>
+              <td class="num">{r.scored}</td>
+              <td class="num">{r.articles}</td>
+              <td class="num">{r.first}</td>
+              <td class="num">{r.silent}</td>
+              <td class="num">{r.gap ?? "—"}</td>
+              <td class="num mono">{r.reviewed}</td>
+            </tr>
+            <tr class="rs-more" hidden><td colspan="11">
+              <div class="rs-missing">
+                {["profile", "history"].map((part) => (
+                  <div>
+                    <h3>{part === "profile" ? "Profile" : "History"}</h3>
+                    <ul>
+                      {r.checks.filter((c) => c.part === part).map((c) => (
+                        <li class={c.got >= c.of ? "ok" : ""}>
+                          <span>{c.got >= c.of ? "✓" : "·"} {c.label} <small>{Math.round(c.got * 10) / 10} of {c.of}</small></span>
+                          {c.missing.length > 0 && <em>{c.missing.slice(0, 12).join("; ")}{c.missing.length > 12 && `; and ${c.missing.length - 12} more`}</em>}
+                        </li>
+                      ))}
+                    </ul>
+                  </div>
+                ))}
+              </div>
+              <p class="rs-sign">{r.signed ? `Signed off as complete on ${r.signed}.` : "Not signed off as complete."} <a href={`${base}country/${r.slug}/`}>Open the country page</a></p>
+            </td></tr>
+          </tbody>
+        ))}
+      </table></div>
+    </section>
+  </main>
+
+  <script is:inline>
+    (() => {
+      const table = document.getElementById("rsTable");
+      const bodies = () => [...table.tBodies];
+      const open = (row) => {
+        const more = row.nextElementSibling;
+        more.hidden = !more.hidden;
+        row.setAttribute("aria-expanded", String(!more.hidden));
+      };
+      table.addEventListener("click", (e) => {
+        const th = e.target.closest("th[data-sort]");
+        if (th) {
+          const key = th.dataset.sort, text = key === "name" || key === "reviewed";
+          const now = th.getAttribute("aria-sort");
+          const down = now ? now === "ascending" : !text; // numbers start from the top, names from A
+          table.querySelectorAll("th[aria-sort]").forEach((x) => x.removeAttribute("aria-sort"));
+          th.setAttribute("aria-sort", down ? "descending" : "ascending");
+          const val = (b) => (text ? b.dataset[key] : +b.dataset[key]);
+          bodies().sort((a, b) => (val(a) < val(b) ? -1 : val(a) > val(b) ? 1 : 0) * (down ? -1 : 1)).forEach((b) => table.appendChild(b));
+          return;
+        }
+        const row = e.target.closest(".rs-row");
+        if (row && !e.target.closest("a")) open(row);
+      });
+      table.addEventListener("keydown", (e) => { if (e.key === "Enter" && e.target.classList.contains("rs-row")) open(e.target); });
+      document.getElementById("rsFind").addEventListener("input", (e) => {
+        const q = e.target.value.trim().toLowerCase();
+        for (const b of bodies()) b.hidden = Boolean(q) && !b.dataset.name.includes(q) && !b.dataset.region.includes(q);
+      });
+    })();
+  </script>
+</Base>

@@ -1619,2 +1619,41 @@ button.sv-row .sv-chip { font: 400 10.5px var(--mono); text-transform: uppercase
 .about .numbers-grid { grid-template-columns: repeat(4, minmax(0, 1fr)); }
 @media (max-width: 760px) { .about .numbers-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
+
+/* ── Research progress (/research/, internal) ──────────────────────────────── */
+.research-page code { font: 400 .9em var(--mono); }
+.rs-checks { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 40px; }
+#rsFind { align-self: end; min-width: 240px; }
+.rs-scroll { overflow-x: auto; }
+.rs-table th, .rs-table td { padding-right: 10px; }
+.rs-table .num { padding-left: 12px; }
+.rs-table thead th[data-sort] { cursor: pointer; white-space: nowrap; }
+.rs-table thead th[aria-sort="descending"]::after { content: " ↓"; }
+.rs-table thead th[aria-sort="ascending"]::after { content: " ↑"; }
+.rs-table tbody th small { color: var(--muted); font: 400 10.5px var(--mono); text-transform: uppercase; letter-spacing: .06em; }
+.rs-row { cursor: pointer; }
+.rs-row:hover td, .rs-row:hover th, .rs-row[aria-expanded="true"] td, .rs-row[aria-expanded="true"] th { background: var(--surface); }
+.rs-bar { position: relative; display: inline-block; width: 110px; padding: 1px 6px; text-align: right; background: linear-gradient(to right, var(--border) var(--p), transparent var(--p)); border: 1px solid var(--border); }
+.rs-bar b { font-weight: 500; }
+.rs-more td { padding: 4px 0 22px; }
+.rs-missing { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 40px; width: min(100%, calc(100vw - 80px)); }
+.rs-missing h3 { font: 400 10.5px var(--mono); text-transform: uppercase; letter-spacing: .06em; color: var(--muted); margin: 12px 0 6px; }
+.rs-missing ul { list-style: none; margin: 0; padding: 0; }
+.rs-missing li { padding: 6px 0; border-top: 1px solid var(--border); font-size: 13.5px; }
+.rs-missing li.ok { color: var(--muted); }
+.rs-missing li small { font: 400 11px var(--mono); color: var(--muted); margin-left: 6px; }
+.rs-missing li em { display: block; font-style: normal; font-size: 12.5px; color: var(--text-2); margin-top: 2px; }
+.rs-sign { margin: 12px 0 0; font-size: 12.5px; color: var(--muted); }
+.rs-sign a { text-decoration: underline; text-underline-offset: 3px; }
+@media (max-width: 900px) {
+  .rs-checks, .rs-missing { grid-template-columns: 1fr; }
+  .rs-table th:nth-child(n+5), .rs-table td:nth-child(n+5) { display: none; }
+  .rs-bar { width: 80px; }
+  #rsFind { min-width: 0; width: 100%; }
+}
+/* the same figure, small, at the top right of the History page */
+.hero-progress { position: relative; }
+.rs-pill { position: absolute; top: clamp(28px, 5vw, 52px); right: 0; display: flex; align-items: center; gap: 10px; padding: 6px 10px; border: 1px solid var(--border); font: 400 10.5px var(--mono); text-transform: uppercase; letter-spacing: .06em; color: var(--muted); }
+.rs-pill:hover { border-color: var(--muted); color: var(--text); }
+.rs-pill i { width: 64px; height: 4px; background: linear-gradient(to right, var(--text) var(--p), var(--border) var(--p)); }
+.rs-pill b { font-weight: 500; font-size: 12px; color: var(--text); }
+@media (max-width: 700px) { .rs-pill { position: static; width: fit-content; margin-bottom: 16px; } }