Forget archive records of sources no longer cited, and count only cited ones

@@ -1001,4 +1001,5 @@ script. The file is written by a script, never by hand:
 ```
 node scripts/archive-sources.cjs status      # how many sources have a recorded copy
+node scripts/archive-sources.cjs prune       # forget addresses no source cites any more (no network)
 node scripts/archive-sources.cjs check 200   # record the copies the Archive already holds
 node scripts/archive-sources.cjs save 20     # ask the Archive to save what it does not hold

@@ -1,3 +1,3 @@
-// Archived copies of the sources: node scripts/archive-sources.cjs [status|check|save|probe] [limit] [--random] [--include-dead]
+// Archived copies of the sources: node scripts/archive-sources.cjs [status|check|save|probe|prune] [limit] [--random] [--include-dead]
 // Run from the project root. Needs js-yaml (already in node_modules) and Node 18+ (fetch).
 //
@@ -7,4 +7,8 @@
 //   probe   open the sources that have no copy yet, as a browser would, and record the ones whose
 //           site refuses or has lost them (see "dead" below)
+//   prune   forget the addresses no source cites any more (no network); check, save and probe do
+//           this too whenever they write. A copy recorded under an older spelling of an address
+//           that is still cited (http or https, with or without www, a trailing slash) moves to
+//           that address when it has no copy of its own.
 //
 // `limit` caps how many addresses one run handles (default 50), so a run stays short and polite;
@@ -82,6 +86,27 @@ const markDead = (url, reason, status) => {
 };
 
+// Addresses in the file that no source cites any more: a source removed or rewritten. They would
+// otherwise stay forever and count as archived sources they no longer are.
+const spelling = (u) => { try { u = decodeURI(u); } catch {} return u.replace(/^https?:\/\/(www\.)?/i, "").replace(/\/$/, "").toLowerCase(); };
+const uncited = () => Object.keys(archive).filter((u) => !urls.has(u));
+const prune = () => {
+  const cited = new Map(all.map((u) => [spelling(u), u]));
+  let moved = 0;
+  const gone = uncited();
+  for (const u of gone) {
+    const now = cited.get(spelling(u));
+    if (now && archive[u].archived && !archive[now]?.archived) {
+      const { dead, ...copy } = archive[u];
+      archive[now] = copy;
+      moved++;
+    }
+    delete archive[u];
+  }
+  return { removed: gone.length, moved };
+};
+
 // written to a temporary file first and then renamed, so a full disk or a crash cannot leave half a file
 const write = () => {
+  prune();
   fs.writeFileSync(FILE + ".tmp", JSON.stringify(Object.fromEntries(Object.entries(archive).sort(([a], [b]) => a.localeCompare(b))), null, 1) + "\n");
   fs.renameSync(FILE + ".tmp", FILE);
@@ -155,8 +180,15 @@ async function saveLoggedIn(url) {
 }
 
+if (uncited().length) console.log(`no longer cited: ${uncited().length} addresses in ${FILE} (\`prune\` forgets them)`);
+
 (async () => {
   if (what === "status") return;
+  if (what === "prune") {
+    const { removed, moved } = prune();
+    write();
+    return console.log(`prune: forgot ${removed} addresses no source cites${moved ? `, ${moved} copies moved to the address now cited` : ""}`);
+  }
   if (what === "save") console.log(KEYS ? "save: logged in" : "save: anonymous (no IA_ACCESS_KEY / IA_SECRET_KEY in .env)");
-  if (!["check", "save", "probe"].includes(what)) return console.log("unknown command: use status, check, save or probe");
+  if (!["check", "save", "probe"].includes(what)) return console.log("unknown command: use status, check, save, probe or prune");
   if (what === "probe") return probe();
   const pending = all.filter(what === "check" ? stale : (u) => !archive[u]?.archived && !resting(u));

Not shown: a picture, a generated file, or past the length shown per change.

@@ -5,5 +5,5 @@ import { SEGMENTS, DATA_SNAPSHOT, places, countries, changes, articles, subunits
 import { MAP_VIEWS } from "../lib/map-views.js";
 import { cultureIn } from "../lib/culture.js";
-import { archivedCount } from "../lib/archive.js";
+import { archivedCount } from "../lib/citations.js";
 import { sourceKey } from "../lib/sources.js";
 

@@ -6,3 +6,3 @@ import archive from "../../source-archive.json";
 
 export const archivedUrl = (url) => archive[url]?.archived ?? null;
-export const archivedCount = Object.values(archive).filter((x) => x.archived).length;
+// how many of the cited sources have a copy: archivedCount in citations.js

@@ -8,4 +8,5 @@ import { getCollection } from "astro:content";
 import { places, changes, SEGMENTS, CONTEXT } from "./countries.js";
 import { cultureEntries, cultureSection } from "./culture.js";
+import { archivedUrl } from "./archive.js";
 
 const placeOf = new Map(places.map((c) => [c.slug, c]));
@@ -49,2 +50,9 @@ export const isBook = (s) => s.type === "book" || s.type === "chapter" || Boolea
 export const bookSlug = (t) => t.normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
 export const domainOf = (url) => new URL(url).hostname.replace(/^www\./, "");
+
+// Every web page cited, once, and how many have a copy in the Internet Archive (an archive.org page
+// counts as one already). Only cited pages count: an address left in source-archive.json for a
+// source that is no longer cited adds nothing (scripts/archive-sources.cjs prune forgets those).
+export const citedUrls = new Set(cites.filter((x) => x.s.url).map((x) => x.s.url));
+export const isArchived = (u) => Boolean(archivedUrl(u)) || /^https?:\/\/([a-z0-9-]+\.)?archive\.org\//i.test(u);
+export const archivedCount = [...citedUrls].filter(isArchived).length;

@@ -8,5 +8,5 @@ import { MAP_VIEWS } from "../lib/map-views.js";
 import { cultureIn } from "../lib/culture.js";
 import { getFacts } from "../lib/facts.js";
-import { archivedCount } from "../lib/archive.js";
+import { archivedCount } from "../lib/citations.js";
 import { sourceKey } from "../lib/sources.js";
 import { survey } from "../lib/survey.js";

@@ -4,5 +4,5 @@ import Base from "../../layouts/Base.astro";
 import { scoreRows, historyRows, priceRows, panelRows, PANEL_FROM } from "../../lib/exports.js";
 import { lastUpdated, formatDate, DATA_SNAPSHOT, ESTIMATE_WINDOW } from "../../lib/countries.js";
-import { archivedCount } from "../../lib/archive.js";
+import { archivedCount } from "../../lib/citations.js";
 import { SHOW_PRICES } from "../../lib/config.js";
 

@@ -6,6 +6,5 @@
 import Base from "../../layouts/Base.astro";
 import { places } from "../../lib/countries.js";
-import { archivedUrl } from "../../lib/archive.js";
-import { cites, isBook, bookSlug, domainOf } from "../../lib/citations.js";
+import { cites, citedUrls, isArchived, isBook, bookSlug, domainOf } from "../../lib/citations.js";
 import { Image } from "astro:assets";
 import { DATASETS } from "../../lib/datasets.js";
@@ -66,10 +65,10 @@ const localRows = [...local]
 // Site icons come from DuckDuckGo's icon service and are not stored here; a missing one is hidden.
 const icon = (d) => `https://icons.duckduckgo.com/ip3/${d}.ico`;
-const urls = new Set(cites.filter((x) => x.s.url).map((x) => x.s.url));
+const urls = citedUrls;
 const urlCount = urls.size;
 // pages with a copy recorded in the Internet Archive (see source-archive.json), counting a source
 // cited by its Archive address already; scripts/archive-sources.cjs status counts the same pages
 // an Internet Archive page (the Wayback Machine or an archive.org item) counts as archived already
-const archivedOf = (set) => [...set].filter((u) => archivedUrl(u) || /^https?:\/\/([a-z0-9-]+\.)?archive\.org\//i.test(u)).length;
+const archivedOf = (set) => [...set].filter(isArchived).length;
 const archived = archivedOf(urls);
 const pct = Math.round((archived / urlCount) * 100);