Cannabis Freedom Index

Changelog d9fe15btree, with Claude Opus 5.5

archive-sources: save and probe skip addresses tried without a copy in the last 2 days

1 file of code, design or docs +9 −4 lines

Code, design and documents 1 file

scripts/archive-sources.cjs +9 −4
@@ -15,5 +15,6 @@
 // run it again to continue. Results go to source-archive.json, which the site reads:
 //   { "<url>": { "archived": "https://web.archive.org/web/<timestamp>/<url>", "checked": "YYYY-MM-DD" } }
-// An address with no copy is recorded with "archived": null and is tried again after 30 days.
+// An address with no copy is recorded with "archived": null. `check` looks it up again after 30
+// days; `save` and `probe` leave it alone while that try is 2 days old or newer.
 //
 // An address whose site cannot be reached is marked, so that runs skip it instead of failing on it
@@ -75,4 +76,8 @@ const DEAD_DAYS = 90;
 const resting = (u) => !includeDead && archive[u]?.dead && Date.now() - Date.parse(archive[u].dead.seen) < DEAD_DAYS * 864e5;
 const stale = (u) => !resting(u) && (!archive[u] || (!archive[u].archived && Date.now() - Date.parse(archive[u].checked) > 30 * 864e5));
+// tried without a copy 2 days ago or later: `save` and `probe` skip it, the Archive just answered
+const RECENT_DAYS = 2;
+const recent = (u) => archive[u] && !archive[u].archived && (Date.parse(today) - Date.parse(archive[u].checked)) / 864e5 <= RECENT_DAYS;
+const unsaved = (u) => !archive[u]?.archived && !resting(u) && !recent(u);
 const have = all.filter((u) => archive[u]?.archived).length;
 const dead = {};
@@ -192,5 +197,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse
   if (!["check", "save", "probe"].includes(what)) return console.log("unknown command: use status, check, save, probe or prune");
   if (what === "probe") return probe();
-  const pending = all.filter(what === "check" ? stale : (u) => !archive[u]?.archived && !resting(u));
+  const pending = all.filter(what === "check" ? stale : unsaved);
   const todo = (random ? shuffle(pending) : pending).slice(0, limit);
   let found = 0, failed = 0;
@@ -222,5 +227,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse
     await sleep(what === "check" ? 3000 : 1000);
   }
-  console.log(`${what}: handled ${todo.length} | copies recorded ${found} | failed ${failed} | still to do ${all.filter(what === "check" ? stale : (u) => !archive[u]?.archived && !resting(u)).length}`);
+  console.log(`${what}: handled ${todo.length} | copies recorded ${found} | failed ${failed} | still to do ${all.filter(what === "check" ? stale : unsaved).length}`);
 })();
 
@@ -229,5 +234,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse
 async function probe() {
   const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128 Safari/537.36";
-  const todo = all.filter((u) => !archive[u]?.archived && !resting(u)).slice(0, limit);
+  const todo = all.filter(unsaved).slice(0, limit);
   const counts = { ok: 0, blocked: 0, gone: 0, unreachable: 0 };
   for (const url of todo) {

    ↑↓ to move, ↵ to open.