Changelog d9fe15btree, with Claude Opus 5.5
archive-sources: save and probe skip addresses tried without a copy in the last 2 days
1 file of code, design or docs +9 −4 lines
Code, design and documents 1 file
scripts/archive-sources.cjs +9 −4
@@ -15,5 +15,6 @@ // run it again to continue. Results go to source-archive.json, which the site reads: // { "<url>": { "archived": "https://web.archive.org/web/<timestamp>/<url>", "checked": "YYYY-MM-DD" } } -// An address with no copy is recorded with "archived": null and is tried again after 30 days. +// An address with no copy is recorded with "archived": null. `check` looks it up again after 30 +// days; `save` and `probe` leave it alone while that try is 2 days old or newer. // // An address whose site cannot be reached is marked, so that runs skip it instead of failing on it @@ -75,4 +76,8 @@ const DEAD_DAYS = 90; const resting = (u) => !includeDead && archive[u]?.dead && Date.now() - Date.parse(archive[u].dead.seen) < DEAD_DAYS * 864e5; const stale = (u) => !resting(u) && (!archive[u] || (!archive[u].archived && Date.now() - Date.parse(archive[u].checked) > 30 * 864e5)); +// tried without a copy 2 days ago or later: `save` and `probe` skip it, the Archive just answered +const RECENT_DAYS = 2; +const recent = (u) => archive[u] && !archive[u].archived && (Date.parse(today) - Date.parse(archive[u].checked)) / 864e5 <= RECENT_DAYS; +const unsaved = (u) => !archive[u]?.archived && !resting(u) && !recent(u); const have = all.filter((u) => archive[u]?.archived).length; const dead = {}; @@ -192,5 +197,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse if (!["check", "save", "probe"].includes(what)) return console.log("unknown command: use status, check, save, probe or prune"); if (what === "probe") return probe(); - const pending = all.filter(what === "check" ? stale : (u) => !archive[u]?.archived && !resting(u)); + const pending = all.filter(what === "check" ? stale : unsaved); const todo = (random ? shuffle(pending) : pending).slice(0, limit); let found = 0, failed = 0; @@ -222,5 +227,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse await sleep(what === "check" ? 3000 : 1000); } - console.log(`${what}: handled ${todo.length} | copies recorded ${found} | failed ${failed} | still to do ${all.filter(what === "check" ? stale : (u) => !archive[u]?.archived && !resting(u)).length}`); + console.log(`${what}: handled ${todo.length} | copies recorded ${found} | failed ${failed} | still to do ${all.filter(what === "check" ? stale : unsaved).length}`); })(); @@ -229,5 +234,5 @@ if (uncited().length) console.log(`no longer cited: ${uncited().length} addresse async function probe() { const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128 Safari/537.36"; - const todo = all.filter((u) => !archive[u]?.archived && !resting(u)).slice(0, limit); + const todo = all.filter(unsaved).slice(0, limit); const counts = { ok: 0, blocked: 0, gone: 0, unreachable: 0 }; for (const url of todo) {