diff --git a/.agents/scripts/rendered-text-diff.mjs b/.agents/scripts/rendered-text-diff.mjs index 870775a..6e239e4 100644 --- a/.agents/scripts/rendered-text-diff.mjs +++ b/.agents/scripts/rendered-text-diff.mjs @@ -21,6 +21,7 @@ import { spawn } from 'node:child_process'; import { cpSync, mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { createServer } from 'node:net'; import { chromium } from 'playwright'; // `index` is the landing page: it lives at the repository root, not in a @@ -43,9 +44,26 @@ const astroPath = route === 'index' ? '' : `${route}/`; const staging = mkdtempSync(join(tmpdir(), 'af-rtd-')); cpSync('dist', join(staging, 'ai-for-dummies'), { recursive: true }); +// Ask the kernel for a free port rather than pinning one. Back-to-back runs +// used to collide: the previous run's server was still holding the fixed port +// while its staging directory had already been deleted, so every page came +// back as a 404 and the diff reported the whole route missing. +const freePort = () => + new Promise((resolve, reject) => { + const probe = createServer(); + probe.on('error', reject); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address(); + probe.close(() => resolve(port)); + }); + }); + +const legacyPort = await freePort(); +const astroPort = await freePort(); + const serve = (dir, port) => spawn('python3', ['-m', 'http.server', String(port), '-d', dir], { stdio: 'ignore' }); -const servers = [serve('.', 4197), serve(staging, 4196)]; +const servers = [serve('.', legacyPort), serve(staging, astroPort)]; const stop = () => { servers.forEach((s) => s.kill()); rmSync(staging, { recursive: true, force: true }); @@ -77,7 +95,13 @@ try { const browser = await chromium.launch(); const grab = async (url) => { const page = await browser.newPage({ viewport: { width: 1400, height: 1000 } }); - await page.goto(url, { waitUntil: 'load' }); + const response = await page.goto(url, { waitUntil: 'load' }); + // A 404 renders as four spans of python's error page and the diff then + // reports the entire route as missing, which reads exactly like a real + // regression. Fail loudly instead. + if (!response || !response.ok()) { + throw new Error(`${url} returned ${response ? response.status() : 'no response'}`); + } // The islands hydrate and render their initial panel on load; without this // every panel's copy reads as missing. await page.waitForTimeout(1200); @@ -87,23 +111,58 @@ try { await toggle.click(); await page.waitForTimeout(1200); } - const spans = await page.evaluate(visibleText); + // Islands hydrate at their own pace, and the language toggle repaints in + // more than one frame. A single read after a fixed wait is flaky, so read + // until two consecutive reads agree. + let spans = await page.evaluate(visibleText); + for (let i = 0; i < 10; i += 1) { + await page.waitForTimeout(300); + const next = await page.evaluate(visibleText); + if (next.length === spans.length && next.every((span, j) => span === spans[j])) { + spans = next; + break; + } + spans = next; + } await page.close(); return spans; }; - const legacy = await grab(`http://localhost:4197/${legacyPath}`); - const astro = await grab(`http://localhost:4196/ai-for-dummies/${astroPath}`); + const legacy = await grab(`http://localhost:${legacyPort}/${legacyPath}`); + const astro = await grab(`http://localhost:${astroPort}/ai-for-dummies/${astroPath}`); await browser.close(); - const rendered = new Set(astro); - const missing = legacy.filter((span) => !rendered.has(span)); + // Count occurrences, not membership. A set comparison reports zero when a + // string the legacy page paints four times is painted three times here -- + // exactly the kind of near-miss that got past the earlier checks. + const tally = (spans) => { + const counts = new Map(); + for (const span of spans) counts.set(span, (counts.get(span) || 0) + 1); + return counts; + }; + + const legacyCounts = tally(legacy); + const astroCounts = tally(astro); + + const missing = []; + for (const [span, count] of legacyCounts) { + const short = count - (astroCounts.get(span) || 0); + for (let i = 0; i < short; i += 1) missing.push(span); + } // Both directions. A string the Astro page paints and the legacy page does // not is just as wrong: it means a translation was invented, or an English // string was left standing where the legacy page swaps it. - const legacySpans = new Set(legacy); - const extra = astro.filter((span) => !legacySpans.has(span)); + const extra = []; + for (const [span, count] of astroCounts) { + const over = count - (legacyCounts.get(span) || 0); + for (let i = 0; i < over; i += 1) extra.push(span); + } + + // Order counts too. Both pages can paint the same strings while a block + // sits in the wrong place -- the Portuguese eyebrow, or a reordered card + // deck -- and a count-only comparison calls that clean. + const firstOutOfOrder = legacy.findIndex((span, i) => astro[i] !== span); const mode = portuguese ? 'pt' : 'en'; console.log( @@ -111,7 +170,12 @@ try { ); for (const span of missing) console.log(` - ${span}`); for (const span of extra) console.log(` + ${span}`); - process.exitCode = missing.length === 0 && extra.length === 0 ? 0 : 1; + if (firstOutOfOrder !== -1) { + console.log(` order diverges at span ${firstOutOfOrder}`); + console.log(` legacy: ${legacy[firstOutOfOrder]}`); + console.log(` astro: ${astro[firstOutOfOrder]}`); + } + process.exitCode = missing.length === 0 && extra.length === 0 && firstOutOfOrder === -1 ? 0 : 1; } finally { stop(); }