From c9796a3e7f6adb5a853088f4daed6ffd896fe1f8 Mon Sep 17 00:00:00 2001 From: Marcos Paulo Date: Sun, 6 Sep 2026 00:33:44 +0000 Subject: [PATCH] test: count and order rendered spans, not just their presence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rendered-text diff compared two *sets* of strings, so it stayed at "missing 0 ยท extra 0" while the built page painted a string a different number of times, or in a different place. That is the same shape of hole that let task 15d ship a full-guide missing a fifth of its content behind a green gate. Three changes: - tally occurrences instead of set membership, so a string the legacy page paints twice has to be painted twice here; - compare the sequences positionally and report the first divergence, which is what caught the Portuguese eyebrow and the reordered skill deck fixed in the next commit; - fail loudly on a non-200 response. A 404 rendered as four spans of python's error page and the diff then reported the entire route as missing, which reads exactly like a real regression. Two robustness fixes behind those: ask the kernel for a free port rather than pinning 4196/4197 (back-to-back runs collided with the previous run's server, which was still holding the port after its staging directory had been deleted), and read the DOM until two consecutive reads agree instead of once after a fixed wait. Co-Authored-By: Claude Opus 5 --- .agents/scripts/rendered-text-diff.mjs | 84 +++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/.agents/scripts/rendered-text-diff.mjs b/.agents/scripts/rendered-text-diff.mjs index 870775a..6e239e4 100644 --- a/.agents/scripts/rendered-text-diff.mjs +++ b/.agents/scripts/rendered-text-diff.mjs @@ -21,6 +21,7 @@ import { spawn } from 'node:child_process'; import { cpSync, mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { createServer } from 'node:net'; import { chromium } from 'playwright'; // `index` is the landing page: it lives at the repository root, not in a @@ -43,9 +44,26 @@ const astroPath = route === 'index' ? '' : `${route}/`; const staging = mkdtempSync(join(tmpdir(), 'af-rtd-')); cpSync('dist', join(staging, 'ai-for-dummies'), { recursive: true }); +// Ask the kernel for a free port rather than pinning one. Back-to-back runs +// used to collide: the previous run's server was still holding the fixed port +// while its staging directory had already been deleted, so every page came +// back as a 404 and the diff reported the whole route missing. +const freePort = () => + new Promise((resolve, reject) => { + const probe = createServer(); + probe.on('error', reject); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address(); + probe.close(() => resolve(port)); + }); + }); + +const legacyPort = await freePort(); +const astroPort = await freePort(); + const serve = (dir, port) => spawn('python3', ['-m', 'http.server', String(port), '-d', dir], { stdio: 'ignore' }); -const servers = [serve('.', 4197), serve(staging, 4196)]; +const servers = [serve('.', legacyPort), serve(staging, astroPort)]; const stop = () => { servers.forEach((s) => s.kill()); rmSync(staging, { recursive: true, force: true }); @@ -77,7 +95,13 @@ try { const browser = await chromium.launch(); const grab = async (url) => { const page = await browser.newPage({ viewport: { width: 1400, height: 1000 } }); - await page.goto(url, { waitUntil: 'load' }); + const response = await page.goto(url, { waitUntil: 'load' }); + // A 404 renders as four spans of python's error page and the diff then + // reports the entire route as missing, which reads exactly like a real + // regression. Fail loudly instead. + if (!response || !response.ok()) { + throw new Error(`${url} returned ${response ? response.status() : 'no response'}`); + } // The islands hydrate and render their initial panel on load; without this // every panel's copy reads as missing. await page.waitForTimeout(1200); @@ -87,23 +111,58 @@ try { await toggle.click(); await page.waitForTimeout(1200); } - const spans = await page.evaluate(visibleText); + // Islands hydrate at their own pace, and the language toggle repaints in + // more than one frame. A single read after a fixed wait is flaky, so read + // until two consecutive reads agree. + let spans = await page.evaluate(visibleText); + for (let i = 0; i < 10; i += 1) { + await page.waitForTimeout(300); + const next = await page.evaluate(visibleText); + if (next.length === spans.length && next.every((span, j) => span === spans[j])) { + spans = next; + break; + } + spans = next; + } await page.close(); return spans; }; - const legacy = await grab(`http://localhost:4197/${legacyPath}`); - const astro = await grab(`http://localhost:4196/ai-for-dummies/${astroPath}`); + const legacy = await grab(`http://localhost:${legacyPort}/${legacyPath}`); + const astro = await grab(`http://localhost:${astroPort}/ai-for-dummies/${astroPath}`); await browser.close(); - const rendered = new Set(astro); - const missing = legacy.filter((span) => !rendered.has(span)); + // Count occurrences, not membership. A set comparison reports zero when a + // string the legacy page paints four times is painted three times here -- + // exactly the kind of near-miss that got past the earlier checks. + const tally = (spans) => { + const counts = new Map(); + for (const span of spans) counts.set(span, (counts.get(span) || 0) + 1); + return counts; + }; + + const legacyCounts = tally(legacy); + const astroCounts = tally(astro); + + const missing = []; + for (const [span, count] of legacyCounts) { + const short = count - (astroCounts.get(span) || 0); + for (let i = 0; i < short; i += 1) missing.push(span); + } // Both directions. A string the Astro page paints and the legacy page does // not is just as wrong: it means a translation was invented, or an English // string was left standing where the legacy page swaps it. - const legacySpans = new Set(legacy); - const extra = astro.filter((span) => !legacySpans.has(span)); + const extra = []; + for (const [span, count] of astroCounts) { + const over = count - (legacyCounts.get(span) || 0); + for (let i = 0; i < over; i += 1) extra.push(span); + } + + // Order counts too. Both pages can paint the same strings while a block + // sits in the wrong place -- the Portuguese eyebrow, or a reordered card + // deck -- and a count-only comparison calls that clean. + const firstOutOfOrder = legacy.findIndex((span, i) => astro[i] !== span); const mode = portuguese ? 'pt' : 'en'; console.log( @@ -111,7 +170,12 @@ try { ); for (const span of missing) console.log(` - ${span}`); for (const span of extra) console.log(` + ${span}`); - process.exitCode = missing.length === 0 && extra.length === 0 ? 0 : 1; + if (firstOutOfOrder !== -1) { + console.log(` order diverges at span ${firstOutOfOrder}`); + console.log(` legacy: ${legacy[firstOutOfOrder]}`); + console.log(` astro: ${astro[firstOutOfOrder]}`); + } + process.exitCode = missing.length === 0 && extra.length === 0 && firstOutOfOrder === -1 ? 0 : 1; } finally { stop(); }