ai-for-dummies/.agents/scripts/rendered-text-diff.mjs

#!/usr/bin/env node
// Compare the *rendered* text of a legacy page against its Astro replacement.
//
//   node .agents/scripts/rendered-text-diff.mjs full-guide
//   node .agents/scripts/rendered-text-diff.mjs full-guide --pt
//
// Why this exists: scripts/verify.mjs reads the legacy files, so a migrated
// page can drop half its content and still pass the gate. Task 15d shipped
// /full-guide/ missing 86 rendered spans -- the entire verification section,
// the hands-on exercise brief, and both "Clone from Gitea" links -- and every
// check was green.
//
// Static HTML comparison is useless here: the guide's tab panels are injected
// by an island at runtime, so half the legacy page's markup has no static
// counterpart. This walks the live DOM instead and skips anything the browser
// is not painting -- which also drops the hidden Portuguese half of each
// bilingual pair, so the two sides line up.
//
// Requires playwright (devDependency) and two static servers; it starts both.
import { spawn } from 'node:child_process';
import { cpSync, mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createServer } from 'node:net';
import { chromium } from 'playwright';

// `index` is the landing page: it lives at the repository root, not in a
// directory of its own, so it needs a different path on the legacy side.
const route = process.argv[2];
if (!route) {
  console.error('usage: rendered-text-diff.mjs <route> [--pt]   e.g. full-guide, or index');
  process.exit(2);
}

// `--pt` clicks the language toggle on both pages first. English parity is
// only half the contract: a page can render every English string and still
// leave a restored block untranslated, because the Portuguese half is a
// separate set of nodes. Only /full-guide/ and /rules/ have a toggle.
const portuguese = process.argv.includes('--pt');
const legacyPath = route === 'index' ? 'index.html' : `${route}/index.html`;
const astroPath = route === 'index' ? '' : `${route}/`;

// The built site expects to be served under the configured base path.
const staging = mkdtempSync(join(tmpdir(), 'af-rtd-'));
cpSync('dist', join(staging, 'ai-for-dummies'), { recursive: true });

// Ask the kernel for a free port rather than pinning one. Back-to-back runs
// used to collide: the previous run's server was still holding the fixed port
// while its staging directory had already been deleted, so every page came
// back as a 404 and the diff reported the whole route missing.
const freePort = () =>
  new Promise((resolve, reject) => {
    const probe = createServer();
    probe.on('error', reject);
    probe.listen(0, '127.0.0.1', () => {
      const { port } = probe.address();
      probe.close(() => resolve(port));
    });
  });

const legacyPort = await freePort();
const astroPort = await freePort();

const serve = (dir, port) =>
  spawn('python3', ['-m', 'http.server', String(port), '-d', dir], { stdio: 'ignore' });
const servers = [serve('.', legacyPort), serve(staging, astroPort)];
const stop = () => {
  servers.forEach((s) => s.kill());
  rmSync(staging, { recursive: true, force: true });
};

// Visible text nodes, in document order, whitespace collapsed.
const visibleText = () => {
  const out = [];
  const walk = (node) => {
    for (const child of node.childNodes) {
      if (child.nodeType === Node.TEXT_NODE) {
        const text = child.textContent.replace(/\s+/g, ' ').trim();
        if (text) out.push(text);
        continue;
      }
      if (child.nodeType !== Node.ELEMENT_NODE) continue;
      if (child.tagName === 'SCRIPT' || child.tagName === 'STYLE') continue;
      const style = getComputedStyle(child);
      if (child.hidden || style.display === 'none' || style.visibility === 'hidden') continue;
      walk(child);
    }
  };
  walk(document.body);
  return out;
};

try {
  await new Promise((r) => setTimeout(r, 1500));
  const browser = await chromium.launch();
  const grab = async (url) => {
    const page = await browser.newPage({ viewport: { width: 1400, height: 1000 } });
    const response = await page.goto(url, { waitUntil: 'load' });
    // A 404 renders as four spans of python's error page and the diff then
    // reports the entire route as missing, which reads exactly like a real
    // regression. Fail loudly instead.
    if (!response || !response.ok()) {
      throw new Error(`${url} returned ${response ? response.status() : 'no response'}`);
    }
    // The islands hydrate and render their initial panel on load; without this
    // every panel's copy reads as missing.
    await page.waitForTimeout(1200);
    if (portuguese) {
      const toggle = await page.$('[data-lang="pt"]');
      if (!toggle) throw new Error(`no language toggle on ${url}`);
      await toggle.click();
      await page.waitForTimeout(1200);
    }
    // Islands hydrate at their own pace, and the language toggle repaints in
    // more than one frame. A single read after a fixed wait is flaky, so read
    // until two consecutive reads agree.
    let spans = await page.evaluate(visibleText);
    for (let i = 0; i < 10; i += 1) {
      await page.waitForTimeout(300);
      const next = await page.evaluate(visibleText);
      if (next.length === spans.length && next.every((span, j) => span === spans[j])) {
        spans = next;
        break;
      }
      spans = next;
    }
    await page.close();
    return spans;
  };

  const legacy = await grab(`http://localhost:${legacyPort}/${legacyPath}`);
  const astro = await grab(`http://localhost:${astroPort}/ai-for-dummies/${astroPath}`);
  await browser.close();

  // Count occurrences, not membership. A set comparison reports zero when a
  // string the legacy page paints four times is painted three times here --
  // exactly the kind of near-miss that got past the earlier checks.
  const tally = (spans) => {
    const counts = new Map();
    for (const span of spans) counts.set(span, (counts.get(span) || 0) + 1);
    return counts;
  };

  const legacyCounts = tally(legacy);
  const astroCounts = tally(astro);

  const missing = [];
  for (const [span, count] of legacyCounts) {
    const short = count - (astroCounts.get(span) || 0);
    for (let i = 0; i < short; i += 1) missing.push(span);
  }

  // Both directions. A string the Astro page paints and the legacy page does
  // not is just as wrong: it means a translation was invented, or an English
  // string was left standing where the legacy page swaps it.
  const extra = [];
  for (const [span, count] of astroCounts) {
    const over = count - (legacyCounts.get(span) || 0);
    for (let i = 0; i < over; i += 1) extra.push(span);
  }

  // Order counts too. Both pages can paint the same strings while a block
  // sits in the wrong place -- the Portuguese eyebrow, or a reordered card
  // deck -- and a count-only comparison calls that clean.
  const firstOutOfOrder = legacy.findIndex((span, i) => astro[i] !== span);

  const mode = portuguese ? 'pt' : 'en';
  console.log(
    `${mode} · legacy ${legacy.length} spans · astro ${astro.length} spans · missing ${missing.length} · extra ${extra.length}`,
  );
  for (const span of missing) console.log(`  - ${span}`);
  for (const span of extra) console.log(`  + ${span}`);
  if (firstOutOfOrder !== -1) {
    console.log(`  order diverges at span ${firstOutOfOrder}`);
    console.log(`    legacy: ${legacy[firstOutOfOrder]}`);
    console.log(`    astro:  ${astro[firstOutOfOrder]}`);
  }
  process.exitCode = missing.length === 0 && extra.length === 0 && firstOutOfOrder === -1 ? 0 : 1;
} finally {
  stop();
}