Files
indd-utils/tools/fetch_fixtures.py
aiongg bf72b93d23 test: download fixtures from pinned sources instead of committing them
Remove the sample documents, PDF and licence copies from tests/fixtures/.
tests/fixtures/manifest.json lists each file's download URL (pinned to a
commit for GitHub sources), size and SHA-256. tools/fetch_fixtures.py
downloads them into the git-ignored tests/fixtures/files/ and verifies
them. The fixture tests read from there and skip when it is absent.
2026-10-07 05:47:17 +00:00

91 lines
3.0 KiB
Python

#!/usr/bin/env python3
"""Download the test fixtures listed in tests/fixtures/manifest.json.
Each file is saved under tests/fixtures/files/ (git-ignored) and checked
against the size and SHA-256 in the manifest. Files that are already
present and match are not downloaded again. Exits non-zero if any file
cannot be downloaded or does not match.
Usage: python3 -I tools/fetch_fixtures.py
"""
import hashlib
import json
import os
import sys
import tempfile
import urllib.request
from pathlib import Path, PurePosixPath
ROOT = Path(__file__).resolve().parent.parent
MANIFEST = ROOT / "tests" / "fixtures" / "manifest.json"
DEST = ROOT / "tests" / "fixtures" / "files"
TIMEOUT = 60
def sha256_of(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1 << 16), b""):
h.update(chunk)
return h.hexdigest()
def matches(path: Path, size: int, sha256: str) -> bool:
return path.is_file() and path.stat().st_size == size and sha256_of(path) == sha256
def target(rel: str) -> Path:
"""Resolve a manifest path, refusing anything outside DEST."""
p = PurePosixPath(rel)
if p.is_absolute() or ".." in p.parts or not p.parts:
raise ValueError(f"bad path in manifest: {rel!r}")
return DEST.joinpath(*p.parts)
def download(url: str, path: Path, size: int, sha256: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".part-")
try:
h = hashlib.sha256()
n = 0
req = urllib.request.Request(url, headers={"User-Agent": "indd-fetch-fixtures"})
with os.fdopen(fd, "wb") as out, urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
while chunk := resp.read(1 << 16):
n += len(chunk)
if n > size:
raise ValueError(f"more than the expected {size} bytes")
h.update(chunk)
out.write(chunk)
if n != size:
raise ValueError(f"got {n} bytes, expected {size}")
if h.hexdigest() != sha256:
raise ValueError(f"SHA-256 {h.hexdigest()}, expected {sha256}")
os.replace(tmp, path)
except BaseException:
Path(tmp).unlink(missing_ok=True)
raise
def main() -> int:
entries = json.loads(MANIFEST.read_text(encoding="utf-8"))["files"]
failures = 0
for e in entries:
rel, size, sha256 = e["path"], e["size"], e["sha256"]
try:
path = target(rel)
if matches(path, size, sha256):
print(f"present {rel}")
continue
download(e["url"], path, size, sha256)
print(f"fetched {rel}")
except Exception as err:
failures += 1
print(f"FAILED {rel}: {err}", file=sys.stderr)
print(f"{len(entries) - failures} of {len(entries)} fixtures ready in {DEST.relative_to(ROOT)}/")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())