README.md

@@ -60,21 +60,40 @@ mysterious.
 
 The pilot runs on `glm-5.2:cloud` through the on-network headroom proxy, so today it
 bills nothing per token — but the token *work* is real, and `pilot/cost_model.py`
-prices it against published API rates. The factory's prompt sizes are measured from
+prices it against published API rates. Factory prompt sizes are measured from the
-the files in this repo; the per-tier workloads come from the `attention-tiering`
+files in this repo; the per-tier workloads are calibrated against runs actually
-budgets. Blended over a 5/35/55/5 tier mix, prompt caching on:
+measured through the `AI-USAGE` label (`OBSERVED_RUNS` in that file).
 
-| Model | per PR | 350 PRs/month |
+**The measured anchor.** The hardening PR (`#7`, 16 files / ~1100 changed lines,
-|---|---:|---:|
+tier `full`) took 28 agent steps and 348s, and consumed **2,071,025 input** and
-| Claude Opus 5 / GPT-5.6 Sol | ~$0.61 | ~$212 |
+**17,303 output** tokens — with **zero cache reads or writes**, because the current
-| Claude Sonnet 5 / GPT-5.6 Terra | ~$0.24 | ~$85 |
+headroom/glm path does no prompt caching. Priced elsewhere, that single review is:
-| Claude Haiku 4.5 | ~$0.12 | ~$43 |
-| GPT-5.6 Luna | ~$0.02 | ~$8.5 |
 
-Run `python3 pilot/cost_model.py --help` for other mixes and PR volumes. The
+| Model | that review | blended per PR | 350 PRs/month |
-dominant cost is the agent loop resending its own context each step, not the diff —
+|---|---:|---:|---:|
-turning prompt caching off multiplies the bill by ~2.3x, which is why the tiering
+| Claude Opus 5 | $10.79 | ~$4.97 | ~$1,740 |
-skill caps steps, file reads, and subagent fan-out per tier.
+| GPT-5.6 Sol | $10.87 | ~$5.02 | ~$1,755 |
+| Claude Sonnet 5 | $4.32 | ~$1.99 | ~$696 |
+| GPT-5.6 Terra | $4.35 | ~$2.01 | ~$702 |
+| Claude Haiku 4.5 | $2.16 | ~$0.99 | ~$348 |
+| GPT-5.6 Luna | $0.43 | ~$0.20 | ~$70 |
+
+Blended figures use a 5/35/55/5 tier mix with caching off, matching what is
+actually observed. Run `python3 pilot/cost_model.py --help` for other mixes and
+volumes.
+
+Two things dominate, and neither is the diff:
+
+1. **The loop resends its context every step.** 28 steps over a ~17k-token diff
+   produced 2M input tokens. Cost is roughly quadratic in step count, which is why
+   `attention-tiering` caps steps, file reads and subagent fan-out per tier.
+2. **Prompt caching is worth about a third of the bill** and is currently not
+   happening. Any move to a paid provider should confirm the `cache_read` column
+   goes nonzero before budgeting.
+
+An earlier version of this model assumed 12 steps and caching on, and was ~15x
+low. The lesson is in the file: budget from `OBSERVED_RUNS`, not from the tier
+table, and append a row every time a real review reports usage.
 
 ## Extension points
 

pilot/cost_model.py

@@ -38,6 +38,8 @@ Usage:
     python3 pilot/cost_model.py --no-cache            # what caching is worth
 """
 
+from __future__ import annotations
+
 import argparse
 import os
 from dataclasses import dataclass, field
@@ -158,13 +160,46 @@ class Tier:
 
 DEFAULT_TIERS = [
     # diff_tok  steps  reads  tok/read  output  subs  share
-    Tier("trivial",     400,  2,  0,    0,      500,  0, share=0.05),
+    Tier("trivial",     400,  2,  0,     0,      600,  0, share=0.05),
-    Tier("lite",       1500,  5,  3,  700,     1800,  0, share=0.35),
+    Tier("lite",       1500,  6,  4,  2000,     2500,  0, share=0.35),
-    Tier("full",       6000, 12, 10, 1200,     5000,  0, share=0.55),
+    Tier("full",       6000, 24, 20,  3300,    12000,  0, share=0.55),
-    Tier("oversized", 25000, 20, 15, 1500,     9000,  2, share=0.05),
+    Tier("oversized", 25000, 35, 30,  3500,    20000,  2, share=0.05),
 ]
 
 
+# ---------------------------------------------------------------------------
+# Observed runs — the calibration anchor
+# ---------------------------------------------------------------------------
+
+# Real usage reported by the AI-USAGE label, summed from opencode's step_finish
+# events. Keep this list append-only: it is the only thing separating this model
+# from a guess, and the first entry corrected the tier assumptions by ~15x.
+OBSERVED_RUNS: list[dict] = [
+    {
+        "label": "gitea_admin/pragent#7 (the hardening PR)",
+        "date": "2026-08-18",
+        "tier": "full",
+        "diff_tokens": 17_600,     # 16 files, 1020 insertions / 91 deletions
+        "steps": 28,
+        "duration_s": 348.3,
+        "input": 2_071_025,
+        "output": 17_303,
+        "cache_read": 0,
+        "cache_write": 0,
+        "subagents": 0,
+    },
+]
+
+
+def observed_usage(run: dict) -> Usage:
+    return Usage(
+        uncached_input=run["input"] - run.get("cache_read", 0),
+        cached_input=run.get("cache_read", 0),
+        cache_writes=run.get("cache_write", 0),
+        output=run["output"],
+    )
+
+
 @dataclass
 class Usage:
     uncached_input: int = 0
@@ -300,6 +335,34 @@ def report(tiers: list[Tier], prs_per_month: int, caching: bool, models: list[st
     lines.append("")
     lines.append("Batch column applies the 50% async discount; it is shown for scale only —")
     lines.append("PR review is latency-sensitive and a stateful agent loop is not batchable.")
+    lines.append("")
+    lines.append(observed_report(models))
+    return "\n".join(lines)
+
+
+def observed_report(models: list[str]) -> str:
+    """Price the runs actually measured through the AI-USAGE label."""
+    if not OBSERVED_RUNS:
+        return "No observed runs recorded yet."
+    lines = ["Observed runs (measured via the AI-USAGE label)"]
+    for run in OBSERVED_RUNS:
+        u = observed_usage(run)
+        lines.append(
+            f"  {run['label']} — tier {run['tier']}, {run['steps']} steps, "
+            f"{run['duration_s']:.0f}s, {run['input']:,} in / {run['output']:,} out, "
+            f"cache {run['cache_read']:,} read / {run['cache_write']:,} write"
+        )
+        row = "   "
+        for key in models:
+            p = PRICES[key]
+            row += f" {p.name}: ${cost(u, p):.2f} "
+        lines.append(row)
+    lines.append("")
+    lines.append("  NOTE: the pilot's headroom/glm-5.2 path reports zero cache read and zero")
+    lines.append("  cache write, i.e. prompt caching is NOT in play today. On a provider where")
+    lines.append("  it is, the stable prefix (agent + skills + brief + diff, resent every step)")
+    lines.append("  drops to 0.1x — worth roughly a third of the bill on a run like the one")
+    lines.append("  above. Budget with caching OFF until the measured cache columns are nonzero.")
     return "\n".join(lines)
 
 

tests/pilot/test_cost_model.py

@@ -196,3 +196,49 @@ def test_main_rejects_bad_mix():
 def test_main_runs(capsys):
     assert cm.main(["--models", "claude-sonnet-5", "--prs-per-month", "10"]) == 0
     assert "per month" in capsys.readouterr().out
+
+
+# ---------------------------------------------------------------------------
+# observed runs — the calibration anchor
+# ---------------------------------------------------------------------------
+
+
+def test_observed_runs_are_well_formed():
+    assert cm.OBSERVED_RUNS, "the model is a guess without at least one measurement"
+    for run in cm.OBSERVED_RUNS:
+        for key in ("label", "date", "tier", "steps", "input", "output",
+                    "cache_read", "cache_write"):
+            assert key in run, f"{run.get('label')} missing {key}"
+        assert run["input"] > 0 and run["output"] > 0
+        assert run["tier"] in {t.name for t in cm.DEFAULT_TIERS}
+
+
+def test_observed_usage_splits_cached_from_uncached():
+    run = {"input": 1000, "output": 100, "cache_read": 400, "cache_write": 50}
+    u = cm.observed_usage(run)
+    assert u.cached_input == 400
+    assert u.uncached_input == 600
+    assert u.cache_writes == 50
+    assert u.total_input == 1000
+
+
+def test_observed_report_prices_every_model():
+    text = cm.observed_report(["claude-opus-5", "gpt-5.6-luna"])
+    assert "Claude Opus 5" in text
+    assert "GPT-5.6 Luna" in text
+    assert "pragent#7" in text
+
+
+def test_model_is_within_an_order_of_magnitude_of_the_measurement():
+    # The first measurement corrected the tier assumptions by ~15x. This guards
+    # against drifting that far out again: predict the observed run's tier at
+    # its actual diff size and step count, and compare to what was measured.
+    run = cm.OBSERVED_RUNS[0]
+    base = _tier(run["tier"])
+    modelled = cm.Tier(
+        base.name, run["diff_tokens"], run["steps"], base.file_reads,
+        base.tokens_per_read, run["output"], run["subagents"],
+    )
+    predicted = cm.tier_usage(modelled, FACTORY, caching=False).total_input
+    measured = run["input"]
+    assert 0.4 < predicted / measured < 2.5, (predicted, measured)