fix(cost-model): calibrate against the first measured review

PR #7 ran under the AI-USAGE label and reported real numbers: 28 agent steps,
348s, 2,071,025 input / 17,303 output tokens, and zero cache reads or writes.
The model predicted ~$0.73 on Opus 5 for that tier. The measurement prices it
at $10.79 — the model was ~15x low.

Two wrong assumptions:

- Step count and per-step growth. `full` assumed 12 steps and 1,200 tokens per
  tool result; the run did 28 steps averaging ~3,300. Cost is roughly quadratic
  in steps, so this compounds. Tier defaults are re-derived from the measured
  per-step growth rather than from guesses.
- Caching. The model defaulted to prompt caching on. The headroom/glm-5.2 path
  reports 0 read / 0 write, so the stable prefix is paid at full input price on
  every step. Budget with caching off until that column is nonzero.

Adds OBSERVED_RUNS as an append-only calibration anchor, an observed-runs
section in the report, and a regression test asserting the model stays within
2.5x of the measurement — so the next drift is caught by the suite rather than
by a surprising invoice.

Corrected blended figures at 350 PRs/month: ~$1,740 Opus 5, ~$1,755 GPT-5.6
Sol, ~$696 Sonnet 5, ~$348 Haiku 4.5, ~$70 GPT-5.6 Luna.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B11e8TZZxJyzHW7jj7KWUN
This commit is contained in:
Marcos
2026-08-18 05:08:22 +00:00
parent 30d2a3d7da
commit 2613b3e3af
3 changed files with 145 additions and 17 deletions
+67 -4
View File
@@ -38,6 +38,8 @@ Usage:
python3 pilot/cost_model.py --no-cache # what caching is worth
"""
from __future__ import annotations
import argparse
import os
from dataclasses import dataclass, field
@@ -158,13 +160,46 @@ class Tier:
DEFAULT_TIERS = [
# diff_tok steps reads tok/read output subs share
Tier("trivial", 400, 2, 0, 0, 500, 0, share=0.05),
Tier("lite", 1500, 5, 3, 700, 1800, 0, share=0.35),
Tier("full", 6000, 12, 10, 1200, 5000, 0, share=0.55),
Tier("oversized", 25000, 20, 15, 1500, 9000, 2, share=0.05),
Tier("trivial", 400, 2, 0, 0, 600, 0, share=0.05),
Tier("lite", 1500, 6, 4, 2000, 2500, 0, share=0.35),
Tier("full", 6000, 24, 20, 3300, 12000, 0, share=0.55),
Tier("oversized", 25000, 35, 30, 3500, 20000, 2, share=0.05),
]
# ---------------------------------------------------------------------------
# Observed runs — the calibration anchor
# ---------------------------------------------------------------------------
# Real usage reported by the AI-USAGE label, summed from opencode's step_finish
# events. Keep this list append-only: it is the only thing separating this model
# from a guess, and the first entry corrected the tier assumptions by ~15x.
OBSERVED_RUNS: list[dict] = [
{
"label": "gitea_admin/pragent#7 (the hardening PR)",
"date": "2026-08-18",
"tier": "full",
"diff_tokens": 17_600, # 16 files, 1020 insertions / 91 deletions
"steps": 28,
"duration_s": 348.3,
"input": 2_071_025,
"output": 17_303,
"cache_read": 0,
"cache_write": 0,
"subagents": 0,
},
]
def observed_usage(run: dict) -> Usage:
return Usage(
uncached_input=run["input"] - run.get("cache_read", 0),
cached_input=run.get("cache_read", 0),
cache_writes=run.get("cache_write", 0),
output=run["output"],
)
@dataclass
class Usage:
uncached_input: int = 0
@@ -300,6 +335,34 @@ def report(tiers: list[Tier], prs_per_month: int, caching: bool, models: list[st
lines.append("")
lines.append("Batch column applies the 50% async discount; it is shown for scale only —")
lines.append("PR review is latency-sensitive and a stateful agent loop is not batchable.")
lines.append("")
lines.append(observed_report(models))
return "\n".join(lines)
def observed_report(models: list[str]) -> str:
"""Price the runs actually measured through the AI-USAGE label."""
if not OBSERVED_RUNS:
return "No observed runs recorded yet."
lines = ["Observed runs (measured via the AI-USAGE label)"]
for run in OBSERVED_RUNS:
u = observed_usage(run)
lines.append(
f" {run['label']} — tier {run['tier']}, {run['steps']} steps, "
f"{run['duration_s']:.0f}s, {run['input']:,} in / {run['output']:,} out, "
f"cache {run['cache_read']:,} read / {run['cache_write']:,} write"
)
row = " "
for key in models:
p = PRICES[key]
row += f" {p.name}: ${cost(u, p):.2f} "
lines.append(row)
lines.append("")
lines.append(" NOTE: the pilot's headroom/glm-5.2 path reports zero cache read and zero")
lines.append(" cache write, i.e. prompt caching is NOT in play today. On a provider where")
lines.append(" it is, the stable prefix (agent + skills + brief + diff, resent every step)")
lines.append(" drops to 0.1x — worth roughly a third of the bill on a run like the one")
lines.append(" above. Budget with caching OFF until the measured cache columns are nonzero.")
return "\n".join(lines)