fix(cost-model): calibrate against the first measured review
PR #7 ran under the AI-USAGE label and reported real numbers: 28 agent steps, 348s, 2,071,025 input / 17,303 output tokens, and zero cache reads or writes. The model predicted ~$0.73 on Opus 5 for that tier. The measurement prices it at $10.79 — the model was ~15x low. Two wrong assumptions: - Step count and per-step growth. `full` assumed 12 steps and 1,200 tokens per tool result; the run did 28 steps averaging ~3,300. Cost is roughly quadratic in steps, so this compounds. Tier defaults are re-derived from the measured per-step growth rather than from guesses. - Caching. The model defaulted to prompt caching on. The headroom/glm-5.2 path reports 0 read / 0 write, so the stable prefix is paid at full input price on every step. Budget with caching off until that column is nonzero. Adds OBSERVED_RUNS as an append-only calibration anchor, an observed-runs section in the report, and a regression test asserting the model stays within 2.5x of the measurement — so the next drift is caught by the suite rather than by a surprising invoice. Corrected blended figures at 350 PRs/month: ~$1,740 Opus 5, ~$1,755 GPT-5.6 Sol, ~$696 Sonnet 5, ~$348 Haiku 4.5, ~$70 GPT-5.6 Luna. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B11e8TZZxJyzHW7jj7KWUN
This commit is contained in:
@@ -196,3 +196,49 @@ def test_main_rejects_bad_mix():
|
||||
def test_main_runs(capsys):
|
||||
assert cm.main(["--models", "claude-sonnet-5", "--prs-per-month", "10"]) == 0
|
||||
assert "per month" in capsys.readouterr().out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# observed runs — the calibration anchor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_observed_runs_are_well_formed():
|
||||
assert cm.OBSERVED_RUNS, "the model is a guess without at least one measurement"
|
||||
for run in cm.OBSERVED_RUNS:
|
||||
for key in ("label", "date", "tier", "steps", "input", "output",
|
||||
"cache_read", "cache_write"):
|
||||
assert key in run, f"{run.get('label')} missing {key}"
|
||||
assert run["input"] > 0 and run["output"] > 0
|
||||
assert run["tier"] in {t.name for t in cm.DEFAULT_TIERS}
|
||||
|
||||
|
||||
def test_observed_usage_splits_cached_from_uncached():
|
||||
run = {"input": 1000, "output": 100, "cache_read": 400, "cache_write": 50}
|
||||
u = cm.observed_usage(run)
|
||||
assert u.cached_input == 400
|
||||
assert u.uncached_input == 600
|
||||
assert u.cache_writes == 50
|
||||
assert u.total_input == 1000
|
||||
|
||||
|
||||
def test_observed_report_prices_every_model():
|
||||
text = cm.observed_report(["claude-opus-5", "gpt-5.6-luna"])
|
||||
assert "Claude Opus 5" in text
|
||||
assert "GPT-5.6 Luna" in text
|
||||
assert "pragent#7" in text
|
||||
|
||||
|
||||
def test_model_is_within_an_order_of_magnitude_of_the_measurement():
|
||||
# The first measurement corrected the tier assumptions by ~15x. This guards
|
||||
# against drifting that far out again: predict the observed run's tier at
|
||||
# its actual diff size and step count, and compare to what was measured.
|
||||
run = cm.OBSERVED_RUNS[0]
|
||||
base = _tier(run["tier"])
|
||||
modelled = cm.Tier(
|
||||
base.name, run["diff_tokens"], run["steps"], base.file_reads,
|
||||
base.tokens_per_read, run["output"], run["subagents"],
|
||||
)
|
||||
predicted = cm.tier_usage(modelled, FACTORY, caching=False).total_input
|
||||
measured = run["input"]
|
||||
assert 0.4 < predicted / measured < 2.5, (predicted, measured)
|
||||
|
||||
Reference in New Issue
Block a user