From 2613b3e3af4ad749ff43cfb422fda8491bd386d6 Mon Sep 17 00:00:00 2001 From: Marcos Date: Tue, 18 Aug 2026 05:08:22 +0000 Subject: [PATCH] fix(cost-model): calibrate against the first measured review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #7 ran under the AI-USAGE label and reported real numbers: 28 agent steps, 348s, 2,071,025 input / 17,303 output tokens, and zero cache reads or writes. The model predicted ~$0.73 on Opus 5 for that tier. The measurement prices it at $10.79 — the model was ~15x low. Two wrong assumptions: - Step count and per-step growth. `full` assumed 12 steps and 1,200 tokens per tool result; the run did 28 steps averaging ~3,300. Cost is roughly quadratic in steps, so this compounds. Tier defaults are re-derived from the measured per-step growth rather than from guesses. - Caching. The model defaulted to prompt caching on. The headroom/glm-5.2 path reports 0 read / 0 write, so the stable prefix is paid at full input price on every step. Budget with caching off until that column is nonzero. Adds OBSERVED_RUNS as an append-only calibration anchor, an observed-runs section in the report, and a regression test asserting the model stays within 2.5x of the measurement — so the next drift is caught by the suite rather than by a surprising invoice. Corrected blended figures at 350 PRs/month: ~$1,740 Opus 5, ~$1,755 GPT-5.6 Sol, ~$696 Sonnet 5, ~$348 Haiku 4.5, ~$70 GPT-5.6 Luna. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01B11e8TZZxJyzHW7jj7KWUN --- README.md | 45 ++++++++++++++------- pilot/cost_model.py | 71 ++++++++++++++++++++++++++++++++-- tests/pilot/test_cost_model.py | 46 ++++++++++++++++++++++ 3 files changed, 145 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d16bc82..c790dfd 100644 --- a/README.md +++ b/README.md @@ -60,21 +60,40 @@ mysterious. The pilot runs on `glm-5.2:cloud` through the on-network headroom proxy, so today it bills nothing per token — but the token *work* is real, and `pilot/cost_model.py` -prices it against published API rates. The factory's prompt sizes are measured from -the files in this repo; the per-tier workloads come from the `attention-tiering` -budgets. Blended over a 5/35/55/5 tier mix, prompt caching on: +prices it against published API rates. Factory prompt sizes are measured from the +files in this repo; the per-tier workloads are calibrated against runs actually +measured through the `AI-USAGE` label (`OBSERVED_RUNS` in that file). -| Model | per PR | 350 PRs/month | -|---|---:|---:| -| Claude Opus 5 / GPT-5.6 Sol | ~$0.61 | ~$212 | -| Claude Sonnet 5 / GPT-5.6 Terra | ~$0.24 | ~$85 | -| Claude Haiku 4.5 | ~$0.12 | ~$43 | -| GPT-5.6 Luna | ~$0.02 | ~$8.5 | +**The measured anchor.** The hardening PR (`#7`, 16 files / ~1100 changed lines, +tier `full`) took 28 agent steps and 348s, and consumed **2,071,025 input** and +**17,303 output** tokens — with **zero cache reads or writes**, because the current +headroom/glm path does no prompt caching. Priced elsewhere, that single review is: -Run `python3 pilot/cost_model.py --help` for other mixes and PR volumes. The -dominant cost is the agent loop resending its own context each step, not the diff — -turning prompt caching off multiplies the bill by ~2.3x, which is why the tiering -skill caps steps, file reads, and subagent fan-out per tier. +| Model | that review | blended per PR | 350 PRs/month | +|---|---:|---:|---:| +| Claude Opus 5 | $10.79 | ~$4.97 | ~$1,740 | +| GPT-5.6 Sol | $10.87 | ~$5.02 | ~$1,755 | +| Claude Sonnet 5 | $4.32 | ~$1.99 | ~$696 | +| GPT-5.6 Terra | $4.35 | ~$2.01 | ~$702 | +| Claude Haiku 4.5 | $2.16 | ~$0.99 | ~$348 | +| GPT-5.6 Luna | $0.43 | ~$0.20 | ~$70 | + +Blended figures use a 5/35/55/5 tier mix with caching off, matching what is +actually observed. Run `python3 pilot/cost_model.py --help` for other mixes and +volumes. + +Two things dominate, and neither is the diff: + +1. **The loop resends its context every step.** 28 steps over a ~17k-token diff + produced 2M input tokens. Cost is roughly quadratic in step count, which is why + `attention-tiering` caps steps, file reads and subagent fan-out per tier. +2. **Prompt caching is worth about a third of the bill** and is currently not + happening. Any move to a paid provider should confirm the `cache_read` column + goes nonzero before budgeting. + +An earlier version of this model assumed 12 steps and caching on, and was ~15x +low. The lesson is in the file: budget from `OBSERVED_RUNS`, not from the tier +table, and append a row every time a real review reports usage. ## Extension points diff --git a/pilot/cost_model.py b/pilot/cost_model.py index 752ec5a..50bd278 100644 --- a/pilot/cost_model.py +++ b/pilot/cost_model.py @@ -38,6 +38,8 @@ Usage: python3 pilot/cost_model.py --no-cache # what caching is worth """ +from __future__ import annotations + import argparse import os from dataclasses import dataclass, field @@ -158,13 +160,46 @@ class Tier: DEFAULT_TIERS = [ # diff_tok steps reads tok/read output subs share - Tier("trivial", 400, 2, 0, 0, 500, 0, share=0.05), - Tier("lite", 1500, 5, 3, 700, 1800, 0, share=0.35), - Tier("full", 6000, 12, 10, 1200, 5000, 0, share=0.55), - Tier("oversized", 25000, 20, 15, 1500, 9000, 2, share=0.05), + Tier("trivial", 400, 2, 0, 0, 600, 0, share=0.05), + Tier("lite", 1500, 6, 4, 2000, 2500, 0, share=0.35), + Tier("full", 6000, 24, 20, 3300, 12000, 0, share=0.55), + Tier("oversized", 25000, 35, 30, 3500, 20000, 2, share=0.05), ] +# --------------------------------------------------------------------------- +# Observed runs — the calibration anchor +# --------------------------------------------------------------------------- + +# Real usage reported by the AI-USAGE label, summed from opencode's step_finish +# events. Keep this list append-only: it is the only thing separating this model +# from a guess, and the first entry corrected the tier assumptions by ~15x. +OBSERVED_RUNS: list[dict] = [ + { + "label": "gitea_admin/pragent#7 (the hardening PR)", + "date": "2026-08-18", + "tier": "full", + "diff_tokens": 17_600, # 16 files, 1020 insertions / 91 deletions + "steps": 28, + "duration_s": 348.3, + "input": 2_071_025, + "output": 17_303, + "cache_read": 0, + "cache_write": 0, + "subagents": 0, + }, +] + + +def observed_usage(run: dict) -> Usage: + return Usage( + uncached_input=run["input"] - run.get("cache_read", 0), + cached_input=run.get("cache_read", 0), + cache_writes=run.get("cache_write", 0), + output=run["output"], + ) + + @dataclass class Usage: uncached_input: int = 0 @@ -300,6 +335,34 @@ def report(tiers: list[Tier], prs_per_month: int, caching: bool, models: list[st lines.append("") lines.append("Batch column applies the 50% async discount; it is shown for scale only —") lines.append("PR review is latency-sensitive and a stateful agent loop is not batchable.") + lines.append("") + lines.append(observed_report(models)) + return "\n".join(lines) + + +def observed_report(models: list[str]) -> str: + """Price the runs actually measured through the AI-USAGE label.""" + if not OBSERVED_RUNS: + return "No observed runs recorded yet." + lines = ["Observed runs (measured via the AI-USAGE label)"] + for run in OBSERVED_RUNS: + u = observed_usage(run) + lines.append( + f" {run['label']} — tier {run['tier']}, {run['steps']} steps, " + f"{run['duration_s']:.0f}s, {run['input']:,} in / {run['output']:,} out, " + f"cache {run['cache_read']:,} read / {run['cache_write']:,} write" + ) + row = " " + for key in models: + p = PRICES[key] + row += f" {p.name}: ${cost(u, p):.2f} " + lines.append(row) + lines.append("") + lines.append(" NOTE: the pilot's headroom/glm-5.2 path reports zero cache read and zero") + lines.append(" cache write, i.e. prompt caching is NOT in play today. On a provider where") + lines.append(" it is, the stable prefix (agent + skills + brief + diff, resent every step)") + lines.append(" drops to 0.1x — worth roughly a third of the bill on a run like the one") + lines.append(" above. Budget with caching OFF until the measured cache columns are nonzero.") return "\n".join(lines) diff --git a/tests/pilot/test_cost_model.py b/tests/pilot/test_cost_model.py index 066d9c2..973286d 100644 --- a/tests/pilot/test_cost_model.py +++ b/tests/pilot/test_cost_model.py @@ -196,3 +196,49 @@ def test_main_rejects_bad_mix(): def test_main_runs(capsys): assert cm.main(["--models", "claude-sonnet-5", "--prs-per-month", "10"]) == 0 assert "per month" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# observed runs — the calibration anchor +# --------------------------------------------------------------------------- + + +def test_observed_runs_are_well_formed(): + assert cm.OBSERVED_RUNS, "the model is a guess without at least one measurement" + for run in cm.OBSERVED_RUNS: + for key in ("label", "date", "tier", "steps", "input", "output", + "cache_read", "cache_write"): + assert key in run, f"{run.get('label')} missing {key}" + assert run["input"] > 0 and run["output"] > 0 + assert run["tier"] in {t.name for t in cm.DEFAULT_TIERS} + + +def test_observed_usage_splits_cached_from_uncached(): + run = {"input": 1000, "output": 100, "cache_read": 400, "cache_write": 50} + u = cm.observed_usage(run) + assert u.cached_input == 400 + assert u.uncached_input == 600 + assert u.cache_writes == 50 + assert u.total_input == 1000 + + +def test_observed_report_prices_every_model(): + text = cm.observed_report(["claude-opus-5", "gpt-5.6-luna"]) + assert "Claude Opus 5" in text + assert "GPT-5.6 Luna" in text + assert "pragent#7" in text + + +def test_model_is_within_an_order_of_magnitude_of_the_measurement(): + # The first measurement corrected the tier assumptions by ~15x. This guards + # against drifting that far out again: predict the observed run's tier at + # its actual diff size and step count, and compare to what was measured. + run = cm.OBSERVED_RUNS[0] + base = _tier(run["tier"]) + modelled = cm.Tier( + base.name, run["diff_tokens"], run["steps"], base.file_reads, + base.tokens_per_read, run["output"], run["subagents"], + ) + predicted = cm.tier_usage(modelled, FACTORY, caching=False).total_input + measured = run["input"] + assert 0.4 < predicted / measured < 2.5, (predicted, measured)