fix(cost-model): calibrate against the first measured review
PR #7 ran under the AI-USAGE label and reported real numbers: 28 agent steps, 348s, 2,071,025 input / 17,303 output tokens, and zero cache reads or writes. The model predicted ~$0.73 on Opus 5 for that tier. The measurement prices it at $10.79 — the model was ~15x low. Two wrong assumptions: - Step count and per-step growth. `full` assumed 12 steps and 1,200 tokens per tool result; the run did 28 steps averaging ~3,300. Cost is roughly quadratic in steps, so this compounds. Tier defaults are re-derived from the measured per-step growth rather than from guesses. - Caching. The model defaulted to prompt caching on. The headroom/glm-5.2 path reports 0 read / 0 write, so the stable prefix is paid at full input price on every step. Budget with caching off until that column is nonzero. Adds OBSERVED_RUNS as an append-only calibration anchor, an observed-runs section in the report, and a regression test asserting the model stays within 2.5x of the measurement — so the next drift is caught by the suite rather than by a surprising invoice. Corrected blended figures at 350 PRs/month: ~$1,740 Opus 5, ~$1,755 GPT-5.6 Sol, ~$696 Sonnet 5, ~$348 Haiku 4.5, ~$70 GPT-5.6 Luna. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B11e8TZZxJyzHW7jj7KWUN
This commit is contained in:
+67
-4
@@ -38,6 +38,8 @@ Usage:
|
||||
python3 pilot/cost_model.py --no-cache # what caching is worth
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
@@ -158,13 +160,46 @@ class Tier:
|
||||
|
||||
DEFAULT_TIERS = [
|
||||
# diff_tok steps reads tok/read output subs share
|
||||
Tier("trivial", 400, 2, 0, 0, 500, 0, share=0.05),
|
||||
Tier("lite", 1500, 5, 3, 700, 1800, 0, share=0.35),
|
||||
Tier("full", 6000, 12, 10, 1200, 5000, 0, share=0.55),
|
||||
Tier("oversized", 25000, 20, 15, 1500, 9000, 2, share=0.05),
|
||||
Tier("trivial", 400, 2, 0, 0, 600, 0, share=0.05),
|
||||
Tier("lite", 1500, 6, 4, 2000, 2500, 0, share=0.35),
|
||||
Tier("full", 6000, 24, 20, 3300, 12000, 0, share=0.55),
|
||||
Tier("oversized", 25000, 35, 30, 3500, 20000, 2, share=0.05),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Observed runs — the calibration anchor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Real usage reported by the AI-USAGE label, summed from opencode's step_finish
|
||||
# events. Keep this list append-only: it is the only thing separating this model
|
||||
# from a guess, and the first entry corrected the tier assumptions by ~15x.
|
||||
OBSERVED_RUNS: list[dict] = [
|
||||
{
|
||||
"label": "gitea_admin/pragent#7 (the hardening PR)",
|
||||
"date": "2026-08-18",
|
||||
"tier": "full",
|
||||
"diff_tokens": 17_600, # 16 files, 1020 insertions / 91 deletions
|
||||
"steps": 28,
|
||||
"duration_s": 348.3,
|
||||
"input": 2_071_025,
|
||||
"output": 17_303,
|
||||
"cache_read": 0,
|
||||
"cache_write": 0,
|
||||
"subagents": 0,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def observed_usage(run: dict) -> Usage:
|
||||
return Usage(
|
||||
uncached_input=run["input"] - run.get("cache_read", 0),
|
||||
cached_input=run.get("cache_read", 0),
|
||||
cache_writes=run.get("cache_write", 0),
|
||||
output=run["output"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
uncached_input: int = 0
|
||||
@@ -300,6 +335,34 @@ def report(tiers: list[Tier], prs_per_month: int, caching: bool, models: list[st
|
||||
lines.append("")
|
||||
lines.append("Batch column applies the 50% async discount; it is shown for scale only —")
|
||||
lines.append("PR review is latency-sensitive and a stateful agent loop is not batchable.")
|
||||
lines.append("")
|
||||
lines.append(observed_report(models))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def observed_report(models: list[str]) -> str:
|
||||
"""Price the runs actually measured through the AI-USAGE label."""
|
||||
if not OBSERVED_RUNS:
|
||||
return "No observed runs recorded yet."
|
||||
lines = ["Observed runs (measured via the AI-USAGE label)"]
|
||||
for run in OBSERVED_RUNS:
|
||||
u = observed_usage(run)
|
||||
lines.append(
|
||||
f" {run['label']} — tier {run['tier']}, {run['steps']} steps, "
|
||||
f"{run['duration_s']:.0f}s, {run['input']:,} in / {run['output']:,} out, "
|
||||
f"cache {run['cache_read']:,} read / {run['cache_write']:,} write"
|
||||
)
|
||||
row = " "
|
||||
for key in models:
|
||||
p = PRICES[key]
|
||||
row += f" {p.name}: ${cost(u, p):.2f} "
|
||||
lines.append(row)
|
||||
lines.append("")
|
||||
lines.append(" NOTE: the pilot's headroom/glm-5.2 path reports zero cache read and zero")
|
||||
lines.append(" cache write, i.e. prompt caching is NOT in play today. On a provider where")
|
||||
lines.append(" it is, the stable prefix (agent + skills + brief + diff, resent every step)")
|
||||
lines.append(" drops to 0.1x — worth roughly a third of the bill on a run like the one")
|
||||
lines.append(" above. Budget with caching OFF until the measured cache columns are nonzero.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user