From 33e4c16782403d898ad1a59dd24137d137a01231 Mon Sep 17 00:00:00 2001 From: claude Date: Sat, 22 Aug 2026 01:06:02 +0000 Subject: [PATCH] feat(usage): multi-provider equivalent cost table --- pilot/ai_review.py | 63 ++++++++++++++++++++++------ tests/pilot/test_ai_review.py | 78 +++++++++++++++++++++++++++++------ 2 files changed, 115 insertions(+), 26 deletions(-) diff --git a/pilot/ai_review.py b/pilot/ai_review.py index 4112cc3..ce8a773 100644 --- a/pilot/ai_review.py +++ b/pilot/ai_review.py @@ -91,6 +91,14 @@ STYLE_DEFAULTS: dict[str, tuple[int, str]] = { # (env) or `.pr-review.json:cost_target` (per repo). DEFAULT_PRICE_TARGET = "claude-sonnet-5" +# Default roster of paid providers shown in the equivalent-cost table when +# `.pr-review.json` does not pin `compare_against`. The pilot is free-tier only, +# so this list is the operator's budgeting signal — it answers "what would this +# have cost on a mainstream paid API?". Override per-repo via +# `.pr-review.json:compare_against` (capped at 12 entries; unknown keys are +# dropped with a stderr line at parse time). +DEFAULT_COMPARE_AGAINST = ("claude-sonnet-5", "gpt-5", "gemini-2.5-pro", "grok-4.5") + SYSTEM_PROMPT = """You are a senior, pragmatic code reviewer. Review the pull request diff below. Report ONLY real, actionable issues: correctness bugs, security problems, risky @@ -1211,9 +1219,13 @@ def findings_table(findings: list[dict]) -> str: def _render_collapsible_usage(usage: dict | None, model: str, config: dict | None) -> str: """Render the telemetry as a collapsible ``
`` block. - Empty string when `usage` is None. The cost-equivalent line is always - shown (it's the operator's budgeting signal). The `actual` line is shown - but the FREE-TIER note is collapsed into a single short clause. + Empty string when `usage` is None. The equivalent-cost table is the + operator's budgeting signal — the pilot runs on a free tier, so the + `actual` line is $0.00; the table shows what the same measured tokens + would bill on mainstream paid APIs (configurable via `compare_against`, + defaulting to ``DEFAULT_COMPARE_AGAINST``). The row matching `cost_target` + is bolded so the price target stands out. The whole table is omitted when + every row would be $0 (no work done). """ if not usage: return "" @@ -1222,15 +1234,34 @@ def _render_collapsible_usage(usage: dict | None, model: str, config: dict | Non actual = usage.get("cost") or 0.0 actual_s = f"${actual:.4f}" if actual else "$0.00" actual_note = " (headroom glm-5.2:cloud — free tier)" if not actual else "" - price_key, price_err = _resolve_price_target(config) - from cost_model import PRICES - eq = equivalent_cost(usage, price_key) - eq_s = f"${eq:.4f}" if eq else "$0.00" - eq_label = PRICES[price_key].name - eq_note = ( - f" _(price target: `{price_key}`; {price_err})_" - if price_err else "" - ) + cost_target, price_err = _resolve_price_target(config) + if price_err: + # Surface config typos loudly but do not pollute the posted summary + # body — typos at the table-row level would render as English + # mid-table and look like a model error. + print(f"pragent: {price_err}", file=sys.stderr, flush=True) + # Lazy: cost_model has no dep on ai_review, and the ollama path + # never reaches this branch. + from cost_model import PRICES as _PRICES + cfg = config or {} + compare: list[str] = list(cfg.get("compare_against") or DEFAULT_COMPARE_AGAINST) + # Always include the resolved cost_target (env + config), even when the + # operator pinned a different `compare_against` roster — the price target + # row is the one maintainers eyeball against. Skip silently if the key + # isn't a known Price (e.g. a typo that slipped past stderr earlier). + if cost_target in _PRICES and cost_target not in compare: + compare.append(cost_target) + eq_rows: list[str] = [] + for key in compare: + if key not in _PRICES: + continue + c = equivalent_cost(usage, key) + if c <= 0: + continue + label = _PRICES[key].name + cost_str = f"${c:.4f}" if c < 0.01 else f"${c:.2f}" + bold = "**" if key == cost_target else "" + eq_rows.append(f"| {bold}{label}{bold} | {cost_str} |") in_tok = usage.get("input", 0) out_tok = usage.get("output", 0) reason_tok = usage.get("reasoning", 0) @@ -1251,10 +1282,16 @@ def _render_collapsible_usage(usage: dict | None, model: str, config: dict | Non f"- **Total Tokens**: {fmt_tokens(in_tok)} in / {fmt_tokens(out_tok)} out " f"({fmt_tokens(reason_tok)} reasoning, cache {fmt_tokens(cache_r)} read / " f"{fmt_tokens(cache_w)} write, {fmt_tokens(total)} total)", - f"- **Est. cost on {eq_label}**: {eq_s}{eq_note}", f"- **Actual**: {actual_s}{actual_note}", f"- **Scope**: {scope}", ] + if eq_rows: + lines.append("") + lines.append("- **Equivalent cost on paid providers** (this run's tokens):") + lines.append("") + lines.append("| Provider | Cost |") + lines.append("|---|---:|") + lines.extend(eq_rows) # Multi-lens fan-out: surface the lens roster + summed steps so the user # can see which lenses contributed (and that triage didn't drop them all). lenses = usage.get("lenses") diff --git a/tests/pilot/test_ai_review.py b/tests/pilot/test_ai_review.py index eb3c131..bcb967e 100644 --- a/tests/pilot/test_ai_review.py +++ b/tests/pilot/test_ai_review.py @@ -1165,19 +1165,22 @@ def test_usage_block_shows_equivalent_provider_cost(): "cache_read": 0, "cache_write": 0, "total": 204000, "cost": 0.0, "steps": 6, "duration_s": 100.0} sec = ai_review._render_collapsible_usage(usage, "glm-5.2:cloud", config=None) - # Two cost lines now: an equivalent (default Sonnet 5) AND the $0 actual. + # New layout: equivalent-cost table instead of a single "Est. cost on …" + # line. The default compare_against is sonnet-5, gpt-5, gemini-2.5-pro, + # grok-4.5; cost_target defaults to sonnet-5 (bolded). assert "🔋 AI Usage & Run Details" in sec - assert "**Est. cost on Claude Sonnet 5**" in sec assert "**Actual**: $0.00" in sec assert "free tier" in sec - # Equivalent should be > 0 for non-trivial token counts. assert "$0.00" in sec # the actual line - # And a non-zero one for the equivalent. - import re - cost_lines = [ln for ln in sec.splitlines() if "cost on" in ln] - assert len(cost_lines) == 1 - assert re.search(r"\$\d", cost_lines[0]) is not None - assert "$0.00" not in cost_lines[0] + # Multi-provider table header present, default roster rendered, default + # cost_target (Sonnet 5) is the bolded row. + assert "| Provider | Cost |" in sec + assert "**Claude Sonnet 5**" in sec + assert "GPT-5" in sec + assert "Gemini 2.5 Pro" in sec + assert "Grok 4.5" in sec + # 200k * $2/MTok + 4k * $10/MTok → $0.44 + assert "$0.44" in sec def test_usage_block_honors_cost_target(monkeypatch): @@ -1204,17 +1207,22 @@ def test_usage_block_respects_repo_config_cost_target(monkeypatch): assert "$0.0075" in sec -def test_usage_block_reports_unknown_price_target(): +def test_usage_block_reports_unknown_price_target(capsys): usage = {"input": 100, "output": 100, "reasoning": 0, "cache_read": 0, "cache_write": 0, "total": 200, "cost": 0.0, "steps": 1, "duration_s": 1.0} sec = ai_review._render_collapsible_usage( usage, "glm-5.2:cloud", config={"cost_target": "bogus-model"} ) - # Falls back to default + surfaces the error in the line. + # Falls back to default. The error now goes to stderr (otherwise it would + # land mid-table and look like a model error in the posted summary). assert "Claude Sonnet 5" in sec - assert "unknown price target" in sec - assert "bogus-model" in sec + assert "**Claude Sonnet 5**" in sec # bolded as the resolved cost_target + assert "bogus-model" not in sec + assert "unknown price target" not in sec + err = capsys.readouterr().err + assert "unknown price target" in err + assert "bogus-model" in err # --------------------------------------------------------------------------- @@ -1769,6 +1777,50 @@ def test_collapsible_usage_renders_humanized_tokens(): assert "17,303 (17.3K) out" in block +# --------------------------------------------------------------------------- +# Multi-provider equivalent-cost table — Task 10 +# --------------------------------------------------------------------------- + + +def test_collapsible_usage_renders_multi_provider_table(): + usage = {"input": 1_000_000, "output": 1000, "reasoning": 0, + "cache_read": 0, "cache_write": 0, "total": 1_001_000, + "cost": 0.0, "steps": 1, "duration_s": 10.0} + block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={"compare_against": ["claude-sonnet-5", "gpt-5"]}) + assert "Claude Sonnet 5" in block + assert "GPT-5" in block + assert "| Provider | Cost |" in block + + +def test_collapsible_usage_uses_default_compare_against_when_absent(): + usage = {"input": 1_000_000, "output": 0, "reasoning": 0, + "cache_read": 0, "cache_write": 0, "total": 1_000_000, + "cost": 0.0, "steps": 1, "duration_s": 5.0} + block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={}) + assert "Claude Sonnet 5" in block + assert "GPT-5" in block + assert "Gemini 2.5 Pro" in block + assert "Grok 4.5" in block + + +def test_collapsible_usage_bolds_cost_target_row(): + usage = {"input": 1_000_000, "output": 0, "reasoning": 0, + "cache_read": 0, "cache_write": 0, "total": 1_000_000, + "cost": 0.0, "steps": 1, "duration_s": 5.0} + block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={"cost_target": "gpt-5"}) + assert "**GPT-5**" in block + assert "Claude Sonnet 5" in block # still in default compare set + + +def test_collapsible_usage_skips_zero_cost_rows(): + usage = {"input": 0, "output": 0, "reasoning": 0, + "cache_read": 0, "cache_write": 0, "total": 0, + "cost": 0.0, "steps": 1, "duration_s": 1.0} + block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={}) + # With zero tokens, all costs are $0 — skip the entire table. + assert "| Provider | Cost |" not in block + + def test_inline_comment_body_humanized_tokens(): # Value chosen > 1000 so fmt_tokens actually adds the comma + short suffix; # the plan's 362 would render identically with or without fmt_tokens.