feat(pilot): token-usage reporting gated by AI-USAGE label
Add per-review + per-comment token accounting, surfaced only when a PR carries the new AI-USAGE label (on top of the existing AI-REVIEW trigger). opencode_review: - run_opencode now uses `--format json`; parse_opencode_events reconstructs the assistant text from `text` events and sums tokens/cost/steps from every `step_finish` event (tolerant of noise / missing fields). - run() measures duration_s around the opencode call and returns (text, usage). - changed_files(diff) extracts the `+++ b/` paths; the brief now lists them under a "Changed files" focus block so the agent grounds findings in the diff's neighbourhood instead of unbounded whole-repo walks. ai_review: - format_usage_section renders a `## AI usage` block: measured totals (in/out/reasoning/cache/cost/steps/duration), the whole-repo scope note, and an attributed per-finding table. Per-comment counts are output tokens split by each finding's body weight — labelled "attributed" since one model pass produces all findings. - inline_comment_body appends `🪙 ~N tok (X% · attributed output)` when attribution is present. - review_pr gains report_usage; compute_attribution stashes _tok_attrib/_tok_pct. - format_review_body inserts the usage section between summary and findings. webhook_server: - Fire on every pull_request action except `closed` (denylist, was an allowlist) — the AI-REVIEW gate + sha dedupe keep this safe. - AI-USAGE label detection + PRAGENT_USAGE_ALWAYS env drive report_usage. .opencode factory + review-methodology skill: new "Ground findings in context" step — read callers/imports/sibling functions per changed file (1-3 files per finding), no unbounded walks. Tests: parse_opencode_events (text+usage sum, malformed tolerance, none-usage), changed_files, compute_attribution math, inline 🪙 line, format_usage_section totals/table/cost, format_review_body ordering. 68 passing. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Marcos
@@ -9,7 +9,9 @@ sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
||||
|
||||
from ai_review import ( # noqa: E402
|
||||
build_user_prompt,
|
||||
compute_attribution,
|
||||
format_review_body,
|
||||
format_usage_section,
|
||||
inline_comment_body,
|
||||
parse_diff_anchors,
|
||||
parse_findings,
|
||||
@@ -436,4 +438,114 @@ def test_format_review_body_with_summary_section():
|
||||
assert "- [high] x:1" in body
|
||||
assert "<!-- pragent:sha=abcdef1234567890 -->" in body
|
||||
# summary appears before the findings bullets
|
||||
assert body.index("risky helper") < body.index("[high]")
|
||||
assert body.index("risky helper") < body.index("[high]")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AI-USAGE: compute_attribution + format_usage_section + inline 🪙 line
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compute_attribution_weighted_split():
|
||||
# weights 1 (problem="a") and 3 (problem="aaa"), output 100 → 25 / 75
|
||||
fs = [
|
||||
{"severity": "high", "path": "x", "line": 1, "problem": "a", "fix": "", "suggestion": ""},
|
||||
{"severity": "low", "path": "x", "line": 2, "problem": "aaa", "fix": "", "suggestion": ""},
|
||||
]
|
||||
compute_attribution(fs, 100)
|
||||
assert fs[0]["_tok_attrib"] == 25
|
||||
assert fs[1]["_tok_attrib"] == 75
|
||||
assert abs(fs[0]["_tok_pct"] - 0.25) < 1e-9
|
||||
assert abs(fs[1]["_tok_pct"] - 0.75) < 1e-9
|
||||
|
||||
|
||||
def test_compute_attribution_zero_weights_splits_evenly():
|
||||
fs = [
|
||||
{"severity": "high", "path": "x", "line": 1, "problem": "", "fix": "", "suggestion": ""},
|
||||
{"severity": "low", "path": "x", "line": 2, "problem": "", "fix": "", "suggestion": ""},
|
||||
]
|
||||
compute_attribution(fs, 80)
|
||||
assert fs[0]["_tok_attrib"] == 40
|
||||
assert fs[1]["_tok_attrib"] == 40
|
||||
assert abs(fs[0]["_tok_pct"] - 0.5) < 1e-9
|
||||
|
||||
|
||||
def test_compute_attribution_noop_on_empty_or_zero_budget():
|
||||
fs = [{"severity": "high", "path": "x", "line": 1, "problem": "a", "fix": "", "suggestion": ""}]
|
||||
compute_attribution([], 100)
|
||||
compute_attribution(fs, 0)
|
||||
assert "_tok_attrib" not in fs[0]
|
||||
|
||||
|
||||
def test_inline_comment_body_with_attribution_line():
|
||||
f = {"severity": "high", "path": "a", "line": 1, "problem": "bad", "fix": "swap",
|
||||
"suggestion": "", "_tok_attrib": 180, "_tok_pct": 0.29}
|
||||
body = inline_comment_body(f)
|
||||
assert "🪙 ~180 tok" in body
|
||||
assert "29%" in body
|
||||
assert "attributed output" in body
|
||||
|
||||
|
||||
def test_inline_comment_body_no_attribution_no_coin_line():
|
||||
f = {"severity": "high", "path": "a", "line": 1, "problem": "bad", "fix": "swap",
|
||||
"suggestion": ""}
|
||||
assert "🪙" not in inline_comment_body(f)
|
||||
|
||||
|
||||
def test_format_usage_section_renders_totals_and_table():
|
||||
fs = [
|
||||
{"severity": "critical", "path": "src/Foo.java", "line": 98,
|
||||
"problem": "p"*10, "fix": "f", "suggestion": "", "_tok_attrib": 180, "_tok_pct": 0.29},
|
||||
]
|
||||
usage = {"input": 18420, "output": 612, "reasoning": 0, "cache_read": 15210,
|
||||
"cache_write": 0, "total": 19032, "cost": 0.0, "steps": 7, "duration_s": 142.0}
|
||||
sec = format_usage_section(usage, fs, "glm-5.2:cloud")
|
||||
assert "## 🔋 AI usage" in sec
|
||||
assert "`glm-5.2:cloud`" in sec
|
||||
assert "agent steps: 7" in sec
|
||||
assert "duration: 142.0s" in sec
|
||||
assert "18420 in" in sec and "612 out" in sec and "19032 total" in sec
|
||||
assert "$0.00" in sec
|
||||
assert "whole-repo checkout" in sec
|
||||
assert "attributed" in sec
|
||||
# table
|
||||
assert "| severity | location | ≈out tok | % |" in sec
|
||||
assert "CRITICAL" in sec
|
||||
assert "`src/Foo.java:98`" in sec
|
||||
assert "180" in sec and "29%" in sec
|
||||
|
||||
|
||||
def test_format_usage_section_omits_table_when_no_attributed_rows():
|
||||
usage = {"input": 10, "output": 0, "reasoning": 0, "cache_read": 0,
|
||||
"cache_write": 0, "total": 10, "cost": 0.0, "steps": 1, "duration_s": 1.0}
|
||||
sec = format_usage_section(usage, [], "glm-5.2:cloud")
|
||||
assert "## 🔋 AI usage" in sec
|
||||
assert "severity | location" not in sec # no rows → no table
|
||||
|
||||
|
||||
def test_format_usage_section_none_returns_empty():
|
||||
assert format_usage_section(None, [], "m") == ""
|
||||
|
||||
|
||||
def test_format_usage_section_cost_nonzero():
|
||||
usage = {"input": 10, "output": 0, "reasoning": 0, "cache_read": 0,
|
||||
"cache_write": 0, "total": 10, "cost": 0.0123, "steps": 1, "duration_s": 1.0}
|
||||
sec = format_usage_section(usage, [], "m")
|
||||
assert "$0.0123" in sec
|
||||
assert "billed by provider" in sec
|
||||
|
||||
|
||||
def test_format_review_body_usage_section_between_summary_and_findings():
|
||||
usage_sec = "## 🔋 AI usage\n\n- model: `m`"
|
||||
body = format_review_body("- [high] x:1 — b", "glm-5.2:cloud", "abcdef1234567890",
|
||||
summary="This PR is risky.", usage_section=usage_sec)
|
||||
# order: header < summary < usage < findings < marker
|
||||
assert body.index("risky.") < body.index("AI usage")
|
||||
assert body.index("AI usage") < body.index("[high]")
|
||||
assert body.index("[high]") < body.index("<!-- pragent:sha=")
|
||||
assert "## 🔋 AI usage" in body
|
||||
|
||||
|
||||
def test_format_review_body_no_usage_section_omitted():
|
||||
body = format_review_body("- [high] x:1 — b", "glm-5.2:cloud", "abcdef1234567890")
|
||||
assert "AI usage" not in body
|
||||
@@ -123,4 +123,105 @@ def test_drop_factory_copies_config_and_agents(tmp_path):
|
||||
oc.drop_factory(str(tmp_path))
|
||||
assert os.path.isfile(tmp_path / "opencode.json")
|
||||
assert os.path.isfile(tmp_path / ".opencode" / "agents" / "pragent.md")
|
||||
assert os.path.isfile(tmp_path / ".opencode" / "skills" / "findings-schema" / "SKILL.md")
|
||||
assert os.path.isfile(tmp_path / ".opencode" / "skills" / "findings-schema" / "SKILL.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# changed_files — extract changed paths from a unified diff
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_changed_files_extracts_new_side_paths():
|
||||
diff = (
|
||||
"diff --git a/src/a.py b/src/a.py\n+++ b/src/a.py\n@@ -1 +1 @@\n-x\n+y\n"
|
||||
"diff --git a/README.md b/README.md\n+++ b/README.md\n@@ -1 +1 @@\n+z\n"
|
||||
)
|
||||
assert oc.changed_files(diff) == ["README.md", "src/a.py"]
|
||||
|
||||
|
||||
def test_changed_files_skips_deletions_and_dedups():
|
||||
diff = (
|
||||
"diff --git a/gone.txt b/gone.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-old\n"
|
||||
"diff --git a/dup.go b/dup.go\n+++ b/dup.go\n@@ -1 +1 @@\n+a\n"
|
||||
"diff --git a/dup.go b/dup.go\n+++ b/dup.go\n@@ -1 +1 @@\n+b\n"
|
||||
)
|
||||
assert oc.changed_files(diff) == ["dup.go"]
|
||||
|
||||
|
||||
def test_changed_files_empty():
|
||||
assert oc.changed_files("") == []
|
||||
assert oc.changed_files("no diff headers here") == []
|
||||
|
||||
|
||||
def test_write_brief_lists_changed_files(tmp_path):
|
||||
brief = oc.write_brief(
|
||||
str(tmp_path), repo="o/r", index="1", sha="abcdef1234567890",
|
||||
title="t", description="d",
|
||||
diff="diff --git a/src/x.ts b/src/x.ts\n+++ b/src/x.ts\n@@ -1 +1 @@\n+x",
|
||||
config=None, prior_reviews=None,
|
||||
)
|
||||
text = open(brief, encoding="utf-8").read()
|
||||
assert "Changed files (focus your context research here)" in text
|
||||
assert "`src/x.ts`" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse_opencode_events — NDJSON → (text, usage)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ev(obj):
|
||||
import json
|
||||
return json.dumps(obj)
|
||||
|
||||
|
||||
def test_parse_events_text_and_usage_summed():
|
||||
stdout = "\n".join([
|
||||
_ev({"type": "step_start", "part": {}}),
|
||||
_ev({"type": "text", "part": {"text": "Hello "}}),
|
||||
_ev({"type": "text", "part": {"text": "world"}}),
|
||||
_ev({"type": "step_finish", "part": {
|
||||
"tokens": {"total": 100, "input": 90, "output": 10,
|
||||
"reasoning": 0, "cache": {"write": 0, "read": 5}},
|
||||
"cost": 0.0}}),
|
||||
_ev({"type": "text", "part": {"text": " more"}}),
|
||||
_ev({"type": "step_finish", "part": {
|
||||
"tokens": {"total": 50, "input": 40, "output": 10,
|
||||
"reasoning": 2, "cache": {"write": 1, "read": 0}},
|
||||
"cost": 0.01}}),
|
||||
])
|
||||
text, usage = oc.parse_opencode_events(stdout)
|
||||
assert text == "Hello world more"
|
||||
assert usage is not None
|
||||
assert usage["steps"] == 2
|
||||
assert usage["input"] == 130
|
||||
assert usage["output"] == 20
|
||||
assert usage["reasoning"] == 2
|
||||
assert usage["cache_read"] == 5
|
||||
assert usage["cache_write"] == 1
|
||||
assert usage["total"] == 150
|
||||
assert abs(usage["cost"] - 0.01) < 1e-9
|
||||
|
||||
|
||||
def test_parse_events_no_step_finish_returns_none_usage():
|
||||
stdout = _ev({"type": "text", "part": {"text": "only text"}})
|
||||
text, usage = oc.parse_opencode_events(stdout)
|
||||
assert text == "only text"
|
||||
assert usage is None
|
||||
|
||||
|
||||
def test_parse_events_tolerates_noise_and_malformed():
|
||||
stdout = "\n".join([
|
||||
"not json at all",
|
||||
_ev({"type": "text", "part": {"text": "ok"}}),
|
||||
"{ broken json",
|
||||
_ev({"type": "step_finish", "part": {}}), # no tokens field -> counted, zero
|
||||
_ev({"type": "tool_start", "part": {"text": "ignored"}}),
|
||||
" ",
|
||||
])
|
||||
text, usage = oc.parse_opencode_events(stdout)
|
||||
assert text == "ok"
|
||||
# step_finish with no tokens still counts as a step; usage dict returned
|
||||
assert usage is not None
|
||||
assert usage["steps"] == 1
|
||||
assert usage["input"] == 0 and usage["output"] == 0
|
||||
Reference in New Issue
Block a user