feat(pilot): token-usage reporting gated by AI-USAGE label
Add per-review + per-comment token accounting, surfaced only when a PR carries the new AI-USAGE label (on top of the existing AI-REVIEW trigger). opencode_review: - run_opencode now uses `--format json`; parse_opencode_events reconstructs the assistant text from `text` events and sums tokens/cost/steps from every `step_finish` event (tolerant of noise / missing fields). - run() measures duration_s around the opencode call and returns (text, usage). - changed_files(diff) extracts the `+++ b/` paths; the brief now lists them under a "Changed files" focus block so the agent grounds findings in the diff's neighbourhood instead of unbounded whole-repo walks. ai_review: - format_usage_section renders a `## AI usage` block: measured totals (in/out/reasoning/cache/cost/steps/duration), the whole-repo scope note, and an attributed per-finding table. Per-comment counts are output tokens split by each finding's body weight — labelled "attributed" since one model pass produces all findings. - inline_comment_body appends `🪙 ~N tok (X% · attributed output)` when attribution is present. - review_pr gains report_usage; compute_attribution stashes _tok_attrib/_tok_pct. - format_review_body inserts the usage section between summary and findings. webhook_server: - Fire on every pull_request action except `closed` (denylist, was an allowlist) — the AI-REVIEW gate + sha dedupe keep this safe. - AI-USAGE label detection + PRAGENT_USAGE_ALWAYS env drive report_usage. .opencode factory + review-methodology skill: new "Ground findings in context" step — read callers/imports/sibling functions per changed file (1-3 files per finding), no unbounded walks. Tests: parse_opencode_events (text+usage sum, malformed tolerance, none-usage), changed_files, compute_attribution math, inline 🪙 line, format_usage_section totals/table/cost, format_review_body ordering. 68 passing. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+113
-5
@@ -134,14 +134,16 @@ def parse_text_blocks(content: list) -> str:
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def format_review_body(findings: str, model: str, sha: str, summary: str = "") -> str:
|
||||
def format_review_body(findings: str, model: str, sha: str, summary: str = "", usage_section: str = "") -> str:
|
||||
"""Format the posted review summary body.
|
||||
|
||||
`findings` is the bullet text for findings that could NOT be anchored inline
|
||||
(or, on the legacy/no-inline path, the whole review). Empty -> "No issues
|
||||
found.". `summary` (optional, opencode engine) is rendered as a "Summary"
|
||||
section right under the header. The hidden sha marker is always appended
|
||||
for the dedupe pass.
|
||||
section right under the header. `usage_section` (optional, shown only when
|
||||
the PR carries the `AI-USAGE` label) is rendered between the summary and the
|
||||
findings bullets. The hidden sha marker is always appended for the dedupe
|
||||
pass.
|
||||
"""
|
||||
header = REVIEW_HEADER.format(model=model, sha=sha[:8] if sha else "unknown")
|
||||
findings = (findings or "").strip()
|
||||
@@ -151,6 +153,8 @@ def format_review_body(findings: str, model: str, sha: str, summary: str = "") -
|
||||
parts = [header]
|
||||
if summary:
|
||||
parts.append(summary.strip())
|
||||
if usage_section:
|
||||
parts.append(usage_section.strip())
|
||||
parts.append(findings)
|
||||
body = "\n\n".join(parts)
|
||||
if marker:
|
||||
@@ -158,6 +162,88 @@ def format_review_body(findings: str, model: str, sha: str, summary: str = "") -
|
||||
return body
|
||||
|
||||
|
||||
def _finding_weight(f: dict) -> int:
|
||||
"""Body-weight used to attribute output tokens to a finding (char length of
|
||||
its rendered problem + fix + suggestion). One model pass produces all
|
||||
findings, so per-finding tokens can't be measured directly — we split the
|
||||
measured output total by this weight as an honest attribution."""
|
||||
return (
|
||||
len(f.get("problem") or "")
|
||||
+ len(f.get("fix") or "")
|
||||
+ len(f.get("suggestion") or "")
|
||||
)
|
||||
|
||||
|
||||
def compute_attribution(findings: list[dict], output_tokens: int) -> None:
|
||||
"""Stash `_tok_attrib` (attributed output tokens) and `_tok_pct` (0..1) on
|
||||
each finding, splitting `output_tokens` by each finding's body weight.
|
||||
Mutates in place. No-op when there are no findings or no output budget."""
|
||||
if not findings or not output_tokens:
|
||||
return
|
||||
weights = [_finding_weight(f) for f in findings]
|
||||
total_w = sum(weights)
|
||||
if total_w <= 0:
|
||||
# All-zero weights (no prose): split evenly.
|
||||
share = output_tokens / len(findings)
|
||||
for f in findings:
|
||||
f["_tok_attrib"] = int(round(share))
|
||||
f["_tok_pct"] = 1.0 / len(findings)
|
||||
return
|
||||
for f, w in zip(findings, weights):
|
||||
f["_tok_attrib"] = int(round(output_tokens * w / total_w))
|
||||
f["_tok_pct"] = w / total_w
|
||||
|
||||
|
||||
def format_usage_section(usage: dict | None, findings: list[dict], model: str) -> str:
|
||||
"""Render the `## 🔋 AI usage` block for the review body.
|
||||
|
||||
Only called when the PR carries the `AI-USAGE` label (and the opencode
|
||||
engine produced a usage dict). Reports the MEASURED total
|
||||
(input/output/reasoning/cache/cost/steps/duration) plus an ATTRIBUTED
|
||||
per-finding table — one model pass generates all findings, so per-comment
|
||||
counts are an estimate (output split by body weight), clearly labelled.
|
||||
Returns "" if `usage` is None.
|
||||
"""
|
||||
if not usage:
|
||||
return ""
|
||||
dur = usage.get("duration_s")
|
||||
dur_s = f"{dur}s" if dur is not None else "?"
|
||||
cost = usage.get("cost") or 0.0
|
||||
cost_s = f"${cost:.4f}" if cost else "$0.00"
|
||||
cost_note = (
|
||||
"(on-network glm-5.2:cloud via headroom — no per-token charge)"
|
||||
if not cost else "(billed by provider)"
|
||||
)
|
||||
lines = [
|
||||
"## 🔋 AI usage",
|
||||
"",
|
||||
f"- model: `{model}` · engine: opencode · agent steps: {usage.get('steps', 0)} · duration: {dur_s}",
|
||||
(
|
||||
f"- tokens: {usage.get('input', 0)} in · {usage.get('output', 0)} out · "
|
||||
f"{usage.get('reasoning', 0)} reasoning · cache "
|
||||
f"{usage.get('cache_read', 0)} read / {usage.get('cache_write', 0)} write "
|
||||
f"→ {usage.get('total', 0)} total"
|
||||
),
|
||||
f"- est. cost: {cost_s} {cost_note}",
|
||||
"- scope: whole-repo checkout at head sha (agent can read any file + run linters, not just the diff) — input tokens include files read beyond the diff",
|
||||
"- per-comment tokens are *attributed* (one model pass produces all findings; output split by each finding's body weight)",
|
||||
]
|
||||
# Per-finding attribution table.
|
||||
rows = [f for f in findings if f.get("_tok_attrib") is not None]
|
||||
if rows:
|
||||
lines.append("")
|
||||
lines.append("| severity | location | ≈out tok | % |")
|
||||
lines.append("|---|---|---:|---:|")
|
||||
for f in rows:
|
||||
loc = f"{f['path']}:{f['line']}" if f.get("line") else f.get("path", "?")
|
||||
pct = f.get("_tok_pct", 0.0) * 100
|
||||
lines.append(
|
||||
f"| {f.get('severity', '').upper()} | `{loc}` | "
|
||||
f"{f.get('_tok_attrib', 0)} | {pct:.0f}% |"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_user_prompt(
|
||||
title: str,
|
||||
body: str,
|
||||
@@ -483,6 +569,10 @@ def inline_comment_body(f: dict) -> str:
|
||||
ref = f.get("reference", "")
|
||||
if ref:
|
||||
body += f"\n\n📎 ref: {ref}"
|
||||
tok = f.get("_tok_attrib")
|
||||
if tok is not None:
|
||||
pct = (f.get("_tok_pct", 0.0) or 0.0) * 100
|
||||
body += f"\n\n🪙 ~{tok} tok ({pct:.0f}% · attributed output)"
|
||||
return body
|
||||
|
||||
|
||||
@@ -713,6 +803,7 @@ def review_pr(
|
||||
model: str,
|
||||
max_tokens: int = 8000,
|
||||
max_chars: int = 150000,
|
||||
report_usage: bool = False,
|
||||
) -> bool:
|
||||
"""Run one review and post it as `pragent-bot`.
|
||||
|
||||
@@ -722,6 +813,11 @@ def review_pr(
|
||||
review with inline comments + suggestions (unanchored findings → summary
|
||||
bullets).
|
||||
|
||||
`report_usage`: when True (PR carries the `AI-USAGE` label), the opencode
|
||||
engine's measured token/cost usage is rendered as a `## 🔋 AI usage` section
|
||||
on the review body and an attributed `🪙 ~N tok` line on each inline
|
||||
comment. No-op on the ollama fallback (no usage available).
|
||||
|
||||
Returns True on success (including a deliberate skip), False on failure
|
||||
(failure note posted when possible). Never raises — fail-open by design.
|
||||
Both the CI `run()` entry point and the central webhook server call this.
|
||||
@@ -752,7 +848,7 @@ def review_pr(
|
||||
# `model` here is the bare id (OLLAMA_MODEL). OPENCODE_MODEL overrides
|
||||
# with the full ref; otherwise we prefix the configured provider.
|
||||
oc_model = os.environ.get("OPENCODE_MODEL") or f"headroom/{model}"
|
||||
stdout = opencode_review.run(
|
||||
stdout, usage = opencode_review.run(
|
||||
api=api, repo=repo, index=index, sha=sha, token=token,
|
||||
title=title, body=body, diff=diff, config=config,
|
||||
prior_reviews=prior, model=oc_model,
|
||||
@@ -767,6 +863,15 @@ def review_pr(
|
||||
user_prompt = build_user_prompt(title, body, diff, config, prior)
|
||||
raw_findings = call_model(ollama_url, model, SYSTEM_PROMPT, user_prompt, max_tokens)
|
||||
findings = parse_findings(raw_findings)
|
||||
usage = None
|
||||
|
||||
# Attribute output tokens to each finding (mutates finding dicts) so
|
||||
# inline comments + the usage table can show a per-comment estimate.
|
||||
# Only meaningful when we have measured usage AND the PR asked for it.
|
||||
usage_section = ""
|
||||
if report_usage and usage and usage.get("output"):
|
||||
compute_attribution(findings, usage["output"])
|
||||
usage_section = format_usage_section(usage, findings, model)
|
||||
|
||||
anchors = parse_diff_anchors(diff)
|
||||
anchored, unanchored = split_findings(findings, anchors)
|
||||
@@ -782,7 +887,10 @@ def review_pr(
|
||||
summary_parts.append(bullets)
|
||||
if not summary_parts:
|
||||
summary_parts.append("No issues found.")
|
||||
summary_body = format_review_body("\n\n".join(summary_parts), model, sha, summary=review_summary)
|
||||
summary_body = format_review_body(
|
||||
"\n\n".join(summary_parts), model, sha,
|
||||
summary=review_summary, usage_section=usage_section,
|
||||
)
|
||||
|
||||
post_inline_review(api, repo, index, token, summary_body, anchored)
|
||||
print(
|
||||
|
||||
Reference in New Issue
Block a user