from __future__ import annotations import base64 import json import os import re import sys import urllib.error import urllib.parse import urllib.request from . import pipeline from .pipeline import * from .pipeline import _CONFIDENCE_BADGE, REVIEW_HEADER, SHA_MARKER # Diff parsing — find valid post-change (RIGHT-side) line anchors per file # --------------------------------------------------------------------------- def parse_diff_anchors(diff: str) -> dict[str, set[int]]: """Parse a unified diff into {path: {new_line, ...}} for lines that exist in the post-change version (context + added lines). Removed lines are NOT anchors (they have no RIGHT-side line). Used to validate inline comments. Robust to: - `diff --git a/x b/x` and `+++ b/x` path headers (uses the `b/` side) - hunk headers `@@ -a,b +c,d @@` (new line counter starts at c) - No-newline-at-eof markers, binary files, missing hunks. """ anchors: dict[str, set[int]] = {} current_path: str | None = None new_line = 0 for raw in (diff or "").splitlines(): # File path: prefer the `+++ b/` line (handles renames); fall back to # `diff --git a/x b/x`'s second path. if raw.startswith("+++ "): p = raw[4:].strip() if p == "/dev/null": current_path = None else: current_path = _strip_path_prefix(p) anchors.setdefault(current_path, set()) continue if raw.startswith("diff --git "): # `diff --git a/foo b/foo` — take the second path as a fallback in # case the `+++` line is missing (binary). Split on " b/". m = re.search(r" b/(.+)$", raw) if m: current_path = m.group(1).strip() anchors.setdefault(current_path, set()) continue if raw.startswith("@@"): m = re.search(r"\+(\d+)(?:,\d+)?\s@@", raw) new_line = int(m.group(1)) if m else 0 continue if current_path is None: continue if raw.startswith("\\ No newline"): continue if raw.startswith("-"): # removed line — no RIGHT-side anchor continue if raw.startswith("+"): anchors[current_path].add(new_line) new_line += 1 continue # Context line: normally " text", but an empty context line arrives as # "" whenever something along the way stripped trailing whitespace (some # forges, some patch tools, copy/paste). Treating "" as "not a line" # would desync `new_line` for the whole rest of the hunk and silently # misplace every later inline comment in the file, so count it. if raw.startswith(" ") or raw == "": anchors[current_path].add(new_line) new_line += 1 return anchors def _strip_path_prefix(p: str) -> str: """`b/foo` or `foo` -> `foo`.""" if p.startswith("b/"): return p[2:] return p # --------------------------------------------------------------------------- # Model output parsing — tolerant JSON findings extraction # --------------------------------------------------------------------------- # How many raw findings the last `parse_review_output` / `parse_findings` call # rejected for an unusable path/line. A side channel rather than a return value # because both parsers already return fixed-width tuples that several callers # and their tests unpack positionally; widening them to carry a telemetry # number would be a breaking change for a fail-open signal. _LAST_PARSE_DROPPED: dict[str, int] = {"n": 0} def last_parse_dropped() -> int: """Findings the last parse discarded. Read it immediately after parsing.""" return int(_LAST_PARSE_DROPPED.get("n") or 0) def _normalize_finding(f: dict) -> dict | None: """Validate + normalize one raw finding dict. Returns None if it's unusable (missing path/line). Normalises severity, keeps `reference` (default "").""" if not isinstance(f, dict): return None path = f.get("path") line = f.get("line") if not isinstance(path, str) or not path.strip(): return None if not isinstance(line, int) or line < 1: return None sev = str(f.get("severity", "medium")).strip().lower() if sev not in SEVERITIES: sev = "medium" reference = str(f.get("reference", "") or "").strip() return { "severity": sev, "path": path.strip(), "line": line, "problem": str(f.get("problem", "")).strip(), "fix": str(f.get("fix", "")).strip(), "suggestion": str(f.get("suggestion", "") or "").strip(), "reference": reference, } def _last_json_block(text: str) -> str | None: r"""Return the substring of the last JSON object/array in text, or None. The pragent agent emits ```json fences around its final block, but real outputs drift: * the fence contains nested objects (regex ``\{.*?\}`` only matches the first ``}``, truncating the JSON — the parser then sees ``json.JSONDecodeError``); * the fence is missing or unterminated, but a balanced JSON object sits in the prose tail; * the agent emits a bare array (findings only, no summary wrapper). Strategy: 1. Find each fenced block, take the last. Inside it, walk a balanced ``{...}``/``[...]`` scanner (not a regex) so nested structures survive. 2. Fall back to a balanced scanner over the whole text, picking the LAST balanced object/array (the agent writes its conclusion last). """ s = text or "" if not s: return None # 1. Fenced blocks: take the last ```json ... ``` or ``` ... ``` region. fences = list(re.finditer(r"```(?:json)?\n", s)) for m in reversed(fences): start = m.end() # Find the matching closing fence. end = s.find("```", start) if end < 0: # Unterminated fence — try to salvage the balanced object inside. end = len(s) inner = s[start:end].strip() obj = _balanced_json_substring(inner) if obj is not None: return obj # 2. No (parseable) fence — scan the whole text for the LAST balanced # object/array. The agent's conclusion is at the tail. return _last_balanced_json(s) def parse_findings(text: str) -> list[dict]: """Parse the model's JSON response into a list of finding dicts. Tolerant: strips ```json fences, and if the model wrapped JSON in prose, scans for the first balanced `{...}` and extracts its `findings` array. Drops findings missing path/line or with an unknown severity (normalised). Never raises — returns [] on any parse failure. Also accepts a bare JSON array as the outer value: ``[{...}, {...}]`` — some agents skip the ``{"summary":..., "findings":[...]}`` wrapper. """ _LAST_PARSE_DROPPED["n"] = 0 data = _parse_json_tolerant(text) if isinstance(data, dict): findings = data.get("findings") elif isinstance(data, list): findings = data else: return [] if not isinstance(findings, list): return [] out = [] for f in findings: n = _normalize_finding(f) if n is not None: out.append(n) _LAST_PARSE_DROPPED["n"] = len(findings) - len(out) return out SALVAGE_MAX_CHARS = 4000 def salvage_summary(text: str, max_chars: int = SALVAGE_MAX_CHARS) -> str: """Recover something postable from agent output we could not parse. An opencode run costs minutes and millions of tokens. When the findings JSON is missing or malformed, the analysis itself is usually still there in the prose — discarding it to post "no parseable output" throws away the whole run and tells the maintainer nothing. This keeps the tail of the prose (the conclusion, which is what the agent writes last), drops fenced code blocks so a half-written JSON blob doesn't dominate, and labels it plainly as unstructured so nobody mistakes it for a normal review. Returns "" when there is genuinely nothing to salvage. """ if not text or not text.strip(): return "" # Drop fenced blocks — a truncated ```json block is noise here. prose = re.sub(r"```.*?```", "", text, flags=re.DOTALL) prose = re.sub(r"```.*$", "", prose, flags=re.DOTALL) # unterminated fence prose = prose.strip() if not prose: return "" if len(prose) > max_chars: prose = "…" + prose[-max_chars:] return ( "⚠️ _The reviewer did not emit a parseable findings block, so there are " "no inline comments. Its raw notes are below — treat them as unverified: " "line numbers were not validated against the diff._\n\n" + prose ) def parse_review_output( text: str, ) -> tuple[str, list[dict], list[str], list[str], list[str], str, str]: """Parse the opengine's stdout into a 7-tuple: (summary, findings, summary_changes, risks, walkthrough, risk_verdict, test_coverage) Accepts `{"summary": "...", "summary_changes": [...], "risks": [...], "walkthrough": [...], "risk_verdict": "...", "test_coverage": "...", "findings": [...]}` (the opencode pragent agent), the legacy 4-field shape, or a bare `[...]` of finding dicts. The three new fields (`walkthrough`, `risk_verdict`, `test_coverage`) default to empty list / empty strings when absent — older outputs and the bare-array shape stay backward compatible. Uses the LAST fenced block (the pragent agent emits JSON as the final block), with a tolerant fallback that scans for the last balanced object/array in the prose tail. Never raises. """ _LAST_PARSE_DROPPED["n"] = 0 blob = _last_json_block(text) if blob is None: return "", [], [], [], [], "", "" try: data = json.loads(blob) except json.JSONDecodeError: return "", [], [], [], [], "", "" summary = "" summary_changes: list[str] = [] risks: list[str] = [] walkthrough: list[str] = [] risk_verdict = "" test_coverage = "" findings_raw = None if isinstance(data, dict): summary = str(data.get("summary", "") or "").strip() summary_changes = _string_list(data.get("summary_changes")) risks = _string_list(data.get("risks")) walkthrough = _string_list(data.get("walkthrough")) risk_verdict = str(data.get("risk_verdict", "") or "").strip() test_coverage = str(data.get("test_coverage", "") or "").strip() findings_raw = data.get("findings") elif isinstance(data, list): # Bare array: each item is a finding; no summary/sections. findings_raw = data else: return "", [], [], [], [], "", "" out = [] if isinstance(findings_raw, list): for f in findings_raw: n = _normalize_finding(f) if n is not None: out.append(n) # A model that emits findings at unusable locations is indistinguishable # from one that found nothing, because both end up with an empty `out`. # Stash the delta so the caller can score it (see `eval_scores`). _LAST_PARSE_DROPPED["n"] = len(findings_raw) - len(out) else: _LAST_PARSE_DROPPED["n"] = 0 return summary, out, summary_changes, risks, walkthrough, risk_verdict, test_coverage def _string_list(value) -> list[str]: """Coerce a JSON value into a list of non-empty strings. Accepts a list of strings, a single string (split on lines/bullets), or anything else (returns []). Used for `summary_changes` and `risks`, which some agents emit as one big string instead of a list. """ if isinstance(value, list): return [str(v).strip() for v in value if str(v).strip()] if isinstance(value, str): s = value.strip() if not s: return [] # Split on newlines OR on lines that start with "- " / "* " (markdown # bullets). Strip the bullet markers. out: list[str] = [] for line in s.splitlines(): line = line.strip() if not line: continue if line[:2] in ("- ", "* "): line = line[2:].strip() if line: out.append(line) return out return [] def _parse_json_tolerant(text: str) -> dict | list | None: """Parse a JSON object/array from text: try the last fenced block, then a direct parse, then the first balanced object. Returns None on any failure. Accepts both ``{...}`` (the pragent schema) and bare ``[...]`` arrays (agents that skip the wrapper).""" if not text: return None blob = _last_json_block(text) if blob is not None: try: d = json.loads(blob) if isinstance(d, (dict, list)): return d except json.JSONDecodeError: pass s = text.strip() if s.startswith("```"): s = re.sub(r"^```[a-zA-Z]*\n?", "", s) s = re.sub(r"\n?```$", "", s).strip() try: d = json.loads(s) if isinstance(d, (dict, list)): return d except json.JSONDecodeError: pass obj = _extract_first_json_object(text) if obj is not None: try: d = json.loads(obj) if isinstance(d, (dict, list)): return d except json.JSONDecodeError: pass # Last resort: the JSON lives at the tail of the prose with no fence. # Walk the whole text for the last balanced object/array. last = _last_balanced_json(text) if last is not None: try: d = json.loads(last) if isinstance(d, (dict, list)): return d except json.JSONDecodeError: pass return None def _extract_first_json_object(s: str) -> str | None: """Return the substring of the first balanced top-level `{ ... }` in s.""" start = s.find("{") if start < 0: return None end = _scan_balanced(s, start, "{", "}") if end is None: return None return s[start:end + 1] def _last_balanced_json(s: str) -> str | None: """Return the substring of the LAST balanced ``{...}`` or ``[...]`` in s. Used when the agent emits no fence: the JSON lives in the prose tail. Picks whichever closer (object or array) appears latest in the text. """ if not s: return None last_obj = _find_last_close(s, "{", "}") last_arr = _find_last_close(s, "[", "]") candidates = [] if last_obj is not None: candidates.append(last_obj) if last_arr is not None: candidates.append(last_arr) if not candidates: return None end, opener, start = max(candidates, key=lambda t: t[0]) return s[start:end + 1] def _balanced_json_substring(s: str) -> str | None: """Return the first balanced ``{...}`` or ``[...]`` substring in ``s``. Skips past leading whitespace/non-JSON and returns the full balanced extent (handles nested objects/arrays and string literals with braces). """ if not s: return None # Try object first; the pragent schema is an object on the outer level. for i, c in enumerate(s): if c == "{": end = _scan_balanced(s, i, "{", "}") if end is not None: return s[i:end + 1] break if c == "[": end = _scan_balanced(s, i, "[", "]") if end is not None: return s[i:end + 1] break return None def _scan_balanced(s: str, start: int, opener: str, closer: str) -> int | None: """Return the index of the matching ``closer`` for ``s[start] == opener``. Tracks string literals (with ``\\`` escapes) so braces inside strings don't fool the depth counter. Returns None if no balance is reached. """ depth = 0 in_str = False esc = False for i in range(start, len(s)): c = s[i] if in_str: if esc: esc = False elif c == "\\": esc = True elif c == '"': in_str = False continue if c == '"': in_str = True elif c == opener: depth += 1 elif c == closer: depth -= 1 if depth == 0: return i return None def _find_last_close(s: str, opener: str, closer: str) -> tuple[int, str, int] | None: """Walk ``s`` backwards from the last ``closer`` to find its matching opener. Returns ``(close_idx, opener_char, open_idx)`` for the rightmost balanced structure, or None if no pair exists. """ # Find the last `closer` candidate. last = s.rfind(closer) while last >= 0: # Walk left, tracking depth from the perspective of the opener. depth = 1 in_str = False esc = False for j in range(last - 1, -1, -1): c = s[j] if in_str: if esc: esc = False elif c == "\\": esc = True elif c == '"': in_str = False continue if c == '"': # Approximation: we don't track quotes perfectly walking # backwards, but strings in agent output are short and rare. in_str = not in_str elif c == closer: depth += 1 elif c == opener: depth -= 1 if depth == 0: return (last, opener, j) last = s.rfind(closer, 0, last) return None def split_findings(findings: list[dict], anchors: dict[str, set[int]]) -> tuple[list[dict], list[dict]]: """Split findings into (anchored, unanchored). A finding is anchored if its path is known AND its line is a valid post-change line for that path. Lines just outside the diff (model off-by-one) are NOT anchored — safer to keep them as summary bullets than to drop or misplace. """ anchored, unanchored = [], [] for f in findings: valid = anchors.get(f["path"]) if valid and f["line"] in valid: anchored.append(f) else: unanchored.append(f) return anchored, unanchored def _lang_for_path(path: str) -> str: """Map a file extension to a chroma language tag for fenced code blocks. Used so the suggested-fix block is syntax-highlighted in Gitea. Gitea 1.26.x has no GitHub-style "Apply suggestion" button (the ```suggestion fence is just an unknown-language code block → plain monospace, no apply), so we tag the block with the file's real language for highlighting instead. """ ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" return { "java": "java", "kt": "kotlin", "scala": "scala", "groovy": "groovy", "ts": "typescript", "tsx": "tsx", "js": "javascript", "jsx": "jsx", "mjs": "javascript", "cjs": "javascript", "py": "python", "pyi": "python", "go": "go", "rs": "rust", "rb": "ruby", "php": "php", "c": "c", "h": "c", "cpp": "cpp", "cc": "cpp", "hpp": "cpp", "cs": "csharp", "swift": "swift", "m": "objc", "sh": "bash", "bash": "bash", "zsh": "bash", "yml": "yaml", "yaml": "yaml", "json": "json", "jsonc": "json", "toml": "toml", "ini": "ini", "cfg": "ini", "html": "html", "htm": "html", "css": "css", "scss": "scss", "xml": "xml", "svg": "xml", "sql": "sql", "md": "markdown", "dockerfile": "dockerfile", }.get(ext, "") _SEVERITY_EMOJI = { "critical": "🔴", "high": "🔴", "medium": "🟡", "low": "🔵", "trivial": "⚪", "info": "⚪", "nit": "⚪", } # Severities whose own name is rendered verbatim (uppercased) in the badge. # Anything outside this set falls back to "INFO" so the badge label stays # a clean short token regardless of what the model emits. _BADGED_SEVERITY_LABELS = frozenset({ "critical", "high", "medium", "low", "trivial", "info", "nit", }) def _severity_badge(severity: str) -> str: """Render the severity as emoji + uppercase label (e.g. ``🔴 [HIGH]``).""" sev = (severity or "").lower() emoji = _SEVERITY_EMOJI.get(sev, "⚪") label = sev.upper() if sev in _BADGED_SEVERITY_LABELS else "INFO" return f"{emoji} [{label}]" def _format_reference(ref: str) -> str: """Render a reference URL as a clean Markdown hyperlink. ``"https://example.com/x"`` → ``"[example.com/x](https://example.com/x)"``. Accepts the bare URL form so older findings still render readably; drops anything that doesn't look like a URL rather than embedding raw text in parens (the spec says: never print raw URLs). """ ref = (ref or "").strip() if not ref: return "" if not (ref.startswith("http://") or ref.startswith("https://")): # Non-URL text (e.g. a CVE id, a doc title). Render as plain text — # `[CVE-2024-1](CVE-2024-1)` would render as a broken *relative* link # in Gitea, which is worse than no link at all. return ref # Strip the scheme + www. for the visible label so the link text is short. visible = ref for prefix in ("https://", "http://"): if visible.startswith(prefix): visible = visible[len(prefix):] break if visible.startswith("www."): visible = visible[4:] # Drop trailing slash + truncate any path noise past 60 chars. visible = visible.rstrip("/") if len(visible) > 60: visible = visible[:57] + "…" return f"[{visible}]({ref})" def inline_comment_body(f: dict) -> str: """Render one finding as a positional review-comment body. Shape: * Severity badge with emoji (🔴 HIGH / 🟡 MEDIUM / 🔵 LOW / ⚪ INFO). * 1–2 short paragraphs: ``problem`` + optional ``fix``. * ``suggestion`` block (Gitea/Forgejo apply-on-click) when the model produced replacement code. Language-tagged fences are reserved for cross-file patterns the suggestion block can't carry. * Reference as a Markdown hyperlink (``[label](url)``) — never a raw URL. * Per-comment attributed output tokens (`🪙 ~N tok (P% · attributed)`) when the caller passed `compute_attribution` data. Hidden when the finding has no attributed tokens (e.g. legacy callers / ollama path without usage metering). """ badge = _severity_badge(f.get("severity", "medium")) body = f"{badge} {f.get('problem', '').strip()}" fix = (f.get("fix") or "").strip() if fix: body += f"\n\n**Fix:** {fix}" suggestion = (f.get("suggestion") or "").strip() if suggestion: # `suggestion` fence is the standard one-click-apply block in # Gitea/Forgejo/GitHub. The agent's replacement lines must already be # indented as in the target file. body += f"\n\n```suggestion\n{suggestion}\n```" ref_md = _format_reference(f.get("reference", "")) if ref_md: body += f"\n\n🔗 **Reference:** {ref_md}" tok = f.get("_tok_attrib") if tok is not None: pct = (f.get("_tok_pct", 0.0) or 0.0) * 100 body += f"\n\n🪙 ~{pipeline.fmt_tokens(tok)} tok ({pct:.0f}% · attributed output)" return body def summary_bullets(findings: list[dict]) -> str: """Render unanchored findings as PR-level bullets. Used for findings that couldn't be anchored to a post-change line (no inline comment posted). Each bullet carries severity, location, problem, fix, and a Markdown-linked reference. """ lines = [] for f in findings: loc = f"{f['path']}:{f['line']}" if f["line"] else f["path"] badge = _severity_badge(f.get("severity", "medium")) problem = f.get("problem", "").strip() body = f"- {badge} `{loc}` — {problem}" fix = (f.get("fix") or "").strip() if fix: body += f"\n - **Fix:** {fix}" ref_md = _format_reference(f.get("reference", "")) if ref_md: body += f"\n - 🔗 **Reference:** {ref_md}" lines.append(body) return "\n".join(lines) def findings_table(findings: list[dict]) -> str: """Render ALL findings as a Markdown table for the PR-level comment. Columns: severity emoji, location (path:line), and a one-line summary. Findings with empty location collapse to just the severity + summary. """ if not findings: return "" header = "| Severity | Location | Finding |\n|---|---|---|" rows = [] for f in findings: badge = _severity_badge(f.get("severity", "medium")) path = (f.get("path") or "").strip() line = f.get("line") loc = f"`{path}:{line}`" if line else (f"`{path}`" if path else "_(no location)_") problem = (f.get("problem") or "").strip() # Escape pipes inside the finding text so the table stays valid. problem_esc = problem.replace("|", "\\|").replace("\n", " ") rows.append(f"| {badge} | {loc} | {problem_esc} |") return "\n".join([header, *rows]) def _render_collapsible_usage(usage: dict | None, model: str, config: dict | None) -> str: """Render the telemetry as a collapsible ``
`` block. Empty string when `usage` is None. The equivalent-cost table is the operator's budgeting signal — the pilot runs on a free tier, so the `actual` line is $0.00; the table shows what the same measured tokens would bill on mainstream paid APIs (configurable via `compare_against`, defaulting to ``DEFAULT_COMPARE_AGAINST``). The row matching `cost_target` is bolded so the price target stands out. The whole table is omitted when every row would be $0 (no work done). The `actual` parenthetical clause reflects the *actually-routed* model (`model` arg, resolved by caller from `OPENCODE_MODEL` env or `headroom/{OLLAMA_MODEL}`) — cost == 0 → "free tier", nonzero → "billed". """ if not usage: return "" dur = usage.get("duration_s") dur_s = f"{dur}s" if dur is not None else "?" actual = usage.get("cost") or 0.0 actual_s = f"${actual:.4f}" if actual else "$0.00" actual_note = f" ({model} — {'free tier' if not actual else 'billed'})" cost_target, price_err = pipeline._resolve_price_target(config) if price_err: # Surface config typos loudly but do not pollute the posted summary # body — typos at the table-row level would render as English # mid-table and look like a model error. print(f"pragent: {price_err}", file=sys.stderr, flush=True) # Lazy: cost_model has no dep on ai_review, and the ollama path # never reaches this branch. from cost_model import PRICES as _PRICES cfg = config or {} compare: list[str] = list(cfg.get("compare_against") or DEFAULT_COMPARE_AGAINST) # Always include the resolved cost_target (env + config), even when the # operator pinned a different `compare_against` roster — the price target # row is the one maintainers eyeball against. Skip silently if the key # isn't a known Price (e.g. a typo that slipped past stderr earlier). if cost_target in _PRICES and cost_target not in compare: compare.append(cost_target) eq_rows: list[str] = [] for key in compare: if key not in _PRICES: continue c = pipeline.equivalent_cost(usage, key) if c <= 0: continue label = _PRICES[key].name cost_str = f"${c:.4f}" if c < 0.01 else f"${c:.2f}" bold = "**" if key == cost_target else "" eq_rows.append(f"| {bold}{label}{bold} | {cost_str} |") in_tok = usage.get("input", 0) out_tok = usage.get("output", 0) reason_tok = usage.get("reasoning", 0) cache_r = usage.get("cache_read", 0) cache_w = usage.get("cache_write", 0) total = usage.get("total", 0) scope = ( "Whole-repo checkout at head sha (agent can read any file + run " "linters, not just the diff) — input tokens include files read " "beyond the diff. Per-comment output is *attributed* (one model pass " "produces all findings; output split by each finding's body weight)." ) lines = [ "
", "🔋 AI Usage & Run Details", "", f"- **Model / Engine**: `{model}` · opencode · {usage.get('steps', 0)} steps · {dur_s}", f"- **Total Tokens**: {pipeline.fmt_tokens(in_tok)} in / {pipeline.fmt_tokens(out_tok)} out " f"({pipeline.fmt_tokens(reason_tok)} reasoning, cache {pipeline.fmt_tokens(cache_r)} read / " f"{pipeline.fmt_tokens(cache_w)} write, {pipeline.fmt_tokens(total)} total)", f"- **Actual**: {actual_s}{actual_note}", f"- **Scope**: {scope}", ] if eq_rows: lines.append("") lines.append("- **Equivalent cost on paid providers** (this run's tokens):") lines.append("") lines.append("| Provider | Cost |") lines.append("|---|---:|") lines.extend(eq_rows) # Multi-lens fan-out: surface the lens roster + summed steps so the user # can see which lenses contributed (and that triage didn't drop them all). lenses = usage.get("lenses") if lenses: ls = usage.get("lens_steps", usage.get("steps", 0)) lines.append( f"- **Lenses**: {', '.join(f'`{x}`' for x in lenses)} " f"({len(lenses)} parallel subprocesses, {ls} summed steps)" ) lines += ["", "
"] return "\n".join(lines) # ---------------------------------------------------------------------------