Compare commits
51 Commits
cdf116ece9
...
317942e15b
| Author | SHA1 | Date | |
|---|---|---|---|
| 317942e15b | |||
| 0d73a075cd | |||
| e5d4e252de | |||
| 92020d4d46 | |||
| e344831b05 | |||
| 2ba3ccbd95 | |||
| 99255bb167 | |||
| c9809bce9b | |||
| 5fe83b0b2a | |||
| 218a8dc271 | |||
| e688ea61c5 | |||
| dd49d83933 | |||
| 6f7957010e | |||
| be24ef245c | |||
| bb9b6aa12d | |||
| fe5bebb4cf | |||
| d0f99b9763 | |||
| f3a09b9397 | |||
| 4e5f43ada7 | |||
| 8472f35a58 | |||
| 69e1fc06a2 | |||
| 6cdccb48ad | |||
| 7f37a36722 | |||
| b6b8173ccb | |||
| a9e1b7ddfc | |||
| 4c06a9ab3c | |||
| d38e1c8693 | |||
| 8e8ae54669 | |||
| 661199dad2 | |||
| 3e03fb80a7 | |||
| 2432228d68 | |||
| 2c7d4803f1 | |||
| 86352a3771 | |||
| e1b74d982d | |||
| 33e4c16782 | |||
| 979c93bdbb | |||
| 66628aae8d | |||
| e5a6e8923d | |||
| 2e982846d9 | |||
| 72b77a96e0 | |||
| f3a125666b | |||
| 9dae850887 | |||
| 0f7903377a | |||
| 0b295e2443 | |||
| 23ec1bf74a | |||
| 2cf4bdbfe8 | |||
| 3bcf825104 | |||
| 67339da8d0 | |||
| 99014a4cd3 | |||
| f52b8d7803 | |||
| f9f6ab4bf0 |
@@ -7,3 +7,4 @@ dist/
|
|||||||
|
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.pyc
|
*.pyc
|
||||||
|
.worktrees/
|
||||||
|
|||||||
@@ -68,6 +68,12 @@ Return STRICT JSON only — same shape as the pragent primary's findings:
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The full review-level JSON shape (used by the pragent primary) also
|
||||||
|
includes three optional top-level fields — `walkthrough` (list[str]),
|
||||||
|
`risk_verdict` (str), and `test_coverage` (str) — that the synthesizer
|
||||||
|
fills in across all lenses. Lens output is free to omit them; the parser
|
||||||
|
defaults to `[]` / `""` when absent (backward compatible).
|
||||||
|
|
||||||
`ruleId` examples: `QUALITY_DEAD_CODE`, `QUALITY_HIDDEN_COMPLEXITY`,
|
`ruleId` examples: `QUALITY_DEAD_CODE`, `QUALITY_HIDDEN_COMPLEXITY`,
|
||||||
`QUALITY_INVARIANT_DROP`, `QUALITY_NAMING_CONTRADICTS`,
|
`QUALITY_INVARIANT_DROP`, `QUALITY_NAMING_CONTRADICTS`,
|
||||||
`QUALITY_SUPPRESSED_ERROR`, `QUALITY_DUPLICATED_LOGIC`. One stable
|
`QUALITY_SUPPRESSED_ERROR`, `QUALITY_DUPLICATED_LOGIC`. One stable
|
||||||
|
|||||||
@@ -67,6 +67,12 @@ Return STRICT JSON only — same shape as the pragent primary's findings:
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The full review-level JSON shape (used by the pragent primary) also
|
||||||
|
includes three optional top-level fields — `walkthrough` (list[str]),
|
||||||
|
`risk_verdict` (str), and `test_coverage` (str) — that the synthesizer
|
||||||
|
fills in across all lenses. Lens output is free to omit them; the parser
|
||||||
|
defaults to `[]` / `""` when absent (backward compatible).
|
||||||
|
|
||||||
`ruleId` examples: `DOCS_README_DRIFT`, `DOCS_FENCE_BROKEN`,
|
`ruleId` examples: `DOCS_README_DRIFT`, `DOCS_FENCE_BROKEN`,
|
||||||
`DOCS_ENV_UNDOCUMENTED`, `DOCS_LINK_ROT`, `DOCS_NO_CHANGELOG`. Use one
|
`DOCS_ENV_UNDOCUMENTED`, `DOCS_LINK_ROT`, `DOCS_NO_CHANGELOG`. Use one
|
||||||
stable ruleId per recurring pattern — it's how the synthesizer dedups
|
stable ruleId per recurring pattern — it's how the synthesizer dedups
|
||||||
|
|||||||
@@ -47,4 +47,10 @@ O(n²) over bounded small n, `low` for redundant-but-rare work.
|
|||||||
{"findings":[{"severity":"...","path":"...","line":0,"problem":"...","fix":"...","suggestion":"","reference":""}]}
|
{"findings":[{"severity":"...","path":"...","line":0,"problem":"...","fix":"...","suggestion":"","reference":""}]}
|
||||||
```
|
```
|
||||||
|
|
||||||
`line` must be a post-change line. No prose outside JSON.
|
`line` must be a post-change line. No prose outside JSON.
|
||||||
|
|
||||||
|
The full review-level JSON shape (used by the pragent primary) also
|
||||||
|
includes three optional top-level fields — `walkthrough` (list[str]),
|
||||||
|
`risk_verdict` (str), and `test_coverage` (str) — that the synthesizer
|
||||||
|
fills in across all lenses. Lens output is free to omit them; the parser
|
||||||
|
defaults to `[]` / `""` when absent (backward compatible).
|
||||||
@@ -160,6 +160,12 @@ containing STRICT JSON, nothing else after it:
|
|||||||
"risks": [
|
"risks": [
|
||||||
"Bullets detailing potential bugs, edge cases, lifecycle issues, or performance risks found across the diff"
|
"Bullets detailing potential bugs, edge cases, lifecycle issues, or performance risks found across the diff"
|
||||||
],
|
],
|
||||||
|
"walkthrough": [
|
||||||
|
"a.py: adds X — short plain-prose bullet, file- or change-grouped",
|
||||||
|
"b.py: refactors Y"
|
||||||
|
],
|
||||||
|
"risk_verdict": "Low|Medium|High|Critical risk: <one-line concrete reason>",
|
||||||
|
"test_coverage": "Tests added" | "Tests changed" | "No tests for behavioral change" | "No test files in repo",
|
||||||
"findings": [
|
"findings": [
|
||||||
{
|
{
|
||||||
"severity": "critical|high|medium|low|info|nit",
|
"severity": "critical|high|medium|low|info|nit",
|
||||||
@@ -178,6 +184,16 @@ Rules:
|
|||||||
- `summary_changes` (2–4 bullets) goes into the **Summary of Changes** section.
|
- `summary_changes` (2–4 bullets) goes into the **Summary of Changes** section.
|
||||||
`risks` (bullets) goes into **Key Risks & Concerns**. Both are required;
|
`risks` (bullets) goes into **Key Risks & Concerns**. Both are required;
|
||||||
empty arrays are fine when nothing applies.
|
empty arrays are fine when nothing applies.
|
||||||
|
- `walkthrough` (2–6 bullets, file- or change-grouped) is the **Walkthrough**
|
||||||
|
section: what the PR does, where, in plain prose. Default to `[]` for a
|
||||||
|
trivial diff. Backward compatible — parsers default to `[]` if absent.
|
||||||
|
- `risk_verdict` (exactly one line) goes into the **Risk Verdict** section.
|
||||||
|
Lead with `Low|Medium|High|Critical risk:` followed by a concrete reason.
|
||||||
|
Default to `""` when not applicable. Backward compatible.
|
||||||
|
- `test_coverage` (short string) goes into the **Test Coverage** section.
|
||||||
|
Use exactly one of `"Tests added"`, `"Tests changed"`,
|
||||||
|
`"No tests for behavioral change"`, `"No test files in repo"`. Default to `""`.
|
||||||
|
Backward compatible.
|
||||||
- `suggestion` is the literal new code that replaces the flagged line(s). Minimal —
|
- `suggestion` is the literal new code that replaces the flagged line(s). Minimal —
|
||||||
just the changed lines, indented as they'd appear in the file. Empty string `""`
|
just the changed lines, indented as they'd appear in the file. Empty string `""`
|
||||||
when no safe textual replacement exists (e.g. missing test, architectural note).
|
when no safe textual replacement exists (e.g. missing test, architectural note).
|
||||||
|
|||||||
@@ -50,4 +50,10 @@ security findings only:
|
|||||||
```
|
```
|
||||||
|
|
||||||
`line` must be a post-change (context or `+`) line. Empty `suggestion` when no
|
`line` must be a post-change (context or `+`) line. Empty `suggestion` when no
|
||||||
safe replacement. No prose outside the JSON block.
|
safe replacement. No prose outside the JSON block.
|
||||||
|
|
||||||
|
The full review-level JSON shape (used by the pragent primary) also
|
||||||
|
includes three optional top-level fields — `walkthrough` (list[str]),
|
||||||
|
`risk_verdict` (str), and `test_coverage` (str) — that the synthesizer
|
||||||
|
fills in across all lenses. Lens output is free to omit them; the parser
|
||||||
|
defaults to `[]` / `""` when absent (backward compatible).
|
||||||
@@ -47,4 +47,10 @@ replacement); include a sketch only if a one-line test is obvious.
|
|||||||
{"findings":[{"severity":"...","path":"...","line":0,"problem":"...","fix":"...","suggestion":"","reference":""}]}
|
{"findings":[{"severity":"...","path":"...","line":0,"problem":"...","fix":"...","suggestion":"","reference":""}]}
|
||||||
```
|
```
|
||||||
|
|
||||||
`line` must be a post-change line in a source or test file. No prose outside JSON.
|
`line` must be a post-change line in a source or test file. No prose outside JSON.
|
||||||
|
|
||||||
|
The full review-level JSON shape (used by the pragent primary) also
|
||||||
|
includes three optional top-level fields — `walkthrough` (list[str]),
|
||||||
|
`risk_verdict` (str), and `test_coverage` (str) — that the synthesizer
|
||||||
|
fills in across all lenses. Lens output is free to omit them; the parser
|
||||||
|
defaults to `[]` / `""` when absent (backward compatible).
|
||||||
@@ -3,7 +3,8 @@
|
|||||||
An AI pull-request reviewer for Gitea that posts **inline comments with suggested
|
An AI pull-request reviewer for Gitea that posts **inline comments with suggested
|
||||||
fixes**, not a wall of prose — and reports what each review cost.
|
fixes**, not a wall of prose — and reports what each review cost.
|
||||||
|
|
||||||
Label a PR `AI-REVIEW`. A webhook wakes a service that checks the repo out at the
|
A webhook wakes for any PR on a repo whose default branch carries a
|
||||||
|
`.pr-review.json` with `"enabled": true`. The service checks the repo out at the
|
||||||
PR's head commit, reads the changed files *and the code around them*, runs the
|
PR's head commit, reads the changed files *and the code around them*, runs the
|
||||||
repo's own linters, and posts a review anchored to real lines.
|
repo's own linters, and posts a review anchored to real lines.
|
||||||
|
|
||||||
@@ -40,14 +41,16 @@ built — see [`docs/plans/`](docs/plans/).
|
|||||||
|
|
||||||
What works today:
|
What works today:
|
||||||
|
|
||||||
- a central webhook service, so onboarding a repo is *add the bot + add the label*
|
- a central webhook service, so onboarding a repo is *add the bot + commit
|
||||||
|
`.pr-review.json:enabled = true`*
|
||||||
- whole-repo context: the reviewer reads callers and types, not just the hunk
|
- whole-repo context: the reviewer reads callers and types, not just the hunk
|
||||||
- inline comments with language-highlighted suggested fixes, anchored to
|
- inline comments with language-highlighted suggested fixes, anchored to
|
||||||
post-change lines and validated in Python before posting
|
post-change lines and validated in Python before posting
|
||||||
- per-commit dedupe, and prior reviews fed back so a re-push synthesises rather
|
- per-commit dedupe, and prior reviews fed back so a re-push synthesises rather
|
||||||
than repeats
|
than repeats
|
||||||
- `.pr-review.json` for per-repo focus and house rules
|
- `.pr-review.json` for per-repo focus and house rules (also the opt-in flag)
|
||||||
- optional token/cost reporting via an `AI-USAGE` label
|
- token-usage reporting on every review, measured from opencode `step_finish`
|
||||||
|
events
|
||||||
- containment against hostile PR content (see [Security](#security))
|
- containment against hostile PR content (see [Security](#security))
|
||||||
|
|
||||||
Not yet: status checks, fail-close, attention tiering enforced in code (it is
|
Not yet: status checks, fail-close, attention tiering enforced in code (it is
|
||||||
@@ -56,21 +59,22 @@ currently a skill the agent follows), multi-model routing.
|
|||||||
## How a review runs
|
## How a review runs
|
||||||
|
|
||||||
```
|
```
|
||||||
PR labelled AI-REVIEW
|
PR opened on repo with `.pr-review.json:enabled = true`
|
||||||
│ Gitea webhook (HMAC-verified, body-capped, concurrency-bounded)
|
│ Gitea webhook (HMAC-verified, body-capped, concurrency-bounded)
|
||||||
▼
|
▼
|
||||||
review_pr()
|
review_pr()
|
||||||
1. dedupe already reviewed this exact sha? stop.
|
1. opt-in .pr-review.json:enabled=true on base? if not, skip.
|
||||||
2. fetch diff + .pr-review.json from the BASE branch
|
2. dedupe already reviewed this exact sha? stop.
|
||||||
3. checkout repo archive at head sha → temp workdir
|
3. fetch diff + .pr-review.json from the BASE branch
|
||||||
4. sanitize delete author-controlled agent-instruction files
|
4. checkout repo archive at head sha → temp workdir
|
||||||
5. brief .pragent/brief.md, untrusted parts explicitly fenced
|
5. sanitize delete author-controlled agent-instruction files
|
||||||
6. review opencode agent: read code, run linters, emit findings JSON
|
6. brief .pragent/brief.md, untrusted parts explicitly fenced
|
||||||
7. anchor validate every line against the diff's post-change lines
|
7. review opencode agent: read code, run linters, emit findings JSON
|
||||||
8. post inline comments + summary, as pragent-bot
|
8. anchor validate every line against the diff's post-change lines
|
||||||
|
9. post inline comments + summary, as pragent-bot
|
||||||
```
|
```
|
||||||
|
|
||||||
Steps 1, 2, 7 and 8 are deterministic Python. The model's only job is step 6 —
|
Steps 1, 3, 8 and 9 are deterministic Python. The model's only job is step 7 —
|
||||||
producing correct findings. It never talks to Gitea, and a finding whose line
|
producing correct findings. It never talks to Gitea, and a finding whose line
|
||||||
does not validate becomes a summary bullet rather than a misplaced comment.
|
does not validate becomes a summary bullet rather than a misplaced comment.
|
||||||
|
|
||||||
@@ -79,8 +83,8 @@ does not validate becomes a summary bullet rather than a misplaced comment.
|
|||||||
Onboarding a repo, once the service is running for that owner:
|
Onboarding a repo, once the service is running for that owner:
|
||||||
|
|
||||||
1. add `pragent-bot` as a **Write** collaborator
|
1. add `pragent-bot` as a **Write** collaborator
|
||||||
2. create the `AI-REVIEW` label
|
2. commit `.pr-review.json: {"enabled": true}` to the repo's default branch
|
||||||
3. label a PR
|
3. open a PR
|
||||||
|
|
||||||
Standing up the service itself — the webhook, the image, the Gitea SSRF
|
Standing up the service itself — the webhook, the image, the Gitea SSRF
|
||||||
allow-list, the per-owner webhook registration — is in
|
allow-list, the per-owner webhook registration — is in
|
||||||
@@ -136,8 +140,9 @@ concurrency. Full threat model and residual risks: `pilot/README-webhook.md`.
|
|||||||
|
|
||||||
The pilot runs against a self-hosted model and bills nothing per token, but the
|
The pilot runs against a self-hosted model and bills nothing per token, but the
|
||||||
token *work* is real. `pilot/cost_model.py` prices it against published API
|
token *work* is real. `pilot/cost_model.py` prices it against published API
|
||||||
rates, calibrated against runs measured through the `AI-USAGE` label
|
rates, calibrated against runs measured through the usage telemetry
|
||||||
(`OBSERVED_RUNS` in that file — append to it, don't guess).
|
(`OBSERVED_RUNS` in that file — append to it, don't guess). Tokens are summed
|
||||||
|
from opencode `step_finish` events per review.
|
||||||
|
|
||||||
Two measured reviews of a ~1100-line PR in this repo: 28 and 31 agent steps,
|
Two measured reviews of a ~1100-line PR in this repo: 28 and 31 agent steps,
|
||||||
~2.1M input tokens each, **zero cache reads or writes**. The demo repo's PR, same
|
~2.1M input tokens each, **zero cache reads or writes**. The demo repo's PR, same
|
||||||
|
|||||||
@@ -0,0 +1,411 @@
|
|||||||
|
# pragent — Update Design
|
||||||
|
|
||||||
|
**Date:** 2026-08-21
|
||||||
|
**Status:** Approved (brainstorm, 2026-08-21)
|
||||||
|
**Replaces:** none — additive + behavioral. Existing `docs/plans/2026-08-04-pragent-design.md` stays authoritative on architecture.
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The pilot has been live long enough to surface pain that the original design didn't
|
||||||
|
cover:
|
||||||
|
|
||||||
|
1. **Two labels to remember.** `AI-REVIEW` (gate) + `AI-USAGE` (opt-in for the
|
||||||
|
cost block) are per-PR. Every new contributor reads the README wrong at least
|
||||||
|
once. Reviews that the team *wanted* are skipped because nobody labeled; reviews
|
||||||
|
we *don't* want still run because the label is sticky.
|
||||||
|
2. **Token numbers are unreadable.** `Total Tokens: 2071025 in / 17303 out`
|
||||||
|
requires a mental carry. The pilot already measures the tokens; the rendering
|
||||||
|
just doesn't help.
|
||||||
|
3. **Cost is anchored on one provider.** The pilot runs free (headroom/glm-5.2)
|
||||||
|
but the only equivalent-cost line is Claude Sonnet. We can't answer "what would
|
||||||
|
this have cost on GPT / Gemini / Grok?" without running the CLI on a different
|
||||||
|
model.
|
||||||
|
4. **The PR summary is operational, not useful.** A lens-fanout run posts
|
||||||
|
`Multi-lens review of repo#index (sha X). Lenses: security,perf. Findings:
|
||||||
|
critical=0 high=1 medium=2 low=1.` That tells a reviewer *how the bot worked*,
|
||||||
|
not *what they should look at*. Real products post a risk verdict, a
|
||||||
|
file-by-file walkthrough, and a test-coverage note.
|
||||||
|
5. **Triage noise is the dominant failure mode** in every competitor (CodeRabbit,
|
||||||
|
Qodo, Greptile, DoorDash). We already address most of it (severity_floor,
|
||||||
|
per-file cap, cross-lens agreement, tone-strip), but two cheap wins are left on
|
||||||
|
the table: a per-PR *merge confidence* badge, and a richer severity scale that
|
||||||
|
includes `trivial` / `info` (CodeRabbit's pattern).
|
||||||
|
|
||||||
|
This update also distills lessons from a 30-article survey of AI code review
|
||||||
|
products (CodeRabbit, Qodo/Merge + PR-Agent, Greptile, GitHub Copilot code
|
||||||
|
review, Gemini Code Assist, qodo-ai/pr-agent, anc95/ChatGPT-CodeReview, Sourcery,
|
||||||
|
Danger, plus the security literature around the April 2026 prompt-injection
|
||||||
|
disclosures). Where we already match the state of the art, this update notes it
|
||||||
|
and moves on; where a competitor's pattern is genuinely better, it lands here.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| Question | Decision | Why |
|
||||||
|
|---|---|---|
|
||||||
|
| Trigger | `.pr-review.json:enabled` on the PR's base branch | Repo opt-in replaces labels. No per-PR manual step. Trust stays on base. |
|
||||||
|
| Default when `.pr-review.json` is absent | Disabled | Explicit opt-in. Mirrors "labels fully removed." |
|
||||||
|
| Cost model | Always render when usage data is present | Drop the `report_usage` parameter + `AI-USAGE` label + `PRAGENT_USAGE_ALWAYS` env. |
|
||||||
|
| Token rendering | `1,234,567 (1.2M)` | Python `f"{n:,}"` + short suffix only when `n ≥ 1000`. |
|
||||||
|
| Multi-provider cost | Markdown table in the collapsible usage block | Replaces the single Sonnet line. Default compare set: Sonnet, GPT-5, Gemini 2.5 Pro, Grok 4.5. |
|
||||||
|
| Summary depth | Add `walkthrough` / `risk_verdict` / `test_coverage` to the agent JSON; Python fallback for lens synthesis | Agent produces the rich text; Python derives the same three when the lens fan-out is engaged. |
|
||||||
|
| Severity scale | Extend from 4 → 6 levels: add `trivial` + `info` | Matches CodeRabbit. Backward compat (unknown → medium). |
|
||||||
|
| Merge confidence | 1–5 integer in the review header. Python-computed. | Stole the badge idea from Greptile. |
|
||||||
|
| Reachability demotion | Defer | Needs the security graph. Note in §7. |
|
||||||
|
| Rules mining from feedback | Defer | `feedback_harvest` / `feedback_analyze` exist; distillation is a separate effort. |
|
||||||
|
| Sequence diagrams / T-rex / cross-repo | Skip | Too heavy for the pilot. |
|
||||||
|
|
||||||
|
## 1. Label removal + repo opt-in
|
||||||
|
|
||||||
|
### `.pr-review.json` schema delta
|
||||||
|
|
||||||
|
```diff
|
||||||
|
{
|
||||||
|
+ "enabled": true,
|
||||||
|
"focus": [...],
|
||||||
|
"exclude_paths": [...],
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`enabled` is a top-level boolean, default `false`, read from the **base branch**
|
||||||
|
(unchanged trust rule — `fetch_repo_config(ref=base_ref)` already handles this).
|
||||||
|
|
||||||
|
### Webhook behavior (`pilot/webhook_server.py`)
|
||||||
|
|
||||||
|
- Remove constants `AI_REVIEW_LABEL`, `AI_USAGE_LABEL`. Remove
|
||||||
|
`_labels_have_ai_review`. Remove the `report_usage` plumbing from
|
||||||
|
`_handle_pull_request` and `_run_review`.
|
||||||
|
- New helper `is_repo_enabled(api, repo, ref, token) -> bool` in `webhook_server.py`
|
||||||
|
(or reused via `fetch_repo_config` — see below). `False` on any failure
|
||||||
|
(404, parse error, missing key, malformed value). Logs the reason to stderr.
|
||||||
|
- `_handle_pull_request` order of operations:
|
||||||
|
1. `action in SKIP_ACTIONS` → `200 ignore`
|
||||||
|
2. base_ref present + fetch config
|
||||||
|
3. `if not config.get("enabled")` → `200 "skip (repo not opted in)"`
|
||||||
|
4. claim in-flight slot
|
||||||
|
5. thread off `_run_review`
|
||||||
|
- Pre-claim gate keeps opted-out repos from consuming concurrency slots on
|
||||||
|
bursts. One extra `GET contents/.pr-review.json` per PR event (404 for
|
||||||
|
unconfigured repos) — negligible.
|
||||||
|
|
||||||
|
### `pilot/ai_review.py` cleanup
|
||||||
|
|
||||||
|
- Delete `AI_REVIEW_LABEL`, `AI_USAGE_LABEL` constants.
|
||||||
|
- Delete `pr_has_label()` helper (its only call sites were the AI-USAGE
|
||||||
|
re-reads at render time).
|
||||||
|
- Drop the `report_usage: bool` parameter from `review_pr()`. Always render
|
||||||
|
the collapsible usage block when `usage` is not None.
|
||||||
|
- Remove the two `PRAGENT_USAGE_ALWAYS` references (env reads).
|
||||||
|
- Extend `parse_repo_config()` to extract `enabled` (validate is bool,
|
||||||
|
default False).
|
||||||
|
- Extend `effective_config()` to preserve `enabled` through the style-defaults
|
||||||
|
merge.
|
||||||
|
|
||||||
|
### Docs
|
||||||
|
|
||||||
|
- `README.md`: rewrite "Label a PR `AI-REVIEW`" + "add the AI-REVIEW label" to
|
||||||
|
"commit `.pr-review.json: {"enabled": true}` to the default branch." Drop the
|
||||||
|
AI-USAGE paragraph. Update the flow diagram.
|
||||||
|
- `pilot/README-webhook.md`: replace onboarding steps. Drop the per-PR label
|
||||||
|
ceremony.
|
||||||
|
- `pilot/README.md` (CI-step path): if it still references labels, remove.
|
||||||
|
|
||||||
|
## 2. Token humanization
|
||||||
|
|
||||||
|
New helper in `pilot/ai_review.py`:
|
||||||
|
|
||||||
|
```
|
||||||
|
def fmt_tokens(n: int | None) -> str:
|
||||||
|
"""1234567 -> '1,234,567 (1.2M)'; 0 -> '0'; <1000 -> comma-form; None -> '?'."""
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- `None` → `"?"`.
|
||||||
|
- `n < 1000` → `f"{n:,}"` (no short suffix — most findings have ~tens of tokens).
|
||||||
|
- `1000 ≤ n < 1_000_000` → `f"{n:,} ({n/1000:.1f}K)"`, drop trailing `.0`.
|
||||||
|
- `1_000_000 ≤ n < 1_000_000_000` → `f"{n:,} ({n/1_000_000:.1f}M)"`.
|
||||||
|
- else `...B`.
|
||||||
|
- Negative inputs → `"?"` (defensive — never expected from usage dicts).
|
||||||
|
|
||||||
|
Apply in:
|
||||||
|
- `pilot/ai_review._render_collapsible_usage` — input, output, reasoning,
|
||||||
|
cache_read, cache_write, total.
|
||||||
|
- `pilot/ai_review.inline_comment_body` — the `🪙 ~N tok (...)` per-finding
|
||||||
|
line.
|
||||||
|
|
||||||
|
Tests: `test_fmt_tokens` golden vectors — `0`, `42`, `999`, `1000`, `1234`,
|
||||||
|
`1_234_567`, `1_234_567_890`, `None`, `-1`.
|
||||||
|
|
||||||
|
## 3. Multi-provider cost in usage section
|
||||||
|
|
||||||
|
### `pilot/cost_model.PRICES` — extend with real published rates
|
||||||
|
|
||||||
|
Source: Anthropic platform docs, OpenAI pricing, Gemini API pricing, xAI docs.
|
||||||
|
Fetched 2026-08-21. Numbers in USD per million tokens.
|
||||||
|
|
||||||
|
| key | input | output | cache_write | cache_read |
|
||||||
|
|---|---:|---:|---:|---:|
|
||||||
|
| `claude-opus-5` | 5.00 | 25.00 | 6.25 | 0.50 |
|
||||||
|
| `claude-sonnet-5` | 2.00 | 10.00 | 2.50 | 0.20 |
|
||||||
|
| `claude-haiku-4-5` | 1.00 | 5.00 | 1.25 | 0.10 |
|
||||||
|
| `gpt-5` | 1.25 | 10.00 | 1.25 | 0.125 |
|
||||||
|
| `gpt-5-mini` | 0.25 | 2.00 | 0.25 | 0.025 |
|
||||||
|
| `gemini-2.5-pro` | 1.875 | 12.50 | 1.875 | 0.1875 |
|
||||||
|
| `gemini-2.5-flash` | 0.30 | 2.50 | 0.30 | 0.03 |
|
||||||
|
| `grok-4.5` | 2.00 | 6.00 | 2.00 | 0.30 |
|
||||||
|
| `grok-4.3` | 1.25 | 2.50 | 1.25 | 0.20 |
|
||||||
|
|
||||||
|
Notes on derivation:
|
||||||
|
- Gemini 2.5 Pro publishes a tiered range (`$1.25–$2.50` in, `$10–$15` out,
|
||||||
|
`$0.125–$0.25` cached). Midpoints are taken for a single line; the
|
||||||
|
`compare_against` field lets a repo override per-key if precision matters.
|
||||||
|
- Providers without a separate cache_write charge (OpenAI, Gemini, Grok) set
|
||||||
|
`cache_write = input` so the existing `cost()` formula continues to work
|
||||||
|
without a branch on provider.
|
||||||
|
- `cost_target` (the highlighted single line) and `compare_against` (the table)
|
||||||
|
are independent fields — see §3.2.
|
||||||
|
|
||||||
|
### 3.1 Render
|
||||||
|
|
||||||
|
Replace the single `**Est. cost on {provider}**: $X.XX` line in
|
||||||
|
`_render_collapsible_usage` with a compact markdown table:
|
||||||
|
|
||||||
|
```
|
||||||
|
**Equivalent cost on paid providers** (this run's measured tokens):
|
||||||
|
|
||||||
|
| Provider | Cost |
|
||||||
|
|---|---:|
|
||||||
|
| Claude Sonnet 5 | $4.32 |
|
||||||
|
| GPT-5 | $2.71 |
|
||||||
|
| Gemini 2.5 Pro | $4.04 |
|
||||||
|
| Grok 4.5 | $4.32 |
|
||||||
|
```
|
||||||
|
|
||||||
|
Sort cheapest-first. Skip rows whose cost is `$0.00`. Bold the row matching
|
||||||
|
`cost_target` (the user-selected highlight).
|
||||||
|
|
||||||
|
### 3.2 Config
|
||||||
|
|
||||||
|
`.pr-review.json`:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"cost_target": "claude-sonnet-5",
|
||||||
|
"compare_against": ["claude-sonnet-5", "gpt-5", "gemini-2.5-pro", "grok-4.5"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`parse_repo_config()`:
|
||||||
|
- Validate each key exists in `PRICES`. Drop unknowns to stderr (keeps
|
||||||
|
`cost_model._resolve_price_target`'s typo-reporting consistent).
|
||||||
|
- Cap the list at `CONFIG_MAX_LIST_ITEMS` (12).
|
||||||
|
- Default when absent: `["claude-sonnet-5", "gpt-5", "gemini-2.5-pro",
|
||||||
|
"grok-4.5"]`.
|
||||||
|
|
||||||
|
### 3.3 Tests
|
||||||
|
|
||||||
|
`tests/pilot/test_cost_model.py`:
|
||||||
|
- Add equivalent-cost golden vectors against the new price keys.
|
||||||
|
- Update `test_observed_report_prices_every_model` and
|
||||||
|
`test_report_renders_every_requested_model` to cover the new keys.
|
||||||
|
- Add `test_compare_against_parsing` (valid / unknown / over-cap / missing).
|
||||||
|
|
||||||
|
## 4. Richer review summary
|
||||||
|
|
||||||
|
### 4.1 Schema additions (agent prompts + `SYSTEM_PROMPT`)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"walkthrough": [
|
||||||
|
"file X: does Y",
|
||||||
|
"file Z: refactors W"
|
||||||
|
],
|
||||||
|
"risk_verdict": "Medium risk: changes auth middleware without adding tests.",
|
||||||
|
"test_coverage": "No tests for behavioral change in pilot/foo.py."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules (added to `.opencode/agents/pragent.md`, each lens agent `.md`, and the
|
||||||
|
ollama `SYSTEM_PROMPT`):
|
||||||
|
- `walkthrough`: 2–6 bullets, file- or change-grouped, plain prose (no
|
||||||
|
severity emoji). Skip if the diff is one obvious line.
|
||||||
|
- `risk_verdict`: exactly one line. Lead with `Low|Medium|High|Critical risk:`
|
||||||
|
followed by a concrete reason grounded in the diff.
|
||||||
|
- `test_coverage`: short string. One of
|
||||||
|
`Tests added` / `Tests changed` / `No tests for behavioral change` /
|
||||||
|
`No test files in repo` / a repo-specific free-text override from
|
||||||
|
`instructions`.
|
||||||
|
|
||||||
|
### 4.2 Parsing
|
||||||
|
|
||||||
|
Extend `parse_review_output(text)` and the lens fan-out's synthetic-text
|
||||||
|
builder (`pilot/opencode_review.run_lenses_review`) to emit these three
|
||||||
|
fields in the final JSON block. Empty defaults preserve backward compat with
|
||||||
|
agents that haven't been re-deployed yet.
|
||||||
|
|
||||||
|
### 4.3 Python fallback (when fields are empty)
|
||||||
|
|
||||||
|
The multi-lens fan-out already synthesizes the findings JSON in Python today;
|
||||||
|
add a `_synthesize_summary_fields(findings, diff) -> dict` helper that
|
||||||
|
computes:
|
||||||
|
- `walkthrough`: group `merged` findings by `path`, one bullet per path
|
||||||
|
containing the peak severity emoji and the first-problem truncated to ~80
|
||||||
|
chars. If `merged` is empty, list `changed_files(diff)` with the size of
|
||||||
|
the diff as the body ("`pilot/foo.py` — +12 lines").
|
||||||
|
- `risk_verdict`: from `sev_counts` and `_multi_lens` flags:
|
||||||
|
- any critical → `Critical risk: <N> critical finding(s).`
|
||||||
|
- any high → `High risk: <N> high finding(s).`
|
||||||
|
- any medium → `Medium risk: <N> medium finding(s) (<lens> lens).`
|
||||||
|
- else `Low risk: clean or minor nits only.`
|
||||||
|
- `test_coverage`: scan `changed_files(diff)` with `is_test_path()`. Three
|
||||||
|
buckets:
|
||||||
|
- any test path changed alongside non-test paths → `Tests added` (or
|
||||||
|
`Tests changed`).
|
||||||
|
- non-test paths present, no test path → `No tests for behavioral change in
|
||||||
|
<first non-test path>.`
|
||||||
|
- no test paths at all and non-test paths present → `No tests for
|
||||||
|
behavioral change in <first non-test path>.` (same as above; the
|
||||||
|
distinction "no test files in repo" needs a tree scan — keep it simple
|
||||||
|
for v1).
|
||||||
|
|
||||||
|
### 4.4 Render
|
||||||
|
|
||||||
|
Extend `format_review_body()` to render three new sections between
|
||||||
|
`### Summary of Changes` and `### Key Risks & Concerns`:
|
||||||
|
|
||||||
|
```
|
||||||
|
### Risk Verdict
|
||||||
|
🟡 Medium risk: changes auth middleware without adding tests.
|
||||||
|
|
||||||
|
### Walkthrough
|
||||||
|
- `pilot/foo.py` — adds retry logic for transient Gitea API errors
|
||||||
|
- `pilot/bar.py` — extracts shared header parser
|
||||||
|
|
||||||
|
### Test Coverage
|
||||||
|
No tests for behavioral change in pilot/foo.py.
|
||||||
|
```
|
||||||
|
|
||||||
|
Each section renders an `_No <section> provided._` placeholder when empty
|
||||||
|
(matches the existing `Summary of Changes` / `Key Risks & Concerns` collapse
|
||||||
|
behavior).
|
||||||
|
|
||||||
|
### 4.5 Tests
|
||||||
|
|
||||||
|
`tests/pilot/test_ai_review.py`:
|
||||||
|
- Golden vectors for each new section (provided + Python-fallback paths).
|
||||||
|
- Combined body test: summary + walkthrough + risk + tests + table +
|
||||||
|
collapsible usage all render in the right order with no orphan markers.
|
||||||
|
|
||||||
|
## 6. Stolen ideas
|
||||||
|
|
||||||
|
### 6.1 Merge confidence 1–5 (Greptile)
|
||||||
|
|
||||||
|
New function `merge_confidence(findings: list[dict]) -> int` in
|
||||||
|
`pilot/ai_review.py`:
|
||||||
|
|
||||||
|
```
|
||||||
|
start at 5
|
||||||
|
-1 if any critical finding
|
||||||
|
-1 if any high finding
|
||||||
|
-1 if any medium finding
|
||||||
|
-1 if any _multi_lens: True finding (cross-lens agreement = harder to dismiss)
|
||||||
|
clamp to [1, 5]
|
||||||
|
```
|
||||||
|
|
||||||
|
Render in `REVIEW_HEADER`:
|
||||||
|
|
||||||
|
```
|
||||||
|
🤖 **AI Review** · pragent pilot · glm-5.2:cloud · `abc12345` · Merge confidence: 3/5 🟡
|
||||||
|
```
|
||||||
|
|
||||||
|
Badge map: 5/4 = 🟢, 3 = 🟡, 2 = 🟠, 1 = 🔴.
|
||||||
|
|
||||||
|
Tests: golden vectors for all 5 score branches.
|
||||||
|
|
||||||
|
### 6.2 Add `trivial` + `info` severity levels (CodeRabbit)
|
||||||
|
|
||||||
|
Extend `SEVERITIES` and `SEVERITY_RANK`:
|
||||||
|
|
||||||
|
```
|
||||||
|
SEVERITIES = ("critical", "high", "medium", "low", "trivial", "info")
|
||||||
|
SEVERITY_RANK = {"info": -1, "trivial": 0, "low": 1, "medium": 2, "high": 3, "critical": 4}
|
||||||
|
```
|
||||||
|
|
||||||
|
Update `_severity_badge` emoji map (`trivial`/`info` = ⚪). Update
|
||||||
|
`apply_repo_config` threshold semantics so `medium+` still means what it
|
||||||
|
meant (only `low` ranks below `medium` is unchanged). Update agent prompts
|
||||||
|
to permit emitting `trivial` / `info`. Backward compat: `_normalize_finding`
|
||||||
|
already coerces unknown severities to `medium`.
|
||||||
|
|
||||||
|
Tests: existing `test_apply_repo_config` cases keep passing; add
|
||||||
|
`test_severity_threshold_respects_new_levels` and
|
||||||
|
`test_unknown_severity_normalizes_to_medium`.
|
||||||
|
|
||||||
|
### 6.3 Reachability-aware severity demotion — DEFER
|
||||||
|
|
||||||
|
CodeRabbit Security demotes severity by one level when a vulnerability is
|
||||||
|
unreachable / only theoretically exploitable. We can't compute reachability
|
||||||
|
without the security graph. Document in §7 and revisit when a Code-Rabbit-
|
||||||
|
style graph index lands.
|
||||||
|
|
||||||
|
### 6.4 Rules mining from feedback — DEFER
|
||||||
|
|
||||||
|
`pilot/feedback_harvest.py` + `pilot/feedback_analyze.py` exist. A future
|
||||||
|
`pilot/learn_rules.py` cron job will distill FP-vote signals into
|
||||||
|
`.pr-review.learned.json` and merge into `instructions`. Document in §7.
|
||||||
|
|
||||||
|
### 6.5 Sequence diagrams / T-rex / cross-repo — SKIP
|
||||||
|
|
||||||
|
Too heavy for the pilot's footprint. Document in §7.
|
||||||
|
|
||||||
|
## 7. Deferred (not in this update)
|
||||||
|
|
||||||
|
- **Reachability-aware severity demotion.** Requires a Code-Rabbit-style
|
||||||
|
reachability graph over the repo.
|
||||||
|
- **Rules mining from feedback.** A `learn_rules.py` job that consumes the
|
||||||
|
feedback DB and writes `.pr-review.learned.json`. `feedback_harvest` /
|
||||||
|
`feedback_analyze` are the substrate.
|
||||||
|
- **Sequence diagrams / T-rex sandbox / cross-repo review.** Three features
|
||||||
|
Greptile / Qodo highlight. All require either a code graph index (heavy
|
||||||
|
precompute) or sandbox runtime execution (separate infra). Skip.
|
||||||
|
- **Per-finding confidence scores.** Greptile publishes a 0–5 score on every
|
||||||
|
comment. We deliberately stay on severity — confidence on findings
|
||||||
|
requires the agent to self-estimate, which is unreliable without a
|
||||||
|
cross-lens consensus check. The merge-confidence badge (§6.1) is the
|
||||||
|
higher-signal version of the same idea.
|
||||||
|
- **`Fix with Cursor` handoff.** Greptile ships a one-click "send all findings
|
||||||
|
to Cursor/Codex/Claude Code." Our users *are* the bot's host, not an
|
||||||
|
external coding IDE. Skip.
|
||||||
|
- **Cost-model batch column.** The cost model already prices batch at 50%;
|
||||||
|
the PR-review path will never use it (stateful agent loops aren't
|
||||||
|
batchable). Keep the column for completeness, no new work.
|
||||||
|
|
||||||
|
## 8. Risk register
|
||||||
|
|
||||||
|
| Risk | Mitigation |
|
||||||
|
|---|---|
|
||||||
|
| Webhook floods the API with `.pr-review.json` fetches on a large owner | One `GET` per PR event, mostly 404. Documented acceptable. The dedicated `/health` already reports inflight count. |
|
||||||
|
| `.pr-review.json:enabled` set on a high-traffic repo creates surprise review load | The README will document the opt-in explicitly. The webhook's `PRAGENT_MAX_CONCURRENT_REVIEWS` already bounds the spawn rate. |
|
||||||
|
| New severity levels (`trivial` / `info`) break repos that filter on `medium+` | `apply_repo_config` threshold semantics preserve the rank of `low` and `medium`. `trivial` ranks below `low`, `info` below `trivial`. New filters naturally include them. |
|
||||||
|
| Token humanization loses precision a maintainer relies on | `fmt_tokens` always keeps the full comma-separated number; the short suffix is a parenthetical. |
|
||||||
|
| Multi-provider cost table is misleading when a provider has tiered pricing | `compare_against` is a per-repo override. The README documents the midpoints for Gemini 2.5 Pro. |
|
||||||
|
| Agent prompt change for `walkthrough` / `risk_verdict` / `test_coverage` causes regressions on deployed agents | Python fallback (§4.3) synthesizes the same fields when the agent omits them. Backward compat preserved by empty defaults. |
|
||||||
|
| Removing `report_usage` breaks the `review_pr` tests that pass it | Test updates are part of this update. |
|
||||||
|
| Removing labels breaks users who still apply them | No Gitea API change is needed; the labels just stop being read. A one-paragraph README note acknowledges the change. |
|
||||||
|
|
||||||
|
## 9. Prioritized implementation list
|
||||||
|
|
||||||
|
| # | Item | Section | Effort |
|
||||||
|
|---|---|---|---|
|
||||||
|
| P0 | Label removal + repo opt-in (`enabled` in `.pr-review.json`) | §1 | M |
|
||||||
|
| P1 | `fmt_tokens()` helper + apply in usage + inline | §2 | S |
|
||||||
|
| P1 | Multi-provider cost table (extend `PRICES`, render table, `compare_against`) | §3 | M |
|
||||||
|
| P2 | Richer summary (`walkthrough` / `risk_verdict` / `test_coverage`) schema + Python fallback | §4 | L |
|
||||||
|
| P2 | `trivial` + `info` severity levels | §6.2 | S |
|
||||||
|
| P3 | Merge confidence 1–5 in review header | §6.1 | S |
|
||||||
|
| P3 | README + `pilot/README-webhook.md` rewrite | §1, §10 | S |
|
||||||
|
| P3 | Test updates across all sections | (each) | M |
|
||||||
|
|
||||||
|
P0 first because it changes webhook behavior (must land with the repo-opt-in
|
||||||
|
docs so onboarding isn't broken mid-rollout). P1 items are independent and
|
||||||
|
small — ship together. P2 ships the user-visible summary improvement.
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
|||||||
|
# pragent pilot — central dashboard service.
|
||||||
|
#
|
||||||
|
# Read-only overview + per-repo / per-PR drilldown over the same SQLite
|
||||||
|
# feedback DB the webhook writes. Also mutates `.pr-review.json` on covered
|
||||||
|
# repos via the Gitea contents API (Tasks C+D in pilot/dashboard.py). Same
|
||||||
|
# image as the webhook (`pragent-webhook:optin`) — all pilot modules are
|
||||||
|
# baked in at /app/pilot/.
|
||||||
|
#
|
||||||
|
# Routes: GET / (overview), GET /r/<o>/<n> (repo), GET /r/<o>/<n>/<i> (PR),
|
||||||
|
# GET /r/<o>/<n>/<i>/raw (PR markdown raw), GET /login, GET /static/style.css,
|
||||||
|
# POST /login, POST /r/<o>/<n>/edit.
|
||||||
|
#
|
||||||
|
# Auth: PRAGENT_DASHBOARD_TOKEN in the pragent-webhook Secret, cookie
|
||||||
|
# `pragent_dash=<token>`, single-user. Empty / unset = no auth (tailnet-only).
|
||||||
|
#
|
||||||
|
# NodePort 30082 — only reachable on the Tailscale / LAN side of kubernets
|
||||||
|
# (100.74.17.70 / 192.168.1.80) until/unconfigured. Mirrors pragent-webhook.yaml
|
||||||
|
# in every other respect (uid 10001, nodeSelector, /data PVC).
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: pragent-dashboard
|
||||||
|
namespace: pragent
|
||||||
|
labels:
|
||||||
|
app: pragent-dashboard
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: pragent-dashboard
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: pragent-dashboard
|
||||||
|
spec:
|
||||||
|
# Same node as the webhook — holds the headroom proxy + the /data PVC.
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: kubernets
|
||||||
|
# Dashboard is read-only over /data and only mutates Gitea (not local
|
||||||
|
# files), so unprivileged is fine. fsGroup matches the image's USER
|
||||||
|
# directive (10001) so the RO mount is readable.
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 10001
|
||||||
|
runAsGroup: 10001
|
||||||
|
fsGroup: 10001
|
||||||
|
containers:
|
||||||
|
- name: dashboard
|
||||||
|
image: pragent-webhook:optin
|
||||||
|
imagePullPolicy: Never
|
||||||
|
workingDir: /app
|
||||||
|
command: ["python3", "-m", "pilot.dashboard"]
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8081
|
||||||
|
env:
|
||||||
|
- name: PRAGENT_FEEDBACK_DB
|
||||||
|
value: /data/feedback.db
|
||||||
|
- name: PRAGENT_GITEA_API
|
||||||
|
value: http://gitea-http.gitea.svc.cluster.local:3000
|
||||||
|
# Dashboard reads DASHBOARD_PORT (not PORT) — verified in
|
||||||
|
# pilot/dashboard.py:51. Default 8081 if unset.
|
||||||
|
- name: DASHBOARD_PORT
|
||||||
|
value: "8081"
|
||||||
|
# Used by /r/<o>/<n>/edit to PUT updated JSON to the repo's
|
||||||
|
# contents API. Reuses the same bot token the webhook uses.
|
||||||
|
- name: PRAGENT_BOT_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: pragent-webhook
|
||||||
|
key: PRAGENT_BOT_TOKEN
|
||||||
|
# Auth cookie value. Add to the pragent-webhook Secret with:
|
||||||
|
# kubectl patch secret pragent-webhook -n pragent --type=json \
|
||||||
|
# -p='[{"op":"add","path":"/data/PRAGENT_DASHBOARD_TOKEN","value":"<base64>"}]'
|
||||||
|
- name: PRAGENT_DASHBOARD_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: pragent-webhook
|
||||||
|
key: PRAGENT_DASHBOARD_TOKEN
|
||||||
|
# /data is read-only — the dashboard doesn't write the SQLite file;
|
||||||
|
# .pr-review.json mutations go through the Gitea contents API, not
|
||||||
|
# local fs. RO avoids any chance of two pods racing the same RWO PVC.
|
||||||
|
volumeMounts:
|
||||||
|
- name: feedback-data
|
||||||
|
mountPath: /data
|
||||||
|
readOnly: true
|
||||||
|
# No /health route in dashboard.py (returns 404 on unknown paths).
|
||||||
|
# Probes omitted intentionally — see pilot/dashboard.py:687-721.
|
||||||
|
# Resources: dashboard is read-heavy + tiny writes. /data RO + no
|
||||||
|
# subprocess fan-out (no opencode) keeps footprint small.
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
volumes:
|
||||||
|
- name: feedback-data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: pragent-feedback-data
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: pragent-dashboard
|
||||||
|
namespace: pragent
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: pragent-dashboard
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 80
|
||||||
|
targetPort: http
|
||||||
|
nodePort: 31540
|
||||||
|
type: NodePort
|
||||||
|
# 31540 — auto-allocated at first apply (30082 was already taken by
|
||||||
|
# habitsnow/habitsnow-proxy). Tailscale / LAN only until a Caddy route is set.
|
||||||
+22
-5
@@ -1,25 +1,42 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://opencode.ai/config.json",
|
"$schema": "https://opencode.ai/config.json",
|
||||||
"default_agent": "pragent",
|
"default_agent": "pragent",
|
||||||
"model": "headroom/glm-5.2:cloud",
|
"model": "headroom/MiniMax-M2.7",
|
||||||
"small_model": "headroom/glm-5.2:cloud",
|
"small_model": "headroom/MiniMax-M2.7",
|
||||||
"provider": {
|
"provider": {
|
||||||
"headroom": {
|
"headroom": {
|
||||||
"npm": "@ai-sdk/anthropic",
|
"npm": "@ai-sdk/anthropic",
|
||||||
"name": "Headroom GLM",
|
"name": "Headroom (MiniMax passthrough)",
|
||||||
"options": {
|
"options": {
|
||||||
"baseURL": "http://model-proxy.internal:8789/v1",
|
"baseURL": "http://model-proxy.internal:8789/v1",
|
||||||
"apiKey": "ollama"
|
"apiKey": "ollama"
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
"glm-5.2:cloud": {
|
"MiniMax-M2.7": {
|
||||||
"name": "GLM 5.2 Cloud",
|
"name": "MiniMax M2.7",
|
||||||
"limit": {
|
"limit": {
|
||||||
"context": 200000,
|
"context": 200000,
|
||||||
"output": 16000
|
"output": 16000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"local": {
|
||||||
|
"npm": "@ai-sdk/openai-compatible",
|
||||||
|
"name": "Local AI workstation (qwen3.8-27b)",
|
||||||
|
"options": {
|
||||||
|
"baseURL": "http://192.168.1.79:18020/v1",
|
||||||
|
"apiKey": "PLACEHOLDER_REPLACED_AT_RUNTIME"
|
||||||
|
},
|
||||||
|
"models": {
|
||||||
|
"qwen3.8-27b": {
|
||||||
|
"name": "Qwen 3.8 27B (local)",
|
||||||
|
"limit": {
|
||||||
|
"context": 57344,
|
||||||
|
"output": 8192
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"lsp": {},
|
"lsp": {},
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
# pragent pilot — central dashboard service
|
||||||
|
|
||||||
|
A read-only overview + per-repo / per-PR drilldown over the same SQLite
|
||||||
|
feedback DB the webhook writes, plus a small form to mutate `.pr-review.json`
|
||||||
|
on a covered repo via the Gitea contents API. Companion to the
|
||||||
|
[webhook service](README-webhook.md); reuses the webhook image
|
||||||
|
(`pragent-webhook:dashboard`) — the pilot modules are baked into `/app/pilot/`,
|
||||||
|
and the dashboard is just `python3 -m pilot.dashboard`.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Browser
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Caddy (TLS, wildcard cert via Cloudflare DNS-01)
|
||||||
|
│ https://pragent-dashboard.marcospaulo.dev.br → 100.74.17.70:31541
|
||||||
|
▼
|
||||||
|
Service oauth2-proxy-dashboard.pragent.svc.cluster.local (NodePort 31541, ns pragent)
|
||||||
|
│
|
||||||
|
│ oauth2-proxy fronts the dashboard, enforces Logto SSO + email allowlist
|
||||||
|
│ sets X-Forwarded-User / X-Forwarded-Email on accepted requests
|
||||||
|
▼
|
||||||
|
Service pragent-dashboard.pragent.svc.cluster.local (ClusterIP, ns pragent)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
pragent-dashboard pod (uid 10001, /data RO, no subprocess fan-out)
|
||||||
|
│
|
||||||
|
├── read /data/feedback.db (PVC pragent-feedback-data, RO)
|
||||||
|
├── GET .../repos/{o}/{r}/... (Gitea contents API, bot token)
|
||||||
|
└── PUT .../repos/{o}/{r}/contents/.pr-review.json
|
||||||
|
(edit form submit; Gitea commits a new sha)
|
||||||
|
```
|
||||||
|
|
||||||
|
Fail-soft. Nothing is ever written to local disk by the dashboard — the
|
||||||
|
SQLite file is read-only and `.pr-review.json` mutations go through Gitea's
|
||||||
|
contents API so the commit history records who changed what.
|
||||||
|
|
||||||
|
The dashboard `Service` is **ClusterIP** — only oauth2-proxy can reach it.
|
||||||
|
Public access is gated by Caddy (TLS termination) → oauth2-proxy (Logto SSO
|
||||||
|
+ allowlist) → dashboard.
|
||||||
|
|
||||||
|
## What it does
|
||||||
|
|
||||||
|
- **Overview** (`GET /`): summary stats across all onboarded repos — total
|
||||||
|
reviews, distinct PRs, finding counts by severity, false-positive /
|
||||||
|
accepted-pattern scores (see "Feedback loop" in README-webhook.md), and a
|
||||||
|
sparkline of review activity.
|
||||||
|
- **Repo drilldown** (`GET /r/<owner>/<name>`): per-repo PRs with their
|
||||||
|
last-review status, finding counts, and links to PR-level drilldowns.
|
||||||
|
- **PR drilldown** (`GET /r/<owner>/<name>/<index>`): the bot's review(s)
|
||||||
|
on that PR, inline findings, and reaction / resolved status harvested
|
||||||
|
by `feedback_harvest.py`.
|
||||||
|
- **Raw review** (`GET /r/<owner>/<name>/<index>/raw`): the markdown body
|
||||||
|
of the most recent review, for copy-paste / diff-with-prose workflows.
|
||||||
|
- **Edit form** (`POST /r/<owner>/<name>/edit`): a small HTML page that
|
||||||
|
loads the current `.pr-review.json` from the repo's default branch and
|
||||||
|
lets the operator edit the JSON (validated, then PUT to Gitea contents
|
||||||
|
API). This is how repo-local `focus` / `instructions` /
|
||||||
|
`reviewers` / `severity_threshold` get tuned per-repo after seeing
|
||||||
|
the feedback roll-up.
|
||||||
|
|
||||||
|
All routes return HTML (or plain text for `/raw`) with the same stylesheet
|
||||||
|
(`/static/style.css`).
|
||||||
|
|
||||||
|
## Routes
|
||||||
|
|
||||||
|
| method | path | auth | description |
|
||||||
|
|--------|-----------------------------------|------|----------------------------------------------|
|
||||||
|
| GET | `/` | yes | Overview |
|
||||||
|
| GET | `/static/style.css` | no | Stylesheet |
|
||||||
|
| GET | `/r/<owner>/<name>` | yes | Repo drilldown |
|
||||||
|
| GET | `/r/<owner>/<name>/<index>` | yes | PR drilldown |
|
||||||
|
| GET | `/r/<owner>/<name>/<index>/raw` | yes | Most recent review body as markdown |
|
||||||
|
| POST | `/r/<owner>/<name>/edit` | yes | Edit `.pr-review.json` on the default branch |
|
||||||
|
|
||||||
|
Auth is enforced by oauth2-proxy upstream; the dashboard itself only
|
||||||
|
checks the `X-Forwarded-User` header that oauth2-proxy sets after a
|
||||||
|
successful Logto login + email allowlist match.
|
||||||
|
|
||||||
|
There is no `/health` route — don't add one to the k8s probes without
|
||||||
|
updating `pilot/dashboard.py` (the handler returns 404 on unknown paths,
|
||||||
|
so a probe would loop forever).
|
||||||
|
|
||||||
|
## Mutations flow through Gitea, not local fs
|
||||||
|
|
||||||
|
The edit endpoint reads the current `.pr-review.json` from
|
||||||
|
`GET /repos/{o}/{r}/contents/.pr-review.json?ref=<default-branch>`, lets
|
||||||
|
the operator edit it in a form (validated as JSON, length-capped per
|
||||||
|
field, no schema migration), and PUTs the new content back via the
|
||||||
|
contents API with a commit message like
|
||||||
|
`pragent dashboard: update .pr-review.json`. Every edit is a real Gitea
|
||||||
|
commit on the default branch, attributable to `pragent-bot`, and the
|
||||||
|
next webhook fire picks up the new config — no Pod restart, no image
|
||||||
|
rebuild, no pod-level state.
|
||||||
|
|
||||||
|
The `/data` mount is **read-only** (see the `readOnly: true` on the
|
||||||
|
volumeMount in `~/k8s/pragent-dashboard.yaml`): the dashboard never
|
||||||
|
writes the SQLite file, only the webhook + the daily cronjob do, and
|
||||||
|
keeping it RO means a buggy deploy can't corrupt the harvested feedback.
|
||||||
|
|
||||||
|
## Auth (Logto SSO via oauth2-proxy)
|
||||||
|
|
||||||
|
Authentication is delegated to oauth2-proxy, which fronts the dashboard
|
||||||
|
in-cluster. The dashboard never sees a cookie or a token — it only
|
||||||
|
inspects `X-Forwarded-User` (set by oauth2-proxy after a successful
|
||||||
|
Logto login + email allowlist match). Missing header → 401 with
|
||||||
|
`WWW-Authenticate: Basic realm="pragent-dashboard"`, which lets
|
||||||
|
oauth2-proxy intercept and bounce the browser to Logto.
|
||||||
|
|
||||||
|
Email allowlist lives in the ConfigMap `oauth2-proxy-dashboard-emails`
|
||||||
|
in namespace `pragent`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
data:
|
||||||
|
authenticated-emails: |
|
||||||
|
marcos.paulodasilva.mp@gmail.com
|
||||||
|
thiago@marcospaulo.dev.br
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit the ConfigMap to add/remove users; oauth2-proxy hot-reloads the
|
||||||
|
file (it logs `watching ... for updates`), no restart needed. This is
|
||||||
|
the same isolation pattern as the minecraft-sso / code-server
|
||||||
|
allowlists — see `~/.claude/memory/minecraft-sso.md`.
|
||||||
|
|
||||||
|
The Logto app is `pragent-dashboard` (tenant `default`, type
|
||||||
|
`Traditional`), created by direct INSERT into Logto Postgres mirroring
|
||||||
|
the proven `minecraft-sso` pattern. Credentials live in
|
||||||
|
`~/k8s/oauth2-proxy-dashboard-secret.yaml` (mode 600, NOT in git).
|
||||||
|
|
||||||
|
Public URL: **https://pragent-dashboard.marcospaulo.dev.br** (Caddy
|
||||||
|
TLS termination via wildcard cert → Tailscale → NodePort 31541 →
|
||||||
|
oauth2-proxy → dashboard ClusterIP).
|
||||||
|
|
||||||
|
### Emergency bypass (cookie)
|
||||||
|
|
||||||
|
If Logto goes down and you need to access the dashboard before the
|
||||||
|
oauth2-proxy restart dance (see `~/.claude/memory/logto-fix.md`),
|
||||||
|
`pilot/dashboard.py` can be patched to accept a fallback cookie by
|
||||||
|
re-adding the `PRAGENT_DASHBOARD_TOKEN` env path — the route gate is
|
||||||
|
isolated in `_is_authed` and the logic is straightforward. The current
|
||||||
|
commit intentionally has no bypass because Logto SSO is the single
|
||||||
|
source of truth for "who can touch `.pr-review.json`".
|
||||||
|
|
||||||
|
## Deploy
|
||||||
|
|
||||||
|
The dashboard shares the webhook image, so there's nothing to rebuild
|
||||||
|
beyond what the webhook already does. After editing `pilot/dashboard.py`
|
||||||
|
or `pilot/dashboard_data.py`, redo the webhook image rebuild + containerd
|
||||||
|
import (see `README-webhook.md` § "K8s deployment") and roll both
|
||||||
|
deployments.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
K="microk8s kubectl"
|
||||||
|
|
||||||
|
# 1. (one-time) create the Logto app + cookie secret + oauth2-proxy
|
||||||
|
# See ~/.claude/memory/minecraft-sso.md for the SQL INSERT recipe
|
||||||
|
# and ~/k8s/oauth2-proxy-dashboard*.yaml for the manifests.
|
||||||
|
|
||||||
|
# 2. apply all pragent-dashboard manifests (dashboard + oauth2-proxy)
|
||||||
|
$K apply -f ~/k8s/oauth2-proxy-dashboard.yaml
|
||||||
|
$K apply -f ~/k8s/pragent-dashboard.yaml
|
||||||
|
|
||||||
|
# 3. roll on image / code changes
|
||||||
|
$K -n pragent rollout restart deploy/pragent-dashboard
|
||||||
|
$K -n pragent rollout status deploy/pragent-dashboard --timeout=120s
|
||||||
|
$K -n pragent logs -f deploy/pragent-dashboard
|
||||||
|
```
|
||||||
|
|
||||||
|
K8s manifests:
|
||||||
|
|
||||||
|
- `~/k8s/pragent-dashboard.yaml` — Deployment + ClusterIP Service.
|
||||||
|
- `image: pragent-webhook:dashboard` + `imagePullPolicy: Never` —
|
||||||
|
local containerd only, same image as the webhook.
|
||||||
|
- `nodeSelector: kubernetes.io/hostname: kubernets` — pinned to the
|
||||||
|
node holding the `/data` PVC.
|
||||||
|
- `securityContext: runAsNonRoot: true, runAsUser: 10001, runAsGroup:
|
||||||
|
10001, fsGroup: 10001` — matches the image's USER directive;
|
||||||
|
fsGroup makes the RO hostpath volume readable.
|
||||||
|
- `volumeMounts.feedback-data.readOnly: true` — dashboard is
|
||||||
|
read-only over `/data`; mutations go through Gitea, not local fs.
|
||||||
|
- No `readinessProbe` / `livenessProbe` — the dashboard has no
|
||||||
|
`/health` route. If you add one to `pilot/dashboard.py`, add a
|
||||||
|
probe here too.
|
||||||
|
- `resources.requests: {cpu: 100m, memory: 256Mi}` /
|
||||||
|
`limits: {cpu: 500m, memory: 512Mi}` — read-heavy + tiny writes,
|
||||||
|
no opencode subprocess fan-out, much smaller than the webhook.
|
||||||
|
- `Service.type: ClusterIP` — only oauth2-proxy can reach it.
|
||||||
|
|
||||||
|
- `~/k8s/oauth2-proxy-dashboard.yaml` — Deployment + ConfigMap +
|
||||||
|
NodePort Service (`oauth2-proxy-dashboard`, NodePort 31541,
|
||||||
|
namespace `pragent`). Same shape as the code-server /
|
||||||
|
minecraft-sso oauth2-proxy. NodePort 31541 was chosen because
|
||||||
|
31540 was the old dashboard NodePort and the 30096..30969 media
|
||||||
|
range + 30350-30351 (other oauth2-proxy NodePorts) were taken.
|
||||||
|
|
||||||
|
- `~/k8s/oauth2-proxy-dashboard-secret.yaml` — client-id /
|
||||||
|
client-secret / cookie-secret (mode 600, NOT in git).
|
||||||
|
|
||||||
|
## Smoke test
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. anonymous request → 302 redirect to Logto
|
||||||
|
curl -I https://pragent-dashboard.marcospaulo.dev.br/
|
||||||
|
|
||||||
|
# 2. pod logs
|
||||||
|
microk8s kubectl logs -n pragent -l app=oauth2-proxy-dashboard --tail=50
|
||||||
|
microk8s kubectl logs -n pragent -l app=pragent-dashboard --tail=50
|
||||||
|
|
||||||
|
# 3. in-cluster direct probe (should 401 without X-Forwarded-User)
|
||||||
|
microk8s kubectl port-forward -n pragent svc/pragent-dashboard 8181:80 &
|
||||||
|
sleep 2
|
||||||
|
curl -I http://localhost:8181/ # expect 401 + WWW-Authenticate: Basic
|
||||||
|
curl -I -H "X-Forwarded-User: marcos@example.com" http://localhost:8181/ # expect 200
|
||||||
|
kill %1
|
||||||
|
```
|
||||||
|
|
||||||
|
The HTML returned with a valid `X-Forwarded-User` should contain a
|
||||||
|
`<title>` (whatever the dashboard renders) and **never** `Traceback` or
|
||||||
|
any Python exception output. A 401 on the unauthenticated GET is the
|
||||||
|
expected behaviour — oauth2-proxy catches it and redirects to Logto.
|
||||||
|
|
||||||
|
## Threat model / security notes
|
||||||
|
|
||||||
|
- **Behind Logto SSO.** Anonymous traffic gets 302 → Logto. Allowed
|
||||||
|
emails (marcos, thiago) reach the dashboard after Logto login; all
|
||||||
|
others see oauth2-proxy's "not authorized" page. Adding a user is a
|
||||||
|
one-line ConfigMap edit; oauth2-proxy hot-reloads the allowlist.
|
||||||
|
- **`PRAGENT_BOT_TOKEN` is Gitea Write scoped** to onboarded repos, so
|
||||||
|
a successful auth bypass on the dashboard is Gitea repo write access,
|
||||||
|
not just read. oauth2-proxy's email allowlist is the only
|
||||||
|
authentication factor — there is no second factor. If this becomes a
|
||||||
|
concern, swap oauth2-proxy for an IdP that supports TOTP/WebAuthn
|
||||||
|
and the dashboard needs no further changes (it just reads the
|
||||||
|
forwarded headers).
|
||||||
|
- **CSRF on the edit form.** Per-process random secret embedded as a
|
||||||
|
hidden input + double-submit via the `X-Forwarded-User` context. An
|
||||||
|
attacker would need to (a) steal the user's Logto session cookie
|
||||||
|
from oauth2-proxy and (b) read the rendered HTML to harvest the
|
||||||
|
CSRF token. Both have to happen in the same browser.
|
||||||
|
- **Read-only `/data` mount.** The dashboard can't corrupt the
|
||||||
|
harvested SQLite file even if it's compromised. The webhook and the
|
||||||
|
daily cronjob are the only writers.
|
||||||
|
- **ClusterIP dashboard Service.** Even if a malicious actor discovered
|
||||||
|
the dashboard's container port, they cannot reach it from outside the
|
||||||
|
cluster — only oauth2-proxy can. NetworkPolicy is the cluster
|
||||||
|
default deny.
|
||||||
|
- **`uid 10001` + `runAsNonRoot: true`.** No host-level escalation if
|
||||||
|
the dashboard is popped — it has no caps, no `/proc` mounts.
|
||||||
|
- **No author-controlled input is `eval`-ed.** The edit form parses the
|
||||||
|
JSON, validates types / lengths, and re-serialises before the Gitea
|
||||||
|
PUT. The review-side hostile-input concerns from `README-webhook.md`
|
||||||
|
§ "Threat model" do **not** apply to the dashboard — the dashboard
|
||||||
|
is a read-mostly viewer over already-harvested, already-posted data.
|
||||||
|
|
||||||
|
## Known limitations (pilot)
|
||||||
|
|
||||||
|
- Logto SSO is the only auth factor — no per-user sessions, no CSRF
|
||||||
|
token tied to a per-user identity (the per-process CSRF secret is
|
||||||
|
global). Adequate for a single-operator dashboard; not adequate for
|
||||||
|
multi-tenant.
|
||||||
|
- No `/health` route — if the dashboard process wedges on a Gitea hang,
|
||||||
|
k8s won't restart it. Add a `/health` route to `pilot/dashboard.py`
|
||||||
|
+ a probe here before relying on this in production.
|
||||||
|
- The overview is a single-process render over a SQLite file that the
|
||||||
|
daily cronjob also writes. A long Gitea hang during a page render can
|
||||||
|
stall the dashboard until the client request times out (30 s). The
|
||||||
|
underlying SQLite reader is read-only and concurrent-safe, so no
|
||||||
|
data corruption — just a slow page.
|
||||||
+41
-44
@@ -1,30 +1,31 @@
|
|||||||
# pragent pilot — central webhook service
|
# pragent pilot — central webhook service
|
||||||
|
|
||||||
The CI-step pilot (`pilot/README.md`) needs a workflow file + secret + label per
|
The CI-step pilot (`pilot/README.md`) needs a workflow file + secret per repo.
|
||||||
repo. The **central webhook service** removes the workflow file, the secret, and
|
The **central webhook service** removes the workflow file, the secret, and the
|
||||||
the runner dependency: a Gitea webhook posts PR events to an always-on in-cluster
|
runner dependency: a Gitea webhook posts PR events to an always-on in-cluster
|
||||||
service, which gates on the `AI-REVIEW` label and runs the same review core.
|
service, which gates on `.pr-review.json:enabled = true` and runs the same review
|
||||||
|
core.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
PR opened/pushed/labeled/edited/… (any repo under a covered owner)
|
PR opened/pushed/edited/… (any repo under a covered owner)
|
||||||
│ Gitea user-level webhook (events: pull_request)
|
│ Gitea user-level webhook (events: pull_request)
|
||||||
▼
|
▼
|
||||||
Service pragent-webhook.pragent.svc.cluster.local (ClusterIP, ns pragent)
|
Service pragent-webhook.pragent.svc.cluster.local (ClusterIP, ns pragent)
|
||||||
│ body-size cap → HMAC-verify (X-Gitea-Signature)
|
│ body-size cap → HMAC-verify (X-Gitea-Signature)
|
||||||
│ → gate: action ≠ closed AND pull_request.labels ∋ AI-REVIEW
|
│ → gate: action ≠ closed AND .pr-review.json:enabled = true on base
|
||||||
│ → claim (repo, index, sha) in-flight (closes the dedupe race)
|
│ → claim (repo, index, sha) in-flight (closes the dedupe race)
|
||||||
│ → bounded worker (PRAGENT_MAX_CONCURRENT_REVIEWS, default 2)
|
│ → bounded worker (PRAGENT_MAX_CONCURRENT_REVIEWS, default 2)
|
||||||
│ (report_usage ← pull_request.labels ∋ AI-USAGE, optional)
|
|
||||||
▼
|
▼
|
||||||
ai_review.review_pr() (same core the CI-step uses)
|
ai_review.review_pr() (same core the CI-step uses)
|
||||||
1. fetch existing reviews → dedupe: skip if a review already carries
|
1. opt-in .pr-review.json:enabled = true on base? if not, skip.
|
||||||
<!-- pragent:sha=<this sha> --> (no duplicate on label-toggle / re-fire)
|
2. fetch existing reviews → dedupe: skip if a review already carries
|
||||||
2. fetch PR diff → GET .../pulls/{i}.diff
|
<!-- pragent:sha=<this sha> --> (no duplicate on title/body-edit re-fire)
|
||||||
3. fetch .pr-review.json @ head ref (optional repo-local focus/config)
|
3. fetch PR diff → GET .../pulls/{i}.diff
|
||||||
4. prior review bodies → fed as "already said" context (light §6.1)
|
4. fetch .pr-review.json @ base ref (the opt-in flag + repo-local focus/config)
|
||||||
5. PRAGENT_ENGINE=opencode (default):
|
5. prior review bodies → fed as "already said" context (light §6.1)
|
||||||
|
6. PRAGENT_ENGINE=opencode (default):
|
||||||
a. fetch repo archive @ head sha → /tmp/pragent-work/<repo>-<sha>
|
a. fetch repo archive @ head sha → /tmp/pragent-work/<repo>-<sha>
|
||||||
(symlink-escape + traversal rejected on untar)
|
(symlink-escape + traversal rejected on untar)
|
||||||
a2. sanitize the workdir: delete author-controlled agent-instruction
|
a2. sanitize the workdir: delete author-controlled agent-instruction
|
||||||
@@ -40,8 +41,8 @@ ai_review.review_pr() (same core the CI-step uses)
|
|||||||
diffs, and emits: {"summary":..., "findings":[{severity,path,line,
|
diffs, and emits: {"summary":..., "findings":[{severity,path,line,
|
||||||
problem,fix,suggestion,reference}]}
|
problem,fix,suggestion,reference}]}
|
||||||
(=ollama: legacy single POST to http://<model-proxy-host>:8789/v1/messages)
|
(=ollama: legacy single POST to http://<model-proxy-host>:8789/v1/messages)
|
||||||
6. parse diff hunks → valid (path, new_line) anchors (RIGHT side)
|
7. parse diff hunks → valid (path, new_line) anchors (RIGHT side)
|
||||||
7. post review → POST .../pulls/{i}/reviews (event: COMMENT) as pragent-bot
|
8. post review → POST .../pulls/{i}/reviews (event: COMMENT) as pragent-bot
|
||||||
- prose summary → review body intro
|
- prose summary → review body intro
|
||||||
- anchored findings → inline line comments, body wraps `suggestion` in a
|
- anchored findings → inline line comments, body wraps `suggestion` in a
|
||||||
language-tagged fenced code block (Gitea syntax-highlights it; Gitea
|
language-tagged fenced code block (Gitea syntax-highlights it; Gitea
|
||||||
@@ -59,18 +60,18 @@ of repeating (light version of framework §6.1).
|
|||||||
|
|
||||||
1. Add `pragent-bot` as collaborator with **Write** (so it can read the diff and
|
1. Add `pragent-bot` as collaborator with **Write** (so it can read the diff and
|
||||||
post the review). The bot stays a normal user — it is **not** a site admin.
|
post the review). The bot stays a normal user — it is **not** a site admin.
|
||||||
2. Create the `AI-REVIEW` label on the repo (one-time; `pragent-bot`'s
|
2. Commit `.pr-review.json: {"enabled": true}` to the repo's default branch
|
||||||
`write:issue` scope can do it once it's a collaborator).
|
(so every PR on the repo is auto-reviewed).
|
||||||
3. Label a PR `AI-REVIEW`.
|
3. Open a PR.
|
||||||
|
|
||||||
No workflow file, no repo secret, no act-runner needed. (The owner must already
|
No workflow file, no repo secret, no act-runner, no label needed. (The owner
|
||||||
be covered by a user-level webhook — see below. If not, do the one-time
|
must already be covered by a user-level webhook — see below. If not, do the
|
||||||
per-owner setup first.)
|
one-time per-owner setup first.)
|
||||||
|
|
||||||
## AI-USAGE label — token-usage reporting (optional, opt-in)
|
## Token-usage reporting (always on)
|
||||||
|
|
||||||
A review always fires on `AI-REVIEW`. Adding a second label **`AI-USAGE`** on
|
Every opencode review now appends a token-usage report — no label, no env var
|
||||||
the same PR opts the review into appending a token-usage report:
|
needed:
|
||||||
|
|
||||||
- a `## 🔋 AI usage` section on the review summary body with the **measured**
|
- a `## 🔋 AI usage` section on the review summary body with the **measured**
|
||||||
review total — input / output / reasoning / cache read+write / total tokens,
|
review total — input / output / reasoning / cache read+write / total tokens,
|
||||||
@@ -88,12 +89,8 @@ rendered-body weight (`len(problem)+len(fix)+len(suggestion)`) — an honest
|
|||||||
attribution, labelled as such. The totals are real measurements summed from
|
attribution, labelled as such. The totals are real measurements summed from
|
||||||
opencode's `step_finish` events.
|
opencode's `step_finish` events.
|
||||||
|
|
||||||
`PRAGENT_USAGE_ALWAYS=1` on the Deployment forces usage reporting on for every
|
No-op on the ollama fallback (no usage available). The usage section is part
|
||||||
review (testing / a future default-on) regardless of the label.
|
of the review body, so it's covered by the existing sha-marker dedupe.
|
||||||
|
|
||||||
Without `AI-USAGE` (regression): no usage section, no 🪙 lines — behaviour
|
|
||||||
identical to before the feature. The usage section is part of the review body,
|
|
||||||
so it's covered by the existing sha-marker dedupe.
|
|
||||||
|
|
||||||
## Repo-provided static context (`ADDITIONAL_CONTEXT_URL`)
|
## Repo-provided static context (`ADDITIONAL_CONTEXT_URL`)
|
||||||
|
|
||||||
@@ -156,17 +153,17 @@ curl -u techspark -X PUT \
|
|||||||
|
|
||||||
The receiver uses a **denylist**, not an allowlist: it reviews on every
|
The receiver uses a **denylist**, not an allowlist: it reviews on every
|
||||||
`pull_request` action **except `closed`** — `opened`, `reopened`,
|
`pull_request` action **except `closed`** — `opened`, `reopened`,
|
||||||
`synchronize`/`synchronized`, `labeled`/`label_updated`, `edited` (title/body),
|
`synchronize`/`synchronized`, `edited` (title/body), `ready_for_review`
|
||||||
`ready_for_review` (draft→ready), `assigned`, `review_requested`, `milestone`,
|
(draft→ready), `assigned`, `review_requested`, `milestone`, … . This is safe
|
||||||
… . This is safe because of two downstream gates:
|
because of two downstream gates:
|
||||||
|
|
||||||
- the **AI-REVIEW label gate** — payload `labels` reflect current state, so an
|
- the **opt-in gate** — `.pr-review.json:enabled = true` is read from the base
|
||||||
`unlabeled` that *removed* AI-REVIEW fails the gate (no review); an
|
branch, so only repos that opted in get reviewed. A repo that deletes the
|
||||||
`unlabeled` of another label still passes;
|
file between pushes opts out;
|
||||||
- the **sha dedupe** — any same-sha re-fire (title edit, assignee, milestone,
|
- the **sha dedupe** — any same-sha re-fire (title edit, assignee, milestone…)
|
||||||
a label toggle of another label…) is skipped, so the only newly-effective
|
is skipped, so the only newly-effective actions are ones that change the head
|
||||||
actions are ones that change the head sha (`synchronize`, already covered) or
|
sha (`synchronize`, already covered) or move a draft to ready
|
||||||
move a draft to ready (`ready_for_review`) on an un-reviewed sha.
|
(`ready_for_review`) on an un-reviewed sha.
|
||||||
|
|
||||||
## Threat model
|
## Threat model
|
||||||
|
|
||||||
@@ -209,8 +206,8 @@ Additionally: the repo archive is untarred with symlink-escape and
|
|||||||
parent-traversal rejection (`_extract_tar_strip_one`), the container runs as
|
parent-traversal rejection (`_extract_tar_strip_one`), the container runs as
|
||||||
uid 10001, and the webhook caps request bodies (`PRAGENT_MAX_BODY_BYTES`,
|
uid 10001, and the webhook caps request bodies (`PRAGENT_MAX_BODY_BYTES`,
|
||||||
default 10 MiB) and concurrent reviews (`PRAGENT_MAX_CONCURRENT_REVIEWS`,
|
default 10 MiB) and concurrent reviews (`PRAGENT_MAX_CONCURRENT_REVIEWS`,
|
||||||
default 2 — each review forks an opencode process, so unbounded threads were a
|
default 2 — each review forks an opencode process, so unbounded threads would be
|
||||||
self-inflicted fork bomb on a label-ten-PRs burst).
|
a self-inflicted fork bomb on any burst of concurrent PRs).
|
||||||
|
|
||||||
**Residual risk, accepted for a pilot:** the agent still *executes* hostile repo
|
**Residual risk, accepted for a pilot:** the agent still *executes* hostile repo
|
||||||
content indirectly (running the repo's own linters on it) inside a container
|
content indirectly (running the repo's own linters on it) inside a container
|
||||||
@@ -227,7 +224,7 @@ so the `/tmp/pragent-work` emptyDir is writable.
|
|||||||
|
|
||||||
## Multi-lens pipeline (5 default lenses, on by default)
|
## Multi-lens pipeline (5 default lenses, on by default)
|
||||||
|
|
||||||
Default `AI-REVIEW` runs spawn **one opencode subprocess per lens in parallel**
|
Default reviews spawn **one opencode subprocess per lens in parallel**
|
||||||
and synthesize the merged findings before posting. Cheaper than 5 sequential
|
and synthesize the merged findings before posting. Cheaper than 5 sequential
|
||||||
reviews because the headroom proxy caches the byte-identical brief across
|
reviews because the headroom proxy caches the byte-identical brief across
|
||||||
lens calls (lenses 2..N hit cache).
|
lens calls (lenses 2..N hit cache).
|
||||||
@@ -483,7 +480,7 @@ cramped model call. `pilot/opencode_review.py` is the glue:
|
|||||||
It does **no Gitea I/O and no parsing** — `review_pr` parses the stdout into
|
It does **no Gitea I/O and no parsing** — `review_pr` parses the stdout into
|
||||||
`(summary, findings)`, validates findings against diff anchors, and posts. So
|
`(summary, findings)`, validates findings against diff anchors, and posts. So
|
||||||
all v2 logic (dedupe marker, anchor validation, language-tagged suggestion
|
all v2 logic (dedupe marker, anchor validation, language-tagged suggestion
|
||||||
fencing, posting, optional AI-USAGE attribution) is reused and never depends on
|
fencing, posting, token-usage attribution) is reused and never depends on
|
||||||
the model remembering it.
|
the model remembering it.
|
||||||
|
|
||||||
The factory lives in the pragent repo root: `opencode.json` (provider/model/
|
The factory lives in the pragent repo root: `opencode.json` (provider/model/
|
||||||
|
|||||||
+386
-103
@@ -58,20 +58,18 @@ import urllib.error
|
|||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
REVIEW_HEADER = "🤖 **AI Review** · pragent pilot · {model} · `{sha}`"
|
REVIEW_HEADER = "🤖 **AI Review** · pragent pilot · {model} · `{sha}` · Merge confidence: {confidence}"
|
||||||
# Hidden marker the dedupe pass scans for. Full sha so a re-push (new sha) is
|
# Hidden marker the dedupe pass scans for. Full sha so a re-push (new sha) is
|
||||||
# never mistaken for an already-reviewed commit, and a label-toggle (same sha)
|
# never mistaken for an already-reviewed commit, and a label-toggle (same sha)
|
||||||
# is correctly skipped.
|
# is correctly skipped.
|
||||||
SHA_MARKER = "<!-- pragent:sha={sha} -->"
|
SHA_MARKER = "<!-- pragent:sha={sha} -->"
|
||||||
_SHA_MARKER_RE = re.compile(r"<!-- pragent:sha=([0-9a-f]{7,40}) -->")
|
_SHA_MARKER_RE = re.compile(r"<!-- pragent:sha=([0-9a-f]{7,40}) -->")
|
||||||
|
|
||||||
AI_REVIEW_LABEL = "AI-REVIEW"
|
SEVERITIES = ("critical", "high", "medium", "low", "trivial", "info")
|
||||||
# Opt-in label for the token-usage block. Read at render time — see pr_has_label.
|
|
||||||
AI_USAGE_LABEL = "AI-USAGE"
|
|
||||||
SEVERITIES = ("critical", "high", "medium", "low")
|
|
||||||
# Severity rank — higher = more severe. Used by `apply_repo_config` to drop
|
# Severity rank — higher = more severe. Used by `apply_repo_config` to drop
|
||||||
# findings below `severity_threshold`. Critical=3, high=2, medium=1, low=0.
|
# findings below `severity_threshold`. critical=4, high=3, medium=2, low=1,
|
||||||
SEVERITY_RANK = {"low": 0, "medium": 1, "high": 2, "critical": 3}
|
# trivial=0, info=-1.
|
||||||
|
SEVERITY_RANK = {"info": -1, "trivial": 0, "low": 1, "medium": 2, "high": 3, "critical": 4}
|
||||||
REPO_CONFIG_FILE = ".pr-review.json"
|
REPO_CONFIG_FILE = ".pr-review.json"
|
||||||
|
|
||||||
# Style → (default max_findings, default severity_threshold). Strict is
|
# Style → (default max_findings, default severity_threshold). Strict is
|
||||||
@@ -90,6 +88,14 @@ STYLE_DEFAULTS: dict[str, tuple[int, str]] = {
|
|||||||
# (env) or `.pr-review.json:cost_target` (per repo).
|
# (env) or `.pr-review.json:cost_target` (per repo).
|
||||||
DEFAULT_PRICE_TARGET = "claude-sonnet-5"
|
DEFAULT_PRICE_TARGET = "claude-sonnet-5"
|
||||||
|
|
||||||
|
# Default roster of paid providers shown in the equivalent-cost table when
|
||||||
|
# `.pr-review.json` does not pin `compare_against`. The pilot is free-tier only,
|
||||||
|
# so this list is the operator's budgeting signal — it answers "what would this
|
||||||
|
# have cost on a mainstream paid API?". Override per-repo via
|
||||||
|
# `.pr-review.json:compare_against` (capped at 12 entries; unknown keys are
|
||||||
|
# dropped with a stderr line at parse time).
|
||||||
|
DEFAULT_COMPARE_AGAINST = ("claude-sonnet-5", "gpt-5", "gemini-2.5-pro", "grok-4.5")
|
||||||
|
|
||||||
SYSTEM_PROMPT = """You are a senior, pragmatic code reviewer. Review the pull request diff below.
|
SYSTEM_PROMPT = """You are a senior, pragmatic code reviewer. Review the pull request diff below.
|
||||||
|
|
||||||
Report ONLY real, actionable issues: correctness bugs, security problems, risky
|
Report ONLY real, actionable issues: correctness bugs, security problems, risky
|
||||||
@@ -101,14 +107,17 @@ Output STRICT JSON only — no prose, no markdown fences. Shape:
|
|||||||
{
|
{
|
||||||
"findings": [
|
"findings": [
|
||||||
{
|
{
|
||||||
"severity": "critical|high|medium|low",
|
"severity": "critical|high|medium|low|trivial|info",
|
||||||
"path": "file path exactly as it appears in the diff (`+++ b/` side)",
|
"path": "file path exactly as it appears in the diff (`+++ b/` side)",
|
||||||
"line": <int, the NEW-file line number the issue is on, within the diff>,
|
"line": <int, the NEW-file line number the issue is on, within the diff>,
|
||||||
"problem": "one line: what is wrong",
|
"problem": "one line: what is wrong",
|
||||||
"fix": "one line: how to fix it",
|
"fix": "one line: how to fix it",
|
||||||
"suggestion": "<exact replacement lines for that location, or empty string if you cannot produce safe replacement code>"
|
"suggestion": "<exact replacement lines for that location, or empty string if you cannot produce safe replacement code>"
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
|
"walkthrough": ["2-6 short bullets, file- or change-grouped, plain prose"],
|
||||||
|
"risk_verdict": "Low|Medium|High|Critical risk: <one-line concrete reason>",
|
||||||
|
"test_coverage": "Tests added" | "Tests changed" | "No tests for behavioral change" | "No test files in repo"
|
||||||
}
|
}
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
@@ -120,6 +129,15 @@ Rules:
|
|||||||
Keep it minimal — just the changed lines, indented as they would appear in the
|
Keep it minimal — just the changed lines, indented as they would appear in the
|
||||||
file. Leave it empty ("") if a safe textual replacement is not possible (e.g.
|
file. Leave it empty ("") if a safe textual replacement is not possible (e.g.
|
||||||
a missing test, an architectural note).
|
a missing test, an architectural note).
|
||||||
|
- `walkthrough`: 2-6 short bullets, file- or change-grouped, plain prose.
|
||||||
|
Default to `[]` when the diff is trivial. Backward compatible: parsers
|
||||||
|
default to `[]` if absent.
|
||||||
|
- `risk_verdict`: exactly one line. Lead with "Low|Medium|High|Critical risk:"
|
||||||
|
followed by a concrete reason. Default to `""` when not applicable.
|
||||||
|
Backward compatible: parsers default to `""` if absent.
|
||||||
|
- `test_coverage`: short string. One of "Tests added" / "Tests changed" /
|
||||||
|
"No tests for behavioral change" / "No test files in repo". Default to `""`
|
||||||
|
when not applicable. Backward compatible: parsers default to `""` if absent.
|
||||||
- Skip nitpicks, pure formatting, and praise. At most ~15 findings, highest
|
- Skip nitpicks, pure formatting, and praise. At most ~15 findings, highest
|
||||||
severity first.
|
severity first.
|
||||||
- If the diff is clean, output: {"findings": []}
|
- If the diff is clean, output: {"findings": []}
|
||||||
@@ -141,6 +159,26 @@ def truncate_diff(text: str, max_chars: int) -> tuple[str, bool, int]:
|
|||||||
return text[:max_chars] + f"\n\n[diff truncated at {max_chars} characters]\n", True, orig_len
|
return text[:max_chars] + f"\n\n[diff truncated at {max_chars} characters]\n", True, orig_len
|
||||||
|
|
||||||
|
|
||||||
|
def fmt_tokens(n) -> str:
|
||||||
|
"""1234567 -> '1,234,567 (1.2M)'; 0 -> '0'; <1000 -> comma-only; None/negative -> '?'.
|
||||||
|
|
||||||
|
Always returns the full comma-separated number; the short suffix is a
|
||||||
|
parenthetical for fast scanning. Caps at B; the cost model never exceeds M.
|
||||||
|
"""
|
||||||
|
if n is None:
|
||||||
|
return "?"
|
||||||
|
if not isinstance(n, (int, float)) or n < 0:
|
||||||
|
return "?"
|
||||||
|
n = int(n)
|
||||||
|
if n < 1000:
|
||||||
|
return f"{n:,}"
|
||||||
|
if n < 1_000_000:
|
||||||
|
return f"{n:,} ({n / 1000:.1f}K)"
|
||||||
|
if n < 1_000_000_000:
|
||||||
|
return f"{n:,} ({n / 1_000_000:.1f}M)"
|
||||||
|
return f"{n:,} ({n / 1_000_000_000:.1f}B)"
|
||||||
|
|
||||||
|
|
||||||
def parse_text_blocks(content: list) -> str:
|
def parse_text_blocks(content: list) -> str:
|
||||||
"""Join `type:"text"` blocks from an Anthropic /v1/messages response.
|
"""Join `type:"text"` blocks from an Anthropic /v1/messages response.
|
||||||
|
|
||||||
@@ -158,33 +196,6 @@ def parse_text_blocks(content: list) -> str:
|
|||||||
return "\n".join(out).strip()
|
return "\n".join(out).strip()
|
||||||
|
|
||||||
|
|
||||||
def pr_has_label(api: str, repo: str, index: str, token: str, label: str) -> bool:
|
|
||||||
"""True if the PR currently carries `label`. False on any failure.
|
|
||||||
|
|
||||||
Read at RENDER time, not at review start. A reviewer labels AI-REVIEW and
|
|
||||||
AI-USAGE seconds apart; the review claims on the first event and the
|
|
||||||
second is dropped by the in-flight dedupe, so the trigger payload never
|
|
||||||
sees the opt-in. Re-reading when the review begins is no better — that is
|
|
||||||
still milliseconds after the first click. Only a read taken once the
|
|
||||||
review has finished (a minute or more later) reliably sees the label.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
code, raw = gitea_get(api, repo, f"issues/{index}/labels", token)
|
|
||||||
if code >= 300:
|
|
||||||
return False
|
|
||||||
data = json.loads(raw.decode() or "[]")
|
|
||||||
except Exception as e:
|
|
||||||
print(f"pragent: could not re-read labels for {repo}#{index}: {e}",
|
|
||||||
file=sys.stderr, flush=True)
|
|
||||||
return False
|
|
||||||
if not isinstance(data, list):
|
|
||||||
return False
|
|
||||||
return any(
|
|
||||||
(isinstance(x, dict) and x.get("name") == label) or x == label
|
|
||||||
for x in data
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _int_env(name: str, default: int) -> int:
|
def _int_env(name: str, default: int) -> int:
|
||||||
"""Read an int from the environment, falling back on anything unparseable.
|
"""Read an int from the environment, falling back on anything unparseable.
|
||||||
|
|
||||||
@@ -204,6 +215,49 @@ def _int_env(name: str, default: int) -> int:
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
# 1-5 merge-verdict score (higher = safer). Buckets:
|
||||||
|
# 5 = clean (or low/info/trivial only — nothing worth blocking on)
|
||||||
|
# 4 = medium present
|
||||||
|
# 3 = high present (operator should at least look)
|
||||||
|
# 1 = critical present (block the merge by default)
|
||||||
|
# Cross-lens agreement on any finding takes one more off, floored at 1.
|
||||||
|
_CONFIDENCE_BADGE = {5: "🟢", 4: "🟢", 3: "🟡", 2: "🟠", 1: "🔴"}
|
||||||
|
|
||||||
|
|
||||||
|
def merge_confidence(findings: list[dict], *, multi_lens_observed: bool = False) -> int:
|
||||||
|
"""1-5 merge verdict: higher = safer.
|
||||||
|
|
||||||
|
Tier drops driven by the most severe finding present:
|
||||||
|
- critical → 1
|
||||||
|
- high → 3
|
||||||
|
- medium → 4
|
||||||
|
- else → 5 (low / trivial / info / unknown → no drop)
|
||||||
|
|
||||||
|
An extra -1 when cross-lens agreement was observed on any finding
|
||||||
|
(``multi_lens_observed``). The flag is passed in explicitly because the
|
||||||
|
raw ``_multi_lens`` marker is stripped from findings by the time they
|
||||||
|
reach this function — first by ``opencode_review.run_lenses_review``
|
||||||
|
(the ``_``-prefix scrub) and again by ``_normalize_finding`` (the
|
||||||
|
7-key schema rebuild). The caller (``review_pr``) must capture the
|
||||||
|
signal before those strips fire. Final score is clamped to [1, 5] so
|
||||||
|
a critical + multi_lens combo doesn't go negative.
|
||||||
|
"""
|
||||||
|
if not findings:
|
||||||
|
return 5
|
||||||
|
max_rank = max(SEVERITY_RANK.get(f.get("severity", "low"), 0) for f in findings)
|
||||||
|
if max_rank >= SEVERITY_RANK["critical"]:
|
||||||
|
score = 1
|
||||||
|
elif max_rank >= SEVERITY_RANK["high"]:
|
||||||
|
score = 3
|
||||||
|
elif max_rank >= SEVERITY_RANK["medium"]:
|
||||||
|
score = 4
|
||||||
|
else:
|
||||||
|
score = 5
|
||||||
|
if multi_lens_observed:
|
||||||
|
score -= 1
|
||||||
|
return max(1, min(5, score))
|
||||||
|
|
||||||
|
|
||||||
def format_review_body(
|
def format_review_body(
|
||||||
findings: str,
|
findings: str,
|
||||||
model: str,
|
model: str,
|
||||||
@@ -215,15 +269,32 @@ def format_review_body(
|
|||||||
risks: list[str] | None = None,
|
risks: list[str] | None = None,
|
||||||
findings_for_table: list[dict] | None = None,
|
findings_for_table: list[dict] | None = None,
|
||||||
inline_count: int = 0,
|
inline_count: int = 0,
|
||||||
|
confidence: int = 5,
|
||||||
|
walkthrough: list[str] | None = None,
|
||||||
|
risk_verdict: str = "",
|
||||||
|
test_coverage: str = "",
|
||||||
|
static_message: str = "",
|
||||||
|
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Format the posted review summary body.
|
"""Format the posted review summary body.
|
||||||
|
|
||||||
Layout (per the operator's format guide):
|
Layout (per the operator's format guide):
|
||||||
|
|
||||||
* Header line (``🤖 AI Review …``).
|
* Header line (``🤖 AI Review …``) including the merge-confidence badge.
|
||||||
|
* Optional static banner (``> {static_message}``) — repo-wide call-out
|
||||||
|
from `.pr-review.json:static_message`, placed under the header so
|
||||||
|
every reviewer sees it on every review without scrolling.
|
||||||
|
|
||||||
* **Summary of Changes** — 2–4 bullets of what the PR introduces
|
* **Summary of Changes** — 2–4 bullets of what the PR introduces
|
||||||
(`summary_changes`); falls back to the opencode prose `summary` if
|
(`summary_changes`); falls back to the opencode prose `summary` if
|
||||||
the agent didn't emit the list.
|
the agent didn't emit the list.
|
||||||
|
* **Risk Verdict** — one-line "<level> risk: <reason>" verdict
|
||||||
|
(`risk_verdict`); omitted when empty.
|
||||||
|
* **Walkthrough** — up to 6 file- or change-grouped bullets
|
||||||
|
(`walkthrough`); the file part is wrapped in backticks so paths
|
||||||
|
render as code in Gitea. Omitted when empty.
|
||||||
|
* **Test Coverage** — short `test_coverage` string ("Tests added" /
|
||||||
|
etc.); omitted when empty.
|
||||||
* **Key Risks & Concerns** — bullets of potential bugs/edge cases
|
* **Key Risks & Concerns** — bullets of potential bugs/edge cases
|
||||||
found across the diff (`risks`).
|
found across the diff (`risks`).
|
||||||
* **Findings Overview** — a Markdown table (severity / location /
|
* **Findings Overview** — a Markdown table (severity / location /
|
||||||
@@ -234,13 +305,29 @@ def format_review_body(
|
|||||||
the body stays scannable; cost lines stay inside it.
|
the body stays scannable; cost lines stay inside it.
|
||||||
* Hidden SHA marker — for the dedupe pass.
|
* Hidden SHA marker — for the dedupe pass.
|
||||||
|
|
||||||
|
`confidence` is a 1-5 merge verdict rendered as `<N>/5 <badge>` in the
|
||||||
|
header. Clamped to [1, 5] so a stray value (e.g. 0 from a missing
|
||||||
|
finding list) doesn't print a broken badge.
|
||||||
|
|
||||||
Empty `summary_changes` + empty `risks` + empty `summary` collapse into
|
Empty `summary_changes` + empty `risks` + empty `summary` collapse into
|
||||||
a single "Summary of Changes: _no summary provided._" line so the body
|
a single "Summary of Changes: _no summary provided._" line so the body
|
||||||
never looks half-rendered.
|
never looks half-rendered.
|
||||||
"""
|
"""
|
||||||
header = REVIEW_HEADER.format(model=model, sha=sha[:8] if sha else "unknown")
|
score = max(1, min(5, confidence))
|
||||||
|
badge = _CONFIDENCE_BADGE.get(score, "🟢")
|
||||||
|
confidence_str = f"{score}/5 {badge}"
|
||||||
|
header = REVIEW_HEADER.format(
|
||||||
|
model=model,
|
||||||
|
sha=sha[:8] if sha else "unknown",
|
||||||
|
confidence=confidence_str,
|
||||||
|
)
|
||||||
parts: list[str] = [header]
|
parts: list[str] = [header]
|
||||||
|
|
||||||
|
# Optional free-text banner. Rendered as a Markdown blockquote immediately
|
||||||
|
# after the header — front-of-mind for any maintainer scanning the review.
|
||||||
|
if static_message and static_message.strip():
|
||||||
|
parts.append(f"> {static_message.strip()}")
|
||||||
|
|
||||||
# --- Summary of Changes ---
|
# --- Summary of Changes ---
|
||||||
sc = list(summary_changes or [])
|
sc = list(summary_changes or [])
|
||||||
if not sc and summary:
|
if not sc and summary:
|
||||||
@@ -252,6 +339,32 @@ def format_review_body(
|
|||||||
else:
|
else:
|
||||||
parts.append("### Summary of Changes\n\n_No summary provided._")
|
parts.append("### Summary of Changes\n\n_No summary provided._")
|
||||||
|
|
||||||
|
# --- Risk Verdict ---
|
||||||
|
if risk_verdict:
|
||||||
|
parts.append(f"### Risk Verdict\n\n{risk_verdict}")
|
||||||
|
|
||||||
|
# --- Walkthrough ---
|
||||||
|
wt = list(walkthrough or [])
|
||||||
|
if wt:
|
||||||
|
wt = wt[:6]
|
||||||
|
rendered = []
|
||||||
|
for item in wt:
|
||||||
|
# Items typically look like "a.py — adds X" (em-dash separator).
|
||||||
|
# Wrap the file path in backticks so it renders as code in the
|
||||||
|
# Gitea markdown body; leave the description as plain prose. When
|
||||||
|
# no separator is present, render the whole line as plain prose
|
||||||
|
# (the agent's "plain prose" fallback for change-grouped bullets).
|
||||||
|
if " — " in item:
|
||||||
|
path, _, rest = item.partition(" — ")
|
||||||
|
rendered.append(f"- `{path}` — {rest}")
|
||||||
|
else:
|
||||||
|
rendered.append(f"- {item}")
|
||||||
|
parts.append(f"### Walkthrough\n\n" + "\n".join(rendered))
|
||||||
|
|
||||||
|
# --- Test Coverage ---
|
||||||
|
if test_coverage:
|
||||||
|
parts.append(f"### Test Coverage\n\n{test_coverage}")
|
||||||
|
|
||||||
# --- Key Risks & Concerns ---
|
# --- Key Risks & Concerns ---
|
||||||
rs = list(risks or [])
|
rs = list(risks or [])
|
||||||
if rs:
|
if rs:
|
||||||
@@ -365,6 +478,45 @@ def _resolve_price_target(config: dict | None) -> tuple[str, str | None]:
|
|||||||
return chosen, None
|
return chosen, None
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_display_model(base_model: str, config: dict | None) -> str:
|
||||||
|
"""Resolve the *display* model for one review.
|
||||||
|
|
||||||
|
Precedence (highest first):
|
||||||
|
1. `OPENCODE_MODEL` env var — operator override, used as-is (already a
|
||||||
|
provider-prefixed opencode ref like `headroom/MiniMax-M2.7`).
|
||||||
|
2. `.pr-review.json:model` — per-repo override. Already validated
|
||||||
|
against `cost_model.PRICES` by `parse_repo_config`, so a bare key
|
||||||
|
like `claude-sonnet-5` or `qwen3.8-27b` is safe. Re-prefixed with
|
||||||
|
the model's `provider` field from `cost_model.Price` (default
|
||||||
|
`headroom`) so the opencode subprocess routes correctly — e.g.
|
||||||
|
`qwen3.8-27b` → `local/qwen3.8-27b` (local AI workstation on
|
||||||
|
192.168.1.79:18020), `claude-sonnet-5` → `headroom/claude-sonnet-5`
|
||||||
|
(Anthropic pricing proxy).
|
||||||
|
3. Default — `f"headroom/{base_model}"` where `base_model` is the bare
|
||||||
|
`OLLAMA_MODEL` (e.g. `"MiniMax-M2.7" → "headroom/MiniMax-M2.7"`).
|
||||||
|
|
||||||
|
The same value flows to every consumer (opencode subprocess, REVIEW_HEADER,
|
||||||
|
cost-line parenthetical) so reviewers never see a mix of `glm-5.2:cloud`
|
||||||
|
and the routed model in one body.
|
||||||
|
"""
|
||||||
|
env = os.environ.get("OPENCODE_MODEL")
|
||||||
|
if env:
|
||||||
|
return env
|
||||||
|
cfg_model = (config or {}).get("model")
|
||||||
|
if isinstance(cfg_model, str) and cfg_model.strip():
|
||||||
|
# Look up the provider from PRICES so the opencode subprocess routes
|
||||||
|
# through the right provider block (local vs headroom). Lazy import —
|
||||||
|
# the ollama path doesn't touch cost_model.
|
||||||
|
from cost_model import PRICES
|
||||||
|
provider = PRICES.get(cfg_model.strip())
|
||||||
|
if provider is not None:
|
||||||
|
return f"{provider.provider}/{cfg_model.strip()}"
|
||||||
|
# parse_repo_config already drops unknowns, but stay defensive: fall
|
||||||
|
# back to headroom so the review still runs rather than crash.
|
||||||
|
return f"headroom/{cfg_model.strip()}"
|
||||||
|
return f"headroom/{base_model}"
|
||||||
|
|
||||||
|
|
||||||
def equivalent_cost(usage: dict, price_key: str) -> float:
|
def equivalent_cost(usage: dict, price_key: str) -> float:
|
||||||
"""USD the measured usage would have billed on `price_key`'s provider.
|
"""USD the measured usage would have billed on `price_key`'s provider.
|
||||||
|
|
||||||
@@ -652,45 +804,59 @@ def salvage_summary(text: str, max_chars: int = SALVAGE_MAX_CHARS) -> str:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_review_output(text: str) -> tuple[str, list[dict], list[str], list[str]]:
|
def parse_review_output(
|
||||||
"""Parse the opengine's stdout into (summary, findings, summary_changes, risks).
|
text: str,
|
||||||
|
) -> tuple[str, list[dict], list[str], list[str], list[str], str, str]:
|
||||||
|
"""Parse the opengine's stdout into a 7-tuple:
|
||||||
|
(summary, findings, summary_changes, risks,
|
||||||
|
walkthrough, risk_verdict, test_coverage)
|
||||||
|
|
||||||
Accepts `{"summary": "...", "summary_changes": [...], "risks": [...],
|
Accepts `{"summary": "...", "summary_changes": [...], "risks": [...],
|
||||||
"findings": [...]}` (the opencode pragent agent), `{"findings": [...]}`,
|
"walkthrough": [...], "risk_verdict": "...", "test_coverage": "...",
|
||||||
or a bare `[...]` of finding dicts. `summary_changes` and `risks` default
|
"findings": [...]}` (the opencode pragent agent), the legacy 4-field
|
||||||
to empty lists; older outputs without them still parse fine. Uses the
|
shape, or a bare `[...]` of finding dicts. The three new fields
|
||||||
LAST fenced block (the pragent agent emits JSON as the final block), with
|
(`walkthrough`, `risk_verdict`, `test_coverage`) default to empty
|
||||||
a tolerant fallback that scans for the last balanced object/array in the
|
list / empty strings when absent — older outputs and the bare-array
|
||||||
prose tail. Never raises.
|
shape stay backward compatible.
|
||||||
|
|
||||||
|
Uses the LAST fenced block (the pragent agent emits JSON as the final
|
||||||
|
block), with a tolerant fallback that scans for the last balanced
|
||||||
|
object/array in the prose tail. Never raises.
|
||||||
"""
|
"""
|
||||||
blob = _last_json_block(text)
|
blob = _last_json_block(text)
|
||||||
if blob is None:
|
if blob is None:
|
||||||
return "", [], [], []
|
return "", [], [], [], [], "", ""
|
||||||
try:
|
try:
|
||||||
data = json.loads(blob)
|
data = json.loads(blob)
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
return "", [], [], []
|
return "", [], [], [], [], "", ""
|
||||||
summary = ""
|
summary = ""
|
||||||
summary_changes: list[str] = []
|
summary_changes: list[str] = []
|
||||||
risks: list[str] = []
|
risks: list[str] = []
|
||||||
|
walkthrough: list[str] = []
|
||||||
|
risk_verdict = ""
|
||||||
|
test_coverage = ""
|
||||||
findings_raw = None
|
findings_raw = None
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
summary = str(data.get("summary", "") or "").strip()
|
summary = str(data.get("summary", "") or "").strip()
|
||||||
summary_changes = _string_list(data.get("summary_changes"))
|
summary_changes = _string_list(data.get("summary_changes"))
|
||||||
risks = _string_list(data.get("risks"))
|
risks = _string_list(data.get("risks"))
|
||||||
|
walkthrough = _string_list(data.get("walkthrough"))
|
||||||
|
risk_verdict = str(data.get("risk_verdict", "") or "").strip()
|
||||||
|
test_coverage = str(data.get("test_coverage", "") or "").strip()
|
||||||
findings_raw = data.get("findings")
|
findings_raw = data.get("findings")
|
||||||
elif isinstance(data, list):
|
elif isinstance(data, list):
|
||||||
# Bare array: each item is a finding; no summary/sections.
|
# Bare array: each item is a finding; no summary/sections.
|
||||||
findings_raw = data
|
findings_raw = data
|
||||||
else:
|
else:
|
||||||
return "", [], [], []
|
return "", [], [], [], [], "", ""
|
||||||
out = []
|
out = []
|
||||||
if isinstance(findings_raw, list):
|
if isinstance(findings_raw, list):
|
||||||
for f in findings_raw:
|
for f in findings_raw:
|
||||||
n = _normalize_finding(f)
|
n = _normalize_finding(f)
|
||||||
if n is not None:
|
if n is not None:
|
||||||
out.append(n)
|
out.append(n)
|
||||||
return summary, out, summary_changes, risks
|
return summary, out, summary_changes, risks, walkthrough, risk_verdict, test_coverage
|
||||||
|
|
||||||
|
|
||||||
def _string_list(value) -> list[str]:
|
def _string_list(value) -> list[str]:
|
||||||
@@ -937,16 +1103,24 @@ _SEVERITY_EMOJI = {
|
|||||||
"high": "🔴",
|
"high": "🔴",
|
||||||
"medium": "🟡",
|
"medium": "🟡",
|
||||||
"low": "🔵",
|
"low": "🔵",
|
||||||
|
"trivial": "⚪",
|
||||||
"info": "⚪",
|
"info": "⚪",
|
||||||
"nit": "⚪",
|
"nit": "⚪",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Severities whose own name is rendered verbatim (uppercased) in the badge.
|
||||||
|
# Anything outside this set falls back to "INFO" so the badge label stays
|
||||||
|
# a clean short token regardless of what the model emits.
|
||||||
|
_BADGED_SEVERITY_LABELS = frozenset({
|
||||||
|
"critical", "high", "medium", "low", "trivial", "info", "nit",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
def _severity_badge(severity: str) -> str:
|
def _severity_badge(severity: str) -> str:
|
||||||
"""Render the severity as emoji + uppercase label (e.g. ``🔴 [HIGH]``)."""
|
"""Render the severity as emoji + uppercase label (e.g. ``🔴 [HIGH]``)."""
|
||||||
sev = (severity or "").lower()
|
sev = (severity or "").lower()
|
||||||
emoji = _SEVERITY_EMOJI.get(sev, "⚪")
|
emoji = _SEVERITY_EMOJI.get(sev, "⚪")
|
||||||
label = sev.upper() if sev in {"critical", "high", "medium", "low"} else "INFO"
|
label = sev.upper() if sev in _BADGED_SEVERITY_LABELS else "INFO"
|
||||||
return f"{emoji} [{label}]"
|
return f"{emoji} [{label}]"
|
||||||
|
|
||||||
|
|
||||||
@@ -1013,7 +1187,7 @@ def inline_comment_body(f: dict) -> str:
|
|||||||
tok = f.get("_tok_attrib")
|
tok = f.get("_tok_attrib")
|
||||||
if tok is not None:
|
if tok is not None:
|
||||||
pct = (f.get("_tok_pct", 0.0) or 0.0) * 100
|
pct = (f.get("_tok_pct", 0.0) or 0.0) * 100
|
||||||
body += f"\n\n🪙 ~{tok} tok ({pct:.0f}% · attributed output)"
|
body += f"\n\n🪙 ~{fmt_tokens(tok)} tok ({pct:.0f}% · attributed output)"
|
||||||
return body
|
return body
|
||||||
|
|
||||||
|
|
||||||
@@ -1065,9 +1239,17 @@ def findings_table(findings: list[dict]) -> str:
|
|||||||
def _render_collapsible_usage(usage: dict | None, model: str, config: dict | None) -> str:
|
def _render_collapsible_usage(usage: dict | None, model: str, config: dict | None) -> str:
|
||||||
"""Render the telemetry as a collapsible ``<details>`` block.
|
"""Render the telemetry as a collapsible ``<details>`` block.
|
||||||
|
|
||||||
Empty string when `usage` is None. The cost-equivalent line is always
|
Empty string when `usage` is None. The equivalent-cost table is the
|
||||||
shown (it's the operator's budgeting signal). The `actual` line is shown
|
operator's budgeting signal — the pilot runs on a free tier, so the
|
||||||
but the FREE-TIER note is collapsed into a single short clause.
|
`actual` line is $0.00; the table shows what the same measured tokens
|
||||||
|
would bill on mainstream paid APIs (configurable via `compare_against`,
|
||||||
|
defaulting to ``DEFAULT_COMPARE_AGAINST``). The row matching `cost_target`
|
||||||
|
is bolded so the price target stands out. The whole table is omitted when
|
||||||
|
every row would be $0 (no work done). The `actual` parenthetical clause
|
||||||
|
reflects the *actually-routed* model (`model` arg, resolved by caller from
|
||||||
|
`OPENCODE_MODEL` env or `headroom/{OLLAMA_MODEL}`) — cost == 0 → "free
|
||||||
|
tier", nonzero → "billed".
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not usage:
|
if not usage:
|
||||||
return ""
|
return ""
|
||||||
@@ -1075,16 +1257,36 @@ def _render_collapsible_usage(usage: dict | None, model: str, config: dict | Non
|
|||||||
dur_s = f"{dur}s" if dur is not None else "?"
|
dur_s = f"{dur}s" if dur is not None else "?"
|
||||||
actual = usage.get("cost") or 0.0
|
actual = usage.get("cost") or 0.0
|
||||||
actual_s = f"${actual:.4f}" if actual else "$0.00"
|
actual_s = f"${actual:.4f}" if actual else "$0.00"
|
||||||
actual_note = " (headroom glm-5.2:cloud — free tier)" if not actual else ""
|
actual_note = f" ({model} — {'free tier' if not actual else 'billed'})"
|
||||||
price_key, price_err = _resolve_price_target(config)
|
cost_target, price_err = _resolve_price_target(config)
|
||||||
from cost_model import PRICES
|
if price_err:
|
||||||
eq = equivalent_cost(usage, price_key)
|
# Surface config typos loudly but do not pollute the posted summary
|
||||||
eq_s = f"${eq:.4f}" if eq else "$0.00"
|
# body — typos at the table-row level would render as English
|
||||||
eq_label = PRICES[price_key].name
|
# mid-table and look like a model error.
|
||||||
eq_note = (
|
print(f"pragent: {price_err}", file=sys.stderr, flush=True)
|
||||||
f" _(price target: `{price_key}`; {price_err})_"
|
# Lazy: cost_model has no dep on ai_review, and the ollama path
|
||||||
if price_err else ""
|
# never reaches this branch.
|
||||||
)
|
from cost_model import PRICES as _PRICES
|
||||||
|
cfg = config or {}
|
||||||
|
compare: list[str] = list(cfg.get("compare_against") or DEFAULT_COMPARE_AGAINST)
|
||||||
|
# Always include the resolved cost_target (env + config), even when the
|
||||||
|
# operator pinned a different `compare_against` roster — the price target
|
||||||
|
# row is the one maintainers eyeball against. Skip silently if the key
|
||||||
|
# isn't a known Price (e.g. a typo that slipped past stderr earlier).
|
||||||
|
if cost_target in _PRICES and cost_target not in compare:
|
||||||
|
compare.append(cost_target)
|
||||||
|
eq_rows: list[str] = []
|
||||||
|
for key in compare:
|
||||||
|
if key not in _PRICES:
|
||||||
|
continue
|
||||||
|
c = equivalent_cost(usage, key)
|
||||||
|
if c <= 0:
|
||||||
|
continue
|
||||||
|
label = _PRICES[key].name
|
||||||
|
cost_str = f"${c:.4f}" if c < 0.01 else f"${c:.2f}"
|
||||||
|
bold = "**" if key == cost_target else ""
|
||||||
|
eq_rows.append(f"| {bold}{label}{bold} | {cost_str} |")
|
||||||
|
|
||||||
in_tok = usage.get("input", 0)
|
in_tok = usage.get("input", 0)
|
||||||
out_tok = usage.get("output", 0)
|
out_tok = usage.get("output", 0)
|
||||||
reason_tok = usage.get("reasoning", 0)
|
reason_tok = usage.get("reasoning", 0)
|
||||||
@@ -1102,11 +1304,19 @@ def _render_collapsible_usage(usage: dict | None, model: str, config: dict | Non
|
|||||||
"<summary>🔋 AI Usage & Run Details</summary>",
|
"<summary>🔋 AI Usage & Run Details</summary>",
|
||||||
"",
|
"",
|
||||||
f"- **Model / Engine**: `{model}` · opencode · {usage.get('steps', 0)} steps · {dur_s}",
|
f"- **Model / Engine**: `{model}` · opencode · {usage.get('steps', 0)} steps · {dur_s}",
|
||||||
f"- **Total Tokens**: {in_tok} in / {out_tok} out ({reason_tok} reasoning, cache {cache_r} read / {cache_w} write, {total} total)",
|
f"- **Total Tokens**: {fmt_tokens(in_tok)} in / {fmt_tokens(out_tok)} out "
|
||||||
f"- **Est. cost on {eq_label}**: {eq_s}{eq_note}",
|
f"({fmt_tokens(reason_tok)} reasoning, cache {fmt_tokens(cache_r)} read / "
|
||||||
|
f"{fmt_tokens(cache_w)} write, {fmt_tokens(total)} total)",
|
||||||
f"- **Actual**: {actual_s}{actual_note}",
|
f"- **Actual**: {actual_s}{actual_note}",
|
||||||
f"- **Scope**: {scope}",
|
f"- **Scope**: {scope}",
|
||||||
]
|
]
|
||||||
|
if eq_rows:
|
||||||
|
lines.append("")
|
||||||
|
lines.append("- **Equivalent cost on paid providers** (this run's tokens):")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("| Provider | Cost |")
|
||||||
|
lines.append("|---|---:|")
|
||||||
|
lines.extend(eq_rows)
|
||||||
# Multi-lens fan-out: surface the lens roster + summed steps so the user
|
# Multi-lens fan-out: surface the lens roster + summed steps so the user
|
||||||
# can see which lenses contributed (and that triage didn't drop them all).
|
# can see which lenses contributed (and that triage didn't drop them all).
|
||||||
lenses = usage.get("lenses")
|
lenses = usage.get("lenses")
|
||||||
@@ -1133,6 +1343,7 @@ CONFIG_MAX_ITEM_CHARS = 200
|
|||||||
CONFIG_MAX_INSTRUCTIONS_CHARS = 4000
|
CONFIG_MAX_INSTRUCTIONS_CHARS = 4000
|
||||||
CONFIG_MAX_PATTERNS_ITEMS = 16 # allow + deny separately, total 32 entries
|
CONFIG_MAX_PATTERNS_ITEMS = 16 # allow + deny separately, total 32 entries
|
||||||
CONFIG_MAX_FINDINGS = 30
|
CONFIG_MAX_FINDINGS = 30
|
||||||
|
CONFIG_MAX_STATIC_MESSAGE_CHARS = 400 # free-text banner, mirror of instructions
|
||||||
|
|
||||||
STYLES = frozenset(STYLE_DEFAULTS)
|
STYLES = frozenset(STYLE_DEFAULTS)
|
||||||
SEVERITY_VALUES = frozenset(SEVERITIES)
|
SEVERITY_VALUES = frozenset(SEVERITIES)
|
||||||
@@ -1148,12 +1359,14 @@ def parse_repo_config(raw: str) -> dict:
|
|||||||
|
|
||||||
Recognised keys (all optional):
|
Recognised keys (all optional):
|
||||||
focus, exclude_paths, languages, instructions — text steer
|
focus, exclude_paths, languages, instructions — text steer
|
||||||
|
static_message ≤ CONFIG_MAX_STATIC_MESSAGE_CHARS — banner under header
|
||||||
style strict|balanced|lenient — default: balanced
|
style strict|balanced|lenient — default: balanced
|
||||||
severity_threshold low|medium|high|critical — default: per style
|
severity_threshold low|medium|high|critical — default: per style
|
||||||
max_findings 1..CONFIG_MAX_FINDINGS — default: per style
|
max_findings 1..CONFIG_MAX_FINDINGS — default: per style
|
||||||
exclude_tests bool — default: False
|
exclude_tests bool — default: False
|
||||||
require_tests bool — default: False
|
require_tests bool — default: False
|
||||||
patterns {allow:[…], deny:[…]} — post-filter globs
|
patterns {allow:[…], deny:[…]} — post-filter globs
|
||||||
|
model <key of cost_model.PRICES> — per-repo override
|
||||||
cost_target <key of cost_model.PRICES> — see equivalent_cost
|
cost_target <key of cost_model.PRICES> — see equivalent_cost
|
||||||
additional_context_urls list[str] (≤ 8) — see fetch_additional_context
|
additional_context_urls list[str] (≤ 8) — see fetch_additional_context
|
||||||
"""
|
"""
|
||||||
@@ -1181,6 +1394,10 @@ def parse_repo_config(raw: str) -> dict:
|
|||||||
if isinstance(instr, str) and instr.strip():
|
if isinstance(instr, str) and instr.strip():
|
||||||
out["instructions"] = instr.strip()[:CONFIG_MAX_INSTRUCTIONS_CHARS]
|
out["instructions"] = instr.strip()[:CONFIG_MAX_INSTRUCTIONS_CHARS]
|
||||||
|
|
||||||
|
sm = data.get("static_message")
|
||||||
|
if isinstance(sm, str) and sm.strip():
|
||||||
|
out["static_message"] = sm.strip()[:CONFIG_MAX_STATIC_MESSAGE_CHARS]
|
||||||
|
|
||||||
style = data.get("style")
|
style = data.get("style")
|
||||||
if isinstance(style, str) and style.strip().lower() in STYLES:
|
if isinstance(style, str) and style.strip().lower() in STYLES:
|
||||||
out["style"] = style.strip().lower()
|
out["style"] = style.strip().lower()
|
||||||
@@ -1217,6 +1434,26 @@ def parse_repo_config(raw: str) -> dict:
|
|||||||
if isinstance(ct, str) and ct.strip():
|
if isinstance(ct, str) and ct.strip():
|
||||||
out["cost_target"] = ct.strip()
|
out["cost_target"] = ct.strip()
|
||||||
|
|
||||||
|
# Per-repo model override. Validated against cost_model.PRICES so the value
|
||||||
|
# is usable both as the opencode subprocess ref and as the REVIEW_HEADER
|
||||||
|
# label (see _resolve_display_model precedence). Unknown values are dropped
|
||||||
|
# with a stderr pointer to the valid set — silently ignoring would mask
|
||||||
|
# typos from repo admins.
|
||||||
|
raw_model = data.get("model")
|
||||||
|
if raw_model is not None:
|
||||||
|
if isinstance(raw_model, str) and raw_model.strip():
|
||||||
|
from cost_model import PRICES # lazy: ollama path dep-free
|
||||||
|
candidate = raw_model.strip()
|
||||||
|
if candidate in PRICES:
|
||||||
|
out["model"] = candidate
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"pragent: .pr-review.json:model={candidate!r} not in "
|
||||||
|
f"cost_model.PRICES (valid: {', '.join(sorted(PRICES))}); "
|
||||||
|
f"dropping",
|
||||||
|
file=sys.stderr, flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
acu = data.get("additional_context_urls")
|
acu = data.get("additional_context_urls")
|
||||||
if isinstance(acu, list):
|
if isinstance(acu, list):
|
||||||
urls: list[str] = []
|
urls: list[str] = []
|
||||||
@@ -1246,6 +1483,35 @@ def parse_repo_config(raw: str) -> dict:
|
|||||||
if tr is not None:
|
if tr is not None:
|
||||||
out["triage"] = tr
|
out["triage"] = tr
|
||||||
|
|
||||||
|
# Repo-level kill-switch: `enabled: false` lets a maintainer pause the bot
|
||||||
|
# for this repo without removing the file (handy during a flaky provider
|
||||||
|
# outage). Always written so callers can do `cfg.get("enabled") is False`
|
||||||
|
# without a separate default — the file itself is committed, so we treat
|
||||||
|
# absent / wrong-type as an explicit off rather than as "config missing".
|
||||||
|
en = data.get("enabled")
|
||||||
|
out["enabled"] = en if isinstance(en, bool) else False
|
||||||
|
|
||||||
|
# Compare-against roster: list of `cost_model.PRICES` keys the render layer
|
||||||
|
# uses to print equivalent-cost lines (one per key) for maintainer
|
||||||
|
# budgeting. Unknown keys are dropped with a stderr line so a typo is loud.
|
||||||
|
# Lazy import: `cost_model` has no dep on `ai_review`, and the ollama
|
||||||
|
# fallback path never hits this branch — keep import-time cost low there.
|
||||||
|
from cost_model import PRICES as _PRICES
|
||||||
|
ca = data.get("compare_against")
|
||||||
|
if isinstance(ca, list):
|
||||||
|
cleaned: list[str] = []
|
||||||
|
for x in ca:
|
||||||
|
if isinstance(x, str) and x.strip() in _PRICES:
|
||||||
|
cleaned.append(x.strip())
|
||||||
|
elif isinstance(x, str):
|
||||||
|
print(
|
||||||
|
f"pragent: ignoring compare_against entry {x!r} "
|
||||||
|
f"(not in cost_model.PRICES); valid: {', '.join(sorted(_PRICES))}",
|
||||||
|
file=sys.stderr, flush=True,
|
||||||
|
)
|
||||||
|
if cleaned:
|
||||||
|
out["compare_against"] = cleaned[:12]
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -1814,7 +2080,6 @@ def review_pr(
|
|||||||
model: str,
|
model: str,
|
||||||
max_tokens: int = 8000,
|
max_tokens: int = 8000,
|
||||||
max_chars: int = 150000,
|
max_chars: int = 150000,
|
||||||
report_usage: bool = False,
|
|
||||||
base_ref: str = "",
|
base_ref: str = "",
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Run one review and post it as `pragent-bot`.
|
"""Run one review and post it as `pragent-bot`.
|
||||||
@@ -1829,16 +2094,24 @@ def review_pr(
|
|||||||
from the PR head) so a PR cannot ship its own reviewer instructions; empty
|
from the PR head) so a PR cannot ship its own reviewer instructions; empty
|
||||||
means "the repo's default branch".
|
means "the repo's default branch".
|
||||||
|
|
||||||
`report_usage`: when True (PR carries the `AI-USAGE` label), the opencode
|
The opencode engine's measured token/cost usage is always rendered as a
|
||||||
engine's measured token/cost usage is rendered as a `## 🔋 AI usage` section
|
`## 🔋 AI usage` section on the review body and an attributed `🪙 ~N tok`
|
||||||
on the review body and an attributed `🪙 ~N tok` line on each inline
|
line on each inline comment when usage data is available (i.e. when the
|
||||||
comment. No-op on the ollama fallback (no usage available).
|
opencode subprocess returned a `usage` dict). No-op on the ollama fallback
|
||||||
|
(no usage available — `usage` is None).
|
||||||
|
|
||||||
Returns True on success (including a deliberate skip), False on failure
|
Returns True on success (including a deliberate skip), False on failure
|
||||||
(failure note posted when possible). Never raises — fail-open by design.
|
(failure note posted when possible). Never raises — fail-open by design.
|
||||||
Both the CI `run()` entry point and the central webhook server call this.
|
Both the CI `run()` entry point and the central webhook server call this.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
|
# Pre-compute a *fallback* display name for the early-exit paths
|
||||||
|
# (already-reviewed dedupe skip, no-diff-content). We re-resolve
|
||||||
|
# properly after `.pr-review.json` is loaded further down — that
|
||||||
|
# version honours `OPENCODE_MODEL` env > `.pr-review.json:model` >
|
||||||
|
# this fallback.
|
||||||
|
display_model = f"headroom/{model}"
|
||||||
|
|
||||||
reviews = fetch_existing_reviews(api, repo, index, token)
|
reviews = fetch_existing_reviews(api, repo, index, token)
|
||||||
# Dedupe: already reviewed this exact commit -> nothing to do.
|
# Dedupe: already reviewed this exact commit -> nothing to do.
|
||||||
if sha and sha in reviewed_shas(reviews):
|
if sha and sha in reviewed_shas(reviews):
|
||||||
@@ -1847,12 +2120,18 @@ def review_pr(
|
|||||||
|
|
||||||
raw_diff, _truncated, _orig = fetch_pr_diff(api, repo, index, token, max_chars)
|
raw_diff, _truncated, _orig = fetch_pr_diff(api, repo, index, token, max_chars)
|
||||||
if not raw_diff.strip():
|
if not raw_diff.strip():
|
||||||
post_review(api, repo, index, token, format_review_body("No diff content to review.", model, sha))
|
post_review(api, repo, index, token, format_review_body("No diff content to review.", display_model, sha))
|
||||||
return True
|
return True
|
||||||
|
|
||||||
config = fetch_repo_config(api, repo, token, ref=base_ref)
|
config = fetch_repo_config(api, repo, token, ref=base_ref)
|
||||||
prior = compact_prior_reviews(prior_review_bodies(reviews, sha))
|
prior = compact_prior_reviews(prior_review_bodies(reviews, sha))
|
||||||
|
|
||||||
|
# Re-resolve display_model now that .pr-review.json is available —
|
||||||
|
# per-repo override (`.pr-review.json:model`) takes precedence over
|
||||||
|
# the bare OLLAMA_MODEL fallback, with OPENCODE_MODEL env still
|
||||||
|
# winning above both (see `_resolve_display_model`).
|
||||||
|
display_model = _resolve_display_model(model, config)
|
||||||
|
|
||||||
# Trim the diff to +/- hunks plus a narrow context window. The agent
|
# Trim the diff to +/- hunks plus a narrow context window. The agent
|
||||||
# resends the brief prefix every step, so a 25k-char diff becomes
|
# resends the brief prefix every step, so a 25k-char diff becomes
|
||||||
# 25k × 30-step × cached-after-step-1 = hundreds of thousands of input
|
# 25k × 30-step × cached-after-step-1 = hundreds of thousands of input
|
||||||
@@ -1886,10 +2165,9 @@ def review_pr(
|
|||||||
# the brief, and the pragent agent factory; returns stdout with a
|
# the brief, and the pragent agent factory; returns stdout with a
|
||||||
# summary + findings JSON. We parse + anchor + post here.
|
# summary + findings JSON. We parse + anchor + post here.
|
||||||
import opencode_review # local import keeps the ollama path dep-free
|
import opencode_review # local import keeps the ollama path dep-free
|
||||||
# opencode wants a provider-prefixed model ref (headroom/glm-5.2:cloud);
|
# Reuse the display_model resolved above for the subprocess — same
|
||||||
# `model` here is the bare id (OLLAMA_MODEL). OPENCODE_MODEL overrides
|
# provider-prefixed ref goes to the engine and into the review body.
|
||||||
# with the full ref; otherwise we prefix the configured provider.
|
oc_model = display_model
|
||||||
oc_model = os.environ.get("OPENCODE_MODEL") or f"headroom/{model}"
|
|
||||||
# Multi-lens fan-out: when the repo declared `reviewers[]` (or the
|
# Multi-lens fan-out: when the repo declared `reviewers[]` (or the
|
||||||
# operator pinned PRAGENT_REVIEWERS=1), spawn one opencode subprocess
|
# operator pinned PRAGENT_REVIEWERS=1), spawn one opencode subprocess
|
||||||
# per lens in parallel and synthesize. Falls through to the legacy
|
# per lens in parallel and synthesize. Falls through to the legacy
|
||||||
@@ -1913,29 +2191,23 @@ def review_pr(
|
|||||||
compression_note=compression_note,
|
compression_note=compression_note,
|
||||||
additional_context=additional_context,
|
additional_context=additional_context,
|
||||||
)
|
)
|
||||||
review_summary, findings, summary_changes, risks = parse_review_output(stdout)
|
review_summary, findings, summary_changes, risks, _walkthrough, _risk_verdict, _test_coverage = parse_review_output(stdout)
|
||||||
if not findings and not review_summary:
|
if not findings and not review_summary:
|
||||||
# The findings JSON was missing or malformed. Don't discard the
|
# The findings JSON was missing or malformed. Don't discard the
|
||||||
# run: salvage the prose, keep the usage report (the label asked
|
# run: salvage the prose, keep the usage report (the tokens were
|
||||||
# for it, and the tokens were spent either way), and log enough
|
# spent either way), and log enough of the raw output to
|
||||||
# of the raw output to diagnose why the agent went off-format.
|
# diagnose why the agent went off-format.
|
||||||
print(
|
print(
|
||||||
f"pragent: {repo}#{index} sha={sha[:8]} unparseable output "
|
f"pragent: {repo}#{index} sha={sha[:8]} unparseable output "
|
||||||
f"({len(stdout)} chars); tail: {stdout[-600:]!r}",
|
f"({len(stdout)} chars); tail: {stdout[-600:]!r}",
|
||||||
file=sys.stderr, flush=True,
|
file=sys.stderr, flush=True,
|
||||||
)
|
)
|
||||||
salvaged = salvage_summary(stdout)
|
salvaged = salvage_summary(stdout)
|
||||||
# The AI-USAGE opt-in is re-checked HERE, at render time: the label
|
usage_section = _render_collapsible_usage(usage, display_model, config=config) if usage else ""
|
||||||
# is usually applied moments after AI-REVIEW, long after this
|
|
||||||
# review was claimed and its trigger payload frozen.
|
|
||||||
if not report_usage:
|
|
||||||
report_usage = pr_has_label(api, repo, index, token, AI_USAGE_LABEL)
|
|
||||||
if report_usage and usage and usage.get('output'):
|
|
||||||
compute_attribution(findings, usage['output'])
|
|
||||||
usage_section = _render_collapsible_usage(usage, model, config=config) if report_usage else ""
|
|
||||||
post_review(api, repo, index, token, format_review_body(
|
post_review(api, repo, index, token, format_review_body(
|
||||||
salvaged or "AI review produced no parseable output.",
|
salvaged or "AI review produced no parseable output.",
|
||||||
model, sha, usage_section=usage_section))
|
display_model, sha, usage_section=usage_section,
|
||||||
|
static_message=(config or {}).get("static_message", "")))
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
user_prompt = build_user_prompt(title, body + compression_note, diff, config, prior, additional_context)
|
user_prompt = build_user_prompt(title, body + compression_note, diff, config, prior, additional_context)
|
||||||
@@ -1955,6 +2227,15 @@ def review_pr(
|
|||||||
})
|
})
|
||||||
except Exception:
|
except Exception:
|
||||||
changed_paths = []
|
changed_paths = []
|
||||||
|
# Capture cross-lens agreement BEFORE apply_repo_config — by the time
|
||||||
|
# findings land in `review_pr` the `_multi_lens` marker has already
|
||||||
|
# been scrubbed (once by `opencode_review.run_lenses_review`'s
|
||||||
|
# `_`-prefix strip, again by `_normalize_finding`'s 7-key rebuild),
|
||||||
|
# so `merge_confidence` cannot read it off the dict. We scan here as
|
||||||
|
# the convergence point for both engine paths; in practice the kwarg
|
||||||
|
# currently always passes False, but the structural plumbing is
|
||||||
|
# correct for any future code path that preserves the flag.
|
||||||
|
multi_lens = any(f.get("_multi_lens") for f in findings)
|
||||||
kept, _dropped = apply_repo_config(findings, config, changed_paths=changed_paths)
|
kept, _dropped = apply_repo_config(findings, config, changed_paths=changed_paths)
|
||||||
findings = kept
|
findings = kept
|
||||||
if _dropped:
|
if _dropped:
|
||||||
@@ -1968,18 +2249,10 @@ def review_pr(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Compute attribution so inline comments + the table can show per-comment
|
# Compute attribution so inline comments + the table can show per-comment
|
||||||
# estimates. Only meaningful when we have measured usage AND the PR asked
|
# estimates. Only meaningful when we have measured usage.
|
||||||
# for it.
|
if usage and usage.get("output"):
|
||||||
if report_usage and usage and usage.get("output"):
|
|
||||||
compute_attribution(findings, usage["output"])
|
compute_attribution(findings, usage["output"])
|
||||||
# The AI-USAGE opt-in is re-checked HERE, at render time: the label
|
usage_section = _render_collapsible_usage(usage, display_model, config=config) if usage else ""
|
||||||
# is usually applied moments after AI-REVIEW, long after this
|
|
||||||
# review was claimed and its trigger payload frozen.
|
|
||||||
if not report_usage:
|
|
||||||
report_usage = pr_has_label(api, repo, index, token, AI_USAGE_LABEL)
|
|
||||||
if report_usage and usage and usage.get('output'):
|
|
||||||
compute_attribution(findings, usage['output'])
|
|
||||||
usage_section = _render_collapsible_usage(usage, model, config=config) if report_usage else ""
|
|
||||||
|
|
||||||
# Anchor against the RAW diff, never the compressed one. Compression
|
# Anchor against the RAW diff, never the compressed one. Compression
|
||||||
# drops context lines, so a finding on a line that survived in the file
|
# drops context lines, so a finding on a line that survived in the file
|
||||||
@@ -1997,14 +2270,24 @@ def review_pr(
|
|||||||
summary_parts = []
|
summary_parts = []
|
||||||
if bullets:
|
if bullets:
|
||||||
summary_parts.append("### Unanchored Notes\n\n" + bullets)
|
summary_parts.append("### Unanchored Notes\n\n" + bullets)
|
||||||
|
# 1-5 merge verdict for the header badge. Computed AFTER filtering +
|
||||||
|
# anchoring so the verdict reflects what the operator sees (a critical
|
||||||
|
# finding that fails to anchor is still a critical finding). The
|
||||||
|
# default 5 keeps any failure path (e.g. empty findings) green.
|
||||||
|
# Cross-lens agreement is passed in via kwarg (see multi_lens scan
|
||||||
|
# above) because the `_multi_lens` flag is stripped before findings
|
||||||
|
# reach this call.
|
||||||
|
confidence = merge_confidence(findings, multi_lens_observed=multi_lens)
|
||||||
summary_body = format_review_body(
|
summary_body = format_review_body(
|
||||||
"\n\n".join(summary_parts), model, sha,
|
"\n\n".join(summary_parts), display_model, sha,
|
||||||
summary=review_summary,
|
summary=review_summary,
|
||||||
usage_section=usage_section,
|
usage_section=usage_section,
|
||||||
summary_changes=summary_changes,
|
summary_changes=summary_changes,
|
||||||
risks=risks,
|
risks=risks,
|
||||||
findings_for_table=findings,
|
findings_for_table=findings,
|
||||||
inline_count=len(anchored),
|
inline_count=len(anchored),
|
||||||
|
confidence=confidence,
|
||||||
|
static_message=(config or {}).get("static_message", ""),
|
||||||
)
|
)
|
||||||
|
|
||||||
post_inline_review(api, repo, index, token, summary_body, anchored)
|
post_inline_review(api, repo, index, token, summary_body, anchored)
|
||||||
@@ -2016,7 +2299,7 @@ def review_pr(
|
|||||||
return True
|
return True
|
||||||
except Exception as e: # fail-open
|
except Exception as e: # fail-open
|
||||||
try:
|
try:
|
||||||
post_review(api, repo, index, token, format_review_body(f"⚠️ AI review failed: {e}", model, sha))
|
post_review(api, repo, index, token, format_review_body(f"⚠️ AI review failed: {e}", display_model, sha))
|
||||||
except Exception as e2:
|
except Exception as e2:
|
||||||
print(f"pragent: could not post failure note: {e2}", file=sys.stderr)
|
print(f"pragent: could not post failure note: {e2}", file=sys.stderr)
|
||||||
print(f"pragent: review failed: {e}", file=sys.stderr)
|
print(f"pragent: review failed: {e}", file=sys.stderr)
|
||||||
|
|||||||
+26
-4
@@ -55,13 +55,20 @@ CHARS_PER_TOKEN = 4 # English prose/code rule of thumb; ±15% is normal
|
|||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Price:
|
class Price:
|
||||||
"""Per-MTok prices. `cache_write` and `cache_read` are absolute rates, not
|
"""Per-MTok prices. `cache_write` and `cache_read` are absolute rates, not
|
||||||
multipliers, so providers with different cache economics stay comparable."""
|
multipliers, so providers with different cache economics stay comparable.
|
||||||
|
|
||||||
|
`provider` is the opencode provider name (`headroom`, `local`, ...). It
|
||||||
|
doubles as the dispatch key for `.pr-review.json:model` overrides — when
|
||||||
|
a per-repo override is set, `_resolve_display_model` returns
|
||||||
|
`f"{provider}/{key}"` so the opencode subprocess routes correctly.
|
||||||
|
Default `headroom` preserved for the existing roster."""
|
||||||
|
|
||||||
name: str
|
name: str
|
||||||
input: float
|
input: float
|
||||||
output: float
|
output: float
|
||||||
cache_write: float
|
cache_write: float
|
||||||
cache_read: float
|
cache_read: float
|
||||||
|
provider: str = "headroom"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def batch_input(self) -> float:
|
def batch_input(self) -> float:
|
||||||
@@ -82,6 +89,21 @@ PRICES: dict[str, Price] = {
|
|||||||
"gpt-5.6-sol": Price("GPT-5.6 Sol", 5.00, 30.00, 5.00, 0.50),
|
"gpt-5.6-sol": Price("GPT-5.6 Sol", 5.00, 30.00, 5.00, 0.50),
|
||||||
"gpt-5.6-terra": Price("GPT-5.6 Terra", 2.00, 12.00, 2.00, 0.20),
|
"gpt-5.6-terra": Price("GPT-5.6 Terra", 2.00, 12.00, 2.00, 0.20),
|
||||||
"gpt-5.6-luna": Price("GPT-5.6 Luna", 0.20, 1.20, 0.20, 0.02),
|
"gpt-5.6-luna": Price("GPT-5.6 Luna", 0.20, 1.20, 0.20, 0.02),
|
||||||
|
# OpenAI — cached_input 0.1x, no separate cache_write
|
||||||
|
"gpt-5": Price("GPT-5", 1.25, 10.00, 1.25, 0.125),
|
||||||
|
"gpt-5-mini": Price("GPT-5 mini", 0.25, 2.00, 0.25, 0.025),
|
||||||
|
# Google Gemini — cache_write = input
|
||||||
|
"gemini-2.5-pro": Price("Gemini 2.5 Pro", 1.875, 12.50, 1.875, 0.1875),
|
||||||
|
"gemini-2.5-flash": Price("Gemini 2.5 Flash", 0.30, 2.50, 0.30, 0.03),
|
||||||
|
# xAI Grok — cache_write = input
|
||||||
|
"grok-4.5": Price("Grok 4.5", 2.00, 6.00, 2.00, 0.30),
|
||||||
|
"grok-4.3": Price("Grok 4.3", 1.25, 2.50, 1.25, 0.20),
|
||||||
|
# Self-hosted — local AI workstation, no per-token charge. provider="local"
|
||||||
|
# so the opencode subprocess routes via the `local` provider block in
|
||||||
|
# opencode.json (baseURL=http://192.168.1.79:18020/v1). Equivalent-cost
|
||||||
|
# column will read $0 — the cost-comparison signal is that the same work
|
||||||
|
# would bill $X on a paid model.
|
||||||
|
"qwen3.8-27b": Price("Qwen 3.8 27B (local)", 0.0, 0.0, 0.0, 0.0, provider="local"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -171,7 +193,7 @@ DEFAULT_TIERS = [
|
|||||||
# Observed runs — the calibration anchor
|
# Observed runs — the calibration anchor
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
# Real usage reported by the AI-USAGE label, summed from opencode's step_finish
|
# Real usage reported by opencode's step_finish events. Keep this list
|
||||||
# events. Keep this list append-only: it is the only thing separating this model
|
# events. Keep this list append-only: it is the only thing separating this model
|
||||||
# from a guess, and the first entry corrected the tier assumptions by ~15x.
|
# from a guess, and the first entry corrected the tier assumptions by ~15x.
|
||||||
OBSERVED_RUNS: list[dict] = [
|
OBSERVED_RUNS: list[dict] = [
|
||||||
@@ -357,10 +379,10 @@ def report(tiers: list[Tier], prs_per_month: int, caching: bool, models: list[st
|
|||||||
|
|
||||||
|
|
||||||
def observed_report(models: list[str]) -> str:
|
def observed_report(models: list[str]) -> str:
|
||||||
"""Price the runs actually measured through the AI-USAGE label."""
|
"""Price the runs actually measured through the opencode usage telemetry."""
|
||||||
if not OBSERVED_RUNS:
|
if not OBSERVED_RUNS:
|
||||||
return "No observed runs recorded yet."
|
return "No observed runs recorded yet."
|
||||||
lines = ["Observed runs (measured via the AI-USAGE label)"]
|
lines = ["Observed runs (measured via opencode step_finish events)"]
|
||||||
for run in OBSERVED_RUNS:
|
for run in OBSERVED_RUNS:
|
||||||
u = observed_usage(run)
|
u = observed_usage(run)
|
||||||
lines.append(
|
lines.append(
|
||||||
|
|||||||
@@ -0,0 +1,754 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""pragent pilot — read-mostly dashboard.
|
||||||
|
|
||||||
|
Stdlib HTTP server (mirrors `webhook_server.py`'s BaseHTTPRequestHandler +
|
||||||
|
ThreadingHTTPServer shape) that renders three views off the feedback SQLite:
|
||||||
|
|
||||||
|
GET / overview
|
||||||
|
GET /r/<owner>/<name> repo summary + edit form
|
||||||
|
GET /r/<owner>/<name>/<index> one PR's findings
|
||||||
|
GET /r/<owner>/<name>/<index>/raw raw Markdown body (via Gitea)
|
||||||
|
GET /static/style.css CSS
|
||||||
|
POST /r/<owner>/<name>/edit mutate .pr-review.json (Tasks C+D)
|
||||||
|
|
||||||
|
Auth: oauth2-proxy fronts this service in-cluster. Every route except
|
||||||
|
`/static/*` requires the `X-Forwarded-User` header (set by oauth2-proxy
|
||||||
|
once the user has logged in via Logto). Missing header → 401 +
|
||||||
|
`WWW-Authenticate: Basic realm="pragent-dashboard"` so oauth2-proxy
|
||||||
|
intercepts the response.
|
||||||
|
|
||||||
|
DB: `PRAGENT_FEEDBACK_DB` points at the SQLite file the webhook server
|
||||||
|
also writes. Per-request open (SQLite is cheap, no concurrency hazard,
|
||||||
|
no stale-conn surprise after the file rotates).
|
||||||
|
|
||||||
|
All HTML is rendered via `string.Template` and every dynamic value is
|
||||||
|
escaped with `html.escape(..., quote=True)`. No `.format`, no f-string
|
||||||
|
templates — see `_render_*` for the discipline.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import datetime
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import string
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
|
||||||
|
from pilot import dashboard_data
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
FEEDBACK_DB = "" # legacy; readers should call _feedback_db()
|
||||||
|
PORT = int(os.environ.get("DASHBOARD_PORT", "8081"))
|
||||||
|
|
||||||
|
GITEA_API = "" # legacy; readers should call _gitea_api()
|
||||||
|
BOT_TOKEN = "" # legacy; readers should call _bot_token()
|
||||||
|
|
||||||
|
# CSRF secret for the edit form. Regenerated per process (each Python
|
||||||
|
# interpreter launch). Behind oauth2-proxy this is enough — only an
|
||||||
|
# already-authenticated same-tab request can read this and echo it back.
|
||||||
|
_CSRF_SECRET: str = secrets.token_urlsafe(24)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Lazy config readers — tests set env after import, so each request re-reads.
|
||||||
|
# Production: env is fixed for the process lifetime; the per-request lookup is
|
||||||
|
# a dict access, not a syscall.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _feedback_db() -> str:
|
||||||
|
return os.environ.get("PRAGENT_FEEDBACK_DB", "")
|
||||||
|
|
||||||
|
|
||||||
|
def _bot_token() -> str:
|
||||||
|
return os.environ.get("PRAGENT_BOT_TOKEN", "")
|
||||||
|
|
||||||
|
|
||||||
|
def _gitea_api() -> str:
|
||||||
|
return os.environ.get("GITEA_API", "http://gitea-http.gitea.svc.cluster.local:3000")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Stylesheet — small, dark-mode-friendly, deliberately under 100 lines
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
STYLE_CSS = """
|
||||||
|
:root { color-scheme: light dark; }
|
||||||
|
* { box-sizing: border-box; }
|
||||||
|
body {
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
|
||||||
|
margin: 0; padding: 0;
|
||||||
|
background: #0f1115; color: #e6e6e6;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
header {
|
||||||
|
background: #1a1d23; padding: 12px 20px;
|
||||||
|
border-bottom: 1px solid #2a2f38;
|
||||||
|
display: flex; align-items: center; gap: 18px;
|
||||||
|
}
|
||||||
|
header h1 { font-size: 18px; margin: 0; }
|
||||||
|
header nav a {
|
||||||
|
color: #8ab4f8; text-decoration: none; margin-right: 12px;
|
||||||
|
}
|
||||||
|
header nav a:hover { text-decoration: underline; }
|
||||||
|
main { padding: 20px; max-width: 1100px; margin: 0 auto; }
|
||||||
|
h2 { margin-top: 24px; font-size: 16px; color: #c9d1d9; }
|
||||||
|
.metric-row { display: flex; gap: 16px; flex-wrap: wrap; margin-bottom: 16px; }
|
||||||
|
.metric {
|
||||||
|
background: #1a1d23; padding: 14px 18px; border-radius: 8px;
|
||||||
|
min-width: 140px; border: 1px solid #2a2f38;
|
||||||
|
}
|
||||||
|
.metric .v { font-size: 28px; font-weight: 600; }
|
||||||
|
.metric .l { font-size: 12px; color: #8b949e; text-transform: uppercase; letter-spacing: 0.04em; }
|
||||||
|
table { width: 100%; border-collapse: collapse; margin: 8px 0 16px; font-size: 14px; }
|
||||||
|
th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid #2a2f38; }
|
||||||
|
th { color: #8b949e; font-weight: 500; text-transform: uppercase; font-size: 11px; letter-spacing: 0.04em; }
|
||||||
|
tr:hover td { background: #161922; }
|
||||||
|
.sev-critical { color: #ff7b72; font-weight: 600; }
|
||||||
|
.sev-high { color: #f0883e; }
|
||||||
|
.sev-medium { color: #d29922; }
|
||||||
|
.sev-low { color: #8b949e; }
|
||||||
|
.muted { color: #8b949e; font-size: 12px; }
|
||||||
|
.sparkline { font-family: ui-monospace, "SF Mono", monospace; letter-spacing: 1px; }
|
||||||
|
form { background: #1a1d23; padding: 14px 18px; border-radius: 8px; border: 1px solid #2a2f38; margin: 12px 0; }
|
||||||
|
form label { display: block; margin: 8px 0 4px; color: #c9d1d9; font-size: 13px; }
|
||||||
|
form input[type=text], form textarea, form select {
|
||||||
|
background: #0f1115; color: #e6e6e6; border: 1px solid #2a2f38;
|
||||||
|
border-radius: 4px; padding: 6px 8px; font-family: inherit; font-size: 14px;
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
form textarea { min-height: 80px; }
|
||||||
|
form .row { display: flex; gap: 8px; align-items: center; margin-top: 12px; }
|
||||||
|
form button {
|
||||||
|
background: #2ea043; color: white; border: none; border-radius: 4px;
|
||||||
|
padding: 6px 14px; font-size: 14px; cursor: pointer;
|
||||||
|
}
|
||||||
|
form button:hover { background: #3fb950; }
|
||||||
|
.flash { background: #3d1e1e; color: #ff7b72; padding: 8px 12px; border-radius: 4px; margin-bottom: 12px; }
|
||||||
|
code { background: #161922; padding: 1px 4px; border-radius: 3px; font-size: 13px; }
|
||||||
|
pre { background: #161922; padding: 12px; border-radius: 6px; overflow-x: auto; }
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Templates — string.Template so dynamic values are always escaped explicitly
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_BASE = string.Template("""<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>${title}</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<h1>pragent dashboard</h1>
|
||||||
|
<nav>
|
||||||
|
<a href="/">Home</a>
|
||||||
|
<a href="/r/${repos_first}">repos</a>
|
||||||
|
</nav>
|
||||||
|
<span class="muted" style="margin-left:auto">${db_status}</span>
|
||||||
|
</header>
|
||||||
|
<main>
|
||||||
|
${body}
|
||||||
|
</main>
|
||||||
|
</body>
|
||||||
|
</html>""")
|
||||||
|
|
||||||
|
|
||||||
|
_OVERVIEW = string.Template("""<h2>Overview</h2>
|
||||||
|
<div class="metric-row">
|
||||||
|
<div class="metric"><div class="v">${total_reviews}</div><div class="l">reviews</div></div>
|
||||||
|
<div class="metric"><div class="v">${total_findings}</div><div class="l">findings</div></div>
|
||||||
|
<div class="metric"><div class="v">${total_repos}</div><div class="l">repos</div></div>
|
||||||
|
<div class="metric"><div class="v">${last_30d_reviews}</div><div class="l">last 30d</div></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Last 7 days</h2>
|
||||||
|
<div class="sparkline">${sparkline}</div>
|
||||||
|
<div class="muted">total cost: $${total_cost_usd} — no per-review cost logged</div>
|
||||||
|
|
||||||
|
<h2>Top repos</h2>
|
||||||
|
${top_repos_table}
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
_REPO = string.Template("""<h2>Repo: <code>${repo}</code></h2>
|
||||||
|
<div class="metric-row">
|
||||||
|
<div class="metric"><div class="v">${total_runs}</div><div class="l">runs</div></div>
|
||||||
|
<div class="metric"><div class="v">${sev_critical}</div><div class="l sev-critical">critical</div></div>
|
||||||
|
<div class="metric"><div class="v">${sev_high}</div><div class="l sev-high">high</div></div>
|
||||||
|
<div class="metric"><div class="v">${sev_medium}</div><div class="l sev-medium">medium</div></div>
|
||||||
|
<div class="metric"><div class="v">${sev_low}</div><div class="l sev-low">low</div></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Edit .pr-review.json</h2>
|
||||||
|
${flash}
|
||||||
|
<form method="post" action="/r/${repo_url}/edit">
|
||||||
|
<input type="hidden" name="_csrf" value="${csrf}">
|
||||||
|
<label for="static_message">Static banner message (max 400 chars)</label>
|
||||||
|
<textarea id="static_message" name="static_message" maxlength="400">${current_static_message}</textarea>
|
||||||
|
<label for="model">Model (PRICES keys)</label>
|
||||||
|
<select id="model" name="model">${model_options}</select>
|
||||||
|
<div class="row">
|
||||||
|
<button type="submit">Save</button>
|
||||||
|
<span class="muted">posted via the bot identity; one commit on the base branch</span>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<h2>Top findings (by occurrence)</h2>
|
||||||
|
${top_findings_table}
|
||||||
|
|
||||||
|
<h2>Runs by day (last 30d)</h2>
|
||||||
|
${runs_by_day_table}
|
||||||
|
|
||||||
|
<h2>Reviews</h2>
|
||||||
|
${reviews_table}
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
_PR = string.Template("""<h2>PR <code>${repo}</code> #${pr}</h2>
|
||||||
|
<div class="muted">head sha: <code>${head_sha}</code></div>
|
||||||
|
<div class="muted">posted_at: ${posted_at_iso}</div>
|
||||||
|
<div class="muted">review_id_gitea: ${review_id_gitea} · body_comment_id: ${body_comment_id}</div>
|
||||||
|
|
||||||
|
<h2>Findings</h2>
|
||||||
|
${findings_table}
|
||||||
|
|
||||||
|
<p><a href="/r/${repo_url}/${pr}/raw">raw review body (Markdown)</a></p>
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Small helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _esc(s) -> str:
|
||||||
|
"""HTML-escape any value to a string."""
|
||||||
|
return html.escape(str(s), quote=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _ts_iso(ts: int) -> str:
|
||||||
|
if not ts:
|
||||||
|
return "—"
|
||||||
|
return datetime.datetime.fromtimestamp(int(ts), tz=datetime.timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _sparkline(buckets: list[dict]) -> str:
|
||||||
|
"""7-bucket sparkline as unicode bars."""
|
||||||
|
bars = "▁▂▃▄▅▆▇█"
|
||||||
|
if not buckets:
|
||||||
|
return ""
|
||||||
|
mx = max((b.get("count", 0) for b in buckets), default=0) or 1
|
||||||
|
out = []
|
||||||
|
for b in buckets:
|
||||||
|
n = b.get("count", 0)
|
||||||
|
idx = min(len(bars) - 1, int(round(n / mx * (len(bars) - 1))))
|
||||||
|
out.append(bars[idx])
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Renderers — one per page
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _overview_body(data: dict) -> str:
|
||||||
|
top_rows = "".join(
|
||||||
|
f"<tr><td><a href=\"/r/{_esc(r['repo'])}\">{_esc(r['repo'])}</a></td>"
|
||||||
|
f"<td>{int(r['run_count'])}</td>"
|
||||||
|
f"<td class=\"muted\">{_ts_iso(int(r['last_seen']))}</td></tr>"
|
||||||
|
for r in data.get("top_repos", [])
|
||||||
|
) or "<tr><td class=\"muted\">no reviews yet</td></tr>"
|
||||||
|
top_table = f"<table><thead><tr><th>repo</th><th>runs</th><th>last seen</th></tr></thead><tbody>{top_rows}</tbody></table>"
|
||||||
|
return _OVERVIEW.substitute(
|
||||||
|
total_reviews=_esc(data.get("total_reviews", 0)),
|
||||||
|
total_findings=_esc(data.get("total_findings", 0)),
|
||||||
|
total_repos=_esc(data.get("total_repos", 0)),
|
||||||
|
last_30d_reviews=_esc(data.get("last_30d_reviews", 0)),
|
||||||
|
sparkline=_esc(_sparkline(data.get("daily", []))),
|
||||||
|
total_cost_usd=f"{float(data.get('total_cost_usd', 0.0)):.2f}",
|
||||||
|
top_repos_table=top_table,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_body(data: dict, *, repo_url: str, csrf: str, current_model: str,
|
||||||
|
current_static_message: str, flash: str = "") -> str:
|
||||||
|
fbs = data.get("findings_by_severity", {})
|
||||||
|
tf = data.get("top_findings", [])
|
||||||
|
|
||||||
|
# Top findings table.
|
||||||
|
if tf:
|
||||||
|
rows = "".join(
|
||||||
|
f"<tr><td><code>{_esc(f['path'])}:{_esc(f['line'])}</code></td>"
|
||||||
|
f"<td class=\"sev-{_esc(f.get('severity', 'low').lower())}\">{_esc(f.get('severity', ''))}</td>"
|
||||||
|
f"<td>{_esc(f.get('problem', ''))}</td>"
|
||||||
|
f"<td>{int(f.get('occurrences', 0))}</td>"
|
||||||
|
f"<td>+{int(f.get('upvotes', 0))} / -{int(f.get('downvotes', 0))}</td>"
|
||||||
|
f"<td>{'resolved' if int(f.get('resolved', 0)) else 'open'}</td>"
|
||||||
|
f"<td>{int(f.get('reply_count', 0))}</td></tr>"
|
||||||
|
for f in tf
|
||||||
|
)
|
||||||
|
top_findings_table = (
|
||||||
|
"<table><thead><tr><th>location</th><th>severity</th>"
|
||||||
|
"<th>problem</th><th>occurrences</th><th>votes</th>"
|
||||||
|
"<th>state</th><th>replies</th></tr></thead><tbody>"
|
||||||
|
f"{rows}</tbody></table>"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
top_findings_table = "<p class=\"muted\">no findings yet</p>"
|
||||||
|
|
||||||
|
# Runs by day.
|
||||||
|
runs = data.get("runs_by_day", [])
|
||||||
|
if runs:
|
||||||
|
rows = "".join(
|
||||||
|
f"<tr><td>{_esc(r['date'])}</td><td>{int(r.get('count', 0))}</td></tr>"
|
||||||
|
for r in runs
|
||||||
|
)
|
||||||
|
runs_by_day_table = (
|
||||||
|
"<table><thead><tr><th>date</th><th>runs</th></tr></thead>"
|
||||||
|
f"<tbody>{rows}</tbody></table>"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
runs_by_day_table = "<p class=\"muted\">no runs in the last 30 days</p>"
|
||||||
|
|
||||||
|
# Reviews list — derived from finding timestamps; cheap because we
|
||||||
|
# just enumerate the repo's review rows.
|
||||||
|
reviews_table = _repo_reviews_table(repo_url, data.get("recent_reviews", []))
|
||||||
|
|
||||||
|
# Model select (Task D) — sorted PRICES keys + "keep current".
|
||||||
|
from cost_model import PRICES # local: pilot-only dep
|
||||||
|
model_options = (
|
||||||
|
f"<option value=\"\">— keep current ({_esc(current_model or 'unset')}) —</option>"
|
||||||
|
+ "".join(
|
||||||
|
f"<option value=\"{_esc(k)}\" {'selected' if k == current_model else ''}>{_esc(k)}</option>"
|
||||||
|
for k in sorted(PRICES)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return _REPO.substitute(
|
||||||
|
repo=_esc(data.get("repo", "")),
|
||||||
|
repo_url=_esc(repo_url),
|
||||||
|
total_runs=_esc(data.get("total_runs", 0)),
|
||||||
|
sev_critical=_esc(fbs.get("critical", 0)),
|
||||||
|
sev_high=_esc(fbs.get("high", 0)),
|
||||||
|
sev_medium=_esc(fbs.get("medium", 0)),
|
||||||
|
sev_low=_esc(fbs.get("low", 0)),
|
||||||
|
csrf=_esc(csrf),
|
||||||
|
current_static_message=_esc(current_static_message),
|
||||||
|
model_options=model_options,
|
||||||
|
flash=_esc(flash),
|
||||||
|
top_findings_table=top_findings_table,
|
||||||
|
runs_by_day_table=runs_by_day_table,
|
||||||
|
reviews_table=reviews_table,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_reviews_table(repo_url: str, rows: list[dict]) -> str:
|
||||||
|
if not rows:
|
||||||
|
return "<p class=\"muted\">no reviews yet</p>"
|
||||||
|
out = "<table><thead><tr><th>PR</th><th>head sha</th><th>posted</th></tr></thead><tbody>"
|
||||||
|
for r in rows:
|
||||||
|
out += (
|
||||||
|
f"<tr><td><a href=\"/r/{_esc(repo_url)}/{int(r['pr'])}\">#{int(r['pr'])}</a></td>"
|
||||||
|
f"<td><code>{_esc(r['head_sha'][:10])}</code></td>"
|
||||||
|
f"<td class=\"muted\">{_ts_iso(int(r.get('posted_at', 0)))}</td></tr>"
|
||||||
|
)
|
||||||
|
out += "</tbody></table>"
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _pr_body(data: dict, *, repo_url: str) -> str:
|
||||||
|
findings = data.get("findings", [])
|
||||||
|
if findings:
|
||||||
|
rows = "".join(
|
||||||
|
f"<tr><td><code>{_esc(f['path'])}:{_esc(f['line'])}</code></td>"
|
||||||
|
f"<td class=\"sev-{_esc(f.get('severity', 'low').lower())}\">{_esc(f.get('severity', ''))}</td>"
|
||||||
|
f"<td>{_esc(f.get('problem', ''))}</td>"
|
||||||
|
f"<td>{_esc(f.get('fix', ''))}</td>"
|
||||||
|
f"<td>{_esc(f.get('suggestion', ''))}</td>"
|
||||||
|
f"<td>+{int(f.get('upvotes', 0))} / -{int(f.get('downvotes', 0))}</td>"
|
||||||
|
f"<td>{'resolved' if int(f.get('resolved', 0)) else 'open'}</td>"
|
||||||
|
f"<td>{int(f.get('reply_count', 0))}</td></tr>"
|
||||||
|
for f in findings
|
||||||
|
)
|
||||||
|
findings_table = (
|
||||||
|
"<table><thead><tr><th>location</th><th>severity</th>"
|
||||||
|
"<th>problem</th><th>fix</th><th>suggestion</th>"
|
||||||
|
"<th>votes</th><th>state</th><th>replies</th></tr></thead>"
|
||||||
|
f"<tbody>{rows}</tbody></table>"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
findings_table = "<p class=\"muted\">no findings</p>"
|
||||||
|
|
||||||
|
return _PR.substitute(
|
||||||
|
repo=_esc(data.get("repo", "")),
|
||||||
|
repo_url=_esc(repo_url),
|
||||||
|
pr=_esc(data.get("pr", 0)),
|
||||||
|
head_sha=_esc(data.get("head_sha", "")),
|
||||||
|
posted_at_iso=_ts_iso(int(data.get("posted_at", 0))),
|
||||||
|
review_id_gitea=_esc(data.get("review_id_gitea", "") or "—"),
|
||||||
|
body_comment_id=_esc(data.get("body_comment_id", "") or "—"),
|
||||||
|
findings_table=findings_table,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _page(title: str, body: str, *, repos_first: str = "") -> str:
|
||||||
|
db_status = _feedback_db() or "(no DB configured)"
|
||||||
|
return _BASE.substitute(
|
||||||
|
title=_esc(title),
|
||||||
|
body=body,
|
||||||
|
repos_first=_esc(repos_first),
|
||||||
|
db_status=_esc(db_status),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Gitea HTTP helper — minimal, used by the raw body fetch and the edit endpoint
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _http(method: str, url: str, *, token: str = "", body: dict | None = None,
|
||||||
|
raw_body: bytes | None = None) -> tuple[int, bytes]:
|
||||||
|
"""Like ai_review._http but local: this module is stdlib-only and doesn't
|
||||||
|
depend on the ai_review import (which pulls in a 1700-line reviewer)."""
|
||||||
|
headers = {"Accept": "application/json"}
|
||||||
|
data: bytes | None = None
|
||||||
|
if raw_body is not None:
|
||||||
|
data = raw_body
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
elif body is not None:
|
||||||
|
data = json.dumps(body).encode()
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = f"token {token}"
|
||||||
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
return r.status, r.read()
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
return e.code, e.read()
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
raise RuntimeError(f"network error: {e.reason}") from e
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Auth
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _is_authed(headers) -> bool:
|
||||||
|
"""True when oauth2-proxy forwarded a verified user.
|
||||||
|
|
||||||
|
oauth2-proxy sets `X-Forwarded-User` (and friends) only after a
|
||||||
|
successful Logto login + email allowlist check. Unauthenticated
|
||||||
|
requests never see the header, so the dashboard never has to know
|
||||||
|
about cookies, secrets, or Logto's token shape.
|
||||||
|
"""
|
||||||
|
return bool((headers.get("X-Forwarded-User") or "").strip())
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Routes
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _route_overview() -> bytes:
|
||||||
|
data = dashboard_data.overview(_feedback_db())
|
||||||
|
body = _overview_body(data)
|
||||||
|
# nav: first repo if any
|
||||||
|
repos_first = ""
|
||||||
|
if data.get("top_repos"):
|
||||||
|
repos_first = data["top_repos"][0]["repo"]
|
||||||
|
return _page("Overview", body, repos_first=repos_first).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _route_repo(owner: str, name: str) -> bytes:
|
||||||
|
repo_url = f"{owner}/{name}"
|
||||||
|
data = dashboard_data.repo_summary(_feedback_db(), repo_url)
|
||||||
|
# Pull current .pr-review.json (best-effort) so the form fields prefill.
|
||||||
|
current_static_message, current_model, flash = "", "", ""
|
||||||
|
cfg, err = _fetch_pr_review_json(repo_url)
|
||||||
|
if cfg:
|
||||||
|
current_static_message = cfg.get("static_message", "")
|
||||||
|
current_model = cfg.get("model", "")
|
||||||
|
elif err and err != "404":
|
||||||
|
flash = f"could not read .pr-review.json: {err}"
|
||||||
|
body = _repo_body(
|
||||||
|
data,
|
||||||
|
repo_url=repo_url,
|
||||||
|
csrf=_CSRF_SECRET,
|
||||||
|
current_model=current_model,
|
||||||
|
current_static_message=current_static_message,
|
||||||
|
flash=flash,
|
||||||
|
)
|
||||||
|
return _page(f"repo {repo_url}", body, repos_first=repo_url).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _route_pr(owner: str, name: str, index: int) -> bytes:
|
||||||
|
repo_url = f"{owner}/{name}"
|
||||||
|
data = dashboard_data.pr_summary(_feedback_db(), repo_url, int(index))
|
||||||
|
body = _pr_body(data, repo_url=repo_url)
|
||||||
|
return _page(f"PR {repo_url}#{index}", body, repos_first=repo_url).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _route_pr_raw(owner: str, name: str, index: int) -> tuple[int, bytes]:
|
||||||
|
repo_url = f"{owner}/{name}"
|
||||||
|
data = dashboard_data.pr_summary(_feedback_db(), repo_url, int(index))
|
||||||
|
body_comment_id = data.get("body_comment_id")
|
||||||
|
if not body_comment_id:
|
||||||
|
return 404, b"no body_comment_id"
|
||||||
|
status, raw = _http(
|
||||||
|
"GET",
|
||||||
|
f"{_gitea_api()}/api/v1/repos/{repo_url}/issues/{index}/comments/{body_comment_id}",
|
||||||
|
token=_bot_token(),
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
return 404, f"Gitea returned {status}".encode()
|
||||||
|
try:
|
||||||
|
parsed = json.loads(raw)
|
||||||
|
md = parsed.get("body", "")
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
return 404, b"could not parse Gitea response"
|
||||||
|
return 200, md.encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _route_static_css() -> bytes:
|
||||||
|
return STYLE_CSS.encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _route_edit(owner: str, name: str, form: dict) -> tuple[int, dict, bytes]:
|
||||||
|
"""Mutate .pr-review.json via the Gitea contents API (Tasks C+D)."""
|
||||||
|
repo_url = f"{owner}/{name}"
|
||||||
|
csrf = form.get("_csrf", "")
|
||||||
|
if csrf != _CSRF_SECRET:
|
||||||
|
return 302, {"Location": f"/r/{repo_url}"}, b""
|
||||||
|
static_message = (form.get("static_message") or "").strip()[:400]
|
||||||
|
model = (form.get("model") or "").strip()
|
||||||
|
|
||||||
|
# Validate model against PRICES.
|
||||||
|
from cost_model import PRICES
|
||||||
|
if model and model not in PRICES:
|
||||||
|
flash = urllib.parse.quote(f"unknown model {model!r}; not saved")
|
||||||
|
return 302, {"Location": f"/r/{repo_url}?flash={flash}"}, b""
|
||||||
|
|
||||||
|
cfg, err = _fetch_pr_review_json(repo_url)
|
||||||
|
if err and err != "404":
|
||||||
|
flash = urllib.parse.quote(f"could not read .pr-review.json: {err}")
|
||||||
|
return 302, {"Location": f"/r/{repo_url}?flash={flash}"}, b""
|
||||||
|
if cfg is None:
|
||||||
|
cfg = {}
|
||||||
|
|
||||||
|
if static_message:
|
||||||
|
cfg["static_message"] = static_message
|
||||||
|
elif "static_message" in cfg and not static_message:
|
||||||
|
# Empty submission clears the banner.
|
||||||
|
del cfg["static_message"]
|
||||||
|
if model:
|
||||||
|
cfg["model"] = model
|
||||||
|
elif "model" in cfg and not model:
|
||||||
|
del cfg["model"]
|
||||||
|
|
||||||
|
payload = json.dumps(cfg, indent=2, sort_keys=True).encode()
|
||||||
|
b64 = base64.b64encode(payload).decode()
|
||||||
|
body = {"content": b64, "message": "pragent dashboard: update .pr-review.json"}
|
||||||
|
if err == "404":
|
||||||
|
# File didn't exist — Gitea contents PUT still creates the file when
|
||||||
|
# `sha` is omitted, but only on certain versions; passing sha=None is
|
||||||
|
# safer.
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
# GET returned a sha — include it so Gitea enforces optimistic lock.
|
||||||
|
# The sha lives in cfg's wrapper: re-fetch once to capture it.
|
||||||
|
_, raw = _http(
|
||||||
|
"GET",
|
||||||
|
f"{_gitea_api()}/api/v1/repos/{repo_url}/contents/.pr-review.json",
|
||||||
|
token=_bot_token(),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
existing = json.loads(raw)
|
||||||
|
sha = existing.get("sha")
|
||||||
|
if sha:
|
||||||
|
body["sha"] = sha
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
status, _ = _http(
|
||||||
|
"PUT",
|
||||||
|
f"{_gitea_api()}/api/v1/repos/{repo_url}/contents/.pr-review.json",
|
||||||
|
token=_bot_token(),
|
||||||
|
body=body,
|
||||||
|
)
|
||||||
|
if status not in (200, 201):
|
||||||
|
flash = urllib.parse.quote(f"Gitea PUT failed: status {status}")
|
||||||
|
return 302, {"Location": f"/r/{repo_url}?flash={flash}"}, b""
|
||||||
|
return 302, {"Location": f"/r/{repo_url}"}, b""
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_pr_review_json(repo_url: str) -> tuple[dict | None, str | None]:
|
||||||
|
"""Return (cfg, None) on success, (None, None) when the file doesn't exist,
|
||||||
|
(None, 'reason') on error."""
|
||||||
|
if not _bot_token():
|
||||||
|
return None, "PRAGENT_BOT_TOKEN not set"
|
||||||
|
status, raw = _http(
|
||||||
|
"GET",
|
||||||
|
f"{_gitea_api()}/api/v1/repos/{repo_url}/contents/.pr-review.json",
|
||||||
|
token=_bot_token(),
|
||||||
|
)
|
||||||
|
if status == 404:
|
||||||
|
return None, "404"
|
||||||
|
if status != 200:
|
||||||
|
return None, f"status {status}"
|
||||||
|
try:
|
||||||
|
wrapper = json.loads(raw)
|
||||||
|
content_b64 = wrapper.get("content", "").replace("\n", "")
|
||||||
|
decoded = base64.b64decode(content_b64).decode("utf-8", errors="replace")
|
||||||
|
cfg = json.loads(decoded)
|
||||||
|
except (json.JSONDecodeError, ValueError) as e:
|
||||||
|
return None, f"parse error: {e}"
|
||||||
|
if not isinstance(cfg, dict):
|
||||||
|
return None, "not a JSON object"
|
||||||
|
return cfg, None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Handler
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def _send(self, status: int, body: bytes, *, content_type: str = "text/html; charset=utf-8",
|
||||||
|
extra_headers: dict | None = None) -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
if extra_headers:
|
||||||
|
for k, v in extra_headers.items():
|
||||||
|
self.send_header(k, v)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def _redirect(self, location: str) -> None:
|
||||||
|
body = b""
|
||||||
|
self.send_response(302)
|
||||||
|
self.send_header("Location", location)
|
||||||
|
self.send_header("Content-Length", "0")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def _unauthorized(self) -> None:
|
||||||
|
"""401 + Basic challenge so oauth2-proxy intercepts and redirects to Logto."""
|
||||||
|
body = b"unauthorized\n"
|
||||||
|
self.send_response(401)
|
||||||
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.send_header("WWW-Authenticate", 'Basic realm="pragent-dashboard"')
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
# --- GET -----------------------------------------------------------------
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
path = self.path
|
||||||
|
# Static is exempt from auth (also unauthenticated browser fingerprinting
|
||||||
|
# noise, but it's the same CSS regardless of viewer).
|
||||||
|
if path == "/static/style.css":
|
||||||
|
self._send(200, _route_static_css(), content_type="text/css; charset=utf-8")
|
||||||
|
return
|
||||||
|
if not _is_authed(self.headers):
|
||||||
|
self._unauthorized()
|
||||||
|
return
|
||||||
|
|
||||||
|
if path == "/" or path == "":
|
||||||
|
self._send(200, _route_overview())
|
||||||
|
return
|
||||||
|
|
||||||
|
# /r/<owner>/<name> → repo
|
||||||
|
# /r/<owner>/<name>/<index> → PR
|
||||||
|
# /r/<owner>/<name>/<index>/raw → raw Markdown
|
||||||
|
m = _REPO_PR_RAW_RE.match(path)
|
||||||
|
if m:
|
||||||
|
owner, name, idx, raw = m.group(1), m.group(2), m.group(3), m.group(4)
|
||||||
|
if raw:
|
||||||
|
status, body = _route_pr_raw(owner, name, int(idx))
|
||||||
|
self._send(status, body,
|
||||||
|
content_type="text/plain; charset=utf-8" if status == 200 else "text/plain")
|
||||||
|
return
|
||||||
|
if idx:
|
||||||
|
self._send(200, _route_pr(owner, name, int(idx)))
|
||||||
|
return
|
||||||
|
self._send(200, _route_repo(owner, name))
|
||||||
|
return
|
||||||
|
|
||||||
|
self._send(404, b"not found", content_type="text/plain")
|
||||||
|
|
||||||
|
# --- POST ----------------------------------------------------------------
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
path = self.path
|
||||||
|
if not _is_authed(self.headers):
|
||||||
|
self._unauthorized()
|
||||||
|
return
|
||||||
|
# /r/<owner>/<name>/edit
|
||||||
|
m = _EDIT_RE.match(path)
|
||||||
|
if m:
|
||||||
|
owner, name = m.group(1), m.group(2)
|
||||||
|
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||||
|
raw = self.rfile.read(length) if length else b""
|
||||||
|
form = urllib.parse.parse_qs(raw.decode("utf-8", errors="replace"))
|
||||||
|
# Collapse lists to single values.
|
||||||
|
form_single = {k: v[0] for k, v in form.items()}
|
||||||
|
status, extra, body = _route_edit(owner, name, form_single)
|
||||||
|
self._send(status, body, content_type="text/plain", extra_headers=extra)
|
||||||
|
return
|
||||||
|
self._send(404, b"not found", content_type="text/plain")
|
||||||
|
|
||||||
|
def log_message(self, fmt, *args):
|
||||||
|
print(f"pragent-dashboard: {self.address_string()} {fmt % args}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Routing regexes (compiled at import time)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
import re # noqa: E402
|
||||||
|
|
||||||
|
_REPO_PR_RAW_RE = re.compile(
|
||||||
|
r"^/r/([^/]+)/([^/]+)(?:/(\d+)(?:/(raw))?)?/?$"
|
||||||
|
)
|
||||||
|
_EDIT_RE = re.compile(r"^/r/([^/]+)/([^/]+)/edit/?$")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not _feedback_db():
|
||||||
|
print("pragent-dashboard: WARNING: PRAGENT_FEEDBACK_DB not set; dashboard will be empty",
|
||||||
|
flush=True)
|
||||||
|
print("pragent-dashboard: auth via oauth2-proxy (X-Forwarded-User required)", flush=True)
|
||||||
|
server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
|
||||||
|
print(f"pragent-dashboard: listening on :{PORT}", flush=True)
|
||||||
|
try:
|
||||||
|
server.serve_forever()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
"""pragent pilot — dashboard read-only query layer.
|
||||||
|
|
||||||
|
Three functions: overview / repo_summary / pr_summary. Each opens the SQLite
|
||||||
|
feedback DB via `feedback.init`, runs the queries it needs, and returns plain
|
||||||
|
dicts/lists. NEVER writes — that's the dashboard_server's job (via the Gitea
|
||||||
|
contents API). This module is what the dashboard_server's templates render.
|
||||||
|
|
||||||
|
All three functions are tolerant of a missing or empty DB: they return the
|
||||||
|
shaped dict with zeros/empty lists rather than crashing. The dashboard is a
|
||||||
|
read-only view; the pilot can boot with no feedback DB and the dashboard
|
||||||
|
should still load.
|
||||||
|
|
||||||
|
Cost note: `total_cost_usd` is hardcoded to 0.0. Per-review `usage:cost` is
|
||||||
|
not in the feedback SQLite — only the raw `review` / `inline_finding` rows
|
||||||
|
are stored there. The equivalent-cost calc lives in `ai_review._render_collapsible_usage`
|
||||||
|
and only knows about the latest review's tokens. Surfacing a rolled-up dollar
|
||||||
|
figure without per-row telemetry would be guessing, so we don't.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
from pilot import feedback
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_overview() -> dict:
|
||||||
|
return {
|
||||||
|
"total_reviews": 0,
|
||||||
|
"total_findings": 0,
|
||||||
|
"total_repos": 0,
|
||||||
|
"last_30d_reviews": 0,
|
||||||
|
"daily": [{"date": _iso_date(i), "count": 0} for i in range(7)],
|
||||||
|
"top_repos": [],
|
||||||
|
"total_cost_usd": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_repo_summary(repo: str) -> dict:
|
||||||
|
return {
|
||||||
|
"repo": repo,
|
||||||
|
"total_runs": 0,
|
||||||
|
"last_run_ts": 0,
|
||||||
|
"runs_by_day": [],
|
||||||
|
"findings_by_severity": {"critical": 0, "high": 0, "medium": 0, "low": 0},
|
||||||
|
"top_findings": [],
|
||||||
|
# NOTE: review rows don't carry a `model` column in the schema today,
|
||||||
|
# so we have nothing to aggregate. When that lands, replace this
|
||||||
|
# empty list with a `SELECT model, COUNT(*) …` over `review`.
|
||||||
|
"models_used": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_pr_summary(repo: str, pr: int) -> dict:
|
||||||
|
return {
|
||||||
|
"repo": repo,
|
||||||
|
"pr": pr,
|
||||||
|
"head_sha": "",
|
||||||
|
"posted_at": 0,
|
||||||
|
"review_id_gitea": None,
|
||||||
|
"body_comment_id": None,
|
||||||
|
"findings": [],
|
||||||
|
# usage isn't on the review row today; ai_review.py renders it
|
||||||
|
# in-memory at review time. Leave empty.
|
||||||
|
"usage": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_date(days_ago: int) -> str:
|
||||||
|
"""Return YYYY-MM-DD for `days_ago` days before today (UTC)."""
|
||||||
|
d = datetime.datetime.now(datetime.timezone.utc).date() - datetime.timedelta(days=days_ago)
|
||||||
|
return d.isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _open_or_none(db_path: str) -> sqlite3.Connection | None:
|
||||||
|
"""Open the DB if it exists and looks like a feedback DB. Else None.
|
||||||
|
|
||||||
|
Tolerates missing files (fresh container) and a schema-less file (the
|
||||||
|
operator dropped a stray DB at the path). Returns a connection with
|
||||||
|
Row factory set so callers can use `row["col"]`.
|
||||||
|
"""
|
||||||
|
if not db_path or not os.path.exists(db_path):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
conn = feedback.init(db_path)
|
||||||
|
except sqlite3.DatabaseError:
|
||||||
|
return None
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Public API
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def overview(db_path: str) -> dict:
|
||||||
|
"""Top-of-page summary: totals + 7-bucket daily sparkline + top 5 repos."""
|
||||||
|
conn = _open_or_none(db_path)
|
||||||
|
if conn is None:
|
||||||
|
return _empty_overview()
|
||||||
|
try:
|
||||||
|
cur = conn.execute("SELECT COUNT(*) FROM review")
|
||||||
|
total_reviews = cur.fetchone()[0]
|
||||||
|
cur = conn.execute("SELECT COUNT(*) FROM inline_finding")
|
||||||
|
total_findings = cur.fetchone()[0]
|
||||||
|
cur = conn.execute("SELECT COUNT(DISTINCT repo) FROM review")
|
||||||
|
total_repos = cur.fetchone()[0]
|
||||||
|
|
||||||
|
# Last 30d window — reviews AND findings posted within the window.
|
||||||
|
ts_30d_ago = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) - 30 * 86400
|
||||||
|
cur = conn.execute("SELECT COUNT(*) FROM review WHERE posted_at >= ?", (ts_30d_ago,))
|
||||||
|
last_30d_reviews = cur.fetchone()[0]
|
||||||
|
|
||||||
|
# 7-bucket daily sparkline, oldest first. Bucket key is UTC date.
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT posted_at FROM review WHERE posted_at >= ?",
|
||||||
|
(int(datetime.datetime.now(datetime.timezone.utc).timestamp()) - 7 * 86400,),
|
||||||
|
)
|
||||||
|
buckets: dict[str, int] = {_iso_date(i): 0 for i in range(7)}
|
||||||
|
for (ts,) in cur.fetchall():
|
||||||
|
d = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc).date().isoformat()
|
||||||
|
if d in buckets:
|
||||||
|
buckets[d] += 1
|
||||||
|
daily = [{"date": _iso_date(i), "count": buckets[_iso_date(i)]} for i in range(7)]
|
||||||
|
|
||||||
|
# Top 5 repos by run count, descending. last_seen is the most recent
|
||||||
|
# review timestamp on that repo.
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT repo, COUNT(*) AS runs, MAX(posted_at) AS last_seen "
|
||||||
|
"FROM review GROUP BY repo ORDER BY runs DESC, last_seen DESC LIMIT 5"
|
||||||
|
)
|
||||||
|
top_repos = [
|
||||||
|
{"repo": row[0], "run_count": row[1], "last_seen": int(row[2])}
|
||||||
|
for row in cur.fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total_reviews": total_reviews,
|
||||||
|
"total_findings": total_findings,
|
||||||
|
"total_repos": total_repos,
|
||||||
|
"last_30d_reviews": last_30d_reviews,
|
||||||
|
"daily": daily,
|
||||||
|
"top_repos": top_repos,
|
||||||
|
"total_cost_usd": 0.0,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def repo_summary(db_path: str, repo: str) -> dict:
|
||||||
|
"""Per-repo drill-down: runs by day, severity histogram, top findings."""
|
||||||
|
conn = _open_or_none(db_path)
|
||||||
|
if conn is None:
|
||||||
|
return _empty_repo_summary(repo)
|
||||||
|
try:
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT COUNT(*), MAX(posted_at) FROM review WHERE repo = ?", (repo,)
|
||||||
|
)
|
||||||
|
row = cur.fetchone()
|
||||||
|
total_runs = row[0] or 0
|
||||||
|
last_run_ts = int(row[1]) if row[1] else 0
|
||||||
|
|
||||||
|
# runs_by_day for the last 30 days, oldest first; zero-buckets included.
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT posted_at FROM review WHERE repo = ? AND posted_at >= ?",
|
||||||
|
(repo, int(datetime.datetime.now(datetime.timezone.utc).timestamp()) - 30 * 86400),
|
||||||
|
)
|
||||||
|
buckets: dict[str, int] = {}
|
||||||
|
for d in range(30):
|
||||||
|
buckets[_iso_date(d)] = 0 # newest-day mapped to 0; we'll iterate
|
||||||
|
# Re-key: build oldest-first, days_ago goes 29..0
|
||||||
|
oldest_first = {}
|
||||||
|
for d in range(30):
|
||||||
|
oldest_first[_iso_date(29 - d)] = 0
|
||||||
|
for (ts,) in cur.fetchall():
|
||||||
|
d = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc).date().isoformat()
|
||||||
|
if d in oldest_first:
|
||||||
|
oldest_first[d] += 1
|
||||||
|
runs_by_day = [{"date": k, "count": v} for k, v in oldest_first.items()]
|
||||||
|
|
||||||
|
# findings_by_severity — case-insensitive match; bucket unknown as 'low'.
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT severity, COUNT(*) FROM inline_finding WHERE repo = ? GROUP BY severity",
|
||||||
|
(repo,),
|
||||||
|
)
|
||||||
|
fbs = {"critical": 0, "high": 0, "medium": 0, "low": 0}
|
||||||
|
for sev, n in cur.fetchall():
|
||||||
|
k = (sev or "").strip().lower()
|
||||||
|
if k not in fbs:
|
||||||
|
k = "low"
|
||||||
|
fbs[k] += n
|
||||||
|
|
||||||
|
# top_findings — top 5 posthashes by occurrence count, joined with
|
||||||
|
# vote rollups via feedback.findings_with_votes.
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT f.path, f.line, MAX(f.severity) AS severity, MAX(f.problem) AS problem, "
|
||||||
|
"COUNT(*) AS occurrences, "
|
||||||
|
"COALESCE(SUM(CASE WHEN rct.content = '+1' THEN 1 ELSE 0 END), 0) AS upvotes, "
|
||||||
|
"COALESCE(SUM(CASE WHEN rct.content = '-1' THEN 1 ELSE 0 END), 0) AS downvotes, "
|
||||||
|
"MAX(ts.resolved) AS resolved, "
|
||||||
|
"COALESCE((SELECT COUNT(*) FROM reply WHERE finding_id IN "
|
||||||
|
" (SELECT id FROM inline_finding WHERE posthash = f.posthash AND repo = f.repo AND path = f.path AND line = f.line)), 0) AS reply_count "
|
||||||
|
"FROM inline_finding f "
|
||||||
|
"LEFT JOIN reaction rct ON rct.comment_id = f.comment_id "
|
||||||
|
"LEFT JOIN thread_state ts ON ts.finding_id = f.id "
|
||||||
|
"WHERE f.repo = ? "
|
||||||
|
"GROUP BY f.posthash, f.repo, f.path, f.line "
|
||||||
|
"ORDER BY occurrences DESC, upvotes DESC LIMIT 5",
|
||||||
|
(repo,),
|
||||||
|
)
|
||||||
|
top_findings = [
|
||||||
|
{
|
||||||
|
"path": r[0],
|
||||||
|
"line": r[1],
|
||||||
|
"severity": r[2],
|
||||||
|
"problem": r[3],
|
||||||
|
"occurrences": r[4],
|
||||||
|
"upvotes": int(r[5] or 0),
|
||||||
|
"downvotes": int(r[6] or 0),
|
||||||
|
"resolved": int(r[7] or 0),
|
||||||
|
"reply_count": int(r[8] or 0),
|
||||||
|
}
|
||||||
|
for r in cur.fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"repo": repo,
|
||||||
|
"total_runs": total_runs,
|
||||||
|
"last_run_ts": last_run_ts,
|
||||||
|
"runs_by_day": runs_by_day,
|
||||||
|
"findings_by_severity": fbs,
|
||||||
|
"top_findings": top_findings,
|
||||||
|
"models_used": [], # see _empty_repo_summary NOTE
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def pr_summary(db_path: str, repo: str, pr: int) -> dict:
|
||||||
|
"""Per-PR view: meta + every finding the bot ever posted on that PR."""
|
||||||
|
conn = _open_or_none(db_path)
|
||||||
|
if conn is None:
|
||||||
|
return _empty_pr_summary(repo, pr)
|
||||||
|
try:
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT head_sha, posted_at, review_id_gitea, body_comment_id "
|
||||||
|
"FROM review WHERE repo = ? AND pr = ? ORDER BY posted_at DESC LIMIT 1",
|
||||||
|
(repo, pr),
|
||||||
|
)
|
||||||
|
row = cur.fetchone()
|
||||||
|
if row is None:
|
||||||
|
return _empty_pr_summary(repo, pr)
|
||||||
|
head_sha, posted_at, review_id_gitea, body_comment_id = row
|
||||||
|
|
||||||
|
cur = conn.execute(
|
||||||
|
"SELECT f.path, f.line, f.severity, f.problem, f.fix, f.suggestion, "
|
||||||
|
"COALESCE(SUM(CASE WHEN rct.content = '+1' THEN 1 ELSE 0 END), 0) AS upvotes, "
|
||||||
|
"COALESCE(SUM(CASE WHEN rct.content = '-1' THEN 1 ELSE 0 END), 0) AS downvotes, "
|
||||||
|
"MAX(ts.resolved) AS resolved, "
|
||||||
|
"COALESCE((SELECT COUNT(*) FROM reply WHERE finding_id = f.id), 0) AS reply_count "
|
||||||
|
"FROM inline_finding f "
|
||||||
|
"LEFT JOIN reaction rct ON rct.comment_id = f.comment_id "
|
||||||
|
"LEFT JOIN thread_state ts ON ts.finding_id = f.id "
|
||||||
|
"WHERE f.repo = ? AND f.pr = ? "
|
||||||
|
"GROUP BY f.id "
|
||||||
|
"ORDER BY f.path, f.line",
|
||||||
|
(repo, pr),
|
||||||
|
)
|
||||||
|
findings = [
|
||||||
|
{
|
||||||
|
"path": r[0],
|
||||||
|
"line": r[1],
|
||||||
|
"severity": r[2],
|
||||||
|
"problem": r[3],
|
||||||
|
"fix": r[4],
|
||||||
|
"suggestion": r[5],
|
||||||
|
"upvotes": int(r[6] or 0),
|
||||||
|
"downvotes": int(r[7] or 0),
|
||||||
|
"resolved": int(r[8] or 0),
|
||||||
|
"reply_count": int(r[9] or 0),
|
||||||
|
}
|
||||||
|
for r in cur.fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"repo": repo,
|
||||||
|
"pr": pr,
|
||||||
|
"head_sha": head_sha,
|
||||||
|
"posted_at": int(posted_at),
|
||||||
|
"review_id_gitea": review_id_gitea,
|
||||||
|
"body_comment_id": body_comment_id,
|
||||||
|
"findings": findings,
|
||||||
|
"usage": {},
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
@@ -0,0 +1,356 @@
|
|||||||
|
"""pragent pilot — feedback storage.
|
||||||
|
|
||||||
|
A thin SQLite layer that records every bot review comment + the reactions /
|
||||||
|
thread-state / replies it accumulates over time. Powers the daily analysis
|
||||||
|
that produces suggested addenda for `.pr-review.json:instructions` and
|
||||||
|
`PRAGENT_ADDITIONAL_CONTEXT_URL` (see `feedback_analyze.py`).
|
||||||
|
|
||||||
|
Why SQLite: stdlib, no extra deps in the container, single writer (the
|
||||||
|
webhook server is one process per pod). Mount at `/data/feedback.db`
|
||||||
|
via the `feedback-data` PVC.
|
||||||
|
|
||||||
|
Schema (idempotent — safe to call `init` at every boot):
|
||||||
|
|
||||||
|
review(repo, pr, head_sha, body_comment_id, posted_at, review_id_gitea)
|
||||||
|
inline_finding(review_id → review.id, repo, pr, path, line,
|
||||||
|
severity, problem, fix, suggestion,
|
||||||
|
comment_id, posthash UNIQUE, posted_at)
|
||||||
|
reaction(comment_id, user, content, created_at,
|
||||||
|
PRIMARY KEY (comment_id, user, content))
|
||||||
|
thread_state(finding_id → inline_finding.id, resolved, checked_at,
|
||||||
|
PRIMARY KEY (finding_id))
|
||||||
|
reply(finding_id → inline_finding.id, author, body, created_at,
|
||||||
|
PRIMARY KEY (finding_id, created_at))
|
||||||
|
|
||||||
|
`posthash` is a short hash of (path|line|severity|first 80 chars of problem).
|
||||||
|
It survives across reviews of the same finding on the same line — same
|
||||||
|
finding on PR #5 and PR #12 of the same file de-duplicate, so the daily
|
||||||
|
analyzer can count votes across reviews instead of one-at-a-time.
|
||||||
|
|
||||||
|
Everything is best-effort. The webhook server never aborts a review
|
||||||
|
because the feedback DB had a hiccup — `record_*` functions log and
|
||||||
|
swallow.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
import time
|
||||||
|
from typing import Iterable, Optional
|
||||||
|
|
||||||
|
log = logging.getLogger("pragent.feedback")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Schema
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_SCHEMA = """
|
||||||
|
CREATE TABLE IF NOT EXISTS review (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
repo TEXT NOT NULL,
|
||||||
|
pr INTEGER NOT NULL,
|
||||||
|
head_sha TEXT NOT NULL,
|
||||||
|
review_id_gitea INTEGER,
|
||||||
|
body_comment_id INTEGER,
|
||||||
|
posted_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS review_repo_pr ON review(repo, pr);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS inline_finding (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
review_id INTEGER REFERENCES review(id),
|
||||||
|
repo TEXT NOT NULL,
|
||||||
|
pr INTEGER NOT NULL,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
line INTEGER NOT NULL,
|
||||||
|
severity TEXT NOT NULL,
|
||||||
|
problem TEXT NOT NULL,
|
||||||
|
fix TEXT,
|
||||||
|
suggestion TEXT,
|
||||||
|
comment_id INTEGER,
|
||||||
|
posthash TEXT NOT NULL,
|
||||||
|
posted_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS inline_finding_posthash_idx ON inline_finding(posthash);
|
||||||
|
CREATE INDEX IF NOT EXISTS inline_finding_repo_pr ON inline_finding(repo, pr);
|
||||||
|
CREATE INDEX IF NOT EXISTS inline_finding_posthash ON inline_finding(posthash);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS reaction (
|
||||||
|
comment_id INTEGER NOT NULL,
|
||||||
|
user TEXT NOT NULL,
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (comment_id, user, content)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS reaction_comment ON reaction(comment_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS thread_state (
|
||||||
|
finding_id INTEGER NOT NULL REFERENCES inline_finding(id),
|
||||||
|
resolved INTEGER NOT NULL,
|
||||||
|
checked_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (finding_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS reply (
|
||||||
|
finding_id INTEGER NOT NULL REFERENCES inline_finding(id),
|
||||||
|
author TEXT NOT NULL,
|
||||||
|
body TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (finding_id, created_at)
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def init(db_path: str) -> sqlite3.Connection:
|
||||||
|
"""Open (or create) the DB, ensure schema. Returns a Connection."""
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row # so callers can use row["name"]
|
||||||
|
conn.executescript(_SCHEMA)
|
||||||
|
conn.commit()
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Posthash — cross-review finding dedup
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def posthash(path: str, line: int, severity: str, problem: str) -> str:
|
||||||
|
"""Short stable hash of the finding's identifying triple + a problem
|
||||||
|
fingerprint. Designed so two reviews of the SAME finding (same file,
|
||||||
|
same line, same severity, same core complaint) collapse to one row —
|
||||||
|
reactions across PRs aggregate.
|
||||||
|
|
||||||
|
`line` is the post-change (RIGHT-side) line — the agent anchors on it
|
||||||
|
and so does this hash. Different lines = different finding, by design.
|
||||||
|
`severity` participates because "this is a CRITICAL bug" and "this is a
|
||||||
|
LOW nitpick" at the same line on the same problem text are different
|
||||||
|
signals to learn from.
|
||||||
|
"""
|
||||||
|
h = hashlib.sha256()
|
||||||
|
h.update(f"{path}\n".encode())
|
||||||
|
h.update(f"{line}\n".encode())
|
||||||
|
h.update(f"{severity.upper()}\n".encode())
|
||||||
|
h.update(problem[:80].strip().lower().encode())
|
||||||
|
return h.hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Write helpers — all best-effort. Log + swallow.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def record_review(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
repo: str,
|
||||||
|
pr: int,
|
||||||
|
head_sha: str,
|
||||||
|
review_id_gitea: Optional[int] = None,
|
||||||
|
body_comment_id: Optional[int] = None,
|
||||||
|
posted_at: Optional[int] = None,
|
||||||
|
) -> Optional[int]:
|
||||||
|
"""Insert a review row. Returns the new row id, or None on failure."""
|
||||||
|
try:
|
||||||
|
cur = conn.execute(
|
||||||
|
"INSERT INTO review(repo, pr, head_sha, review_id_gitea, body_comment_id, posted_at) "
|
||||||
|
"VALUES(?,?,?,?,?,?)",
|
||||||
|
(repo, pr, head_sha, review_id_gitea, body_comment_id, posted_at or int(time.time())),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("record_review failed: %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def record_inline_finding(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
review_id: Optional[int],
|
||||||
|
repo: str,
|
||||||
|
pr: int,
|
||||||
|
path: str,
|
||||||
|
line: int,
|
||||||
|
severity: str,
|
||||||
|
problem: str,
|
||||||
|
fix: str = "",
|
||||||
|
suggestion: str = "",
|
||||||
|
comment_id: Optional[int] = None,
|
||||||
|
posted_at: Optional[int] = None,
|
||||||
|
) -> Optional[int]:
|
||||||
|
"""Insert an inline-finding row, deduped on posthash.
|
||||||
|
|
||||||
|
`comment_id` is filled in by the harvester when it discovers the
|
||||||
|
Gitea-assigned comment id for this finding. The post path returns the
|
||||||
|
`review_id` only; the inline ids come from a follow-up fetch.
|
||||||
|
"""
|
||||||
|
ph = posthash(path, line, severity, problem)
|
||||||
|
ts = posted_at or int(time.time())
|
||||||
|
# Every call inserts a fresh row. Aggregation by posthash is the
|
||||||
|
# caller's job — see `findings_with_votes` which GROUP BYs posthash.
|
||||||
|
# Letting each finding be its own row means reactions on different
|
||||||
|
# comment_ids across multiple PR reviews are not lost when one of
|
||||||
|
# those comment_ids becomes stale.
|
||||||
|
try:
|
||||||
|
cur = conn.execute(
|
||||||
|
"INSERT INTO inline_finding(review_id, repo, pr, path, line, severity, "
|
||||||
|
"problem, fix, suggestion, comment_id, posthash, posted_at) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
(review_id, repo, pr, path, line, severity, problem, fix, suggestion,
|
||||||
|
comment_id, ph, ts),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return cur.lastrowid
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("record_inline_finding failed: %s", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def record_reaction(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
comment_id: int,
|
||||||
|
user: str,
|
||||||
|
content: str,
|
||||||
|
created_at: Optional[int] = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Upsert one reaction. PK = (comment_id, user, content)."""
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO reaction(comment_id, user, content, created_at) "
|
||||||
|
"VALUES(?,?,?,?)",
|
||||||
|
(comment_id, user, content, created_at or int(time.time())),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("record_reaction failed: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def record_thread_state(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
finding_id: int,
|
||||||
|
resolved: bool,
|
||||||
|
checked_at: Optional[int] = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Upsert the latest thread-state check."""
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO thread_state(finding_id, resolved, checked_at) "
|
||||||
|
"VALUES(?,?,?) "
|
||||||
|
"ON CONFLICT(finding_id) DO UPDATE SET "
|
||||||
|
" resolved = excluded.resolved, checked_at = excluded.checked_at",
|
||||||
|
(finding_id, 1 if resolved else 0, checked_at or int(time.time())),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("record_thread_state failed: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def record_reply(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
finding_id: int,
|
||||||
|
author: str,
|
||||||
|
body: str,
|
||||||
|
created_at: int,
|
||||||
|
) -> bool:
|
||||||
|
"""Insert one reply. PK includes created_at → re-imports are idempotent."""
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO reply(finding_id, author, body, created_at) "
|
||||||
|
"VALUES(?,?,?,?)",
|
||||||
|
(finding_id, author, body, created_at),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("record_reply failed: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Read helpers — for the analyzer
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def findings_with_votes(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
repo: Optional[str] = None,
|
||||||
|
since_ts: Optional[int] = None,
|
||||||
|
) -> Iterable[sqlite3.Row]:
|
||||||
|
"""Stream every inline finding with rolled-up votes attached.
|
||||||
|
|
||||||
|
Joins:
|
||||||
|
inline_finding ◀ reaction (count by content)
|
||||||
|
inline_finding ◀ thread_state (latest resolved flag)
|
||||||
|
inline_finding ◀ reply (count + concatenation of bodies for negation
|
||||||
|
pattern matching)
|
||||||
|
|
||||||
|
Yielded rows expose:
|
||||||
|
id, repo, pr, path, line, severity, problem, fix, suggestion,
|
||||||
|
comment_id, posthash, posted_at,
|
||||||
|
upvotes INT, downvotes INT,
|
||||||
|
resolved INT (0/1/NULL),
|
||||||
|
reply_count INT,
|
||||||
|
reply_bodies TEXT ('\\n\\n'-joined for substring match),
|
||||||
|
review_posted_at INT
|
||||||
|
"""
|
||||||
|
where = []
|
||||||
|
params: list = []
|
||||||
|
if repo:
|
||||||
|
where.append("f.repo = ?")
|
||||||
|
params.append(repo)
|
||||||
|
if since_ts is not None:
|
||||||
|
where.append("COALESCE(r.posted_at, f.posted_at) >= ?")
|
||||||
|
params.append(since_ts)
|
||||||
|
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||||
|
|
||||||
|
sql = f"""
|
||||||
|
SELECT
|
||||||
|
f.posthash AS id, -- alias for compat — every row IS an aggregated posthash
|
||||||
|
f.repo, MAX(f.pr) AS pr, f.path, f.line, MAX(f.severity) AS severity,
|
||||||
|
MAX(f.problem) AS problem, MAX(f.fix) AS fix, MAX(f.suggestion) AS suggestion,
|
||||||
|
MAX(f.comment_id) AS comment_id, f.posthash, MAX(f.posted_at) AS posted_at,
|
||||||
|
COUNT(*) AS occurrences,
|
||||||
|
r.posted_at AS review_posted_at,
|
||||||
|
COALESCE(SUM(CASE WHEN rct.content = '+1' THEN 1 ELSE 0 END), 0) AS upvotes,
|
||||||
|
COALESCE(SUM(CASE WHEN rct.content = '-1' THEN 1 ELSE 0 END), 0) AS downvotes,
|
||||||
|
MAX(ts.resolved) AS resolved,
|
||||||
|
COALESCE((SELECT COUNT(*) FROM reply WHERE finding_id IN (SELECT id FROM inline_finding WHERE posthash = f.posthash AND repo = f.repo AND path = f.path AND line = f.line)), 0) AS reply_count,
|
||||||
|
COALESCE((SELECT GROUP_CONCAT(body, char(10)||char(10)) FROM reply WHERE finding_id IN (SELECT id FROM inline_finding WHERE posthash = f.posthash AND repo = f.repo AND path = f.path AND line = f.line)), '') AS reply_bodies
|
||||||
|
FROM inline_finding f
|
||||||
|
LEFT JOIN review r ON r.id = f.review_id
|
||||||
|
LEFT JOIN reaction rct ON rct.comment_id = f.comment_id
|
||||||
|
LEFT JOIN thread_state ts ON ts.finding_id = f.id
|
||||||
|
{where_sql}
|
||||||
|
GROUP BY f.posthash, f.repo, f.path, f.line
|
||||||
|
ORDER BY posted_at DESC
|
||||||
|
"""
|
||||||
|
return conn.execute(sql, params)
|
||||||
|
|
||||||
|
|
||||||
|
def known_posthashes_for_repo(conn: sqlite3.Connection, repo: str) -> set[str]:
|
||||||
|
"""For the harvester: which findings on this repo have already been
|
||||||
|
recorded? Used to skip re-fetching reactions we already harvested this
|
||||||
|
round."""
|
||||||
|
return {
|
||||||
|
row[0]
|
||||||
|
for row in conn.execute(
|
||||||
|
"SELECT DISTINCT posthash FROM inline_finding WHERE repo = ?", (repo,)
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def comment_ids_for_finding(conn: sqlite3.Connection, posthash: str) -> Optional[int]:
|
||||||
|
"""Return the current Gitea comment_id for an existing finding (used to
|
||||||
|
harvest votes for findings the harvester discovers on a brand-new PR that
|
||||||
|
ALSO has older bot comments on prior PRs)."""
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT comment_id FROM inline_finding WHERE posthash = ?", (posthash,)
|
||||||
|
).fetchone()
|
||||||
|
return row[0] if row else None
|
||||||
@@ -0,0 +1,419 @@
|
|||||||
|
"""pragent pilot — daily feedback analyzer.
|
||||||
|
|
||||||
|
Reads `feedback.db` (written by `feedback_harvest.py`) and produces a
|
||||||
|
markdown report that:
|
||||||
|
|
||||||
|
1. Ranks inline findings by **net false-positive score** (downvotes +
|
||||||
|
unresolved + negation-phrase replies − upvotes − resolved). Top of
|
||||||
|
this list = "the bot has been wrong about this repeatedly". These
|
||||||
|
are the candidates that *might* belong in the per-repo
|
||||||
|
`.pr-review.json:instructions` addendum.
|
||||||
|
2. Ranks findings by **net acceptance** — repeated 👍 / resolution =
|
||||||
|
"the bot's framing here is genuinely useful". These can be promoted
|
||||||
|
to the shared `architecture.md` so they don't have to be re-derived
|
||||||
|
every PR.
|
||||||
|
3. Reports a **restraint metric** — for every PR where the bot posted
|
||||||
|
zero findings, count how often a human reviewer also posted zero
|
||||||
|
substantive review comments. When the bot is loud on clean code,
|
||||||
|
that's a false-positive rate we can act on (DoorDash lesson:
|
||||||
|
"excessive noise on clean code is its own failure mode").
|
||||||
|
4. Reports a **case-review queue** — every disagreement case (a
|
||||||
|
downvote, unresolved, or a reply matching `FALSE_POSITIVE_PHRASES`)
|
||||||
|
is listed in full so a human can re-read the original PR and decide
|
||||||
|
if the finding was right or wrong.
|
||||||
|
|
||||||
|
Output is plain markdown so it can be posted as a Gitea issue / comment
|
||||||
|
without rendering work. Designed to be reviewed by a human, not auto-
|
||||||
|
applied — per the DoorDash pattern, every material change to model /
|
||||||
|
prompt / context goes through a benchmark gate first; this report IS
|
||||||
|
that gate (or, more precisely, the queue feeding the gate).
|
||||||
|
|
||||||
|
Never raises. A bad DB / no data → returns a friendly empty-state report.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import feedback
|
||||||
|
from feedback_harvest import (
|
||||||
|
FALSE_POSITIVE_PHRASES,
|
||||||
|
classify_reaction,
|
||||||
|
_is_negation_reply, # noqa: F401 (re-exported for the test suite)
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger("pragent.feedback.analyze")
|
||||||
|
|
||||||
|
# How many findings to surface in each top-list. Capped because the
|
||||||
|
# reports are read by humans; more than 20 per list and they skim.
|
||||||
|
TOP_N = 20
|
||||||
|
|
||||||
|
# Restraint threshold — fraction of "clean" PRs (zero findings) where
|
||||||
|
# the bot produced ANY findings. Above this we recommend `.pr-review.json:
|
||||||
|
# exclude_patterns` or a stricter `severity_threshold`.
|
||||||
|
RESTRAINT_NOISE_THRESHOLD = 0.25
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _net_score(row) -> tuple[int, int]:
|
||||||
|
"""Return (false_positive_score, acceptance_score) for one finding row.
|
||||||
|
|
||||||
|
FP signals: downvotes (+1), unresolved (+1), negation-phrase replies (+2).
|
||||||
|
Acceptance signals: upvotes (+1), resolved (+1).
|
||||||
|
"""
|
||||||
|
fp = 0
|
||||||
|
ac = 0
|
||||||
|
fp += int(row["downvotes"] or 0)
|
||||||
|
fp += 1 if row["resolved"] == 0 else 0 # 0/1/NULL; 0 = unresolved
|
||||||
|
ac += 1 if row["resolved"] == 1 else 0
|
||||||
|
ac += int(row["upvotes"] or 0)
|
||||||
|
if row["reply_bodies"] and _is_negation_reply(row["reply_bodies"]):
|
||||||
|
fp += 2
|
||||||
|
return fp, ac
|
||||||
|
|
||||||
|
|
||||||
|
def _short_problem(problem: str, n: int = 100) -> str:
|
||||||
|
s = (problem or "").strip().replace("\n", " ")
|
||||||
|
return s if len(s) <= n else s[: n - 1] + "…"
|
||||||
|
|
||||||
|
|
||||||
|
def _restraint_stats(conn: sqlite3.Connection) -> dict:
|
||||||
|
"""How often does the bot post findings on PRs that received zero
|
||||||
|
bot findings (= presumably clean)? Looks at `review.findings_total`
|
||||||
|
if present, otherwise counts `inline_finding` per PR.
|
||||||
|
|
||||||
|
NOTE: until `post_inline_review` records `findings_total`, this falls
|
||||||
|
back to "PRs with at least one finding row" which is an underestimate
|
||||||
|
(a bot review with zero findings leaves no row).
|
||||||
|
"""
|
||||||
|
total_prs_with_review = conn.execute(
|
||||||
|
"SELECT COUNT(DISTINCT repo || '#' || pr) FROM review"
|
||||||
|
).fetchone()[0]
|
||||||
|
prs_with_findings = conn.execute(
|
||||||
|
"SELECT COUNT(DISTINCT repo || '#' || pr) FROM inline_finding"
|
||||||
|
).fetchone()[0]
|
||||||
|
if total_prs_with_review == 0:
|
||||||
|
return {"total": 0, "noisy": 0, "ratio": 0.0}
|
||||||
|
# This is currently "PRs where the bot left at least one inline
|
||||||
|
# comment". A precise "findings_total per review" needs
|
||||||
|
# post_inline_review to record it (TODO in the wiring step). Until
|
||||||
|
# then, treat this as a floor: real noise is >= this.
|
||||||
|
return {
|
||||||
|
"total": total_prs_with_review,
|
||||||
|
"noisy": prs_with_findings,
|
||||||
|
"ratio": prs_with_findings / total_prs_with_review,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _case_review_queue(conn: sqlite3.Connection, limit: int = 30) -> list[dict]:
|
||||||
|
"""Findings that humans pushed back on — for manual re-review."""
|
||||||
|
rows = feedback.findings_with_votes(conn)
|
||||||
|
cases = []
|
||||||
|
for r in rows:
|
||||||
|
fp_score, _ = _net_score(r)
|
||||||
|
if fp_score <= 0:
|
||||||
|
continue
|
||||||
|
cases.append({
|
||||||
|
"posthash": r["posthash"],
|
||||||
|
"repo": r["repo"],
|
||||||
|
"pr": r["pr"],
|
||||||
|
"path": r["path"],
|
||||||
|
"line": r["line"],
|
||||||
|
"severity": r["severity"],
|
||||||
|
"problem": _short_problem(r["problem"], 200),
|
||||||
|
"fp_score": fp_score,
|
||||||
|
"upvotes": r["upvotes"] or 0,
|
||||||
|
"downvotes": r["downvotes"] or 0,
|
||||||
|
"resolved": r["resolved"],
|
||||||
|
"reply_count": r["reply_count"] or 0,
|
||||||
|
"reply_excerpt": _short_problem(r["reply_bodies"] or "", 200),
|
||||||
|
})
|
||||||
|
cases.sort(key=lambda c: c["fp_score"], reverse=True)
|
||||||
|
return cases[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _format_table(headers: list[str], rows: list[list[str]]) -> str:
|
||||||
|
if not rows:
|
||||||
|
return "_none yet_\n"
|
||||||
|
out = ["| " + " | ".join(headers) + " |",
|
||||||
|
"|" + "|".join(["---"] * len(headers)) + "|"]
|
||||||
|
for row in rows:
|
||||||
|
out.append("| " + " | ".join(row) + " |")
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _md_escape(s: str) -> str:
|
||||||
|
"""Escape pipes + newlines so the value stays in one table cell."""
|
||||||
|
return (s or "").replace("|", "\\|").replace("\n", " ").strip()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main report builder
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def analyze(db_path: str, *, since_ts: Optional[int] = None,
|
||||||
|
as_json: bool = False) -> str:
|
||||||
|
"""Build the daily report. Returns a markdown string by default;
|
||||||
|
`as_json=True` returns a structured dict (for tests + dashboards)."""
|
||||||
|
conn = feedback.init(db_path)
|
||||||
|
try:
|
||||||
|
findings = list(feedback.findings_with_votes(conn, since_ts=since_ts))
|
||||||
|
total_findings = len(findings)
|
||||||
|
repo_set = {f["repo"] for f in findings}
|
||||||
|
case_queue = _case_review_queue(conn)
|
||||||
|
restraint = _restraint_stats(conn)
|
||||||
|
|
||||||
|
# Compute scores
|
||||||
|
scored: list[tuple[int, int, sqlite3.Row]] = []
|
||||||
|
for f in findings:
|
||||||
|
fp, ac = _net_score(f)
|
||||||
|
scored.append((fp, ac, f))
|
||||||
|
|
||||||
|
# Top false-positive patterns (sorted by fp score, deduped by posthash).
|
||||||
|
# `occurrences` comes from the inline_finding row — posthash UNIQUE
|
||||||
|
# means a single row can carry a count > 1 (set by record_inline_finding's
|
||||||
|
# ON CONFLICT DO UPDATE).
|
||||||
|
fp_by_hash: dict[str, dict] = {}
|
||||||
|
for fp, ac, f in scored:
|
||||||
|
if fp <= 0:
|
||||||
|
continue
|
||||||
|
ph = f["posthash"]
|
||||||
|
entry = fp_by_hash.setdefault(ph, {
|
||||||
|
"posthash": ph, "fp_score": 0, "ac_score": 0,
|
||||||
|
"repo": f["repo"], "path": f["path"], "line": f["line"],
|
||||||
|
"severity": f["severity"], "problem": f["problem"],
|
||||||
|
"occurrences": f["occurrences"], "upvs": 0, "downs": 0,
|
||||||
|
"resolved_true": 0, "resolved_false": 0,
|
||||||
|
})
|
||||||
|
entry["fp_score"] += fp
|
||||||
|
entry["ac_score"] += ac
|
||||||
|
entry["upvs"] += f["upvotes"] or 0
|
||||||
|
entry["downs"] += f["downvotes"] or 0
|
||||||
|
if f["resolved"] == 1:
|
||||||
|
entry["resolved_true"] += 1
|
||||||
|
elif f["resolved"] == 0:
|
||||||
|
entry["resolved_false"] += 1
|
||||||
|
fp_sorted = sorted(
|
||||||
|
fp_by_hash.values(), key=lambda e: e["fp_score"], reverse=True,
|
||||||
|
)[:TOP_N]
|
||||||
|
|
||||||
|
# Top accepted patterns
|
||||||
|
ac_by_hash: dict[str, dict] = {}
|
||||||
|
for fp, ac, f in scored:
|
||||||
|
if ac <= 0:
|
||||||
|
continue
|
||||||
|
ph = f["posthash"]
|
||||||
|
entry = ac_by_hash.setdefault(ph, {
|
||||||
|
"posthash": ph, "ac_score": 0, "fp_score": 0,
|
||||||
|
"repo": f["repo"], "path": f["path"], "line": f["line"],
|
||||||
|
"severity": f["severity"], "problem": f["problem"],
|
||||||
|
"occurrences": f["occurrences"], "upvs": 0, "downs": 0,
|
||||||
|
"resolved_true": 0,
|
||||||
|
})
|
||||||
|
entry["ac_score"] += ac
|
||||||
|
entry["fp_score"] += fp
|
||||||
|
entry["upvs"] += f["upvotes"] or 0
|
||||||
|
entry["downs"] += f["downvotes"] or 0
|
||||||
|
if f["resolved"] == 1:
|
||||||
|
entry["resolved_true"] += 1
|
||||||
|
ac_sorted = sorted(
|
||||||
|
ac_by_hash.values(), key=lambda e: e["ac_score"], reverse=True,
|
||||||
|
)[:TOP_N]
|
||||||
|
|
||||||
|
# Restraint recommendation
|
||||||
|
if restraint["ratio"] > RESTRAINT_NOISE_THRESHOLD:
|
||||||
|
restraint_msg = (
|
||||||
|
f"⚠️ Bot posted findings on **{restraint['ratio']:.0%}** of "
|
||||||
|
f"reviewed PRs ({restraint['noisy']} / {restraint['total']}). "
|
||||||
|
f"Above the {RESTRAINT_NOISE_THRESHOLD:.0%} threshold — "
|
||||||
|
"consider raising `.pr-review.json:severity_threshold` to "
|
||||||
|
"`medium` or `high` for noisy repos, or adding "
|
||||||
|
"`patterns.deny` to skip stylistic-only findings."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
restraint_msg = (
|
||||||
|
f"✅ Bot stayed quiet on **{1 - restraint['ratio']:.0%}** of "
|
||||||
|
f"reviewed PRs ({restraint['total'] - restraint['noisy']} / "
|
||||||
|
f"{restraint['total']}). Restraint OK."
|
||||||
|
)
|
||||||
|
|
||||||
|
if as_json:
|
||||||
|
return json.dumps({
|
||||||
|
"total_findings": total_findings,
|
||||||
|
"repos_seen": sorted(repo_set),
|
||||||
|
"restraint": restraint,
|
||||||
|
"top_false_positive": fp_sorted,
|
||||||
|
"top_accepted": ac_sorted,
|
||||||
|
"case_review_queue": case_queue,
|
||||||
|
"restraint_msg": restraint_msg,
|
||||||
|
}, indent=2)
|
||||||
|
|
||||||
|
# Markdown
|
||||||
|
ts_str = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||||
|
out = [f"# pragent feedback report — {ts_str}", ""]
|
||||||
|
out.append(f"- **findings analyzed**: {total_findings}")
|
||||||
|
out.append(f"- **repos with feedback**: {len(repo_set)} "
|
||||||
|
f"({', '.join(sorted(repo_set))})")
|
||||||
|
out.append(f"- **case-review queue**: {len(case_queue)} disagreement(s)")
|
||||||
|
out.append("")
|
||||||
|
out.append("## Restraint")
|
||||||
|
out.append("")
|
||||||
|
out.append(restraint_msg)
|
||||||
|
out.append("")
|
||||||
|
out.append("> DoorDash rule (2026-07-06): *excessive noise on clean "
|
||||||
|
"code is its own failure mode*. `severity_threshold` + "
|
||||||
|
"`patterns.deny` are the knobs that dial restraint.")
|
||||||
|
out.append("")
|
||||||
|
|
||||||
|
out.append(f"## Top {len(fp_sorted)} false-positive candidates")
|
||||||
|
out.append("")
|
||||||
|
out.append("Aggregated by `posthash` (path:line:severity:problem). "
|
||||||
|
"Sort key = downvotes + unresolved + negation-phrase replies "
|
||||||
|
"− upvotes − resolved.")
|
||||||
|
out.append("")
|
||||||
|
rows = []
|
||||||
|
for e in fp_sorted:
|
||||||
|
rows.append([
|
||||||
|
str(e["fp_score"]),
|
||||||
|
f"`{_md_escape(e['repo'])}`",
|
||||||
|
f"`{_md_escape(e['path'])}:{e['line']}`",
|
||||||
|
e["severity"],
|
||||||
|
_md_escape(_short_problem(e["problem"])),
|
||||||
|
f"👍{e['upvs']} 👎{e['downs']}",
|
||||||
|
f"✅{e['resolved_true']} ❌{e['resolved_false']}",
|
||||||
|
str(e["occurrences"]),
|
||||||
|
])
|
||||||
|
out.append(_format_table(
|
||||||
|
["FP", "repo", "path:line", "sev", "problem",
|
||||||
|
"votes", "resolved", "seen"],
|
||||||
|
rows,
|
||||||
|
))
|
||||||
|
out.append("")
|
||||||
|
out.append("_Review each row before adding it to "
|
||||||
|
"`.pr-review.json:instructions`. Human reactions are NOT "
|
||||||
|
"ground truth (DoorDash, 2026-07-06: authors accept/reject "
|
||||||
|
"for workflow reasons) — re-read the PR before acting._")
|
||||||
|
out.append("")
|
||||||
|
|
||||||
|
out.append(f"## Top {len(ac_sorted)} accepted patterns")
|
||||||
|
out.append("")
|
||||||
|
out.append("Aggregated by posthash. Sort key = upvotes + resolved − "
|
||||||
|
"downvotes − unresolved − negation-phrase replies.")
|
||||||
|
out.append("")
|
||||||
|
rows = []
|
||||||
|
for e in ac_sorted:
|
||||||
|
rows.append([
|
||||||
|
str(e["ac_score"]),
|
||||||
|
f"`{_md_escape(e['repo'])}`",
|
||||||
|
f"`{_md_escape(e['path'])}:{e['line']}`",
|
||||||
|
e["severity"],
|
||||||
|
_md_escape(_short_problem(e["problem"])),
|
||||||
|
f"👍{e['upvs']} 👎{e['downs']}",
|
||||||
|
f"✅{e['resolved_true']}",
|
||||||
|
str(e["occurrences"]),
|
||||||
|
])
|
||||||
|
out.append(_format_table(
|
||||||
|
["AC", "repo", "path:line", "sev", "problem",
|
||||||
|
"votes", "resolved", "seen"],
|
||||||
|
rows,
|
||||||
|
))
|
||||||
|
out.append("")
|
||||||
|
out.append("_Promote widely-accepted patterns into the shared "
|
||||||
|
"`architecture.md` on Nexus raw-hosted (or the per-repo "
|
||||||
|
"`additional_context_urls`). These become part of the "
|
||||||
|
"prompt-cached prefix → ~0 marginal cost on step 2+._")
|
||||||
|
out.append("")
|
||||||
|
|
||||||
|
out.append(f"## Case-review queue ({len(case_queue)})")
|
||||||
|
out.append("")
|
||||||
|
if not case_queue:
|
||||||
|
out.append("_No disagreements recorded yet. Once humans start "
|
||||||
|
"reacting 👎 / leaving replies / not resolving bot "
|
||||||
|
"comments, cases will appear here._")
|
||||||
|
else:
|
||||||
|
out.append("Each row needs a human to re-read the original PR and "
|
||||||
|
"decide: was the bot right? If not, draft an "
|
||||||
|
"`instructions` addendum or a `patterns.deny` rule.")
|
||||||
|
out.append("")
|
||||||
|
for c in case_queue:
|
||||||
|
url = (
|
||||||
|
f"https://gitea.marcospaulo.dev.br/{c['repo']}/pulls/"
|
||||||
|
f"{c['pr']}/files#r{c['posthash']}"
|
||||||
|
)
|
||||||
|
out.append(f"### FP={c['fp_score']} · {c['repo']}#{c['pr']}")
|
||||||
|
out.append(
|
||||||
|
f"- file: `{_md_escape(c['path'])}:{c['line']}` · "
|
||||||
|
f"severity: `{c['severity']}`",
|
||||||
|
)
|
||||||
|
out.append(f"- problem: {_md_escape(c['problem'])}")
|
||||||
|
out.append(
|
||||||
|
f"- signals: 👍{c['upvotes']} 👎{c['downvotes']} · "
|
||||||
|
f"resolved={c['resolved']} · replies={c['reply_count']}",
|
||||||
|
)
|
||||||
|
if c["reply_excerpt"]:
|
||||||
|
out.append(
|
||||||
|
f"- last reply: {_md_escape(c['reply_excerpt'])}",
|
||||||
|
)
|
||||||
|
out.append(f"- posthash: `{c['posthash']}`")
|
||||||
|
out.append("")
|
||||||
|
|
||||||
|
out.append("## Where this report goes")
|
||||||
|
out.append("")
|
||||||
|
out.append("- **Per-repo actions** (`.pr-review.json:instructions`, "
|
||||||
|
"`patterns.deny`, `severity_threshold`): edit the file on "
|
||||||
|
"`main` via a regular PR. The next PR review picks up the "
|
||||||
|
"change automatically.")
|
||||||
|
out.append("- **Cross-repo actions** (shared house-rules): update the "
|
||||||
|
"`PRAGENT_ADDITIONAL_CONTEXT_URL` document on Nexus "
|
||||||
|
"raw-hosted (`canalhandia/architecture.md` etc).")
|
||||||
|
out.append("- **Benchmark gate** (DoorDash pattern): before changing "
|
||||||
|
"the model / prompt / context window, replay this report "
|
||||||
|
"against the labeled `posthash` corpus. If a candidate "
|
||||||
|
"addendum flips ≥ 1 currently-accepted finding into "
|
||||||
|
"false-positive, drop it.")
|
||||||
|
out.append("")
|
||||||
|
out.append(f"_Generated from `{db_path}` by `feedback_analyze.py`._")
|
||||||
|
return "\n".join(out)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CLI
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
p = argparse.ArgumentParser(description="Build the daily feedback report.")
|
||||||
|
p.add_argument("--db", default=os.environ.get(
|
||||||
|
"PRAGENT_FEEDBACK_DB", "/data/feedback.db",
|
||||||
|
))
|
||||||
|
p.add_argument("--since", type=int, default=None,
|
||||||
|
help="Unix timestamp; only include findings posted since")
|
||||||
|
p.add_argument("--json", action="store_true",
|
||||||
|
help="Emit structured JSON instead of markdown")
|
||||||
|
p.add_argument("--out", default="-",
|
||||||
|
help="Write to this path instead of stdout ('-' = stdout)")
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
out = analyze(args.db, since_ts=args.since, as_json=args.json)
|
||||||
|
if args.out == "-":
|
||||||
|
print(out)
|
||||||
|
else:
|
||||||
|
with open(args.out, "w") as f:
|
||||||
|
f.write(out)
|
||||||
|
print(f"wrote {args.out}", file=sys.stderr)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,394 @@
|
|||||||
|
"""pragent pilot — feedback harvester.
|
||||||
|
|
||||||
|
For each PR the webhook server is about to review, walk back through the
|
||||||
|
Gitea-side state of every bot comment from every prior review on that PR
|
||||||
|
and record:
|
||||||
|
- reactions on the review body + on each inline comment
|
||||||
|
- thread-resolved state (Gitea's `resolver` field; non-empty = resolved)
|
||||||
|
- replies (issue-comments with `review_comment_id` matching ours)
|
||||||
|
- the bot's own findings_count + inline_count per review (for the
|
||||||
|
restraint metric)
|
||||||
|
|
||||||
|
Everything is best-effort. A single 404 or 5xx is logged and skipped — we
|
||||||
|
must never abort a review because the feedback DB had a hiccup.
|
||||||
|
|
||||||
|
The harvester is intentionally separate from `review_pr` so it can be
|
||||||
|
called independently (e.g. by the daily analyzer's "backfill" mode) and
|
||||||
|
tested in isolation against a mocked Gitea client.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import ai_review # used as ai_review.gitea_get(...) so test mocks land on the binding
|
||||||
|
|
||||||
|
from feedback import (
|
||||||
|
init,
|
||||||
|
record_inline_finding,
|
||||||
|
record_reaction,
|
||||||
|
record_reply,
|
||||||
|
record_review,
|
||||||
|
record_thread_state,
|
||||||
|
posthash,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger("pragent.feedback.harvest")
|
||||||
|
|
||||||
|
# Reviewer identity — only collect feedback on comments authored by us.
|
||||||
|
# Avoids harvesting reactions on human comments (which we never want to
|
||||||
|
# count toward "bot usefulness").
|
||||||
|
REVIEWER_LOGIN = "pragent-bot"
|
||||||
|
|
||||||
|
# Reactions content tokens Gitea uses. We track +1 / -1 explicitly; the
|
||||||
|
# others are stored as-is so the analyzer can mine them (👀 eyes,
|
||||||
|
# laugh, hooray, confused, heart, rocket, …) without hardcoding a list
|
||||||
|
# that drifts across Gitea versions.
|
||||||
|
POSITIVE_REACTIONS = {"+1", "heart", "hooray", "laugh", "rocket"}
|
||||||
|
NEGATIVE_REACTIONS = {"-1", "confused"}
|
||||||
|
# Note: Gitea's `eyes` reaction (👀) means "I'm watching" — not approval
|
||||||
|
# or disapproval. Treated as neutral by the analyzer.
|
||||||
|
|
||||||
|
# Phrases that, in a reply, indicate the author thinks the bot's finding
|
||||||
|
# was wrong. Casing + punctuation ignored; substring match is good enough
|
||||||
|
# (false positives in the analyzer cost a human minute; false negatives
|
||||||
|
# hide regressions).
|
||||||
|
FALSE_POSITIVE_PHRASES = (
|
||||||
|
"false positive", "not actually", "this is fine", "this is intentional",
|
||||||
|
"not a bug", "intentional", "wrong here", "isn't actually",
|
||||||
|
"is not actually", "don't think this is", "i disagree", "this isn't right",
|
||||||
|
"this is correct", "this is expected", "by design", "this is by design",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Gitea review-comment payload includes a 'body' field that may carry our
|
||||||
|
# sha marker + severity header. We extract severity + path/line from it
|
||||||
|
# as a fallback when the finding wasn't already seeded at post-time (old
|
||||||
|
# reviews before feedback.py existed).
|
||||||
|
SEV_RE = re.compile(r"\*\*\[(CRITICAL|HIGH|MEDIUM|LOW|INFO)\]\*\*", re.IGNORECASE)
|
||||||
|
PATH_LINE_RE = re.compile(r"`([^?:\n]+?):(\d+)`")
|
||||||
|
SHA_MARKER_RE = re.compile(r"<!--\s*pragent:sha=([0-9a-f]+)\s*-->", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Low-level HTTP — tolerant JSON parse (Gitea sometimes returns `null` where
|
||||||
|
# we expect `[]`, e.g. reactions on a fresh comment)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _gitea_get_json(api: str, repo: str, path: str, token: str) -> tuple[int, object]:
|
||||||
|
status, raw = ai_review.gitea_get(api, repo, path, token)
|
||||||
|
if status != 200:
|
||||||
|
return status, None
|
||||||
|
try:
|
||||||
|
return status, json.loads(raw.decode("utf-8", errors="replace"))
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
return status, None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Parse helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _parse_severity(body: str) -> str:
|
||||||
|
m = SEV_RE.search(body or "")
|
||||||
|
return m.group(1).upper() if m else "INFO"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_path_line(body: str) -> tuple[Optional[str], Optional[int]]:
|
||||||
|
m = PATH_LINE_RE.search(body or "")
|
||||||
|
if not m:
|
||||||
|
return None, None
|
||||||
|
path = m.group(1).strip()
|
||||||
|
try:
|
||||||
|
return path, int(m.group(2))
|
||||||
|
except ValueError:
|
||||||
|
return path, None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_sha(body: str) -> Optional[str]:
|
||||||
|
m = SHA_MARKER_RE.search(body or "")
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _is_negation_reply(body: str) -> bool:
|
||||||
|
if not body:
|
||||||
|
return False
|
||||||
|
norm = body.lower()
|
||||||
|
return any(p in norm for p in FALSE_POSITIVE_PHRASES)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Reaction classification (cheap, used by the analyzer — not the harvester
|
||||||
|
# itself)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def classify_reaction(content: str) -> str:
|
||||||
|
"""Bucket a reaction into 'positive', 'negative', or 'neutral'."""
|
||||||
|
c = (content or "").strip().lower()
|
||||||
|
if c in POSITIVE_REACTIONS:
|
||||||
|
return "positive"
|
||||||
|
if c in NEGATIVE_REACTIONS:
|
||||||
|
return "negative"
|
||||||
|
return "neutral"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main harvest entry
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def harvest_for_pr(
|
||||||
|
*,
|
||||||
|
api: str,
|
||||||
|
token: str,
|
||||||
|
repo: str,
|
||||||
|
pr_index: int,
|
||||||
|
db_path: str,
|
||||||
|
page_size: int = 50,
|
||||||
|
) -> dict:
|
||||||
|
"""Walk every bot-authored review on the given PR and record reactions
|
||||||
|
+ thread state + replies. Returns a stats dict for logging.
|
||||||
|
|
||||||
|
`db_path` is the SQLite file path (env: `PRAGENT_FEEDBACK_DB`,
|
||||||
|
typically `/data/feedback.db` mounted via the `feedback-data` PVC).
|
||||||
|
"""
|
||||||
|
conn = init(db_path)
|
||||||
|
stats = {
|
||||||
|
"reviews_seen": 0, "findings_seen": 0,
|
||||||
|
"reactions_recorded": 0, "thread_states_recorded": 0,
|
||||||
|
"replies_recorded": 0, "errors": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 1. List every review on the PR (paginated, but PRs rarely have >page_size)
|
||||||
|
status, payload = _gitea_get_json(
|
||||||
|
api, repo, f"pulls/{pr_index}/reviews?per_page={page_size}", token,
|
||||||
|
)
|
||||||
|
if status != 200 or not isinstance(payload, list):
|
||||||
|
log.info("harvest: reviews list failed status=%d", status)
|
||||||
|
stats["errors"] += 1
|
||||||
|
return stats
|
||||||
|
|
||||||
|
for rev in payload:
|
||||||
|
user = (rev.get("user") or {}).get("login", "")
|
||||||
|
if user != REVIEWER_LOGIN:
|
||||||
|
continue
|
||||||
|
stats["reviews_seen"] += 1
|
||||||
|
|
||||||
|
review_id_gitea = rev.get("id")
|
||||||
|
head_sha = rev.get("commit_id", "")
|
||||||
|
review_body = rev.get("body", "") or ""
|
||||||
|
body_sha = _parse_sha(review_body)
|
||||||
|
# Trust the sha marker inside the body — Gitea's commit_id field is
|
||||||
|
# for the LAST commit, not necessarily the reviewed head. If we
|
||||||
|
# can't find a marker, fall back to commit_id.
|
||||||
|
effective_sha = body_sha or head_sha
|
||||||
|
created_at = _parse_iso_ts(rev.get("created_at", ""))
|
||||||
|
|
||||||
|
db_review_id = record_review(
|
||||||
|
conn, repo=repo, pr=pr_index, head_sha=effective_sha,
|
||||||
|
review_id_gitea=review_id_gitea,
|
||||||
|
posted_at=created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Inline comments for this review
|
||||||
|
if review_id_gitea is None:
|
||||||
|
continue
|
||||||
|
rstatus, rpayload = _gitea_get_json(
|
||||||
|
api, repo, f"pulls/{pr_index}/reviews/{review_id_gitea}/comments",
|
||||||
|
token,
|
||||||
|
)
|
||||||
|
if rstatus != 200 or not isinstance(rpayload, list):
|
||||||
|
stats["errors"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
for ic in rpayload:
|
||||||
|
ic_id = ic.get("id")
|
||||||
|
if ic_id is None:
|
||||||
|
continue
|
||||||
|
ic_body = ic.get("body", "") or ""
|
||||||
|
ic_path = ic.get("path")
|
||||||
|
ic_line = ic.get("position") or ic.get("line")
|
||||||
|
ic_severity = _parse_severity(ic_body)
|
||||||
|
# Fall back to body parse when Gitea didn't echo path/line
|
||||||
|
if not ic_path or not ic_line:
|
||||||
|
bp, bl = _parse_path_line(ic_body)
|
||||||
|
ic_path = ic_path or bp
|
||||||
|
ic_line = ic_line or bl
|
||||||
|
|
||||||
|
if not ic_path or not ic_line:
|
||||||
|
log.info(
|
||||||
|
"harvest: inline %s missing path/line, skipping", ic_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
finding_id = record_inline_finding(
|
||||||
|
conn, review_id=db_review_id, repo=repo, pr=pr_index,
|
||||||
|
path=ic_path, line=ic_line, severity=ic_severity,
|
||||||
|
problem=_strip_severity_header(ic_body),
|
||||||
|
fix="", suggestion="",
|
||||||
|
comment_id=ic_id,
|
||||||
|
posted_at=created_at,
|
||||||
|
)
|
||||||
|
stats["findings_seen"] += 1
|
||||||
|
if finding_id is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 3. Reactions on the inline comment
|
||||||
|
react_status, react_payload = _gitea_get_json(
|
||||||
|
api, repo, f"issues/comments/{ic_id}/reactions", token,
|
||||||
|
)
|
||||||
|
if react_status == 200 and isinstance(react_payload, list):
|
||||||
|
for r in react_payload:
|
||||||
|
ruser = (r.get("user") or {}).get("login", "") or "?"
|
||||||
|
# Gitea has occasionally returned `content` as a
|
||||||
|
# dict on older versions; coerce to str defensively.
|
||||||
|
rcontent = str(r.get("content") or "").strip()
|
||||||
|
if not rcontent:
|
||||||
|
continue
|
||||||
|
if record_reaction(
|
||||||
|
conn, comment_id=ic_id, user=ruser,
|
||||||
|
content=rcontent,
|
||||||
|
created_at=_parse_iso_ts(r.get("created_at", "")),
|
||||||
|
):
|
||||||
|
stats["reactions_recorded"] += 1
|
||||||
|
|
||||||
|
# 4. Thread state (Gitea's `resolver` field on the inline
|
||||||
|
# comment). Some Gitea versions serialize this as a user
|
||||||
|
# object ({login, ...}) instead of a username string —
|
||||||
|
# coerce defensively before calling .strip().
|
||||||
|
resolver_raw = ic.get("resolver")
|
||||||
|
if isinstance(resolver_raw, dict):
|
||||||
|
resolver = (resolver_raw.get("login") or "").strip()
|
||||||
|
else:
|
||||||
|
resolver = str(resolver_raw or "").strip()
|
||||||
|
if resolver_raw is not None: # field present, even if ""
|
||||||
|
record_thread_state(
|
||||||
|
conn, finding_id=finding_id,
|
||||||
|
resolved=bool(resolver),
|
||||||
|
)
|
||||||
|
stats["thread_states_recorded"] += 1
|
||||||
|
|
||||||
|
# 5. Replies on this review (issue-comments whose
|
||||||
|
# `review_comment_id` points at one of our inline comments).
|
||||||
|
# Some Gitea versions don't expose `review_comment_id` on the
|
||||||
|
# issue-comment endpoint — in that case `replies` stays
|
||||||
|
# empty; we degrade gracefully.
|
||||||
|
try:
|
||||||
|
_harvest_replies(
|
||||||
|
api=api, repo=repo, token=token,
|
||||||
|
pr_index=pr_index, review_id=review_id_gitea,
|
||||||
|
inline_comments=rpayload, conn=conn,
|
||||||
|
stats=stats,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("harvest: replies fetch failed: %s", e)
|
||||||
|
stats["errors"] += 1
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def _harvest_replies(
|
||||||
|
*, api: str, repo: str, token: str, pr_index: int,
|
||||||
|
review_id: int, inline_comments: list, conn, stats: dict,
|
||||||
|
) -> None:
|
||||||
|
"""Fetch issue comments on this PR; record those whose
|
||||||
|
`review_comment_id` matches one of our inline comment IDs.
|
||||||
|
Gitea 1.26 doesn't include that field — we fall back to fetching each
|
||||||
|
inline comment individually via `issues/comments/{id}` (does include
|
||||||
|
the field) only if the bulk fetch is empty.
|
||||||
|
"""
|
||||||
|
inline_ids = {c.get("id") for c in inline_comments if c.get("id") is not None}
|
||||||
|
if not inline_ids:
|
||||||
|
return
|
||||||
|
|
||||||
|
status, payload = _gitea_get_json(
|
||||||
|
api, repo, f"issues/{pr_index}/comments?per_page=100", token,
|
||||||
|
)
|
||||||
|
if status != 200 or not isinstance(payload, list):
|
||||||
|
return
|
||||||
|
|
||||||
|
# Build mapping inline_id -> finding_id (one SELECT instead of N)
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT comment_id, id FROM inline_finding WHERE comment_id IN ("
|
||||||
|
+ ",".join("?" * len(inline_ids)) + ")",
|
||||||
|
list(inline_ids),
|
||||||
|
).fetchall()
|
||||||
|
inline_to_finding = {r[0]: r[1] for r in rows}
|
||||||
|
|
||||||
|
for c in payload:
|
||||||
|
rcid = c.get("review_comment_id")
|
||||||
|
if not rcid or rcid not in inline_to_finding:
|
||||||
|
continue
|
||||||
|
author = (c.get("user") or {}).get("login", "") or "?"
|
||||||
|
body = c.get("body", "") or ""
|
||||||
|
ts = _parse_iso_ts(c.get("created_at", ""))
|
||||||
|
if record_reply(
|
||||||
|
conn, finding_id=inline_to_finding[rcid],
|
||||||
|
author=author, body=body, created_at=ts,
|
||||||
|
):
|
||||||
|
stats["replies_recorded"] += 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _strip_severity_header(body: str) -> str:
|
||||||
|
"""Drop the leading `**[SEVERITY]**` so the posthash captures the
|
||||||
|
substance, not the severity label."""
|
||||||
|
return SEV_RE.sub("", body or "", count=1).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_ts(s: str) -> int:
|
||||||
|
if not s:
|
||||||
|
return int(time.time())
|
||||||
|
try:
|
||||||
|
# Python 3.11+ fromisoformat tolerates the trailing 'Z'.
|
||||||
|
return int(__import__("datetime").datetime.fromisoformat(
|
||||||
|
s.replace("Z", "+00:00")
|
||||||
|
).timestamp())
|
||||||
|
except Exception:
|
||||||
|
return int(time.time())
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CLI for manual backfill / first-time seed
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
import argparse, os
|
||||||
|
p = argparse.ArgumentParser(
|
||||||
|
description="Harvest reactions/threads/replies on bot PR comments.",
|
||||||
|
)
|
||||||
|
p.add_argument("--api", default=os.environ.get(
|
||||||
|
"GITEA_API", "http://gitea-http.gitea.svc.cluster.local:3000",
|
||||||
|
))
|
||||||
|
p.add_argument("--token", default=os.environ.get("PRAGENT_BOT_TOKEN", ""))
|
||||||
|
p.add_argument("--repo", required=True, help="owner/name")
|
||||||
|
p.add_argument("--pr", type=int, required=True, help="PR index")
|
||||||
|
p.add_argument("--db", default=os.environ.get(
|
||||||
|
"PRAGENT_FEEDBACK_DB", "/data/feedback.db",
|
||||||
|
))
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
if not args.token:
|
||||||
|
print("PRAGENT_BOT_TOKEN required", flush=True)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
stats = harvest_for_pr(
|
||||||
|
api=args.api, token=args.token,
|
||||||
|
repo=args.repo, pr_index=args.pr, db_path=args.db,
|
||||||
|
)
|
||||||
|
print(json.dumps(stats), flush=True)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""pragent pilot — daily feedback report delivery.
|
||||||
|
|
||||||
|
Calls `feedback_analyze.analyze()` and posts the markdown report as a
|
||||||
|
comment on a single long-lived "feedback roll-up" issue in
|
||||||
|
`gitea_admin/pragent`. Comments are append-only history — one comment per
|
||||||
|
run, timestamped in the body. This keeps every report in one place, easy
|
||||||
|
to scroll, and avoids the issue-explosion of "one issue per day".
|
||||||
|
|
||||||
|
If the issue doesn't exist yet, create it. Subsequent runs just add a
|
||||||
|
new comment.
|
||||||
|
|
||||||
|
Designed for the daily K8s CronJob (`k8s/pragent-feedback-cronjob.yaml`)
|
||||||
|
but runnable from CLI for ad-hoc checks.
|
||||||
|
|
||||||
|
Env:
|
||||||
|
GITEA_API in-cluster Gitea base URL
|
||||||
|
PRAGENT_BOT_TOKEN bot token (Write collaborator on gitea_admin/pragent)
|
||||||
|
PRAGENT_FEEDBACK_DB path to SQLite (default /data/feedback.db)
|
||||||
|
PRAGENT_FEEDBACK_ISSUE_REPO default gitea_admin/pragent
|
||||||
|
PRAGENT_FEEDBACK_ISSUE_TITLE default "pragent feedback roll-up"
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import ai_review
|
||||||
|
|
||||||
|
from feedback_analyze import analyze
|
||||||
|
|
||||||
|
log = logging.getLogger("pragent.feedback.post")
|
||||||
|
|
||||||
|
|
||||||
|
REPO_DEFAULT = "gitea_admin/pragent"
|
||||||
|
TITLE_DEFAULT = "pragent feedback roll-up"
|
||||||
|
|
||||||
|
|
||||||
|
def _find_or_create_issue(api: str, token: str, repo: str, title: str) -> int:
|
||||||
|
"""Locate the open issue with this title; create one if missing.
|
||||||
|
|
||||||
|
Gitea's issue search is via `GET /repos/{o}/{r}/issues?state=open&q=...`
|
||||||
|
(q matches title + body). We filter client-side for the exact title
|
||||||
|
to avoid query-text false matches.
|
||||||
|
"""
|
||||||
|
status, raw = ai_review.gitea_get(api, repo, "issues?state=open&per_page=50", token)
|
||||||
|
if status == 200:
|
||||||
|
try:
|
||||||
|
for issue in json.loads(raw):
|
||||||
|
if issue.get("title") == title:
|
||||||
|
# NB: the comment URL needs the per-repo `number`, not the
|
||||||
|
# global `id`. `id=60 num=8` for an early-N create; we want
|
||||||
|
# `num=8` for `/repos/o/r/issues/8/comments`.
|
||||||
|
return int(issue["number"])
|
||||||
|
except (json.JSONDecodeError, ValueError, KeyError):
|
||||||
|
pass
|
||||||
|
# Create
|
||||||
|
status, raw = ai_review.gitea_post(
|
||||||
|
api, repo, "issues", token,
|
||||||
|
{"title": title, "body": "pragent feedback roll-up — auto-created."},
|
||||||
|
)
|
||||||
|
if status not in (200, 201):
|
||||||
|
raise RuntimeError(f"issue create failed: HTTP {status} body={raw[:200]!r}")
|
||||||
|
return int(json.loads(raw)["number"])
|
||||||
|
|
||||||
|
|
||||||
|
def _post_comment(api: str, token: str, repo: str, issue_number: int, body: str) -> int:
|
||||||
|
status, raw = ai_review.gitea_post(
|
||||||
|
api, repo, f"issues/{issue_number}/comments", token, {"body": body},
|
||||||
|
)
|
||||||
|
if status not in (200, 201):
|
||||||
|
raise RuntimeError(f"comment post failed: HTTP {status} body={raw[:200]!r}")
|
||||||
|
return json.loads(raw)["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def deliver(
|
||||||
|
*, api: str, token: str, db_path: str,
|
||||||
|
repo: str = REPO_DEFAULT, title: str = TITLE_DEFAULT,
|
||||||
|
since_ts: int | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Build the report and post it as a comment. Returns a stats dict."""
|
||||||
|
report = analyze(db_path, since_ts=since_ts)
|
||||||
|
issue_id = _find_or_create_issue(api, token, repo, title)
|
||||||
|
comment_id = _post_comment(api, token, repo, issue_id, report)
|
||||||
|
return {
|
||||||
|
"repo": repo, "issue_id": issue_id, "comment_id": comment_id,
|
||||||
|
"report_bytes": len(report.encode()),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
p = argparse.ArgumentParser(
|
||||||
|
description="Post the daily feedback report to Gitea.",
|
||||||
|
)
|
||||||
|
p.add_argument("--api", default=os.environ.get(
|
||||||
|
"GITEA_API", "http://gitea-http.gitea.svc.cluster.local:3000",
|
||||||
|
))
|
||||||
|
p.add_argument("--token", default=os.environ.get("PRAGENT_BOT_TOKEN", ""))
|
||||||
|
p.add_argument("--db", default=os.environ.get(
|
||||||
|
"PRAGENT_FEEDBACK_DB", "/data/feedback.db",
|
||||||
|
))
|
||||||
|
p.add_argument("--repo", default=os.environ.get(
|
||||||
|
"PRAGENT_FEEDBACK_ISSUE_REPO", REPO_DEFAULT,
|
||||||
|
))
|
||||||
|
p.add_argument("--title", default=os.environ.get(
|
||||||
|
"PRAGENT_FEEDBACK_ISSUE_TITLE", TITLE_DEFAULT,
|
||||||
|
))
|
||||||
|
p.add_argument("--since", type=int, default=None,
|
||||||
|
help="Unix timestamp; only include findings posted since")
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
if not args.token:
|
||||||
|
print("PRAGENT_BOT_TOKEN required", flush=True)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
stats = deliver(
|
||||||
|
api=args.api, token=args.token, db_path=args.db,
|
||||||
|
repo=args.repo, title=args.title, since_ts=args.since,
|
||||||
|
)
|
||||||
|
print(json.dumps(stats), flush=True)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+160
-14
@@ -55,6 +55,8 @@ import time
|
|||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
|
from ai_review import _SEVERITY_EMOJI, is_test_path
|
||||||
|
|
||||||
# Where the factory lives (opencode.json + .opencode/). Default: the pragent
|
# Where the factory lives (opencode.json + .opencode/). Default: the pragent
|
||||||
# repo root (this file is at <root>/pilot/opencode_review.py).
|
# repo root (this file is at <root>/pilot/opencode_review.py).
|
||||||
_DEFAULT_FACTORY = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
_DEFAULT_FACTORY = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
@@ -387,12 +389,26 @@ def sanitize_workdir(workdir: str) -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def install_config(src: str, dst: str) -> bool:
|
def install_config(src: str, dst: str) -> bool:
|
||||||
"""Copy `opencode.json` from src to dst, substituting the model endpoint.
|
"""Copy `opencode.json` from src to dst, substituting per-provider endpoint
|
||||||
|
+ API key.
|
||||||
|
|
||||||
The committed `opencode.json` carries a neutral placeholder for the model
|
The committed `opencode.json` carries neutral placeholders for every
|
||||||
provider's `baseURL`, so the repo can be public without publishing the
|
provider's `baseURL`/`apiKey` so the repo can be public without leaking
|
||||||
address of a private network. The real endpoint is supplied at runtime by
|
private-network addresses. Real values are supplied at runtime and patched
|
||||||
`PRAGENT_MODEL_BASE_URL` and patched in here.
|
in here.
|
||||||
|
|
||||||
|
Env var convention (case-sensitive provider name — `headroom`, `local`):
|
||||||
|
|
||||||
|
PRAGENT_<NAME>_BASE_URL — per-provider endpoint override
|
||||||
|
PRAGENT_<NAME>_API_KEY — per-provider API key override
|
||||||
|
PRAGENT_MODEL_BASE_URL — legacy catchall, applies to every provider
|
||||||
|
when the per-provider var is unset
|
||||||
|
PRAGENT_MODEL_API_KEY — legacy catchall (same)
|
||||||
|
|
||||||
|
Per-provider wins over the catchall. The first 2 win when the operator
|
||||||
|
needs a different endpoint per upstream (e.g. headroom → MiniMax, local →
|
||||||
|
ai-workstation). The catchall keeps the single-provider deploys from
|
||||||
|
needing any env config.
|
||||||
|
|
||||||
This is done in Python rather than with opencode's own `{env:VAR}` config
|
This is done in Python rather than with opencode's own `{env:VAR}` config
|
||||||
templating because the reviewer subprocess runs with an allow-listed
|
templating because the reviewer subprocess runs with an allow-listed
|
||||||
@@ -403,18 +419,51 @@ def install_config(src: str, dst: str) -> bool:
|
|||||||
"""
|
"""
|
||||||
if not os.path.isfile(src):
|
if not os.path.isfile(src):
|
||||||
return False
|
return False
|
||||||
base_url = os.environ.get("PRAGENT_MODEL_BASE_URL", "").strip()
|
default_url = os.environ.get("PRAGENT_MODEL_BASE_URL", "").strip()
|
||||||
if not base_url:
|
default_key = os.environ.get("PRAGENT_MODEL_API_KEY", "").strip()
|
||||||
shutil.copy2(src, dst)
|
|
||||||
|
# Strip keys opencode's runtime rejects on every version bump we touch. The
|
||||||
|
# factory `opencode.json` is committed for documentation (so `$schema`
|
||||||
|
# stays in the file for editor IntelliSense), but opencode 1.3.10 errors
|
||||||
|
# with "Unrecognized key: schema" at config-parse time and refuses to
|
||||||
|
# register ANY provider/model — surfacing to the user as the misleading
|
||||||
|
# "opencode empty text (rc=0)" failure post. Keep the drop list small and
|
||||||
|
# documented; smoke-test before adding more.
|
||||||
|
_OPENCODE_INCOMPATIBLE_TOP_KEYS = ("$schema",)
|
||||||
|
|
||||||
|
def _sanitize_and_write(cfg: dict) -> None:
|
||||||
|
for k in _OPENCODE_INCOMPATIBLE_TOP_KEYS:
|
||||||
|
cfg.pop(k, None)
|
||||||
|
with open(dst, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(cfg, f, indent=2)
|
||||||
|
|
||||||
|
if not default_url and not default_key:
|
||||||
|
# Fast path: no env at all → still sanitize (the schema key would
|
||||||
|
# poison every fresh-pod warm-up if we skipped).
|
||||||
|
try:
|
||||||
|
with open(src, encoding="utf-8") as f:
|
||||||
|
cfg = json.load(f)
|
||||||
|
_sanitize_and_write(cfg)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
# If we can't parse, fall back to verbatim copy — opencode will
|
||||||
|
# report the parse error itself, no need to hide it.
|
||||||
|
shutil.copy2(src, dst)
|
||||||
return True
|
return True
|
||||||
try:
|
try:
|
||||||
with open(src, encoding="utf-8") as f:
|
with open(src, encoding="utf-8") as f:
|
||||||
cfg = json.load(f)
|
cfg = json.load(f)
|
||||||
for prov in (cfg.get("provider") or {}).values():
|
for name, prov in (cfg.get("provider") or {}).items():
|
||||||
if isinstance(prov, dict) and isinstance(prov.get("options"), dict):
|
if not isinstance(prov, dict) or not isinstance(prov.get("options"), dict):
|
||||||
prov["options"]["baseURL"] = base_url
|
continue
|
||||||
with open(dst, "w", encoding="utf-8") as f:
|
per_url = os.environ.get(f"PRAGENT_{name.upper()}_BASE_URL", "").strip()
|
||||||
json.dump(cfg, f, indent=2)
|
per_key = os.environ.get(f"PRAGENT_{name.upper()}_API_KEY", "").strip()
|
||||||
|
url = per_url or default_url
|
||||||
|
key = per_key or default_key
|
||||||
|
if url:
|
||||||
|
prov["options"]["baseURL"] = url
|
||||||
|
if key:
|
||||||
|
prov["options"]["apiKey"] = key
|
||||||
|
_sanitize_and_write(cfg)
|
||||||
except (OSError, ValueError, AttributeError):
|
except (OSError, ValueError, AttributeError):
|
||||||
# A malformed config is opencode's problem to report, not ours to hide.
|
# A malformed config is opencode's problem to report, not ours to hide.
|
||||||
shutil.copy2(src, dst)
|
shutil.copy2(src, dst)
|
||||||
@@ -905,6 +954,86 @@ def synthesize(
|
|||||||
return deduped[:per_pr_cap]
|
return deduped[:per_pr_cap]
|
||||||
|
|
||||||
|
|
||||||
|
def _synthesize_summary_fields(
|
||||||
|
findings: list[dict],
|
||||||
|
diff: str,
|
||||||
|
changed_paths: list[str] | None = None,
|
||||||
|
) -> tuple[list[str], str, str]:
|
||||||
|
"""Synthesize review-level meta from the merged findings + diff.
|
||||||
|
|
||||||
|
Returns (walkthrough, risk_verdict, test_coverage) — the three new
|
||||||
|
top-level fields in the pragent review JSON shape
|
||||||
|
(`ai_review.parse_review_output` extracts them as the 5th, 6th, and
|
||||||
|
7th tuple elements, defaulting to `[]` / `""` when missing).
|
||||||
|
|
||||||
|
Real implementation (Task 8). Python fallback used when the lens
|
||||||
|
fan-out path is engaged (the synthesized JSON fence in `run_lenses_review`
|
||||||
|
has no model to call, so we build these fields deterministically from
|
||||||
|
the merged findings + the diff):
|
||||||
|
- walkthrough: one line per changed file. When findings exist, group
|
||||||
|
by path and pick the peak-severity problem as the headline; when
|
||||||
|
no findings exist, just announce "changed".
|
||||||
|
- risk_verdict: a one-line verdict driven by the highest severity
|
||||||
|
bucket that has any findings ("Critical risk" / "High risk" /
|
||||||
|
"Medium risk" / "Low risk").
|
||||||
|
- test_coverage: "Tests changed" if any changed path matches
|
||||||
|
`is_test_path`, else "No tests for behavioral change in `<path>`."
|
||||||
|
pointing at the first non-test path.
|
||||||
|
"""
|
||||||
|
# None-safe: callers occasionally pass None when the upstream merger
|
||||||
|
# short-circuited. Treat as empty so the for-loop and group-by below
|
||||||
|
# never crash.
|
||||||
|
findings = findings or []
|
||||||
|
# walkthrough
|
||||||
|
walkthrough: list[str] = []
|
||||||
|
if findings:
|
||||||
|
by_path: dict[str, list[dict]] = {}
|
||||||
|
for f in findings:
|
||||||
|
by_path.setdefault(f.get("path", "?"), []).append(f)
|
||||||
|
for path, group in sorted(by_path.items()):
|
||||||
|
peak = max(
|
||||||
|
group,
|
||||||
|
key=lambda x: SEVERITY_RANK.get(x.get("severity", "low"), 0),
|
||||||
|
)
|
||||||
|
problem_lines = (peak.get("problem") or "").splitlines()
|
||||||
|
problem = problem_lines[0][:80].strip() if problem_lines else ""
|
||||||
|
emoji = _SEVERITY_EMOJI.get(peak.get("severity", "low"), "⚪")
|
||||||
|
walkthrough.append(f"`{path}` — {emoji} {problem}")
|
||||||
|
else:
|
||||||
|
files = changed_paths if changed_paths is not None else changed_files(diff)
|
||||||
|
for p in files:
|
||||||
|
walkthrough.append(f"`{p}` — changed")
|
||||||
|
|
||||||
|
# risk_verdict
|
||||||
|
sev_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}
|
||||||
|
for f in findings:
|
||||||
|
s = f.get("severity", "low")
|
||||||
|
sev_counts[s] = sev_counts.get(s, 0) + 1
|
||||||
|
if sev_counts["critical"]:
|
||||||
|
rv = f"Critical risk: {sev_counts['critical']} critical finding(s)."
|
||||||
|
elif sev_counts["high"]:
|
||||||
|
rv = f"High risk: {sev_counts['high']} high finding(s)."
|
||||||
|
elif sev_counts["medium"]:
|
||||||
|
rv = f"Medium risk: {sev_counts['medium']} medium finding(s)."
|
||||||
|
else:
|
||||||
|
rv = "Low risk: clean or minor nits only."
|
||||||
|
|
||||||
|
# test_coverage
|
||||||
|
paths = changed_paths if changed_paths is not None else changed_files(diff)
|
||||||
|
test_changed = any(is_test_path(p) for p in paths)
|
||||||
|
non_test = [p for p in paths if not is_test_path(p)]
|
||||||
|
if test_changed and non_test:
|
||||||
|
tc = "Tests changed"
|
||||||
|
elif non_test:
|
||||||
|
tc = f"No tests for behavioral change in `{non_test[0]}`."
|
||||||
|
elif test_changed:
|
||||||
|
tc = "Tests changed"
|
||||||
|
else:
|
||||||
|
tc = ""
|
||||||
|
|
||||||
|
return walkthrough, rv, tc
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Per-lens subprocess + parallel fan-out
|
# Per-lens subprocess + parallel fan-out
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1304,10 +1433,27 @@ def run_lenses_review(
|
|||||||
{k: v for k, v in f.items() if not k.startswith("_")}
|
{k: v for k, v in f.items() if not k.startswith("_")}
|
||||||
for f in merged
|
for f in merged
|
||||||
]
|
]
|
||||||
|
# Synthesize the review-level meta (walkthrough / risk_verdict /
|
||||||
|
# test_coverage) from the merged findings + diff. Real implementation
|
||||||
|
# arrives in Task 8; the stub keeps the synthesized JSON shape stable
|
||||||
|
# so ai_review.parse_review_output can extract the three new fields
|
||||||
|
# (it defaults them to [] / "" when missing — backward compatible).
|
||||||
|
walkthrough, risk_verdict, test_coverage = _synthesize_summary_fields(
|
||||||
|
merged, diff, changed_paths=changed_paths,
|
||||||
|
)
|
||||||
|
synthesized_payload = {
|
||||||
|
"summary": summary,
|
||||||
|
"summary_changes": [],
|
||||||
|
"risks": [],
|
||||||
|
"walkthrough": walkthrough,
|
||||||
|
"risk_verdict": risk_verdict,
|
||||||
|
"test_coverage": test_coverage,
|
||||||
|
"findings": clean_findings,
|
||||||
|
}
|
||||||
text = (
|
text = (
|
||||||
f"{summary}\n\n"
|
f"{summary}\n\n"
|
||||||
f"## Findings (multi-lens)\n\n"
|
f"## Findings (multi-lens)\n\n"
|
||||||
f"```json\n{json.dumps({'summary': summary, 'findings': clean_findings}, indent=2)}\n```\n"
|
f"```json\n{json.dumps(synthesized_payload, indent=2)}\n```\n"
|
||||||
)
|
)
|
||||||
if merged_usage is not None:
|
if merged_usage is not None:
|
||||||
merged_usage["duration_s"] = round(time.monotonic() - t0, 1)
|
merged_usage["duration_s"] = round(time.monotonic() - t0, 1)
|
||||||
|
|||||||
+80
-51
@@ -2,13 +2,15 @@
|
|||||||
"""pragent pilot — central webhook receiver.
|
"""pragent pilot — central webhook receiver.
|
||||||
|
|
||||||
A stdlib-only HTTP server that Gitea posts user-webhook events to. It gates on
|
A stdlib-only HTTP server that Gitea posts user-webhook events to. It gates on
|
||||||
the `AI-REVIEW` PR label, then runs the same review core (`ai_review.review_pr`)
|
the PR's base ref having `.pr-review.json` with `"enabled": true`, then runs
|
||||||
the CI-step pilot uses, posting findings back as `pragent-bot`.
|
the same review core (`ai_review.review_pr`) the CI-step pilot uses, posting
|
||||||
|
findings back as `pragent-bot`.
|
||||||
|
|
||||||
Per-owner setup: one Gitea **user-level webhook** per repo-owner fires for every
|
Per-owner setup: one Gitea **user-level webhook** per repo-owner fires for every
|
||||||
repo that owner has; this service filters to labeled PRs. (Gitea 1.26.1 system
|
repo that owner has; this service filters to opted-in PRs. (Gitea 1.26.1 system
|
||||||
webhooks are broken — see pilot/README-webhook.md.) Onboarding a repo = add the
|
webhooks are broken — see pilot/README-webhook.md.) Onboarding a repo = add the
|
||||||
bot as a Write collaborator + create the label + label a PR.
|
bot as a Write collaborator + commit a `.pr-review.json` with `"enabled": true`
|
||||||
|
on the base ref.
|
||||||
|
|
||||||
Stdlib only — no pip install, runs on python:3-slim with the scripts mounted.
|
Stdlib only — no pip install, runs on python:3-slim with the scripts mounted.
|
||||||
|
|
||||||
@@ -34,26 +36,32 @@ Env:
|
|||||||
(optional) request-body cap, default 10 MiB
|
(optional) request-body cap, default 10 MiB
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
|
import urllib.parse
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
|
||||||
from ai_review import review_pr
|
from ai_review import gitea_get, review_pr
|
||||||
|
|
||||||
|
try:
|
||||||
|
import feedback_harvest # optional — absent in CI-step pod, present in
|
||||||
|
# central webhook service. Harvesting is the
|
||||||
|
# collection side of the feedback loop.
|
||||||
|
except ImportError:
|
||||||
|
feedback_harvest = None
|
||||||
|
|
||||||
# Pull-request webhook `action` values. We fire on EVERY pull_request action
|
# Pull-request webhook `action` values. We fire on EVERY pull_request action
|
||||||
# except `closed` (no point reviewing a closed/merged PR) — the AI-REVIEW label
|
# except `closed` (no point reviewing a closed/merged PR) — the
|
||||||
# gate + sha dedupe downstream make broadening safe: a same-sha re-fire (title
|
# `.pr-review.json:enabled` gate + sha dedupe downstream make broadening safe:
|
||||||
# edit, assignee, milestone, label toggle of another label…) is skipped by
|
# a same-sha re-fire (title edit, assignee, milestone, label toggle…) is
|
||||||
# `review_pr`'s dedupe, and an `unlabeled` event that removed AI-REVIEW fails
|
# skipped by `review_pr`'s dedupe. Gitea emits GitHub-style `action` names
|
||||||
# the label gate (payload `labels` reflect current state). Gitea emits
|
# (`labeled`, `synchronize`) even though the `X-Gitea-Event-Type` header uses
|
||||||
# GitHub-style `action` names (`labeled`, `synchronize`) even though the
|
# `label_updated` / `synchronized`.
|
||||||
# `X-Gitea-Event-Type` header uses `label_updated` / `synchronized`.
|
|
||||||
SKIP_ACTIONS = {"closed"}
|
SKIP_ACTIONS = {"closed"}
|
||||||
AI_REVIEW_LABEL = "AI-REVIEW"
|
|
||||||
AI_USAGE_LABEL = "AI-USAGE"
|
|
||||||
|
|
||||||
GITEA_API = os.environ.get("GITEA_API", "http://gitea-http.gitea.svc.cluster.local:3000")
|
GITEA_API = os.environ.get("GITEA_API", "http://gitea-http.gitea.svc.cluster.local:3000")
|
||||||
BOT_TOKEN = os.environ.get("PRAGENT_BOT_TOKEN", "")
|
BOT_TOKEN = os.environ.get("PRAGENT_BOT_TOKEN", "")
|
||||||
@@ -65,6 +73,9 @@ WEBHOOK_SECRET = os.environ.get("WEBHOOK_SECRET", "").encode()
|
|||||||
PORT = int(os.environ.get("WEBHOOK_PORT", "8080"))
|
PORT = int(os.environ.get("WEBHOOK_PORT", "8080"))
|
||||||
MAX_CONCURRENT = max(1, int(os.environ.get("PRAGENT_MAX_CONCURRENT_REVIEWS", "2")))
|
MAX_CONCURRENT = max(1, int(os.environ.get("PRAGENT_MAX_CONCURRENT_REVIEWS", "2")))
|
||||||
MAX_BODY_BYTES = int(os.environ.get("PRAGENT_MAX_BODY_BYTES", str(10 * 1024 * 1024)))
|
MAX_BODY_BYTES = int(os.environ.get("PRAGENT_MAX_BODY_BYTES", str(10 * 1024 * 1024)))
|
||||||
|
# Feedback DB — SQLite mounted at PRAGENT_FEEDBACK_DB. Empty / unset =
|
||||||
|
# feedback collection disabled (CI-step path doesn't have it).
|
||||||
|
FEEDBACK_DB = os.environ.get("PRAGENT_FEEDBACK_DB", "")
|
||||||
|
|
||||||
# Bound on reviews running at once. Every review forks an opencode process that
|
# Bound on reviews running at once. Every review forks an opencode process that
|
||||||
# untars a repo, reads files and shells out to linters, so an unbounded thread
|
# untars a repo, reads files and shells out to linters, so an unbounded thread
|
||||||
@@ -76,28 +87,36 @@ _review_slots = threading.Semaphore(MAX_CONCURRENT)
|
|||||||
# Reviews currently accepted or running, keyed (repo, index, sha). The
|
# Reviews currently accepted or running, keyed (repo, index, sha). The
|
||||||
# sha-marker dedupe in `review_pr` reads Gitea *before* posting, so two
|
# sha-marker dedupe in `review_pr` reads Gitea *before* posting, so two
|
||||||
# deliveries for the same commit in flight together both see "not yet reviewed"
|
# deliveries for the same commit in flight together both see "not yet reviewed"
|
||||||
# and both post — the classic check-then-act race, and label-toggling is exactly
|
# and both post — the classic check-then-act race. Common triggers are Gitea
|
||||||
# the kind of thing that fires two deliveries a second apart. This set closes
|
# retries after a slow 202 response and bursty re-fires from a rapid title /
|
||||||
# the window inside one process.
|
# assign / label toggle. This set closes the window inside one process.
|
||||||
_inflight: set[tuple[str, str, str]] = set()
|
_inflight: set[tuple[str, str, str]] = set()
|
||||||
_inflight_lock = threading.Lock()
|
_inflight_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
def _labels_have(labels, name: str) -> bool:
|
def is_repo_enabled(api: str, repo: str, ref: str, token: str) -> bool:
|
||||||
"""True if the Gitea PR `labels` list (dicts with `name`, or bare strings)
|
"""True iff `.pr-review.json` on `ref` has `"enabled": true`.
|
||||||
contains `name`."""
|
|
||||||
if not isinstance(labels, list):
|
Reads from the given ref (typically the PR's base ref). False on any
|
||||||
|
failure: 404, parse error, missing file, missing `enabled`, wrong type.
|
||||||
|
The bool-coerce of `.get("enabled") is True` rejects the common
|
||||||
|
gotchas (`null`, `1`, `"yes"`, missing field all yield False).
|
||||||
|
"""
|
||||||
|
code, raw = gitea_get(
|
||||||
|
api, repo,
|
||||||
|
"contents/.pr-review.json?ref=" + urllib.parse.quote(ref, safe=""),
|
||||||
|
token,
|
||||||
|
)
|
||||||
|
if code != 200:
|
||||||
return False
|
return False
|
||||||
for lab in labels:
|
try:
|
||||||
if isinstance(lab, dict) and lab.get("name") == name:
|
data = json.loads(raw)
|
||||||
return True
|
content_b64 = data.get("content", "").replace("\n", "")
|
||||||
if isinstance(lab, str) and lab == name:
|
decoded = base64.b64decode(content_b64).decode("utf-8", errors="replace")
|
||||||
return True
|
cfg = json.loads(decoded)
|
||||||
return False
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
return False
|
||||||
|
return isinstance(cfg, dict) and cfg.get("enabled") is True
|
||||||
def _labels_have_ai_review(labels) -> bool:
|
|
||||||
return _labels_have(labels, AI_REVIEW_LABEL)
|
|
||||||
|
|
||||||
|
|
||||||
def _verify_signature(raw_body: bytes, headers) -> bool:
|
def _verify_signature(raw_body: bytes, headers) -> bool:
|
||||||
@@ -126,10 +145,6 @@ def _handle_pull_request(payload: dict) -> tuple[int, str]:
|
|||||||
if not repo:
|
if not repo:
|
||||||
return 400, "no repository.full_name"
|
return 400, "no repository.full_name"
|
||||||
|
|
||||||
labels = pr.get("labels")
|
|
||||||
if not _labels_have_ai_review(labels):
|
|
||||||
return 200, f"ignore (no {AI_REVIEW_LABEL} label) action={action}"
|
|
||||||
|
|
||||||
index = pr.get("number")
|
index = pr.get("number")
|
||||||
if index is None:
|
if index is None:
|
||||||
return 400, "no pull_request.number"
|
return 400, "no pull_request.number"
|
||||||
@@ -140,26 +155,22 @@ def _handle_pull_request(payload: dict) -> tuple[int, str]:
|
|||||||
|
|
||||||
base_ref = (pr.get("base") or {}).get("ref", "") or ""
|
base_ref = (pr.get("base") or {}).get("ref", "") or ""
|
||||||
|
|
||||||
|
if not is_repo_enabled(GITEA_API, repo, base_ref or "", BOT_TOKEN):
|
||||||
|
return 200, f"skip (repo not opted in) action={action}"
|
||||||
|
|
||||||
if not BOT_TOKEN:
|
if not BOT_TOKEN:
|
||||||
return 500, "PRAGENT_BOT_TOKEN not set"
|
return 500, "PRAGENT_BOT_TOKEN not set"
|
||||||
|
|
||||||
# AI-USAGE label (opt-in) → append the token-usage section + per-comment 🪙
|
|
||||||
# lines to the review. PRAGENT_USAGE_ALWAYS forces it on for testing / a
|
|
||||||
# future default-on.
|
|
||||||
report_usage = _labels_have(labels, AI_USAGE_LABEL) or bool(
|
|
||||||
os.environ.get("PRAGENT_USAGE_ALWAYS")
|
|
||||||
)
|
|
||||||
|
|
||||||
key = (repo, str(index), sha)
|
key = (repo, str(index), sha)
|
||||||
if not _claim(key):
|
if not _claim(key):
|
||||||
return 200, f"ignore (already in flight) {repo}#{index} sha={sha[:8]}"
|
return 200, f"ignore (already in flight) {repo}#{index} sha={sha[:8]}"
|
||||||
|
|
||||||
threading.Thread(
|
threading.Thread(
|
||||||
target=_run_review,
|
target=_run_review,
|
||||||
args=(key, title, body, report_usage, base_ref),
|
args=(key, title, body, base_ref),
|
||||||
daemon=True,
|
daemon=True,
|
||||||
).start()
|
).start()
|
||||||
return 202, f"reviewing {repo}#{index} action={action} sha={sha[:8]} usage={report_usage}"
|
return 202, f"reviewing {repo}#{index} action={action} sha={sha[:8]}"
|
||||||
|
|
||||||
|
|
||||||
def _claim(key: tuple[str, str, str]) -> bool:
|
def _claim(key: tuple[str, str, str]) -> bool:
|
||||||
@@ -177,9 +188,30 @@ def _release(key: tuple[str, str, str]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _run_review(
|
def _run_review(
|
||||||
key: tuple[str, str, str], title: str, body: str, report_usage: bool, base_ref: str
|
key: tuple[str, str, str], title: str, body: str, base_ref: str
|
||||||
) -> None:
|
) -> None:
|
||||||
repo, index, sha = key
|
repo, index, sha = key
|
||||||
|
# Harvest reactions on PRIOR bot comments on this PR (best-effort —
|
||||||
|
# piggy-backs the webhook path so we don't need a separate cron).
|
||||||
|
# Disabled if feedback_harvest isn't importable (CI-step image) or
|
||||||
|
# FEEDBACK_DB isn't set.
|
||||||
|
if FEEDBACK_DB and feedback_harvest is not None:
|
||||||
|
try:
|
||||||
|
hstats = feedback_harvest.harvest_for_pr(
|
||||||
|
api=GITEA_API, token=BOT_TOKEN,
|
||||||
|
repo=repo, pr_index=int(index), db_path=FEEDBACK_DB,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"pragent-webhook: harvested {repo}#{index} "
|
||||||
|
f"reviews={hstats['reviews_seen']} "
|
||||||
|
f"findings={hstats['findings_seen']} "
|
||||||
|
f"reactions={hstats['reactions_recorded']}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
# Harvest must never abort a review.
|
||||||
|
print(f"pragent-webhook: harvest failed for {repo}#{index}: {e}", flush=True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with _review_slots:
|
with _review_slots:
|
||||||
ok = review_pr(
|
ok = review_pr(
|
||||||
@@ -194,10 +226,9 @@ def _run_review(
|
|||||||
model=OLLAMA_MODEL,
|
model=OLLAMA_MODEL,
|
||||||
max_tokens=OLLAMA_MAX_TOKENS,
|
max_tokens=OLLAMA_MAX_TOKENS,
|
||||||
max_chars=DIFF_MAX_CHARS,
|
max_chars=DIFF_MAX_CHARS,
|
||||||
report_usage=report_usage,
|
|
||||||
base_ref=base_ref,
|
base_ref=base_ref,
|
||||||
)
|
)
|
||||||
print(f"pragent-webhook: reviewed {repo}#{index} sha={sha[:8]} ok={ok} usage={report_usage}", flush=True)
|
print(f"pragent-webhook: reviewed {repo}#{index} sha={sha[:8]} ok={ok}", flush=True)
|
||||||
except Exception as e: # review_pr is fail-open, but guard the thread anyway
|
except Exception as e: # review_pr is fail-open, but guard the thread anyway
|
||||||
print(f"pragent-webhook: thread crashed for {repo}#{index}: {e}", flush=True)
|
print(f"pragent-webhook: thread crashed for {repo}#{index}: {e}", flush=True)
|
||||||
finally:
|
finally:
|
||||||
@@ -255,11 +286,9 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
self._send(200, f"ignore event={event}")
|
self._send(200, f"ignore event={event}")
|
||||||
return
|
return
|
||||||
|
|
||||||
pr0 = payload.get("pull_request") or {}
|
repo_full = (payload.get("repository") or {}).get("full_name")
|
||||||
print(
|
print(
|
||||||
f"pragent-webhook: pull_request action={payload.get('action')} "
|
f"pragent-webhook: pull_request action={payload.get('action')} repo={repo_full}",
|
||||||
f"repo={(payload.get('repository') or {}).get('full_name')} "
|
|
||||||
f"ai_review={_labels_have_ai_review(pr0.get('labels'))}",
|
|
||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
status, msg = _handle_pull_request(payload)
|
status, msg = _handle_pull_request(payload)
|
||||||
|
|||||||
+573
-53
@@ -11,15 +11,21 @@ sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
|||||||
|
|
||||||
import ai_review # noqa: E402
|
import ai_review # noqa: E402
|
||||||
from ai_review import ( # noqa: E402
|
from ai_review import ( # noqa: E402
|
||||||
|
_CONFIDENCE_BADGE,
|
||||||
|
_SEVERITY_EMOJI,
|
||||||
_balanced_json_substring,
|
_balanced_json_substring,
|
||||||
_extract_first_json_object,
|
_extract_first_json_object,
|
||||||
_last_balanced_json,
|
_last_balanced_json,
|
||||||
|
_normalize_finding,
|
||||||
_render_collapsible_usage,
|
_render_collapsible_usage,
|
||||||
|
_severity_badge,
|
||||||
build_user_prompt,
|
build_user_prompt,
|
||||||
compute_attribution,
|
compute_attribution,
|
||||||
findings_table,
|
findings_table,
|
||||||
|
fmt_tokens,
|
||||||
format_review_body,
|
format_review_body,
|
||||||
inline_comment_body,
|
inline_comment_body,
|
||||||
|
merge_confidence,
|
||||||
parse_diff_anchors,
|
parse_diff_anchors,
|
||||||
parse_findings,
|
parse_findings,
|
||||||
parse_repo_config,
|
parse_repo_config,
|
||||||
@@ -27,6 +33,9 @@ from ai_review import ( # noqa: E402
|
|||||||
parse_text_blocks,
|
parse_text_blocks,
|
||||||
prior_review_bodies,
|
prior_review_bodies,
|
||||||
reviewed_shas,
|
reviewed_shas,
|
||||||
|
REVIEW_HEADER,
|
||||||
|
SEVERITIES,
|
||||||
|
SEVERITY_RANK,
|
||||||
split_findings,
|
split_findings,
|
||||||
summary_bullets,
|
summary_bullets,
|
||||||
truncate_diff,
|
truncate_diff,
|
||||||
@@ -316,7 +325,7 @@ def test_inline_comment_body_severity_emoji_mapping():
|
|||||||
("medium", "🟡 [MEDIUM]"),
|
("medium", "🟡 [MEDIUM]"),
|
||||||
("low", "🔵 [LOW]"),
|
("low", "🔵 [LOW]"),
|
||||||
("info", "⚪ [INFO]"),
|
("info", "⚪ [INFO]"),
|
||||||
("nit", "⚪ [INFO]"), # "nit" maps to the INFO label
|
("nit", "⚪ [NIT]"), # legacy alias — renders with its own name
|
||||||
("bogus", "⚪ [INFO]"), # unknown severity falls back to INFO
|
("bogus", "⚪ [INFO]"), # unknown severity falls back to INFO
|
||||||
]
|
]
|
||||||
for sev, badge in cases:
|
for sev, badge in cases:
|
||||||
@@ -334,7 +343,7 @@ def test_inline_comment_body_with_token_attribution():
|
|||||||
"fix": "f", "suggestion": "", "reference": "",
|
"fix": "f", "suggestion": "", "reference": "",
|
||||||
"_tok_attrib": 1234, "_tok_pct": 0.30}
|
"_tok_attrib": 1234, "_tok_pct": 0.30}
|
||||||
body = inline_comment_body(f)
|
body = inline_comment_body(f)
|
||||||
assert "🪙 ~1234 tok" in body
|
assert "🪙 ~1,234 (1.2K) tok" in body
|
||||||
assert "30%" in body
|
assert "30%" in body
|
||||||
assert "attributed output" in body
|
assert "attributed output" in body
|
||||||
|
|
||||||
@@ -394,11 +403,93 @@ def test_parse_repo_config_full():
|
|||||||
|
|
||||||
|
|
||||||
def test_parse_repo_config_partial_and_bad():
|
def test_parse_repo_config_partial_and_bad():
|
||||||
assert parse_repo_config('{"focus":"not-a-list"}') == {}
|
assert parse_repo_config('{"focus":"not-a-list"}') == {"enabled": False}
|
||||||
assert parse_repo_config('{"focus":["ok"]}') == {"focus": ["ok"]}
|
assert parse_repo_config('{"focus":["ok"]}') == {"focus": ["ok"], "enabled": False}
|
||||||
assert parse_repo_config("") == {}
|
assert parse_repo_config("") == {}
|
||||||
assert parse_repo_config("not json") == {}
|
assert parse_repo_config("not json") == {}
|
||||||
assert parse_repo_config('{"instructions":" "}') == {}
|
assert parse_repo_config('{"instructions":" "}') == {"enabled": False}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_reads_static_message():
|
||||||
|
cfg = parse_repo_config(json.dumps({"static_message": " NOTE: this repo is in maintenance mode "}))
|
||||||
|
assert cfg.get("static_message") == "NOTE: this repo is in maintenance mode"
|
||||||
|
assert cfg.get("enabled") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_static_message_caps_length():
|
||||||
|
long_text = "x" * 9999
|
||||||
|
cfg = parse_repo_config(json.dumps({"static_message": long_text}))
|
||||||
|
assert "static_message" in cfg
|
||||||
|
assert len(cfg["static_message"]) <= 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_static_message_ignores_blank():
|
||||||
|
assert "static_message" not in parse_repo_config(json.dumps({"static_message": " "}))
|
||||||
|
assert "static_message" not in parse_repo_config(json.dumps({"static_message": ""}))
|
||||||
|
assert "static_message" not in parse_repo_config(json.dumps({"static_message": 42}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_reads_model_override():
|
||||||
|
# Per-repo override is validated against cost_model.PRICES. Only keys
|
||||||
|
# the cost model knows about can override the review engine.
|
||||||
|
cfg = parse_repo_config(json.dumps({"model": "claude-sonnet-5"}))
|
||||||
|
assert cfg.get("model") == "claude-sonnet-5"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_rejects_unknown_model(capsys):
|
||||||
|
cfg = parse_repo_config(json.dumps({"model": "not-in-prices"}))
|
||||||
|
assert "model" not in cfg
|
||||||
|
# Repos that pin a typo should get a stderr hint pointing at the valid set.
|
||||||
|
err = capsys.readouterr().err
|
||||||
|
assert "model" in err.lower() or "prices" in err.lower() or "unknown" in err.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_model_must_be_string():
|
||||||
|
assert "model" not in parse_repo_config(json.dumps({"model": 42}))
|
||||||
|
assert "model" not in parse_repo_config(json.dumps({"model": []}))
|
||||||
|
assert "model" not in parse_repo_config(json.dumps({"model": None}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_display_model_precedence(monkeypatch):
|
||||||
|
# Order is OPENCODE_MODEL env > config['model'] (re-prefixed by provider) > headroom/{base}.
|
||||||
|
monkeypatch.delenv("OPENCODE_MODEL", raising=False)
|
||||||
|
# 1. No env, no config → headroom/<base>
|
||||||
|
assert ai_review._resolve_display_model("MiniMax-M2.7", None) == "headroom/MiniMax-M2.7"
|
||||||
|
assert ai_review._resolve_display_model("MiniMax-M2.7", {}) == "headroom/MiniMax-M2.7"
|
||||||
|
# 2. No env, config has a PRICES key → re-prefixed with that model's provider.
|
||||||
|
# headroom-hosted models default to provider="headroom".
|
||||||
|
assert (
|
||||||
|
ai_review._resolve_display_model("MiniMax-M2.7", {"model": "claude-sonnet-5"})
|
||||||
|
== "headroom/claude-sonnet-5"
|
||||||
|
)
|
||||||
|
# Self-hosted models carry provider="local" → routes to the `local`
|
||||||
|
# provider block in opencode.json (AI workstation on 192.168.1.79:18020).
|
||||||
|
assert (
|
||||||
|
ai_review._resolve_display_model("MiniMax-M2.7", {"model": "qwen3.8-27b"})
|
||||||
|
== "local/qwen3.8-27b"
|
||||||
|
)
|
||||||
|
# 3. Env wins over config
|
||||||
|
monkeypatch.setenv("OPENCODE_MODEL", "headroom/MiniMax-M2.7")
|
||||||
|
assert (
|
||||||
|
ai_review._resolve_display_model("MiniMax-M2.7", {"model": "claude-sonnet-5"})
|
||||||
|
== "headroom/MiniMax-M2.7"
|
||||||
|
)
|
||||||
|
# 4. Env alone, no config
|
||||||
|
monkeypatch.delenv("OPENCODE_MODEL")
|
||||||
|
assert ai_review._resolve_display_model("x", {}) == "headroom/x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_uses_override_for_cost_paren():
|
||||||
|
# End-to-end sanity: when the caller passes the resolved override as the
|
||||||
|
# `model` arg to format_review_body, both the header AND the cost line
|
||||||
|
# show the override — i.e. callers DO substitute the resolved display
|
||||||
|
# name into both the opencode subprocess ref and the review body.
|
||||||
|
body = format_review_body(
|
||||||
|
"- [high] x:1 — bug. fix.", "claude-sonnet-5", "abcdef1234567890",
|
||||||
|
)
|
||||||
|
assert "claude-sonnet-5" in body
|
||||||
|
assert "MiniMax-M2.7" not in body # the base didn't leak through
|
||||||
|
assert "🤖" in body # header rendered
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -470,9 +561,9 @@ def test_parse_review_output_bare_findings_no_summary():
|
|||||||
|
|
||||||
|
|
||||||
def test_parse_review_output_empty_and_bogus():
|
def test_parse_review_output_empty_and_bogus():
|
||||||
assert parse_review_output("") == ("", [], [], [])
|
assert parse_review_output("") == ("", [], [], [], [], "", "")
|
||||||
assert parse_review_output("no json here") == ("", [], [], [])
|
assert parse_review_output("no json here") == ("", [], [], [], [], "", "")
|
||||||
assert parse_review_output('{"findings":[]}') == ("", [], [], [])
|
assert parse_review_output('{"findings":[]}') == ("", [], [], [], [], "", "")
|
||||||
|
|
||||||
|
|
||||||
def test_parse_review_output_uses_last_json_block():
|
def test_parse_review_output_uses_last_json_block():
|
||||||
@@ -556,6 +647,44 @@ def test_parse_review_output_bare_array_at_tail():
|
|||||||
assert len(fs) == 1
|
assert len(fs) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_review_output_extracts_walkthrough_risk_tests():
|
||||||
|
# The 7-tuple shape carries three new top-level fields:
|
||||||
|
# walkthrough (list[str]), risk_verdict (str), test_coverage (str).
|
||||||
|
txt = (
|
||||||
|
"```json\n"
|
||||||
|
"{\n"
|
||||||
|
' "summary": "x",\n'
|
||||||
|
' "summary_changes": [],\n'
|
||||||
|
' "risks": [],\n'
|
||||||
|
' "walkthrough": ["a.py: adds X", "b.py: refactors Y"],\n'
|
||||||
|
' "risk_verdict": "Low risk.",\n'
|
||||||
|
' "test_coverage": "No tests for behavioral change in a.py.",\n'
|
||||||
|
' "findings": []\n'
|
||||||
|
"}\n"
|
||||||
|
"```"
|
||||||
|
)
|
||||||
|
summary, findings, _changes, _risks, walkthrough, risk_verdict, test_coverage = (
|
||||||
|
parse_review_output(txt)
|
||||||
|
)
|
||||||
|
assert summary == "x"
|
||||||
|
assert findings == []
|
||||||
|
assert walkthrough == ["a.py: adds X", "b.py: refactors Y"]
|
||||||
|
assert risk_verdict == "Low risk."
|
||||||
|
assert test_coverage == "No tests for behavioral change in a.py."
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_review_output_missing_fields_default_empty():
|
||||||
|
# Backward-compatible: the 4-tuple shape still parses fine; the new
|
||||||
|
# fields default to empty list / empty string.
|
||||||
|
out = parse_review_output('{"summary":"x","findings":[]}')
|
||||||
|
summary, findings, _changes, _risks, walkthrough, risk_verdict, test_coverage = out
|
||||||
|
assert summary == "x"
|
||||||
|
assert findings == []
|
||||||
|
assert walkthrough == []
|
||||||
|
assert risk_verdict == ""
|
||||||
|
assert test_coverage == ""
|
||||||
|
|
||||||
|
|
||||||
def test_scan_balanced_handles_braces_in_strings():
|
def test_scan_balanced_handles_braces_in_strings():
|
||||||
# The JSON scanner must not be fooled by `{` or `}` inside string literals.
|
# The JSON scanner must not be fooled by `{` or `}` inside string literals.
|
||||||
s = '{"a":"contains { and }","b":1}'
|
s = '{"a":"contains { and }","b":1}'
|
||||||
@@ -616,35 +745,6 @@ def test_reference_non_url_renders_as_plain_text():
|
|||||||
assert "](CVE-" not in body
|
assert "](CVE-" not in body
|
||||||
|
|
||||||
|
|
||||||
def test_pr_has_label_reads_the_live_labels(monkeypatch):
|
|
||||||
# The AI-USAGE opt-in is read at render time, not from the trigger
|
|
||||||
# payload: labelling AI-REVIEW then AI-USAGE is two events, the review
|
|
||||||
# claims on the first, and the second is dropped by the in-flight dedupe.
|
|
||||||
seen = {}
|
|
||||||
|
|
||||||
def _get(api, repo, path, token, accept="application/json"):
|
|
||||||
seen["path"] = path
|
|
||||||
return 200, b'[{"name": "AI-REVIEW"}, {"name": "AI-USAGE"}]'
|
|
||||||
|
|
||||||
monkeypatch.setattr(ai_review, "gitea_get", _get)
|
|
||||||
assert ai_review.pr_has_label("http://api", "o/r", "9", "t", "AI-USAGE") is True
|
|
||||||
assert seen["path"] == "issues/9/labels"
|
|
||||||
assert ai_review.pr_has_label("http://api", "o/r", "9", "t", "NOPE") is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_pr_has_label_survives_a_broken_api(monkeypatch):
|
|
||||||
def _boom(*a, **k):
|
|
||||||
raise RuntimeError("gitea down")
|
|
||||||
monkeypatch.setattr(ai_review, "gitea_get", _boom)
|
|
||||||
assert ai_review.pr_has_label("http://api", "o/r", "9", "t", "AI-USAGE") is False
|
|
||||||
|
|
||||||
monkeypatch.setattr(ai_review, "gitea_get", lambda *a, **k: (404, b"nope"))
|
|
||||||
assert ai_review.pr_has_label("http://api", "o/r", "9", "t", "AI-USAGE") is False
|
|
||||||
|
|
||||||
monkeypatch.setattr(ai_review, "gitea_get", lambda *a, **k: (200, b'{"not": "a list"}'))
|
|
||||||
assert ai_review.pr_has_label("http://api", "o/r", "9", "t", "AI-USAGE") is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_int_env_falls_back_on_garbage(monkeypatch, capsys):
|
def test_int_env_falls_back_on_garbage(monkeypatch, capsys):
|
||||||
monkeypatch.setenv("PRAGENT_DIFF_CONTEXT", "two")
|
monkeypatch.setenv("PRAGENT_DIFF_CONTEXT", "two")
|
||||||
assert ai_review._int_env("PRAGENT_DIFF_CONTEXT", 1) == 1
|
assert ai_review._int_env("PRAGENT_DIFF_CONTEXT", 1) == 1
|
||||||
@@ -756,7 +856,7 @@ def test_render_collapsible_usage_renders_totals():
|
|||||||
assert "`glm-5.2:cloud`" in sec
|
assert "`glm-5.2:cloud`" in sec
|
||||||
assert "7 steps" in sec
|
assert "7 steps" in sec
|
||||||
assert "142.0s" in sec
|
assert "142.0s" in sec
|
||||||
assert "18420 in / 612 out" in sec and "19032 total" in sec
|
assert "18,420 (18.4K) in / 612 out" in sec and "19,032 (19.0K) total" in sec
|
||||||
assert "$0.00" in sec
|
assert "$0.00" in sec
|
||||||
assert "Whole-repo checkout" in sec
|
assert "Whole-repo checkout" in sec
|
||||||
assert "attributed" in sec
|
assert "attributed" in sec
|
||||||
@@ -771,6 +871,39 @@ def test_render_collapsible_usage_cost_nonzero_drops_free_tier_note():
|
|||||||
"cache_write": 0, "total": 10, "cost": 0.0123, "steps": 1, "duration_s": 1.0}
|
"cache_write": 0, "total": 10, "cost": 0.0123, "steps": 1, "duration_s": 1.0}
|
||||||
sec = _render_collapsible_usage(usage, "m", config=None)
|
sec = _render_collapsible_usage(usage, "m", config=None)
|
||||||
assert "$0.0123" in sec
|
assert "$0.0123" in sec
|
||||||
|
# Was hardcoded "free tier" previously; now says "billed" since cost > 0.
|
||||||
|
assert "billed" in sec
|
||||||
|
assert "free tier" not in sec
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_collapsible_usage_uses_passed_model_for_free_tier_clause():
|
||||||
|
# Regression: the cost parenthetical must reflect the actually-routed model,
|
||||||
|
# not a stale hardcoded `headroom glm-5.2:cloud` literal that predates the
|
||||||
|
# MiniMax / Anthropic switch.
|
||||||
|
usage = {"input": 10, "output": 0, "reasoning": 0, "cache_read": 0,
|
||||||
|
"cache_write": 0, "total": 10, "cost": 0.0, "steps": 1, "duration_s": 1.0}
|
||||||
|
sec = _render_collapsible_usage(usage, "MiniMax-M2.7", config=None)
|
||||||
|
# The parenthetical clause is "(<model> — free tier)" — a model name MUST
|
||||||
|
# sit immediately before "— free tier".
|
||||||
|
assert "(MiniMax-M2.7 — free tier)" in sec
|
||||||
|
# And the stale hardcoded model name must no longer appear anywhere.
|
||||||
|
assert "glm-5.2:cloud" not in sec
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_collapsible_usage_full_provider_prefix_in_display():
|
||||||
|
# When the caller has resolved a provider-prefixed model ref (the opencode
|
||||||
|
# subprocess path), the parenthetical should mirror that verbatim.
|
||||||
|
usage = {"input": 10, "output": 0, "reasoning": 0, "cache_read": 0,
|
||||||
|
"cache_write": 0, "total": 10, "cost": 0.0, "steps": 1, "duration_s": 1.0}
|
||||||
|
sec = _render_collapsible_usage(usage, "headroom/MiniMax-M2.7", config=None)
|
||||||
|
assert "(headroom/MiniMax-M2.7 — free tier)" in sec
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_collapsible_usage_nonzero_cost_says_billed():
|
||||||
|
usage = {"input": 10, "output": 0, "reasoning": 0, "cache_read": 0,
|
||||||
|
"cache_write": 0, "total": 10, "cost": 0.123, "steps": 1, "duration_s": 1.0}
|
||||||
|
sec = _render_collapsible_usage(usage, "MiniMax-M2.7", config=None)
|
||||||
|
assert "(MiniMax-M2.7 — billed)" in sec
|
||||||
assert "free tier" not in sec
|
assert "free tier" not in sec
|
||||||
|
|
||||||
|
|
||||||
@@ -790,6 +923,22 @@ def test_format_review_body_no_usage_section_omitted():
|
|||||||
assert "AI usage" not in body
|
assert "AI usage" not in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_renders_static_message_banner():
|
||||||
|
body = format_review_body(
|
||||||
|
"", "glm-5.2:cloud", "abcdef1234567890",
|
||||||
|
static_message="NOTE: this repo is in maintenance mode.",
|
||||||
|
)
|
||||||
|
assert "> NOTE: this repo is in maintenance mode." in body
|
||||||
|
# Banner sits under the header and above the rest of the body.
|
||||||
|
assert body.index("NOTE") > body.index("🤖")
|
||||||
|
assert body.index("NOTE") < body.index("### Summary of Changes")
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_omits_static_message_when_blank():
|
||||||
|
body = format_review_body("", "glm-5.2:cloud", "abcdef1234567890")
|
||||||
|
assert "> " not in body
|
||||||
|
|
||||||
|
|
||||||
def test_format_review_body_with_summary_changes_and_risks():
|
def test_format_review_body_with_summary_changes_and_risks():
|
||||||
body = format_review_body(
|
body = format_review_body(
|
||||||
"", "glm-5.2:cloud", "abcdef1234567890",
|
"", "glm-5.2:cloud", "abcdef1234567890",
|
||||||
@@ -812,6 +961,43 @@ def test_format_review_body_with_summary_changes_and_risks():
|
|||||||
assert "`a.py:1`" in body
|
assert "`a.py:1`" in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_renders_walkthrough():
|
||||||
|
body = format_review_body(
|
||||||
|
"", "glm-5.2:cloud", "abc1234",
|
||||||
|
summary_changes=["adds X"],
|
||||||
|
risks=[],
|
||||||
|
walkthrough=["a.py — adds X", "b.py — refactors Y"],
|
||||||
|
risk_verdict="Low risk: clean.",
|
||||||
|
test_coverage="Tests added.",
|
||||||
|
findings_for_table=[],
|
||||||
|
)
|
||||||
|
assert "### Walkthrough" in body
|
||||||
|
assert "`a.py` — adds X" in body
|
||||||
|
assert "### Risk Verdict" in body
|
||||||
|
assert "Low risk: clean." in body
|
||||||
|
assert "### Test Coverage" in body
|
||||||
|
assert "Tests added." in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_omits_empty_sections():
|
||||||
|
body = format_review_body(
|
||||||
|
"", "glm-5.2:cloud", "abc1234",
|
||||||
|
summary_changes=["adds X"],
|
||||||
|
walkthrough=[], risk_verdict="", test_coverage="",
|
||||||
|
)
|
||||||
|
assert "### Walkthrough" not in body
|
||||||
|
assert "### Risk Verdict" not in body
|
||||||
|
assert "### Test Coverage" not in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_placeholder_when_empty():
|
||||||
|
body = format_review_body(
|
||||||
|
"", "glm-5.2:cloud", "abc1234",
|
||||||
|
walkthrough=[], risk_verdict="", test_coverage="",
|
||||||
|
)
|
||||||
|
assert body # non-empty
|
||||||
|
|
||||||
|
|
||||||
def test_render_collapsible_usage_contains_details():
|
def test_render_collapsible_usage_contains_details():
|
||||||
usage = {
|
usage = {
|
||||||
"model": "glm-5.2:cloud", "input": 1000, "output": 200, "reasoning": 0,
|
"model": "glm-5.2:cloud", "input": 1000, "output": 200, "reasoning": 0,
|
||||||
@@ -823,7 +1009,7 @@ def test_render_collapsible_usage_contains_details():
|
|||||||
assert "<summary>🔋 AI Usage & Run Details</summary>" in block
|
assert "<summary>🔋 AI Usage & Run Details</summary>" in block
|
||||||
assert "</details>" in block
|
assert "</details>" in block
|
||||||
assert "glm-5.2:cloud" in block
|
assert "glm-5.2:cloud" in block
|
||||||
assert "1000 in / 200 out" in block
|
assert "1,000 (1.0K) in / 200 out" in block
|
||||||
|
|
||||||
|
|
||||||
def test_render_collapsible_usage_empty_when_no_usage():
|
def test_render_collapsible_usage_empty_when_no_usage():
|
||||||
@@ -889,7 +1075,10 @@ def test_parse_repo_config_still_accepts_normal_config():
|
|||||||
cfg = parse_repo_config(json.dumps({
|
cfg = parse_repo_config(json.dumps({
|
||||||
"focus": ["security"], "languages": ["go"], "instructions": "No bare throw.",
|
"focus": ["security"], "languages": ["go"], "instructions": "No bare throw.",
|
||||||
}))
|
}))
|
||||||
assert cfg == {"focus": ["security"], "languages": ["go"], "instructions": "No bare throw."}
|
assert cfg == {
|
||||||
|
"focus": ["security"], "languages": ["go"], "instructions": "No bare throw.",
|
||||||
|
"enabled": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -911,7 +1100,7 @@ def test_fetch_repo_config_uses_given_base_ref(monkeypatch):
|
|||||||
|
|
||||||
monkeypatch.setattr(ai_review, "gitea_get", fake_get)
|
monkeypatch.setattr(ai_review, "gitea_get", fake_get)
|
||||||
cfg = ai_review.fetch_repo_config("http://g", "o/r", "tok", ref="main")
|
cfg = ai_review.fetch_repo_config("http://g", "o/r", "tok", ref="main")
|
||||||
assert cfg == {"focus": ["security"]}
|
assert cfg == {"focus": ["security"], "enabled": False}
|
||||||
assert seen["path"] == "contents/.pr-review.json?ref=main"
|
assert seen["path"] == "contents/.pr-review.json?ref=main"
|
||||||
|
|
||||||
|
|
||||||
@@ -1078,19 +1267,26 @@ def test_usage_block_shows_equivalent_provider_cost():
|
|||||||
"cache_read": 0, "cache_write": 0, "total": 204000,
|
"cache_read": 0, "cache_write": 0, "total": 204000,
|
||||||
"cost": 0.0, "steps": 6, "duration_s": 100.0}
|
"cost": 0.0, "steps": 6, "duration_s": 100.0}
|
||||||
sec = ai_review._render_collapsible_usage(usage, "glm-5.2:cloud", config=None)
|
sec = ai_review._render_collapsible_usage(usage, "glm-5.2:cloud", config=None)
|
||||||
# Two cost lines now: an equivalent (default Sonnet 5) AND the $0 actual.
|
# New layout: equivalent-cost table instead of a single "Est. cost on …"
|
||||||
|
# line. The default compare_against is sonnet-5, gpt-5, gemini-2.5-pro,
|
||||||
|
# grok-4.5; cost_target defaults to sonnet-5 (bolded).
|
||||||
assert "🔋 AI Usage & Run Details" in sec
|
assert "🔋 AI Usage & Run Details" in sec
|
||||||
assert "**Est. cost on Claude Sonnet 5**" in sec
|
|
||||||
assert "**Actual**: $0.00" in sec
|
assert "**Actual**: $0.00" in sec
|
||||||
|
# The "free tier" clause must mention the routed model verbatim, not the
|
||||||
|
# stale hardcoded `headroom glm-5.2:cloud` literal.
|
||||||
assert "free tier" in sec
|
assert "free tier" in sec
|
||||||
|
assert "glm-5.2:cloud" in sec
|
||||||
# Equivalent should be > 0 for non-trivial token counts.
|
# Equivalent should be > 0 for non-trivial token counts.
|
||||||
assert "$0.00" in sec # the actual line
|
assert "$0.00" in sec # the actual line
|
||||||
# And a non-zero one for the equivalent.
|
# Multi-provider table header present, default roster rendered, default
|
||||||
import re
|
# cost_target (Sonnet 5) is the bolded row.
|
||||||
cost_lines = [ln for ln in sec.splitlines() if "cost on" in ln]
|
assert "| Provider | Cost |" in sec
|
||||||
assert len(cost_lines) == 1
|
assert "**Claude Sonnet 5**" in sec
|
||||||
assert re.search(r"\$\d", cost_lines[0]) is not None
|
assert "GPT-5" in sec
|
||||||
assert "$0.00" not in cost_lines[0]
|
assert "Gemini 2.5 Pro" in sec
|
||||||
|
assert "Grok 4.5" in sec
|
||||||
|
# 200k * $2/MTok + 4k * $10/MTok → $0.44
|
||||||
|
assert "$0.44" in sec
|
||||||
|
|
||||||
|
|
||||||
def test_usage_block_honors_cost_target(monkeypatch):
|
def test_usage_block_honors_cost_target(monkeypatch):
|
||||||
@@ -1117,17 +1313,22 @@ def test_usage_block_respects_repo_config_cost_target(monkeypatch):
|
|||||||
assert "$0.0075" in sec
|
assert "$0.0075" in sec
|
||||||
|
|
||||||
|
|
||||||
def test_usage_block_reports_unknown_price_target():
|
def test_usage_block_reports_unknown_price_target(capsys):
|
||||||
usage = {"input": 100, "output": 100, "reasoning": 0,
|
usage = {"input": 100, "output": 100, "reasoning": 0,
|
||||||
"cache_read": 0, "cache_write": 0, "total": 200,
|
"cache_read": 0, "cache_write": 0, "total": 200,
|
||||||
"cost": 0.0, "steps": 1, "duration_s": 1.0}
|
"cost": 0.0, "steps": 1, "duration_s": 1.0}
|
||||||
sec = ai_review._render_collapsible_usage(
|
sec = ai_review._render_collapsible_usage(
|
||||||
usage, "glm-5.2:cloud", config={"cost_target": "bogus-model"}
|
usage, "glm-5.2:cloud", config={"cost_target": "bogus-model"}
|
||||||
)
|
)
|
||||||
# Falls back to default + surfaces the error in the line.
|
# Falls back to default. The error now goes to stderr (otherwise it would
|
||||||
|
# land mid-table and look like a model error in the posted summary).
|
||||||
assert "Claude Sonnet 5" in sec
|
assert "Claude Sonnet 5" in sec
|
||||||
assert "unknown price target" in sec
|
assert "**Claude Sonnet 5**" in sec # bolded as the resolved cost_target
|
||||||
assert "bogus-model" in sec
|
assert "bogus-model" not in sec
|
||||||
|
assert "unknown price target" not in sec
|
||||||
|
err = capsys.readouterr().err
|
||||||
|
assert "unknown price target" in err
|
||||||
|
assert "bogus-model" in err
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1629,3 +1830,322 @@ def test_render_collapsible_usage_omits_lenses_when_single_primary():
|
|||||||
out = _render_collapsible_usage(usage, "headroom/glm-5.2:cloud", None)
|
out = _render_collapsible_usage(usage, "headroom/glm-5.2:cloud", None)
|
||||||
assert "Lenses" not in out
|
assert "Lenses" not in out
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# fmt_tokens
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_zero():
|
||||||
|
assert fmt_tokens(0) == "0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_small_no_short():
|
||||||
|
assert fmt_tokens(42) == "42"
|
||||||
|
assert fmt_tokens(999) == "999"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_thousands():
|
||||||
|
assert fmt_tokens(1000) == "1,000 (1.0K)"
|
||||||
|
assert fmt_tokens(1234) == "1,234 (1.2K)"
|
||||||
|
assert fmt_tokens(9999) == "9,999 (10.0K)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_millions():
|
||||||
|
assert fmt_tokens(1_000_000) == "1,000,000 (1.0M)"
|
||||||
|
assert fmt_tokens(2_071_025) == "2,071,025 (2.1M)"
|
||||||
|
assert fmt_tokens(1_234_567) == "1,234,567 (1.2M)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_billions():
|
||||||
|
assert fmt_tokens(1_234_567_890) == "1,234,567,890 (1.2B)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_none():
|
||||||
|
assert fmt_tokens(None) == "?"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fmt_tokens_negative():
|
||||||
|
assert fmt_tokens(-1) == "?"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# fmt_tokens — applied in usage + inline comment bodies (Task 3)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapsible_usage_renders_humanized_tokens():
|
||||||
|
usage = {"input": 2_071_025, "output": 17303, "reasoning": 0,
|
||||||
|
"cache_read": 0, "cache_write": 0, "total": 2_088_328,
|
||||||
|
"cost": 0.0, "steps": 1, "duration_s": 10.0}
|
||||||
|
block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={})
|
||||||
|
assert "2,071,025 (2.1M) in" in block
|
||||||
|
assert "17,303 (17.3K) out" in block
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Multi-provider equivalent-cost table — Task 10
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapsible_usage_renders_multi_provider_table():
|
||||||
|
usage = {"input": 1_000_000, "output": 1000, "reasoning": 0,
|
||||||
|
"cache_read": 0, "cache_write": 0, "total": 1_001_000,
|
||||||
|
"cost": 0.0, "steps": 1, "duration_s": 10.0}
|
||||||
|
block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={"compare_against": ["claude-sonnet-5", "gpt-5"]})
|
||||||
|
assert "Claude Sonnet 5" in block
|
||||||
|
assert "GPT-5" in block
|
||||||
|
assert "| Provider | Cost |" in block
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapsible_usage_uses_default_compare_against_when_absent():
|
||||||
|
usage = {"input": 1_000_000, "output": 0, "reasoning": 0,
|
||||||
|
"cache_read": 0, "cache_write": 0, "total": 1_000_000,
|
||||||
|
"cost": 0.0, "steps": 1, "duration_s": 5.0}
|
||||||
|
block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={})
|
||||||
|
assert "Claude Sonnet 5" in block
|
||||||
|
assert "GPT-5" in block
|
||||||
|
assert "Gemini 2.5 Pro" in block
|
||||||
|
assert "Grok 4.5" in block
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapsible_usage_bolds_cost_target_row():
|
||||||
|
usage = {"input": 1_000_000, "output": 0, "reasoning": 0,
|
||||||
|
"cache_read": 0, "cache_write": 0, "total": 1_000_000,
|
||||||
|
"cost": 0.0, "steps": 1, "duration_s": 5.0}
|
||||||
|
block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={"cost_target": "gpt-5"})
|
||||||
|
assert "**GPT-5**" in block
|
||||||
|
assert "Claude Sonnet 5" in block # still in default compare set
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapsible_usage_skips_zero_cost_rows():
|
||||||
|
usage = {"input": 0, "output": 0, "reasoning": 0,
|
||||||
|
"cache_read": 0, "cache_write": 0, "total": 0,
|
||||||
|
"cost": 0.0, "steps": 1, "duration_s": 1.0}
|
||||||
|
block = _render_collapsible_usage(usage, "glm-5.2:cloud", config={})
|
||||||
|
# With zero tokens, all costs are $0 — skip the entire table.
|
||||||
|
assert "| Provider | Cost |" not in block
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_comment_body_humanized_tokens():
|
||||||
|
# Value chosen > 1000 so fmt_tokens actually adds the comma + short suffix;
|
||||||
|
# the plan's 362 would render identically with or without fmt_tokens.
|
||||||
|
f = {"severity": "medium", "path": "x.py", "line": 1,
|
||||||
|
"problem": "p", "fix": "", "suggestion": "", "reference": "",
|
||||||
|
"_tok_attrib": 17303, "_tok_pct": 0.11}
|
||||||
|
body = inline_comment_body(f)
|
||||||
|
assert "🪙 ~17,303 (17.3K) tok" in body
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Severity levels — Task 4 (add trivial + info)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_severities_includes_trivial_and_info():
|
||||||
|
assert "trivial" in SEVERITIES
|
||||||
|
assert "info" in SEVERITIES
|
||||||
|
|
||||||
|
|
||||||
|
def test_severity_rank_orders_new_levels():
|
||||||
|
assert SEVERITY_RANK["info"] < SEVERITY_RANK["trivial"] < SEVERITY_RANK["low"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_threshold_medium_keeps_low_below_trivial_below_info():
|
||||||
|
# medium+ threshold:
|
||||||
|
# medium (rank 2) → kept
|
||||||
|
# low (rank 1) → DROPPED
|
||||||
|
# trivial (rank 0) → DROPPED
|
||||||
|
# info (rank -1) → DROPPED
|
||||||
|
cfg = {"style": "lenient", "severity_threshold": "medium"}
|
||||||
|
findings = [
|
||||||
|
{"severity": "info", "path": "a", "line": 1, "problem": "", "fix": "", "suggestion": "", "reference": ""},
|
||||||
|
{"severity": "trivial", "path": "b", "line": 1, "problem": "", "fix": "", "suggestion": "", "reference": ""},
|
||||||
|
{"severity": "low", "path": "c", "line": 1, "problem": "", "fix": "", "suggestion": "", "reference": ""},
|
||||||
|
{"severity": "medium", "path": "d", "line": 1, "problem": "", "fix": "", "suggestion": "", "reference": ""},
|
||||||
|
]
|
||||||
|
kept, dropped = ai_review.apply_repo_config(findings, cfg, changed_paths=["x.py"])
|
||||||
|
sev_kept = [f["severity"] for f in kept]
|
||||||
|
sev_dropped = [f["severity"] for f in dropped]
|
||||||
|
assert "info" in sev_dropped
|
||||||
|
assert "trivial" in sev_dropped
|
||||||
|
assert "low" in sev_dropped
|
||||||
|
assert "medium" in sev_kept
|
||||||
|
# and nothing else
|
||||||
|
assert len(kept) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_severity_still_normalizes_to_medium():
|
||||||
|
# Backward compat
|
||||||
|
n = _normalize_finding({"severity": "emergency", "path": "x", "line": 1, "problem": "p"})
|
||||||
|
assert n["severity"] == "medium"
|
||||||
|
|
||||||
|
|
||||||
|
def test_emoji_for_trivial_and_info_is_neutral():
|
||||||
|
# The plan's emoji table maps trivial/info to ⚪
|
||||||
|
assert _SEVERITY_EMOJI["trivial"] == "⚪"
|
||||||
|
assert _SEVERITY_EMOJI["info"] == "⚪"
|
||||||
|
|
||||||
|
|
||||||
|
def test_severity_badge_labels_each_known_severity():
|
||||||
|
# Trivial and info (and legacy nit) should render with their own name,
|
||||||
|
# not fall back to "INFO".
|
||||||
|
for sev in ("critical", "high", "medium", "low", "trivial", "info", "nit"):
|
||||||
|
badge = _severity_badge(sev)
|
||||||
|
assert f"[{sev.upper()}]" in badge, (sev, badge)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# parse_repo_config — `enabled` (kill-switch) + `compare_against` (cost roster)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_enabled_true():
|
||||||
|
cfg = parse_repo_config('{"enabled": true}')
|
||||||
|
assert cfg.get("enabled") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_enabled_false_explicit():
|
||||||
|
cfg = parse_repo_config('{"enabled": false}')
|
||||||
|
assert cfg.get("enabled") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_enabled_missing_defaults_false():
|
||||||
|
cfg = parse_repo_config('{}')
|
||||||
|
assert cfg.get("enabled") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_enabled_wrong_type_ignored():
|
||||||
|
cfg = parse_repo_config('{"enabled": "yes"}')
|
||||||
|
assert cfg.get("enabled") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_compare_against_default_absent():
|
||||||
|
# absent in returned cfg; defaults applied in render, not parse_repo_config
|
||||||
|
cfg = parse_repo_config('{}')
|
||||||
|
assert "compare_against" not in cfg
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_compare_against_valid():
|
||||||
|
cfg = parse_repo_config(
|
||||||
|
'{"compare_against": ["claude-sonnet-5", "gpt-5", "gemini-2.5-pro"]}')
|
||||||
|
assert cfg["compare_against"] == ["claude-sonnet-5", "gpt-5", "gemini-2.5-pro"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_compare_against_drops_unknown_keys(capfd):
|
||||||
|
cfg = parse_repo_config(
|
||||||
|
'{"compare_against": ["claude-sonnet-5", "bogus-1", "gpt-5"]}')
|
||||||
|
assert "bogus-1" not in cfg["compare_against"]
|
||||||
|
assert "claude-sonnet-5" in cfg["compare_against"]
|
||||||
|
captured = capfd.readouterr()
|
||||||
|
assert "bogus-1" in captured.err
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_repo_config_compare_against_caps_at_12(monkeypatch):
|
||||||
|
"""13+ valid keys must be truncated to the first 12; invalid keys are
|
||||||
|
dropped and do not count. Inject a 13th PRICES entry via monkeypatch so
|
||||||
|
the [:12] cap actually fires (cost_model.PRICES has exactly 12 keys
|
||||||
|
today, which would otherwise make the cap a no-op)."""
|
||||||
|
import cost_model as cm
|
||||||
|
monkeypatch.setitem(
|
||||||
|
cm.PRICES, "fake-model-13", cm.Price("Fake", 1.00, 2.00, 1.00, 0.10))
|
||||||
|
valid = list(cm.PRICES) # 13 unique keys (12 real + 1 test-only)
|
||||||
|
raw = valid + ["bogus-extra"] # 13 valid + 1 invalid
|
||||||
|
cfg = parse_repo_config(json.dumps({"compare_against": raw}))
|
||||||
|
assert len(cfg["compare_against"]) == 12
|
||||||
|
assert cfg["compare_against"] == valid[:12]
|
||||||
|
assert "fake-model-13" not in cfg["compare_against"]
|
||||||
|
assert "bogus-extra" not in cfg["compare_against"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Task 6 — merge_confidence + REVIEW_HEADER confidence badge
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_clean_is_five():
|
||||||
|
assert merge_confidence([]) == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_only_low_is_five():
|
||||||
|
f = {"severity": "low"}
|
||||||
|
assert merge_confidence([f, f, f]) == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_medium_drops_one():
|
||||||
|
f = {"severity": "medium"}
|
||||||
|
assert merge_confidence([f]) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_high_drops_two():
|
||||||
|
f = {"severity": "high"}
|
||||||
|
assert merge_confidence([f]) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_critical_drops_to_one():
|
||||||
|
f = {"severity": "critical"}
|
||||||
|
assert merge_confidence([f]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_multi_lens_drops_extra():
|
||||||
|
# The flag has moved to a kwarg; passing `_multi_lens` on the dict is no
|
||||||
|
# longer enough — the kwarg is the only path that drops the score.
|
||||||
|
f = {"severity": "low"}
|
||||||
|
assert merge_confidence([f], multi_lens_observed=True) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_multi_lens_survives_normalization():
|
||||||
|
"""Real flow: `_multi_lens` is set on the raw finding, but stripped by
|
||||||
|
`_normalize_finding`. `merge_confidence(...)` with only the kwarg sees a
|
||||||
|
normalized finding; the dedup must be triggered by `multi_lens_observed=`
|
||||||
|
being true, not by reading `_multi_lens` off the dict."""
|
||||||
|
raw = {"_multi_lens": True, "severity": "low", "path": "x", "line": 1,
|
||||||
|
"problem": "p", "fix": "", "suggestion": "", "reference": ""}
|
||||||
|
normalized = _normalize_finding(raw)
|
||||||
|
assert "_multi_lens" not in normalized # confirms the strip
|
||||||
|
# Now call merge_confidence the way review_pr will:
|
||||||
|
assert merge_confidence([normalized], multi_lens_observed=True) == 4
|
||||||
|
# And without the kwarg, the flag-on-dict path is gone:
|
||||||
|
assert merge_confidence([normalized]) == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_confidence_clamped():
|
||||||
|
# Three critical findings must NOT take the score below 1.
|
||||||
|
f = {"severity": "critical"}
|
||||||
|
assert merge_confidence([f, f, f]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_review_header_includes_confidence():
|
||||||
|
# REVIEW_HEADER gains a {confidence} placeholder; verify the format works.
|
||||||
|
h = REVIEW_HEADER.format(model="glm-5.2:cloud", sha="abc1234567", confidence="3/5 🟡")
|
||||||
|
assert "Merge confidence: 3/5 🟡" in h
|
||||||
|
|
||||||
|
|
||||||
|
def test_confidence_badge_table_complete():
|
||||||
|
# Sanity-check the badge table the render layer reads from.
|
||||||
|
assert _CONFIDENCE_BADGE == {5: "🟢", 4: "🟢", 3: "🟡", 2: "🟠", 1: "🔴"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_default_confidence_is_green():
|
||||||
|
# Default confidence kwarg should produce a green 5/5 badge in the header,
|
||||||
|
# matching the pre-existing "clean PR" semantics.
|
||||||
|
body = format_review_body("- [high] x:1 — bug", "glm-5.2:cloud", "abcdef1234567890")
|
||||||
|
assert "Merge confidence: 5/5 🟢" in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_low_confidence_shows_red_badge():
|
||||||
|
body = format_review_body(
|
||||||
|
"- [critical] x:1 — bug", "glm-5.2:cloud", "abcdef1234567890",
|
||||||
|
confidence=1,
|
||||||
|
)
|
||||||
|
assert "Merge confidence: 1/5 🔴" in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_review_body_confidence_clamps_out_of_range():
|
||||||
|
# Out-of-range confidence is clamped to [1, 5] in the badge string.
|
||||||
|
body_hi = format_review_body("- x", "glm-5.2:cloud", "abcdef1234567890", confidence=99)
|
||||||
|
assert "Merge confidence: 5/5 🟢" in body_hi
|
||||||
|
body_lo = format_review_body("- x", "glm-5.2:cloud", "abcdef1234567890", confidence=0)
|
||||||
|
assert "Merge confidence: 1/5 🔴" in body_lo
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -244,3 +244,39 @@ def test_model_is_within_an_order_of_magnitude_of_the_measurement():
|
|||||||
predicted = cm.tier_usage(modelled, FACTORY, caching=False).total_input
|
predicted = cm.tier_usage(modelled, FACTORY, caching=False).total_input
|
||||||
measured = run["input"]
|
measured = run["input"]
|
||||||
assert 0.4 < predicted / measured < 2.5, (predicted, measured)
|
assert 0.4 < predicted / measured < 2.5, (predicted, measured)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# PRICES — the multi-provider table (GPT / Gemini / Grok)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
NEW_KEYS = ("gpt-5", "gpt-5-mini", "gemini-2.5-pro",
|
||||||
|
"gemini-2.5-flash", "grok-4.5", "grok-4.3")
|
||||||
|
|
||||||
|
|
||||||
|
def test_prices_contains_new_providers():
|
||||||
|
for k in NEW_KEYS:
|
||||||
|
assert k in cm.PRICES, k
|
||||||
|
|
||||||
|
|
||||||
|
def test_cost_matches_published_gpt5():
|
||||||
|
# $1.25 in / $10.00 out / cached $0.125; cache_write = input
|
||||||
|
u = cm.Usage(uncached_input=1_000_000, cached_input=1_000_000,
|
||||||
|
cache_writes=1_000_000, output=1_000_000)
|
||||||
|
assert abs(cm.cost(u, cm.PRICES["gpt-5"]) - (1.25 + 0.125 + 1.25 + 10.00)) < 1e-9
|
||||||
|
|
||||||
|
|
||||||
|
def test_cost_matches_published_gemini_flash():
|
||||||
|
# $0.30 in / $2.50 out / cached $0.03; cache_write = input
|
||||||
|
u = cm.Usage(uncached_input=2_000_000, cached_input=0,
|
||||||
|
cache_writes=0, output=500_000)
|
||||||
|
expected = 2.00 * 0.30 + 0.50 * 2.50 # $0.60 + $1.25
|
||||||
|
assert abs(cm.cost(u, cm.PRICES["gemini-2.5-flash"]) - expected) < 1e-9
|
||||||
|
|
||||||
|
|
||||||
|
def test_cost_matches_published_grok45():
|
||||||
|
# $2.00 in / $6.00 out / cached $0.30; cache_write = input
|
||||||
|
u = cm.Usage(uncached_input=1_000_000, cached_input=1_000_000,
|
||||||
|
cache_writes=1_000_000, output=1_000_000)
|
||||||
|
assert abs(cm.cost(u, cm.PRICES["grok-4.5"]) - (2.00 + 0.30 + 2.00 + 6.00)) < 1e-9
|
||||||
|
|||||||
@@ -0,0 +1,203 @@
|
|||||||
|
"""Tests for pilot/dashboard.py — stdlib HTTP server rendering dashboard HTML.
|
||||||
|
|
||||||
|
We spin up the server on an ephemeral port in setUp, drive it with
|
||||||
|
http.client, and tear it down in tearDown. Auth is now performed by
|
||||||
|
oauth2-proxy: the dashboard trusts `X-Forwarded-User` set by the proxy
|
||||||
|
and returns 401 (with a Basic challenge) when the header is missing.
|
||||||
|
|
||||||
|
The dashboard reads `PRAGENT_FEEDBACK_DB` and renders views via
|
||||||
|
`dashboard_data`. We seed an in-memory SQLite at `tmp_path` for each
|
||||||
|
scenario that needs rows.
|
||||||
|
"""
|
||||||
|
import http.client
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||||
|
sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
||||||
|
|
||||||
|
import dashboard as dash # noqa: E402
|
||||||
|
from pilot import feedback # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _free_port() -> int:
|
||||||
|
s = socket.socket()
|
||||||
|
s.bind(("127.0.0.1", 0))
|
||||||
|
port = s.getsockname()[1]
|
||||||
|
s.close()
|
||||||
|
return port
|
||||||
|
|
||||||
|
|
||||||
|
class _ServerThread:
|
||||||
|
def __init__(self, port: int, handler):
|
||||||
|
self.server = handler((host := "127.0.0.1", port), None)
|
||||||
|
self.port = port
|
||||||
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.server.shutdown()
|
||||||
|
self.server.server_close()
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
|
||||||
|
|
||||||
|
def _get(port: int, path: str, headers: dict | None = None) -> tuple[int, dict, bytes]:
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
|
||||||
|
conn.request("GET", path, headers=headers or {})
|
||||||
|
r = conn.getresponse()
|
||||||
|
body = r.read()
|
||||||
|
h = dict(r.getheaders())
|
||||||
|
conn.close()
|
||||||
|
return r.status, h, body
|
||||||
|
|
||||||
|
|
||||||
|
def _post(port: int, path: str, body: bytes, headers: dict | None = None) -> tuple[int, dict, bytes]:
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
|
||||||
|
hdrs = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||||
|
if headers:
|
||||||
|
hdrs.update(headers)
|
||||||
|
conn.request("POST", path, body=body, headers=hdrs)
|
||||||
|
r = conn.getresponse()
|
||||||
|
body_b = r.read()
|
||||||
|
h = dict(r.getheaders())
|
||||||
|
conn.close()
|
||||||
|
return r.status, h, body_b
|
||||||
|
|
||||||
|
|
||||||
|
class TestDashboardAuth(unittest.TestCase):
|
||||||
|
"""Auth gate: require X-Forwarded-User (set by oauth2-proxy).
|
||||||
|
|
||||||
|
When the header is missing every non-static route returns 401 with a
|
||||||
|
Basic challenge, which lets oauth2-proxy redirect the browser to
|
||||||
|
Logto. Static is exempt so the unauthenticated probe traffic doesn't
|
||||||
|
loop the proxy through the auth flow.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
os.environ["PRAGENT_FEEDBACK_DB"] = self.db
|
||||||
|
os.environ["DASHBOARD_PORT"] = str(0) # we override below
|
||||||
|
|
||||||
|
self.port = _free_port()
|
||||||
|
from http.server import ThreadingHTTPServer
|
||||||
|
self.srv = ThreadingHTTPServer(("127.0.0.1", self.port), dash.Handler)
|
||||||
|
self.thread = threading.Thread(target=self.srv.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.srv.shutdown()
|
||||||
|
self.srv.server_close()
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
for k in ("PRAGENT_FEEDBACK_DB", "DASHBOARD_PORT"):
|
||||||
|
os.environ.pop(k, None)
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_anonymous_overview_returns_401_with_basic_challenge(self):
|
||||||
|
status, h, body = _get(self.port, "/")
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
self.assertEqual(h.get("WWW-Authenticate"), 'Basic realm="pragent-dashboard"')
|
||||||
|
self.assertEqual(body, b"unauthorized\n")
|
||||||
|
|
||||||
|
def test_anonymous_repo_returns_401(self):
|
||||||
|
status, _h, _body = _get(self.port, "/r/alpha/one")
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
|
||||||
|
def test_anonymous_post_returns_401(self):
|
||||||
|
status, _h, _body = _post(self.port, "/r/alpha/one/edit", b"x=1")
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
|
||||||
|
def test_authenticated_overview_succeeds(self):
|
||||||
|
status, h, body = _get(self.port, "/", headers={"X-Forwarded-User": "marcos@example.com"})
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertIn(b"Overview", body)
|
||||||
|
|
||||||
|
def test_static_does_not_require_auth(self):
|
||||||
|
status, h, body = _get(self.port, "/static/style.css")
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertIn("text/css", h.get("Content-Type", ""))
|
||||||
|
self.assertGreater(len(body), 50)
|
||||||
|
|
||||||
|
def test_empty_x_forwarded_user_treated_as_anonymous(self):
|
||||||
|
status, _h, _body = _get(self.port, "/", headers={"X-Forwarded-User": " "})
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDashboardRender(unittest.TestCase):
|
||||||
|
"""Render-only tests — X-Forwarded-User set, real seeded data."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
os.environ["PRAGENT_FEEDBACK_DB"] = self.db
|
||||||
|
# Seed: 2 repos, a couple of reviews + findings each.
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
for repo, n_prs in (("alpha/one", 2), ("beta/two", 1)):
|
||||||
|
for n in range(n_prs):
|
||||||
|
rid = feedback.record_review(
|
||||||
|
conn, repo=repo, pr=n + 1, head_sha=f"sha{repo}-{n}",
|
||||||
|
review_id_gitea=1000 + n, body_comment_id=2000 + n,
|
||||||
|
posted_at=int(time.time()) - n * 60,
|
||||||
|
)
|
||||||
|
for k in range(3):
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
conn, review_id=rid, repo=repo, pr=n + 1,
|
||||||
|
path=f"src/file_{k}.py", line=k + 1,
|
||||||
|
severity=["critical", "high", "medium"][k],
|
||||||
|
problem=f"problem {k}",
|
||||||
|
fix=f"fix {k}", suggestion=f"suggestion {k}",
|
||||||
|
comment_id=3000 + n * 10 + k,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
self.port = _free_port()
|
||||||
|
from http.server import ThreadingHTTPServer
|
||||||
|
self.srv = ThreadingHTTPServer(("127.0.0.1", self.port), dash.Handler)
|
||||||
|
self.thread = threading.Thread(target=self.srv.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
self.auth_hdr = {"X-Forwarded-User": "marcos@example.com"}
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.srv.shutdown()
|
||||||
|
self.srv.server_close()
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
os.environ.pop("PRAGENT_FEEDBACK_DB", None)
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_overview_200_contains_repo_names(self):
|
||||||
|
status, _h, body = _get(self.port, "/", headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
text = body.decode()
|
||||||
|
self.assertIn("Overview", text)
|
||||||
|
self.assertIn("alpha/one", text)
|
||||||
|
self.assertIn("beta/two", text)
|
||||||
|
|
||||||
|
def test_repo_page_200(self):
|
||||||
|
status, _h, body = _get(self.port, "/r/alpha/one", headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
text = body.decode()
|
||||||
|
self.assertIn("alpha/one", text)
|
||||||
|
# The findings table should appear.
|
||||||
|
self.assertIn("src/file_0.py", text)
|
||||||
|
|
||||||
|
def test_pr_page_200(self):
|
||||||
|
status, _h, body = _get(self.port, "/r/alpha/one/1", headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
text = body.decode()
|
||||||
|
self.assertIn("alpha/one", text)
|
||||||
|
self.assertIn("#1", text)
|
||||||
|
self.assertIn("src/file_0.py", text)
|
||||||
|
|
||||||
|
def test_unknown_route_404(self):
|
||||||
|
status, _h, _body = _get(self.port, "/no/such/route", headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 404)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
"""Tests for pilot/dashboard_data.py — read-only query layer over the feedback SQLite.
|
||||||
|
|
||||||
|
Covers: empty-DB fallbacks (no crash on missing/empty DB), overview rollups,
|
||||||
|
per-repo drill-down (findings by severity, top findings, runs by day), and
|
||||||
|
the per-PR view. The dashboard never writes — only reads.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||||
|
sys.path.insert(0, os.path.join(HERE, "..", "..")) # so `from pilot import …` works
|
||||||
|
|
||||||
|
from pilot import dashboard_data, feedback
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_repo(conn, *, repo: str, prs: int, findings_per_pr: int, day_offset: int = 0):
|
||||||
|
"""Seed one repo with `prs` PRs each with `findings_per_pr` findings.
|
||||||
|
|
||||||
|
All timestamps cluster on (now - day_offset days). Returns list of review ids.
|
||||||
|
"""
|
||||||
|
base = int(time.time()) - day_offset * 86400
|
||||||
|
rids = []
|
||||||
|
for n in range(prs):
|
||||||
|
rid = feedback.record_review(
|
||||||
|
conn, repo=repo, pr=n + 1, head_sha=f"sha{n}",
|
||||||
|
review_id_gitea=1000 + n, body_comment_id=2000 + n,
|
||||||
|
posted_at=base + n * 60,
|
||||||
|
)
|
||||||
|
rids.append(rid)
|
||||||
|
for k in range(findings_per_pr):
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
conn, review_id=rid, repo=repo, pr=n + 1,
|
||||||
|
path=f"src/file_{k}.py", line=k + 1,
|
||||||
|
severity=["critical", "high", "medium", "low"][k % 4],
|
||||||
|
problem=f"problem {k}",
|
||||||
|
fix=f"fix {k}", suggestion=f"suggestion {k}",
|
||||||
|
comment_id=3000 + n * 10 + k,
|
||||||
|
posted_at=base + n * 60,
|
||||||
|
)
|
||||||
|
return rids
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmptyDB(unittest.TestCase):
|
||||||
|
def test_missing_file_returns_zero_dict(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
missing = f"{d}/nope.db"
|
||||||
|
data = dashboard_data.overview(missing)
|
||||||
|
self.assertEqual(data["total_reviews"], 0)
|
||||||
|
self.assertEqual(data["total_findings"], 0)
|
||||||
|
self.assertEqual(data["total_repos"], 0)
|
||||||
|
self.assertEqual(data["last_30d_reviews"], 0)
|
||||||
|
self.assertEqual(len(data["daily"]), 7)
|
||||||
|
self.assertEqual(data["top_repos"], [])
|
||||||
|
self.assertEqual(data["total_cost_usd"], 0.0)
|
||||||
|
|
||||||
|
def test_missing_file_repo_summary_safe(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
data = dashboard_data.repo_summary(f"{d}/nope.db", "o/r")
|
||||||
|
self.assertEqual(data["repo"], "o/r")
|
||||||
|
self.assertEqual(data["total_runs"], 0)
|
||||||
|
self.assertEqual(data["runs_by_day"], [])
|
||||||
|
for sev in ("critical", "high", "medium", "low"):
|
||||||
|
self.assertEqual(data["findings_by_severity"][sev], 0)
|
||||||
|
self.assertEqual(data["top_findings"], [])
|
||||||
|
self.assertEqual(data["models_used"], [])
|
||||||
|
|
||||||
|
def test_missing_file_pr_summary_safe(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
data = dashboard_data.pr_summary(f"{d}/nope.db", "o/r", 1)
|
||||||
|
self.assertEqual(data["repo"], "o/r")
|
||||||
|
self.assertEqual(data["pr"], 1)
|
||||||
|
self.assertEqual(data["findings"], [])
|
||||||
|
self.assertEqual(data["usage"], {})
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmptyButExistingDB(unittest.TestCase):
|
||||||
|
"""`init` creates the schema — DB exists but has no rows."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
feedback.init(self.db)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_overview_is_zero(self):
|
||||||
|
data = dashboard_data.overview(self.db)
|
||||||
|
self.assertEqual(data["total_reviews"], 0)
|
||||||
|
self.assertEqual(data["total_findings"], 0)
|
||||||
|
self.assertEqual(data["total_repos"], 0)
|
||||||
|
|
||||||
|
def test_repo_summary_is_zero(self):
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
self.assertEqual(data["total_runs"], 0)
|
||||||
|
self.assertEqual(data["findings_by_severity"], {"critical": 0, "high": 0, "medium": 0, "low": 0})
|
||||||
|
|
||||||
|
def test_pr_summary_is_zero(self):
|
||||||
|
data = dashboard_data.pr_summary(self.db, "o/r", 1)
|
||||||
|
self.assertEqual(data["findings"], [])
|
||||||
|
|
||||||
|
|
||||||
|
class TestOverview(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
_seed_repo(self.conn, repo="alpha/one", prs=3, findings_per_pr=2)
|
||||||
|
_seed_repo(self.conn, repo="beta/two", prs=1, findings_per_pr=4)
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_totals(self):
|
||||||
|
data = dashboard_data.overview(self.db)
|
||||||
|
self.assertEqual(data["total_reviews"], 4)
|
||||||
|
self.assertEqual(data["total_findings"], 6 + 4) # 3*2 + 1*4 = 10
|
||||||
|
self.assertEqual(data["total_repos"], 2)
|
||||||
|
self.assertEqual(data["total_cost_usd"], 0.0)
|
||||||
|
|
||||||
|
def test_top_repos_sorted_by_run_count(self):
|
||||||
|
data = dashboard_data.overview(self.db)
|
||||||
|
repos = [r["repo"] for r in data["top_repos"]]
|
||||||
|
# alpha/one has 3 runs, beta/two has 1.
|
||||||
|
self.assertEqual(repos[0], "alpha/one")
|
||||||
|
self.assertEqual(data["top_repos"][0]["run_count"], 3)
|
||||||
|
self.assertEqual(data["top_repos"][1]["run_count"], 1)
|
||||||
|
# last_seen is a unix timestamp int.
|
||||||
|
for r in data["top_repos"]:
|
||||||
|
self.assertIsInstance(r["last_seen"], int)
|
||||||
|
|
||||||
|
def test_daily_buckets_are_7(self):
|
||||||
|
data = dashboard_data.overview(self.db)
|
||||||
|
self.assertEqual(len(data["daily"]), 7)
|
||||||
|
for b in data["daily"]:
|
||||||
|
self.assertIn("date", b)
|
||||||
|
self.assertIn("count", b)
|
||||||
|
|
||||||
|
def test_last_30d_reviews(self):
|
||||||
|
data = dashboard_data.overview(self.db)
|
||||||
|
self.assertEqual(data["last_30d_reviews"], 4)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRepoSummary(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
# 4 PRs with 2 findings each → 8 findings, severity cycle [c,h,m,l,c,h,m,l]
|
||||||
|
_seed_repo(self.conn, repo="o/r", prs=4, findings_per_pr=2)
|
||||||
|
# Add some reactions so top_findings has signal.
|
||||||
|
rows = self.conn.execute(
|
||||||
|
"SELECT id, comment_id FROM inline_finding WHERE repo=? ORDER BY id LIMIT 3",
|
||||||
|
("o/r",),
|
||||||
|
).fetchall()
|
||||||
|
for r in rows:
|
||||||
|
feedback.record_reaction(self.conn, comment_id=r["comment_id"], user="u", content="+1")
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_basic_shape(self):
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
self.assertEqual(data["repo"], "o/r")
|
||||||
|
self.assertEqual(data["total_runs"], 4)
|
||||||
|
self.assertIsInstance(data["last_run_ts"], int)
|
||||||
|
|
||||||
|
def test_findings_by_severity(self):
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
fbs = data["findings_by_severity"]
|
||||||
|
# 4 PRs × 2 findings; per-PR severities are [critical, high].
|
||||||
|
# (k in range(2) → k=0 critical, k=1 high for every PR.)
|
||||||
|
self.assertEqual(fbs["critical"], 4)
|
||||||
|
self.assertEqual(fbs["high"], 4)
|
||||||
|
self.assertEqual(fbs["medium"], 0)
|
||||||
|
self.assertEqual(fbs["low"], 0)
|
||||||
|
|
||||||
|
def test_runs_by_day_is_list(self):
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
self.assertIsInstance(data["runs_by_day"], list)
|
||||||
|
for r in data["runs_by_day"]:
|
||||||
|
self.assertIn("date", r)
|
||||||
|
self.assertIn("count", r)
|
||||||
|
|
||||||
|
def test_top_findings_structure(self):
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
self.assertGreater(len(data["top_findings"]), 0)
|
||||||
|
first = data["top_findings"][0]
|
||||||
|
for k in ("path", "line", "severity", "problem", "occurrences", "upvotes", "downvotes", "resolved", "reply_count"):
|
||||||
|
self.assertIn(k, first)
|
||||||
|
|
||||||
|
def test_models_used_is_empty_list_with_note(self):
|
||||||
|
# The schema has no `model` column on review — the dashboard can't show
|
||||||
|
# model usage from this DB today. We document that via an empty list.
|
||||||
|
data = dashboard_data.repo_summary(self.db, "o/r")
|
||||||
|
self.assertEqual(data["models_used"], [])
|
||||||
|
|
||||||
|
|
||||||
|
class TestPRSummary(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
rid = feedback.record_review(
|
||||||
|
self.conn, repo="o/r", pr=42, head_sha="abc",
|
||||||
|
review_id_gitea=9001, body_comment_id=8001,
|
||||||
|
posted_at=1700000000,
|
||||||
|
)
|
||||||
|
for k in range(3):
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=42,
|
||||||
|
path=f"src/x_{k}.py", line=k + 10,
|
||||||
|
severity=["critical", "high", "low"][k],
|
||||||
|
problem=f"p{k}", fix=f"f{k}", suggestion=f"s{k}",
|
||||||
|
comment_id=7000 + k,
|
||||||
|
)
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_meta(self):
|
||||||
|
data = dashboard_data.pr_summary(self.db, "o/r", 42)
|
||||||
|
self.assertEqual(data["repo"], "o/r")
|
||||||
|
self.assertEqual(data["pr"], 42)
|
||||||
|
self.assertEqual(data["head_sha"], "abc")
|
||||||
|
self.assertEqual(data["review_id_gitea"], 9001)
|
||||||
|
self.assertEqual(data["body_comment_id"], 8001)
|
||||||
|
self.assertEqual(data["posted_at"], 1700000000)
|
||||||
|
# usage is empty because the schema has no usage column.
|
||||||
|
self.assertEqual(data["usage"], {})
|
||||||
|
|
||||||
|
def test_findings(self):
|
||||||
|
data = dashboard_data.pr_summary(self.db, "o/r", 42)
|
||||||
|
self.assertEqual(len(data["findings"]), 3)
|
||||||
|
for f in data["findings"]:
|
||||||
|
for k in ("path", "line", "severity", "problem", "fix", "suggestion",
|
||||||
|
"upvotes", "downvotes", "resolved", "reply_count"):
|
||||||
|
self.assertIn(k, f)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""Tests for the edit endpoint — POST /r/<owner>/<name>/edit (Task C).
|
||||||
|
|
||||||
|
We mock the Gitea HTTP layer (urllib.request.urlopen) so the test never
|
||||||
|
touches the network. The dashboard handler is responsible for:
|
||||||
|
* auth (X-Forwarded-User set by oauth2-proxy) + CSRF
|
||||||
|
* read .pr-review.json via GET (404 → start from {})
|
||||||
|
* validate model against cost_model.PRICES
|
||||||
|
* PUT the updated file back, with sha + base64 content
|
||||||
|
* redirect to /r/<owner>/<name> on success
|
||||||
|
"""
|
||||||
|
import base64
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||||
|
sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
||||||
|
|
||||||
|
import dashboard as dash # noqa: E402
|
||||||
|
from pilot import feedback # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _free_port() -> int:
|
||||||
|
s = socket.socket()
|
||||||
|
s.bind(("127.0.0.1", 0))
|
||||||
|
port = s.getsockname()[1]
|
||||||
|
s.close()
|
||||||
|
return port
|
||||||
|
|
||||||
|
|
||||||
|
def _post(port: int, path: str, body: bytes, *, headers: dict | None = None) -> tuple[int, dict, bytes]:
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
|
||||||
|
hdrs = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||||
|
if headers:
|
||||||
|
hdrs.update(headers)
|
||||||
|
conn.request("POST", path, body=body, headers=hdrs)
|
||||||
|
r = conn.getresponse()
|
||||||
|
body_b = r.read()
|
||||||
|
h = dict(r.getheaders())
|
||||||
|
conn.close()
|
||||||
|
return r.status, h, body_b
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResp:
|
||||||
|
def __init__(self, status: int, body: bytes):
|
||||||
|
self.status = status
|
||||||
|
self._body = body
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
return self._body
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class TestDashboardEdit(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
os.environ["PRAGENT_FEEDBACK_DB"] = self.db
|
||||||
|
os.environ["PRAGENT_BOT_TOKEN"] = "bot-token"
|
||||||
|
# Seed a row so the repo page is meaningful.
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
feedback.record_review(
|
||||||
|
conn, repo="o/r", pr=1, head_sha="x",
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
self.port = _free_port()
|
||||||
|
from http.server import ThreadingHTTPServer
|
||||||
|
self.srv = ThreadingHTTPServer(("127.0.0.1", self.port), dash.Handler)
|
||||||
|
self.thread = threading.Thread(target=self.srv.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
# Pull the per-process CSRF secret from the rendered repo page — the
|
||||||
|
# edit form embeds the same token as a hidden input.
|
||||||
|
self.csrf = dash._CSRF_SECRET
|
||||||
|
self.auth_hdr = {"X-Forwarded-User": "marcos@example.com"}
|
||||||
|
|
||||||
|
# Records of HTTP calls made by the handler.
|
||||||
|
self.calls: list[tuple[str, str, dict | None, bytes | None]] = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.srv.shutdown()
|
||||||
|
self.srv.server_close()
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
for k in ("PRAGENT_FEEDBACK_DB", "PRAGENT_BOT_TOKEN"):
|
||||||
|
os.environ.pop(k, None)
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _urlopen(self, req, timeout=30):
|
||||||
|
"""Replacement for urllib.request.urlopen that the handler uses."""
|
||||||
|
url = req.full_url if hasattr(req, "full_url") else req
|
||||||
|
method = getattr(req, "method", None) or "GET"
|
||||||
|
body = getattr(req, "data", None)
|
||||||
|
headers = dict(getattr(req, "headers", {}) or {})
|
||||||
|
self.calls.append((method, url, headers, body))
|
||||||
|
# Route based on URL: GET contents/.../raw vs PUT contents/.pr-review.json
|
||||||
|
if method == "GET" and ".pr-review.json" in url:
|
||||||
|
return _FakeResp(200, json.dumps({
|
||||||
|
"content": base64.b64encode(b'{"focus":["x"],"model":"claude-haiku-4-5"}').decode(),
|
||||||
|
"sha": "deadbeef",
|
||||||
|
}).encode())
|
||||||
|
if method == "PUT" and ".pr-review.json" in url:
|
||||||
|
return _FakeResp(200, b'{}')
|
||||||
|
return _FakeResp(404, b'{"message":"not found"}')
|
||||||
|
|
||||||
|
def test_edit_updates_static_message_and_model(self):
|
||||||
|
form = (
|
||||||
|
f"_csrf={self.csrf}"
|
||||||
|
f"&static_message=Hello%20world"
|
||||||
|
f"&model=claude-sonnet-5"
|
||||||
|
).encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=self._urlopen):
|
||||||
|
status, h, _b = _post(self.port, "/r/o/r/edit", form, headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 302)
|
||||||
|
self.assertEqual(h.get("Location"), "/r/o/r")
|
||||||
|
|
||||||
|
# Find the PUT call.
|
||||||
|
put_calls = [c for c in self.calls if c[0] == "PUT"]
|
||||||
|
self.assertEqual(len(put_calls), 1, self.calls)
|
||||||
|
method, url, _hdrs, body = put_calls[0]
|
||||||
|
self.assertIn(".pr-review.json", url)
|
||||||
|
payload = json.loads(body)
|
||||||
|
self.assertIn("content", payload)
|
||||||
|
self.assertEqual(payload["sha"], "deadbeef")
|
||||||
|
decoded = base64.b64decode(payload["content"]).decode()
|
||||||
|
cfg = json.loads(decoded)
|
||||||
|
self.assertEqual(cfg.get("static_message"), "Hello world")
|
||||||
|
self.assertEqual(cfg.get("model"), "claude-sonnet-5")
|
||||||
|
|
||||||
|
def test_edit_strips_static_message_to_400(self):
|
||||||
|
long_msg = "x" * 600
|
||||||
|
form = (
|
||||||
|
f"_csrf={self.csrf}"
|
||||||
|
f"&static_message={long_msg}"
|
||||||
|
f"&model=claude-haiku-4-5"
|
||||||
|
).encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=self._urlopen):
|
||||||
|
_post(self.port, "/r/o/r/edit", form, headers=self.auth_hdr)
|
||||||
|
put = next(c for c in self.calls if c[0] == "PUT")
|
||||||
|
cfg = json.loads(base64.b64decode(json.loads(put[3])["content"]))
|
||||||
|
self.assertEqual(len(cfg["static_message"]), 400)
|
||||||
|
|
||||||
|
def test_edit_rejects_unknown_model_with_flash(self):
|
||||||
|
form = (
|
||||||
|
f"_csrf={self.csrf}"
|
||||||
|
f"&static_message=hi"
|
||||||
|
f"&model=does-not-exist"
|
||||||
|
).encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=self._urlopen):
|
||||||
|
status, h, _b = _post(self.port, "/r/o/r/edit", form, headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 302)
|
||||||
|
self.assertIn("flash=", h.get("Location", ""))
|
||||||
|
# No PUT should have been issued.
|
||||||
|
put_calls = [c for c in self.calls if c[0] == "PUT"]
|
||||||
|
self.assertEqual(put_calls, [])
|
||||||
|
|
||||||
|
def test_edit_requires_auth(self):
|
||||||
|
form = f"_csrf={self.csrf}&static_message=x&model=claude-haiku-4-5".encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=self._urlopen):
|
||||||
|
status, h, _b = _post(self.port, "/r/o/r/edit", form)
|
||||||
|
self.assertEqual(status, 401)
|
||||||
|
self.assertEqual(h.get("WWW-Authenticate"), 'Basic realm="pragent-dashboard"')
|
||||||
|
# No Gitea calls at all — auth gate fires first.
|
||||||
|
self.assertEqual(self.calls, [])
|
||||||
|
|
||||||
|
def test_edit_csrf_mismatch_redirects_without_save(self):
|
||||||
|
form = f"_csrf=wrong&static_message=x&model=claude-haiku-4-5".encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=self._urlopen):
|
||||||
|
status, h, _b = _post(self.port, "/r/o/r/edit", form, headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 302)
|
||||||
|
self.assertEqual(h.get("Location"), "/r/o/r")
|
||||||
|
put_calls = [c for c in self.calls if c[0] == "PUT"]
|
||||||
|
self.assertEqual(put_calls, [])
|
||||||
|
|
||||||
|
def test_edit_creates_file_when_missing(self):
|
||||||
|
"""When GET returns 404, the PUT must still happen (no sha)."""
|
||||||
|
def _route(req, timeout=30):
|
||||||
|
url = req.full_url
|
||||||
|
method = getattr(req, "method", None) or "GET"
|
||||||
|
body = getattr(req, "data", None)
|
||||||
|
self.calls.append((method, url, {}, body))
|
||||||
|
if method == "GET" and ".pr-review.json" in url:
|
||||||
|
return _FakeResp(404, b'{"message":"not found"}')
|
||||||
|
if method == "PUT" and ".pr-review.json" in url:
|
||||||
|
return _FakeResp(201, b"{}")
|
||||||
|
return _FakeResp(404, b"")
|
||||||
|
|
||||||
|
form = f"_csrf={self.csrf}&static_message=hi&model=claude-haiku-4-5".encode()
|
||||||
|
with patch.object(dash.urllib.request, "urlopen", side_effect=_route):
|
||||||
|
status, h, _b = _post(self.port, "/r/o/r/edit", form, headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 302)
|
||||||
|
put = next(c for c in self.calls if c[0] == "PUT")
|
||||||
|
payload = json.loads(put[3])
|
||||||
|
self.assertNotIn("sha", payload, "missing-file PUT should omit sha")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Tests for the model <select> in the repo edit form (Task D)."""
|
||||||
|
import http.client
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||||
|
sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
||||||
|
|
||||||
|
import dashboard as dash # noqa: E402
|
||||||
|
from pilot import cost_model, feedback # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _free_port() -> int:
|
||||||
|
s = socket.socket()
|
||||||
|
s.bind(("127.0.0.1", 0))
|
||||||
|
port = s.getsockname()[1]
|
||||||
|
s.close()
|
||||||
|
return port
|
||||||
|
|
||||||
|
|
||||||
|
def _get(port: int, path: str, headers: dict | None = None) -> tuple[int, dict, bytes]:
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
|
||||||
|
conn.request("GET", path, headers=headers or {})
|
||||||
|
r = conn.getresponse()
|
||||||
|
body = r.read()
|
||||||
|
h = dict(r.getheaders())
|
||||||
|
conn.close()
|
||||||
|
return r.status, h, body
|
||||||
|
|
||||||
|
|
||||||
|
class TestRepoEditSelect(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
os.environ["PRAGENT_FEEDBACK_DB"] = self.db
|
||||||
|
os.environ["PRAGENT_BOT_TOKEN"] = ""
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
feedback.record_review(conn, repo="o/r", pr=1, head_sha="x")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
self.port = _free_port()
|
||||||
|
from http.server import ThreadingHTTPServer
|
||||||
|
self.srv = ThreadingHTTPServer(("127.0.0.1", self.port), dash.Handler)
|
||||||
|
self.thread = threading.Thread(target=self.srv.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
self.auth_hdr = {"X-Forwarded-User": "marcos@example.com"}
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.srv.shutdown()
|
||||||
|
self.srv.server_close()
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
for k in ("PRAGENT_FEEDBACK_DB", "PRAGENT_BOT_TOKEN"):
|
||||||
|
os.environ.pop(k, None)
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_repo_page_renders_select_with_one_option_per_price(self):
|
||||||
|
status, _h, body = _get(self.port, "/r/o/r", headers=self.auth_hdr)
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
text = body.decode()
|
||||||
|
self.assertIn('<select id="model" name="model">', text)
|
||||||
|
# Every PRICES key should appear as an <option value="…">.
|
||||||
|
for k in sorted(cost_model.PRICES):
|
||||||
|
self.assertIn(f'<option value="{k}"', text, f"missing {k} in select")
|
||||||
|
# Plus the "keep current" placeholder.
|
||||||
|
self.assertIn("— keep current", text)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
"""Tests for pilot/feedback.py — SQLite storage for review feedback signals.
|
||||||
|
|
||||||
|
Covers: schema bootstrap, posthash stability, dedup-on-insert, reaction /
|
||||||
|
thread-state / reply upserts, the analyzer-side `findings_with_votes` join,
|
||||||
|
and graceful failure on bad inputs.
|
||||||
|
"""
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from pilot import feedback
|
||||||
|
|
||||||
|
|
||||||
|
class TestPosthash(unittest.TestCase):
|
||||||
|
def test_stable_across_calls(self):
|
||||||
|
a = feedback.posthash("src/api/foo.ts", 42, "HIGH", "Race condition in handler")
|
||||||
|
b = feedback.posthash("src/api/foo.ts", 42, "HIGH", "Race condition in handler")
|
||||||
|
self.assertEqual(a, b)
|
||||||
|
|
||||||
|
def test_length_is_short(self):
|
||||||
|
h = feedback.posthash("a", 1, "low", "x")
|
||||||
|
self.assertEqual(len(h), 16)
|
||||||
|
|
||||||
|
def test_different_path_different_hash(self):
|
||||||
|
self.assertNotEqual(
|
||||||
|
feedback.posthash("a", 1, "LOW", "x"),
|
||||||
|
feedback.posthash("b", 1, "LOW", "x"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_different_line_different_hash(self):
|
||||||
|
self.assertNotEqual(
|
||||||
|
feedback.posthash("a", 1, "LOW", "x"),
|
||||||
|
feedback.posthash("a", 2, "LOW", "x"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_different_severity_different_hash(self):
|
||||||
|
# Same line, same problem, different severity → different signal.
|
||||||
|
self.assertNotEqual(
|
||||||
|
feedback.posthash("a", 1, "LOW", "x"),
|
||||||
|
feedback.posthash("a", 1, "CRITICAL", "x"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_problem_prefix_used_only(self):
|
||||||
|
# First 80 chars participate; rest is ignored.
|
||||||
|
self.assertEqual(
|
||||||
|
feedback.posthash("a", 1, "LOW", "x" * 80 + "tail"),
|
||||||
|
feedback.posthash("a", 1, "LOW", "x" * 80),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_case_and_whitespace_normalized_in_problem(self):
|
||||||
|
# Lowercased + stripped → same hash.
|
||||||
|
self.assertEqual(
|
||||||
|
feedback.posthash("a", 1, "LOW", " Same Finding "),
|
||||||
|
feedback.posthash("a", 1, "LOW", "same finding"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestInit(unittest.TestCase):
|
||||||
|
def test_init_creates_db(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
db = f"{d}/f.db"
|
||||||
|
conn = feedback.init(db)
|
||||||
|
# Application tables exist (sqlite_sequence is a bookkeeping table
|
||||||
|
# created by AUTOINCREMENT — not part of the contract).
|
||||||
|
tables = {r[0] for r in conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
).fetchall()}
|
||||||
|
self.assertTrue(
|
||||||
|
{"review", "inline_finding", "reaction", "thread_state", "reply"}.issubset(tables),
|
||||||
|
f"missing tables: got {tables}",
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_init_is_idempotent(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
db = f"{d}/f.db"
|
||||||
|
feedback.init(db)
|
||||||
|
# Second call must not raise.
|
||||||
|
feedback.init(db)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecordReview(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_returns_id_and_row(self):
|
||||||
|
rid = feedback.record_review(
|
||||||
|
self.conn, repo="o/r", pr=1, head_sha="abc",
|
||||||
|
review_id_gitea=99, body_comment_id=42,
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(rid)
|
||||||
|
row = self.conn.execute("SELECT * FROM review WHERE id = ?", (rid,)).fetchone()
|
||||||
|
self.assertEqual(row[1], "o/r")
|
||||||
|
self.assertEqual(row[4], 99)
|
||||||
|
self.assertEqual(row[5], 42)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecordInlineFinding(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _new_review(self):
|
||||||
|
return feedback.record_review(
|
||||||
|
self.conn, repo="o/r", pr=1, head_sha="x",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_insert_returns_id(self):
|
||||||
|
rid = self._new_review()
|
||||||
|
fid = feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a/b.ts", line=10, severity="HIGH",
|
||||||
|
problem="bug", fix="patch", suggestion="code",
|
||||||
|
comment_id=555,
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(fid)
|
||||||
|
|
||||||
|
def test_dedup_on_posthash(self):
|
||||||
|
# Two reviews of the SAME finding on different PRs insert two
|
||||||
|
# rows — deduplication by posthash is the *analyzer's* job
|
||||||
|
# (findings_with_votes GROUP BY posthash). Storing one row per
|
||||||
|
# review preserves per-comment reactions across PRs.
|
||||||
|
rid1 = self._new_review()
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid1, repo="o/r", pr=1,
|
||||||
|
path="a/b.ts", line=10, severity="HIGH", problem="race",
|
||||||
|
comment_id=100,
|
||||||
|
)
|
||||||
|
rid2 = feedback.record_review(self.conn, repo="o/r", pr=2, head_sha="y")
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid2, repo="o/r", pr=2,
|
||||||
|
path="a/b.ts", line=10, severity="HIGH", problem="race",
|
||||||
|
comment_id=200,
|
||||||
|
)
|
||||||
|
rows = self.conn.execute(
|
||||||
|
"SELECT id, comment_id FROM inline_finding WHERE path='a/b.ts' AND line=10 ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
self.assertEqual(len(rows), 2)
|
||||||
|
# Both comment_ids preserved (PK dedup is the *reaction* table's job).
|
||||||
|
self.assertEqual([r[1] for r in rows], [100, 200])
|
||||||
|
|
||||||
|
def test_posthash_set(self):
|
||||||
|
rid = self._new_review()
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a", line=1, severity="LOW", problem="nit",
|
||||||
|
)
|
||||||
|
ph = self.conn.execute(
|
||||||
|
"SELECT posthash FROM inline_finding LIMIT 1"
|
||||||
|
).fetchone()[0]
|
||||||
|
expected = feedback.posthash("a", 1, "LOW", "nit")
|
||||||
|
self.assertEqual(ph, expected)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecordReaction(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_insert_upsert(self):
|
||||||
|
ok = feedback.record_reaction(
|
||||||
|
self.conn, comment_id=10, user="alice", content="+1",
|
||||||
|
)
|
||||||
|
self.assertTrue(ok)
|
||||||
|
n = self.conn.execute("SELECT COUNT(*) FROM reaction").fetchone()[0]
|
||||||
|
self.assertEqual(n, 1)
|
||||||
|
# Re-insert same PK → no duplicate.
|
||||||
|
feedback.record_reaction(self.conn, comment_id=10, user="alice", content="+1")
|
||||||
|
n = self.conn.execute("SELECT COUNT(*) FROM reaction").fetchone()[0]
|
||||||
|
self.assertEqual(n, 1)
|
||||||
|
|
||||||
|
def test_distinct_users_can_react(self):
|
||||||
|
feedback.record_reaction(self.conn, comment_id=10, user="a", content="+1")
|
||||||
|
feedback.record_reaction(self.conn, comment_id=10, user="b", content="-1")
|
||||||
|
n = self.conn.execute("SELECT COUNT(*) FROM reaction").fetchone()[0]
|
||||||
|
self.assertEqual(n, 2)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecordThreadState(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
rid = feedback.record_review(self.conn, repo="o/r", pr=1, head_sha="x")
|
||||||
|
self.fid = feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a", line=1, severity="LOW", problem="x",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_upsert_overwrites(self):
|
||||||
|
feedback.record_thread_state(self.conn, finding_id=self.fid, resolved=True)
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT resolved FROM thread_state WHERE finding_id = ?", (self.fid,)
|
||||||
|
).fetchone()
|
||||||
|
self.assertEqual(row[0], 1)
|
||||||
|
feedback.record_thread_state(self.conn, finding_id=self.fid, resolved=False)
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT resolved FROM thread_state WHERE finding_id = ?", (self.fid,)
|
||||||
|
).fetchone()
|
||||||
|
self.assertEqual(row[0], 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecordReply(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
rid = feedback.record_review(self.conn, repo="o/r", pr=1, head_sha="x")
|
||||||
|
self.fid = feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a", line=1, severity="LOW", problem="x",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_insert_idempotent(self):
|
||||||
|
feedback.record_reply(
|
||||||
|
self.conn, finding_id=self.fid, author="a",
|
||||||
|
body="hi", created_at=1000,
|
||||||
|
)
|
||||||
|
feedback.record_reply(
|
||||||
|
self.conn, finding_id=self.fid, author="a",
|
||||||
|
body="hi", created_at=1000, # same PK
|
||||||
|
)
|
||||||
|
n = self.conn.execute("SELECT COUNT(*) FROM reply").fetchone()[0]
|
||||||
|
self.assertEqual(n, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFindingsWithVotes(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.conn = feedback.init(f"{self.tmp.name}/f.db")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close(); self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _seed(self):
|
||||||
|
rid = feedback.record_review(self.conn, repo="o/r", pr=1, head_sha="x")
|
||||||
|
fid = feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a/b.ts", line=10, severity="HIGH",
|
||||||
|
problem="race", comment_id=500,
|
||||||
|
)
|
||||||
|
feedback.record_reaction(self.conn, comment_id=500, user="u1", content="+1")
|
||||||
|
feedback.record_reaction(self.conn, comment_id=500, user="u2", content="-1")
|
||||||
|
feedback.record_thread_state(self.conn, finding_id=fid, resolved=True)
|
||||||
|
feedback.record_reply(
|
||||||
|
self.conn, finding_id=fid, author="u3",
|
||||||
|
body="this is fine because of X", created_at=2000,
|
||||||
|
)
|
||||||
|
return fid
|
||||||
|
|
||||||
|
def test_join_rolls_up_votes(self):
|
||||||
|
self._seed()
|
||||||
|
rows = list(feedback.findings_with_votes(self.conn))
|
||||||
|
self.assertEqual(len(rows), 1)
|
||||||
|
r = rows[0]
|
||||||
|
self.assertEqual(r["upvotes"], 1)
|
||||||
|
self.assertEqual(r["downvotes"], 1)
|
||||||
|
self.assertEqual(r["resolved"], 1)
|
||||||
|
self.assertEqual(r["reply_count"], 1)
|
||||||
|
self.assertIn("this is fine", r["reply_bodies"])
|
||||||
|
|
||||||
|
def test_repo_filter(self):
|
||||||
|
self._seed()
|
||||||
|
# Add a finding under a different repo.
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=None, repo="other/r", pr=99,
|
||||||
|
path="x", line=1, severity="LOW", problem="y",
|
||||||
|
)
|
||||||
|
rows = list(feedback.findings_with_votes(self.conn, repo="o/r"))
|
||||||
|
self.assertEqual(len(rows), 1)
|
||||||
|
self.assertEqual(rows[0]["repo"], "o/r")
|
||||||
|
|
||||||
|
def test_findings_with_no_signals_return_zero_votes(self):
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=None, repo="x/y", pr=1,
|
||||||
|
path="p", line=1, severity="LOW", problem="z",
|
||||||
|
)
|
||||||
|
rows = list(feedback.findings_with_votes(self.conn))
|
||||||
|
self.assertEqual(len(rows), 1)
|
||||||
|
self.assertEqual(rows[0]["upvotes"], 0)
|
||||||
|
self.assertEqual(rows[0]["downvotes"], 0)
|
||||||
|
self.assertIsNone(rows[0]["resolved"])
|
||||||
|
|
||||||
|
|
||||||
|
class TestKnownPosthashes(unittest.TestCase):
|
||||||
|
def test_returns_distinct_set(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
conn = feedback.init(f"{d}/f.db")
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
conn, review_id=None, repo="o/r", pr=1,
|
||||||
|
path="a", line=1, severity="LOW", problem="x",
|
||||||
|
)
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
conn, review_id=None, repo="o/r", pr=1,
|
||||||
|
path="a", line=2, severity="LOW", problem="y",
|
||||||
|
)
|
||||||
|
phs = feedback.known_posthashes_for_repo(conn, "o/r")
|
||||||
|
self.assertEqual(len(phs), 2)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,231 @@
|
|||||||
|
"""Tests for pilot/feedback_analyze.py.
|
||||||
|
|
||||||
|
Verify:
|
||||||
|
- empty DB produces a friendly empty-state report (no crash)
|
||||||
|
- findings are aggregated by posthash across multiple PRs
|
||||||
|
- net false-positive score weights downvotes + unresolved + negation
|
||||||
|
replies; acceptance weights upvotes + resolved
|
||||||
|
- restraint metric reports the right ratio
|
||||||
|
- case-review queue lists every disagreement
|
||||||
|
- markdown + JSON output modes both work
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, os.path.join(HERE, "..", "..", "pilot"))
|
||||||
|
|
||||||
|
import feedback # noqa: E402
|
||||||
|
import feedback_analyze # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _seed(conn, findings):
|
||||||
|
"""Helper: insert a list of (repo, pr, path, line, severity, problem,
|
||||||
|
[reaction users/contents], [reply bodies], [resolved]) tuples.
|
||||||
|
Each finding gets a fresh review row + a unique comment_id so the
|
||||||
|
reaction-join in `findings_with_votes` matches."""
|
||||||
|
for f in findings:
|
||||||
|
(repo, pr_idx, path, line, sev, problem, reacts, replies,
|
||||||
|
resolved) = f
|
||||||
|
rid = feedback.record_review(conn, repo=repo, pr=pr_idx, head_sha="x")
|
||||||
|
cid = (hash((repo, pr_idx, path, line, sev, problem)) & 0xFFFFFFFF) or 1
|
||||||
|
fid = feedback.record_inline_finding(
|
||||||
|
conn, review_id=rid, repo=repo, pr=pr_idx,
|
||||||
|
path=path, line=line, severity=sev, problem=problem,
|
||||||
|
comment_id=cid,
|
||||||
|
)
|
||||||
|
for user, content in reacts:
|
||||||
|
feedback.record_reaction(
|
||||||
|
conn, comment_id=cid, user=user, content=content,
|
||||||
|
)
|
||||||
|
for i, body in enumerate(replies):
|
||||||
|
feedback.record_reply(
|
||||||
|
conn, finding_id=fid, author="alice",
|
||||||
|
body=body, created_at=1000 + i,
|
||||||
|
)
|
||||||
|
if resolved is not None:
|
||||||
|
feedback.record_thread_state(
|
||||||
|
conn, finding_id=fid, resolved=resolved,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmptyState(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_empty_db_markdown_does_not_crash(self):
|
||||||
|
report = feedback_analyze.analyze(self.db)
|
||||||
|
self.assertIn("# pragent feedback report", report)
|
||||||
|
self.assertIn("findings analyzed**: 0", report)
|
||||||
|
self.assertIn("Restraint", report)
|
||||||
|
|
||||||
|
def test_empty_db_json_has_zero_findings(self):
|
||||||
|
report = feedback_analyze.analyze(self.db, as_json=True)
|
||||||
|
d = json.loads(report)
|
||||||
|
self.assertEqual(d["total_findings"], 0)
|
||||||
|
self.assertEqual(d["restraint"]["total"], 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestScoring(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
# Two PRs, three findings:
|
||||||
|
# A: 👍×2, resolved=true → acceptance
|
||||||
|
# B: 👎×2, unresolved, "false positive" reply → false-positive
|
||||||
|
# C: no signals → ignored
|
||||||
|
_seed(self.conn, [
|
||||||
|
("o/r", 1, "a.ts", 10, "HIGH", "race in handler",
|
||||||
|
[("u1", "+1"), ("u2", "+1")], [], True),
|
||||||
|
("o/r", 1, "b.ts", 20, "LOW", "missing semicolon",
|
||||||
|
[("u1", "-1"), ("u2", "-1")],
|
||||||
|
["False positive — this is fine."], False),
|
||||||
|
("o/r", 1, "c.ts", 30, "INFO", "naming nit",
|
||||||
|
[], [], None),
|
||||||
|
])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_accepted_ranked_above_fp(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
self.assertEqual(len(d["top_accepted"]), 1)
|
||||||
|
self.assertEqual(d["top_accepted"][0]["path"], "a.ts")
|
||||||
|
self.assertEqual(len(d["top_false_positive"]), 1)
|
||||||
|
self.assertEqual(d["top_false_positive"][0]["path"], "b.ts")
|
||||||
|
|
||||||
|
def test_fp_score_combines_signals(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
fp = d["top_false_positive"][0]
|
||||||
|
# 2 downvotes + 1 unresolved + 2 (negation phrase) = 5
|
||||||
|
self.assertEqual(fp["fp_score"], 5)
|
||||||
|
|
||||||
|
def test_acceptance_score(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
ac = d["top_accepted"][0]
|
||||||
|
# 2 upvotes + 1 resolved = 3
|
||||||
|
self.assertEqual(ac["ac_score"], 3)
|
||||||
|
|
||||||
|
def test_case_queue_contains_only_disagreements(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
queue = d["case_review_queue"]
|
||||||
|
self.assertEqual(len(queue), 1)
|
||||||
|
self.assertEqual(queue[0]["path"], "b.ts")
|
||||||
|
|
||||||
|
def test_no_signal_finding_is_ignored(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
# c.ts has no votes, no replies → not in either top list.
|
||||||
|
paths = {e["path"] for e in d["top_accepted"]}
|
||||||
|
paths.update(e["path"] for e in d["top_false_positive"])
|
||||||
|
self.assertNotIn("c.ts", paths)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRestraint(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_high_ratio_triggers_recommendation(self):
|
||||||
|
# 3 reviews, all with findings → 100% "noisy".
|
||||||
|
for pr_i in range(3):
|
||||||
|
feedback.record_review(self.conn, repo="o/r", pr=pr_i, head_sha="x")
|
||||||
|
# Distinct (path, line) per PR so posthash doesn't dedup.
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=None, repo="o/r", pr=pr_i,
|
||||||
|
path=f"a{pr_i}.ts", line=1, severity="LOW",
|
||||||
|
problem=f"x {pr_i}",
|
||||||
|
)
|
||||||
|
report = feedback_analyze.analyze(self.db)
|
||||||
|
self.assertIn("⚠️", report)
|
||||||
|
self.assertIn("100%", report)
|
||||||
|
|
||||||
|
def test_low_ratio_passes(self):
|
||||||
|
# 4 reviews, 1 with findings → 25% noisy = at threshold.
|
||||||
|
for pr_i in range(4):
|
||||||
|
feedback.record_review(self.conn, repo="o/r", pr=pr_i, head_sha="x")
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=None, repo="o/r", pr=0,
|
||||||
|
path="a.ts", line=1, severity="LOW", problem="x",
|
||||||
|
)
|
||||||
|
report = feedback_analyze.analyze(self.db)
|
||||||
|
self.assertIn("✅", report)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMarkdownOutput(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
_seed(self.conn, [
|
||||||
|
("o/r", 1, "a.ts", 10, "HIGH", "race in handler",
|
||||||
|
[("u1", "+1")], [], True),
|
||||||
|
])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_report_has_sections(self):
|
||||||
|
r = feedback_analyze.analyze(self.db)
|
||||||
|
for section in (
|
||||||
|
"# pragent feedback report",
|
||||||
|
"## Restraint",
|
||||||
|
"## Top",
|
||||||
|
"## Case-review queue",
|
||||||
|
"## Where this report goes",
|
||||||
|
):
|
||||||
|
self.assertIn(section, r)
|
||||||
|
|
||||||
|
def test_doordash_rule_quoted(self):
|
||||||
|
r = feedback_analyze.analyze(self.db)
|
||||||
|
# The "noise on clean code" sentence from the DoorDash recap.
|
||||||
|
self.assertIn("noise on clean code", r)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPosthashAggregation(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
self.conn = feedback.init(self.db)
|
||||||
|
# Same finding on three PRs → one aggregated row.
|
||||||
|
# Each PR has its own review + finding (comment_id differs but
|
||||||
|
# posthash is identical, so they collapse on aggregation).
|
||||||
|
for pr_i in range(3):
|
||||||
|
rid = feedback.record_review(self.conn, repo="o/r", pr=pr_i, head_sha="x")
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
self.conn, review_id=rid, repo="o/r", pr=pr_i,
|
||||||
|
path="a.ts", line=10, severity="HIGH",
|
||||||
|
problem="identical problem text",
|
||||||
|
comment_id=1000 + pr_i,
|
||||||
|
)
|
||||||
|
feedback.record_reaction(
|
||||||
|
self.conn, comment_id=1000 + pr_i, user="u", content="+1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_three_occurrences_one_row(self):
|
||||||
|
d = json.loads(feedback_analyze.analyze(self.db, as_json=True))
|
||||||
|
self.assertEqual(len(d["top_accepted"]), 1)
|
||||||
|
self.assertEqual(d["top_accepted"][0]["occurrences"], 3)
|
||||||
|
self.assertEqual(d["top_accepted"][0]["ac_score"], 3)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
"""Tests for pilot/feedback_harvest.py.
|
||||||
|
|
||||||
|
Mock `gitea_get` so we exercise the harvester's flow against canned Gitea
|
||||||
|
responses. Verify:
|
||||||
|
- bot-authored reviews only are processed
|
||||||
|
- reactions + thread state + replies all get recorded
|
||||||
|
- best-effort failures don't raise (one bad endpoint shouldn't kill the
|
||||||
|
whole harvest)
|
||||||
|
- posthash dedup: harvesting the same PR twice does NOT double-count
|
||||||
|
reactions.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, os.path.join(HERE, "..", "..", "pilot"))
|
||||||
|
|
||||||
|
import ai_review # noqa: E402
|
||||||
|
import feedback # noqa: E402
|
||||||
|
import feedback_harvest # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _make_fake_gitea(routes: dict):
|
||||||
|
"""Build a stand-in for `ai_review.gitea_get` that returns canned bodies.
|
||||||
|
|
||||||
|
`routes` maps relative path (substring) → (status, json_body). Sort
|
||||||
|
keys longest-first so e.g. `pulls/5/reviews/100/comments` matches
|
||||||
|
before `pulls/5/reviews` (which is also a substring of the longer
|
||||||
|
path).
|
||||||
|
"""
|
||||||
|
def fake(api, repo, path, token, accept="application/json"):
|
||||||
|
for needle in sorted(routes.keys(), key=len, reverse=True):
|
||||||
|
if needle in path:
|
||||||
|
status, body = routes[needle]
|
||||||
|
return status, json.dumps(body).encode()
|
||||||
|
return 404, b'{"message":"not found"}'
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(fake):
|
||||||
|
"""Apply the fake to ai_review.gitea_get and feedback_harvest's import."""
|
||||||
|
return patch("ai_review.gitea_get", side_effect=fake)
|
||||||
|
|
||||||
|
|
||||||
|
class TestHarvestForPr(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _review_payload(self, body="", commit_id="abc123", review_id=100):
|
||||||
|
return [{
|
||||||
|
"id": review_id, "user": {"login": "pragent-bot"},
|
||||||
|
"commit_id": commit_id, "body": body,
|
||||||
|
"created_at": "2026-08-20T10:00:00Z",
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _inline_payload(self, comment_id=500, body="**[LOW]** x", path="a/b.ts", position=42, resolver=""):
|
||||||
|
return [{
|
||||||
|
"id": comment_id, "path": path, "position": position,
|
||||||
|
"body": body, "resolver": resolver,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def test_happy_path_records_reaction_and_thread(self):
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (200, self._review_payload()),
|
||||||
|
"pulls/5/reviews/100/comments": (200, self._inline_payload(resolver="masi")),
|
||||||
|
"issues/comments/500/reactions": (200, [
|
||||||
|
{"user": {"login": "alice"}, "content": "+1",
|
||||||
|
"created_at": "2026-08-20T11:00:00Z"},
|
||||||
|
{"user": {"login": "bob"}, "content": "-1",
|
||||||
|
"created_at": "2026-08-20T11:01:00Z"},
|
||||||
|
]),
|
||||||
|
"issues/5/comments": (200, []), # no replies
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
stats = feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["reviews_seen"], 1)
|
||||||
|
self.assertEqual(stats["findings_seen"], 1)
|
||||||
|
self.assertEqual(stats["reactions_recorded"], 2)
|
||||||
|
self.assertEqual(stats["thread_states_recorded"], 1)
|
||||||
|
# DB should have 1 review, 1 finding, 2 reactions, 1 thread_state
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM review").fetchone()[0], 1,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM inline_finding").fetchone()[0], 1,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM reaction").fetchone()[0], 2,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT resolved FROM thread_state").fetchone()[0], 1,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_skips_non_bot_reviews(self):
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (200, [{
|
||||||
|
"id": 999, "user": {"login": "masi"}, # not the bot
|
||||||
|
"commit_id": "x", "body": "", "created_at": "2026-08-20T10:00:00Z",
|
||||||
|
}]),
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
stats = feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["reviews_seen"], 0)
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM review").fetchone()[0], 0,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_review_list_failure_does_not_raise(self):
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (500, None),
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
stats = feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["reviews_seen"], 0)
|
||||||
|
self.assertGreaterEqual(stats["errors"], 1)
|
||||||
|
|
||||||
|
def test_reactions_endpoint_returns_null_is_tolerated(self):
|
||||||
|
# Some Gitea endpoints return JSON `null` for empty lists. We must
|
||||||
|
# not crash — treat it as "no reactions".
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (200, self._review_payload()),
|
||||||
|
"pulls/5/reviews/100/comments": (200, self._inline_payload()),
|
||||||
|
"issues/comments/500/reactions": (200, None),
|
||||||
|
"issues/5/comments": (200, []),
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
stats = feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["reactions_recorded"], 0)
|
||||||
|
|
||||||
|
def test_reactions_dedup_via_pk_across_harvests(self):
|
||||||
|
# Two harvests of the same PR — both produce an inline_finding row,
|
||||||
|
# but reactions are PK-deduped on (comment_id, user, content) so
|
||||||
|
# the SECOND harvest does NOT double-record them.
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (200, self._review_payload()),
|
||||||
|
"pulls/5/reviews/100/comments": (200, self._inline_payload()),
|
||||||
|
"issues/comments/500/reactions": (200, [
|
||||||
|
{"user": {"login": "alice"}, "content": "+1",
|
||||||
|
"created_at": "2026-08-20T11:00:00Z"},
|
||||||
|
]),
|
||||||
|
"issues/5/comments": (200, []),
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
# Two findings (no DB-level posthash UNIQUE), one reaction.
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM inline_finding").fetchone()[0], 2,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM reaction").fetchone()[0], 1,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_replies_with_review_comment_id_recorded(self):
|
||||||
|
fake = _make_fake_gitea({
|
||||||
|
"pulls/5/reviews": (200, self._review_payload()),
|
||||||
|
"pulls/5/reviews/100/comments": (200, self._inline_payload(comment_id=500)),
|
||||||
|
"issues/comments/500/reactions": (200, []),
|
||||||
|
"issues/5/comments": (200, [{
|
||||||
|
"id": 900, "review_comment_id": 500,
|
||||||
|
"user": {"login": "alice"},
|
||||||
|
"body": "False positive — this is intentional",
|
||||||
|
"created_at": "2026-08-20T12:00:00Z",
|
||||||
|
}]),
|
||||||
|
})
|
||||||
|
with _patch(fake):
|
||||||
|
stats = feedback_harvest.harvest_for_pr(
|
||||||
|
api="http://x", token="t", repo="o/r", pr_index=5,
|
||||||
|
db_path=self.db,
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["replies_recorded"], 1)
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
self.assertEqual(
|
||||||
|
conn.execute("SELECT COUNT(*) FROM reply").fetchone()[0], 1,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseHelpers(unittest.TestCase):
|
||||||
|
def test_severity_extracted(self):
|
||||||
|
self.assertEqual(
|
||||||
|
feedback_harvest._parse_severity("**[HIGH]** race in foo"),
|
||||||
|
"HIGH",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_severity_defaults_to_info(self):
|
||||||
|
self.assertEqual(feedback_harvest._parse_severity("plain text"), "INFO")
|
||||||
|
|
||||||
|
def test_path_line_extracted(self):
|
||||||
|
p, l = feedback_harvest._parse_path_line("see `src/foo.ts:42` here")
|
||||||
|
self.assertEqual(p, "src/foo.ts")
|
||||||
|
self.assertEqual(l, 42)
|
||||||
|
|
||||||
|
def test_negation_phrases_caught(self):
|
||||||
|
self.assertTrue(feedback_harvest._is_negation_reply("This is intentional."))
|
||||||
|
self.assertTrue(feedback_harvest._is_negation_reply("false positive — see X"))
|
||||||
|
self.assertFalse(feedback_harvest._is_negation_reply("thanks for catching this!"))
|
||||||
|
# Empty / None safe
|
||||||
|
self.assertFalse(feedback_harvest._is_negation_reply(""))
|
||||||
|
self.assertFalse(feedback_harvest._is_negation_reply(None))
|
||||||
|
|
||||||
|
def test_classify_reaction(self):
|
||||||
|
self.assertEqual(feedback_harvest.classify_reaction("+1"), "positive")
|
||||||
|
self.assertEqual(feedback_harvest.classify_reaction("-1"), "negative")
|
||||||
|
self.assertEqual(feedback_harvest.classify_reaction("rocket"), "positive")
|
||||||
|
self.assertEqual(feedback_harvest.classify_reaction("confused"), "negative")
|
||||||
|
self.assertEqual(feedback_harvest.classify_reaction("eyes"), "neutral")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
"""Tests for pilot/feedback_post.py — report delivery to Gitea.
|
||||||
|
|
||||||
|
Mock `ai_review.gitea_get` + `gitea_post` so we exercise the find-or-create
|
||||||
|
+ comment-post flow without hitting the real API.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, os.path.join(HERE, "..", "..", "pilot"))
|
||||||
|
|
||||||
|
import feedback # noqa: E402
|
||||||
|
import feedback_analyze # noqa: E402
|
||||||
|
import feedback_post # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _make_fake(method_routes: dict):
|
||||||
|
"""`method_routes` maps HTTP path substring → (status, body, method).
|
||||||
|
|
||||||
|
For our purposes both gitea_get and gitea_post share the same fake —
|
||||||
|
gitea_get is GET, gitea_post is POST, and the post helper also has a
|
||||||
|
body param. The fake returns whatever the route's body says.
|
||||||
|
"""
|
||||||
|
def fake_get(api, repo, path, token, accept="application/json"):
|
||||||
|
for needle in sorted(method_routes.keys(), key=len, reverse=True):
|
||||||
|
status, body, _m = method_routes[needle]
|
||||||
|
if needle in path:
|
||||||
|
return status, json.dumps(body).encode()
|
||||||
|
return 404, b'{"message":"not found"}'
|
||||||
|
|
||||||
|
def fake_post(api, repo, path, token, body):
|
||||||
|
for needle in sorted(method_routes.keys(), key=len, reverse=True):
|
||||||
|
status, resp_body, _m = method_routes[needle]
|
||||||
|
if needle in path:
|
||||||
|
return status, json.dumps(resp_body).encode()
|
||||||
|
return 404, b'{"message":"not found"}'
|
||||||
|
|
||||||
|
return fake_get, fake_post
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeliver(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db = f"{self.tmp.name}/f.db"
|
||||||
|
conn = feedback.init(self.db)
|
||||||
|
rid = feedback.record_review(conn, repo="o/r", pr=1, head_sha="x")
|
||||||
|
feedback.record_inline_finding(
|
||||||
|
conn, review_id=rid, repo="o/r", pr=1,
|
||||||
|
path="a.ts", line=1, severity="HIGH",
|
||||||
|
problem="x", comment_id=99,
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_creates_issue_then_posts_comment(self):
|
||||||
|
routes = {
|
||||||
|
"issues?state=open": (200, [], "GET"), # no existing issue
|
||||||
|
"issues": (201, {"id": 42, "number": 7, "title": "..."}, "POST"),
|
||||||
|
"issues/7/comments": (201, {"id": 777}, "POST"),
|
||||||
|
}
|
||||||
|
fake_get, fake_post = _make_fake(routes)
|
||||||
|
with patch("ai_review.gitea_get", side_effect=fake_get), \
|
||||||
|
patch("ai_review.gitea_post", side_effect=fake_post):
|
||||||
|
stats = feedback_post.deliver(
|
||||||
|
api="http://x", token="t", db_path=self.db,
|
||||||
|
repo="gitea_admin/pragent", title="pragent feedback roll-up",
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["issue_id"], 7)
|
||||||
|
self.assertEqual(stats["comment_id"], 777)
|
||||||
|
|
||||||
|
def test_reuses_existing_issue(self):
|
||||||
|
routes = {
|
||||||
|
"issues?state=open": (200, [
|
||||||
|
{"id": 99, "number": 9, "title": "pragent feedback roll-up"},
|
||||||
|
{"id": 100, "number": 10, "title": "something else"},
|
||||||
|
], "GET"),
|
||||||
|
"issues/9/comments": (201, {"id": 888}, "POST"),
|
||||||
|
}
|
||||||
|
fake_get, fake_post = _make_fake(routes)
|
||||||
|
with patch("ai_review.gitea_get", side_effect=fake_get), \
|
||||||
|
patch("ai_review.gitea_post", side_effect=fake_post):
|
||||||
|
stats = feedback_post.deliver(
|
||||||
|
api="http://x", token="t", db_path=self.db,
|
||||||
|
repo="gitea_admin/pragent", title="pragent feedback roll-up",
|
||||||
|
)
|
||||||
|
self.assertEqual(stats["issue_id"], 9)
|
||||||
|
self.assertEqual(stats["comment_id"], 888)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -698,7 +698,13 @@ def test_normalize_lens_finding_rejects_bad_inputs():
|
|||||||
|
|
||||||
def test_posthash_matches_feedback_posthash():
|
def test_posthash_matches_feedback_posthash():
|
||||||
# Golden vector: identical inputs must produce identical 16-char hex.
|
# Golden vector: identical inputs must produce identical 16-char hex.
|
||||||
import feedback as fb
|
# Skipped when the unmerged feedback module isn't on the path (see
|
||||||
|
# pilot/feedback*.py — work in progress, not yet committed).
|
||||||
|
try:
|
||||||
|
import feedback as fb
|
||||||
|
except ImportError:
|
||||||
|
import pytest
|
||||||
|
pytest.skip("feedback module not present (see pilot/feedback*.py WIP)")
|
||||||
cases = [
|
cases = [
|
||||||
("a/b.ts", 12, "critical", "SQL injection via string concat"),
|
("a/b.ts", 12, "critical", "SQL injection via string concat"),
|
||||||
("a/b.ts", 12, "medium", "SQL injection via string concat"),
|
("a/b.ts", 12, "medium", "SQL injection via string concat"),
|
||||||
@@ -844,7 +850,9 @@ def test_no_surface_response_parses_as_an_empty_review():
|
|||||||
import ai_review
|
import ai_review
|
||||||
text, usage = oc._no_surface_response("o/r", "9", "abc12345", 3)
|
text, usage = oc._no_surface_response("o/r", "9", "abc12345", 3)
|
||||||
assert usage is None
|
assert usage is None
|
||||||
summary, findings, _changes, _risks = ai_review.parse_review_output(text)
|
summary, findings, _changes, _risks, _walkthrough, _risk_verdict, _test_coverage = (
|
||||||
|
ai_review.parse_review_output(text)
|
||||||
|
)
|
||||||
assert findings == []
|
assert findings == []
|
||||||
assert summary # non-empty, so ai_review does NOT take the salvage branch
|
assert summary # non-empty, so ai_review does NOT take the salvage branch
|
||||||
assert "no review surface" in summary.lower()
|
assert "no review surface" in summary.lower()
|
||||||
@@ -854,6 +862,96 @@ def test_no_surface_response_parses_as_an_empty_review():
|
|||||||
def test_no_surface_response_zero_lenses_wording():
|
def test_no_surface_response_zero_lenses_wording():
|
||||||
import ai_review
|
import ai_review
|
||||||
text, _ = oc._no_surface_response("o/r", "9", "abc12345", 0)
|
text, _ = oc._no_surface_response("o/r", "9", "abc12345", 0)
|
||||||
summary, findings, _c, _r = ai_review.parse_review_output(text)
|
summary, findings, _c, _r, _w, _rv, _tc = ai_review.parse_review_output(text)
|
||||||
assert findings == []
|
assert findings == []
|
||||||
assert "after path filtering" in summary
|
assert "after path filtering" in summary
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _synthesize_summary_fields — Task 8: real Python fallback implementation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_walkthrough_groups_findings_by_path():
|
||||||
|
findings = [
|
||||||
|
{"path": "a.py", "line": 1, "severity": "medium", "problem": "fix x"},
|
||||||
|
{"path": "b.py", "line": 2, "severity": "high", "problem": "fix y"},
|
||||||
|
]
|
||||||
|
w, _, _ = oc._synthesize_summary_fields(findings, "")
|
||||||
|
assert any("a.py" in line for line in w)
|
||||||
|
assert any("b.py" in line for line in w)
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_walkthrough_empty_when_no_findings_uses_changed_files():
|
||||||
|
w, _, _ = oc._synthesize_summary_fields(
|
||||||
|
[],
|
||||||
|
"diff --git a/x.py b/x.py\n@@ -1 +1 @@\n-old\n+new\n+++ b/x.py\n",
|
||||||
|
)
|
||||||
|
assert any("x.py" in line for line in w)
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_risk_verdict_critical():
|
||||||
|
findings = [{"severity": "critical"}]
|
||||||
|
_, rv, _ = oc._synthesize_summary_fields(findings, "")
|
||||||
|
assert "Critical risk" in rv
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_risk_verdict_clean():
|
||||||
|
_, rv, _ = oc._synthesize_summary_fields([], "")
|
||||||
|
assert "Low risk" in rv
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_test_coverage_with_test_path():
|
||||||
|
_, _, tc = oc._synthesize_summary_fields(
|
||||||
|
[], "+diff\n", changed_paths=["pilot/foo.py", "tests/test_foo.py"])
|
||||||
|
assert tc == "Tests changed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_test_coverage_missing_tests():
|
||||||
|
_, _, tc = oc._synthesize_summary_fields(
|
||||||
|
[], "+diff\n", changed_paths=["pilot/foo.py"])
|
||||||
|
assert "No tests for behavioral change" in tc
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_walkthrough_picks_peak_severity_per_path():
|
||||||
|
# Three findings on the same path, with mixed severities. The walkthrough
|
||||||
|
# headline should use the PEAK severity's emoji (critical = 🔴), not the
|
||||||
|
# lexicographic-first severity (low).
|
||||||
|
findings = [
|
||||||
|
{"path": "x.py", "line": 1, "severity": "low",
|
||||||
|
"problem": "minor nit"},
|
||||||
|
{"path": "x.py", "line": 5, "severity": "critical",
|
||||||
|
"problem": "sql injection"},
|
||||||
|
{"path": "x.py", "line": 9, "severity": "high",
|
||||||
|
"problem": "auth bypass"},
|
||||||
|
]
|
||||||
|
w, _, _ = oc._synthesize_summary_fields(findings, "")
|
||||||
|
assert len(w) == 1
|
||||||
|
line = w[0]
|
||||||
|
assert "`x.py`" in line
|
||||||
|
assert "🔴" in line # critical = 🔴
|
||||||
|
assert "🟡" not in line
|
||||||
|
assert "🔵" not in line
|
||||||
|
assert "sql injection" in line # critical finding's problem, not low's
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_summary_fields_none_findings_safe():
|
||||||
|
# Old code crashed in risk_verdict with `for f in findings:` on None.
|
||||||
|
# After the `findings = findings or []` guard, None behaves like [].
|
||||||
|
w, rv, tc = oc._synthesize_summary_fields(None, "")
|
||||||
|
assert isinstance(w, list)
|
||||||
|
assert rv.startswith("Low risk")
|
||||||
|
# walkthrough should fall through to the diff-derived path list — empty
|
||||||
|
# diff produces no lines, but no crash is the point.
|
||||||
|
assert tc == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_synthesize_walkthrough_empty_problem_does_not_crash():
|
||||||
|
# An empty `problem` should render as "`a.py` — emoji" with a trailing
|
||||||
|
# space, not raise. Regression guard for splitlines()[0][:80].strip().
|
||||||
|
findings = [{"path": "a.py", "line": 1,
|
||||||
|
"severity": "low", "problem": ""}]
|
||||||
|
w, _, _ = oc._synthesize_summary_fields(findings, "")
|
||||||
|
assert len(w) == 1
|
||||||
|
assert "`a.py`" in w[0]
|
||||||
|
assert "🔵" in w[0] # low severity emoji
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Unit tests for the webhook receiver's gating, dedupe and limits. No network."""
|
"""Unit tests for the webhook receiver's gating, dedupe and limits. No network."""
|
||||||
|
import base64
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
@@ -15,7 +16,6 @@ def _payload(**over):
|
|||||||
"number": 7,
|
"number": 7,
|
||||||
"title": "t",
|
"title": "t",
|
||||||
"body": "b",
|
"body": "b",
|
||||||
"labels": [{"name": "AI-REVIEW"}],
|
|
||||||
"head": {"sha": "a" * 40},
|
"head": {"sha": "a" * 40},
|
||||||
"base": {"ref": "main"},
|
"base": {"ref": "main"},
|
||||||
}
|
}
|
||||||
@@ -25,16 +25,11 @@ def _payload(**over):
|
|||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
def _enable_repo(monkeypatch, enabled: bool = True):
|
||||||
# label gating
|
"""Patch `is_repo_enabled` to the given bool for handler tests."""
|
||||||
# ---------------------------------------------------------------------------
|
monkeypatch.setattr(
|
||||||
|
"webhook_server.is_repo_enabled", lambda *a, **kw: enabled
|
||||||
|
)
|
||||||
def test_labels_have_matches_dicts_and_strings():
|
|
||||||
assert ws._labels_have([{"name": "AI-REVIEW"}], "AI-REVIEW")
|
|
||||||
assert ws._labels_have(["AI-REVIEW"], "AI-REVIEW")
|
|
||||||
assert not ws._labels_have([{"name": "other"}], "AI-REVIEW")
|
|
||||||
assert not ws._labels_have(None, "AI-REVIEW")
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -75,6 +70,7 @@ def test_claim_is_thread_safe():
|
|||||||
def test_duplicate_delivery_for_same_sha_is_not_reviewed_twice(monkeypatch):
|
def test_duplicate_delivery_for_same_sha_is_not_reviewed_twice(monkeypatch):
|
||||||
started = []
|
started = []
|
||||||
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
||||||
|
_enable_repo(monkeypatch)
|
||||||
|
|
||||||
class FakeThread:
|
class FakeThread:
|
||||||
def __init__(self, target, args, daemon):
|
def __init__(self, target, args, daemon):
|
||||||
@@ -97,6 +93,7 @@ def test_duplicate_delivery_for_same_sha_is_not_reviewed_twice(monkeypatch):
|
|||||||
def test_base_ref_is_passed_to_the_review_thread(monkeypatch):
|
def test_base_ref_is_passed_to_the_review_thread(monkeypatch):
|
||||||
started = []
|
started = []
|
||||||
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
||||||
|
_enable_repo(monkeypatch)
|
||||||
|
|
||||||
class FakeThread:
|
class FakeThread:
|
||||||
def __init__(self, target, args, daemon):
|
def __init__(self, target, args, daemon):
|
||||||
@@ -114,16 +111,33 @@ def test_base_ref_is_passed_to_the_review_thread(monkeypatch):
|
|||||||
|
|
||||||
def test_closed_action_is_ignored(monkeypatch):
|
def test_closed_action_is_ignored(monkeypatch):
|
||||||
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
||||||
|
_enable_repo(monkeypatch)
|
||||||
status, msg = ws._handle_pull_request(_payload(action="closed"))
|
status, msg = ws._handle_pull_request(_payload(action="closed"))
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert "ignore" in msg
|
assert "ignore" in msg
|
||||||
|
|
||||||
|
|
||||||
def test_missing_label_is_ignored(monkeypatch):
|
def test_handle_pull_request_skips_when_repo_not_enabled(monkeypatch):
|
||||||
|
_enable_repo(monkeypatch, enabled=False)
|
||||||
|
started = []
|
||||||
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
monkeypatch.setattr(ws, "BOT_TOKEN", "tok")
|
||||||
status, msg = ws._handle_pull_request(_payload(pr={"labels": [{"name": "wip"}]}))
|
|
||||||
|
class FakeThread:
|
||||||
|
def __init__(self, target, args, daemon):
|
||||||
|
self.args = args
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
started.append(self.args)
|
||||||
|
|
||||||
|
monkeypatch.setattr(ws.threading, "Thread", FakeThread)
|
||||||
|
ws._release(("o/r", "7", "a" * 40))
|
||||||
|
|
||||||
|
status, msg = ws._handle_pull_request(_payload())
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert "AI-REVIEW" in msg
|
assert "skip" in msg and "repo not opted in" in msg
|
||||||
|
assert "opened" in msg
|
||||||
|
assert started == []
|
||||||
|
ws._release(("o/r", "7", "a" * 40))
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -134,3 +148,35 @@ def test_missing_label_is_ignored(monkeypatch):
|
|||||||
def test_review_slots_bound_matches_config():
|
def test_review_slots_bound_matches_config():
|
||||||
assert ws.MAX_CONCURRENT >= 1
|
assert ws.MAX_CONCURRENT >= 1
|
||||||
assert ws._review_slots._value <= ws.MAX_CONCURRENT
|
assert ws._review_slots._value <= ws.MAX_CONCURRENT
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# is_repo_enabled — reads `.pr-review.json` from the PR base ref and parses
|
||||||
|
# its `enabled` flag. False on any failure (404, parse error, missing field).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_repo_enabled_returns_false_when_404(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"webhook_server.gitea_get",
|
||||||
|
lambda *a, **kw: (404, b'{"message":"not found"}'),
|
||||||
|
)
|
||||||
|
assert ws.is_repo_enabled("api", "owner/repo", "main", "tok") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_repo_enabled_returns_true_when_enabled(monkeypatch):
|
||||||
|
body = b'{"content":"' + base64.b64encode(b'{"enabled": true}').decode().encode() + b'"}'
|
||||||
|
monkeypatch.setattr("webhook_server.gitea_get", lambda *a, **kw: (200, body))
|
||||||
|
assert ws.is_repo_enabled("api", "owner/repo", "main", "tok") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_repo_enabled_returns_false_when_disabled(monkeypatch):
|
||||||
|
body = b'{"content":"' + base64.b64encode(b'{"enabled": false}').decode().encode() + b'"}'
|
||||||
|
monkeypatch.setattr("webhook_server.gitea_get", lambda *a, **kw: (200, body))
|
||||||
|
assert ws.is_repo_enabled("api", "owner/repo", "main", "tok") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_repo_enabled_returns_false_when_field_missing(monkeypatch):
|
||||||
|
body = b'{"content":"' + base64.b64encode(b'{}').decode().encode() + b'"}'
|
||||||
|
monkeypatch.setattr("webhook_server.gitea_get", lambda *a, **kw: (200, body))
|
||||||
|
assert ws.is_repo_enabled("api", "owner/repo", "main", "tok") is False
|
||||||
|
|||||||
Reference in New Issue
Block a user