gitea_admin/pragent
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: organize pilot packages

Browse Source 
Group review, feedback, evaluation, observability, and entrypoint code into packages. Keep thin top-level compatibility shims for existing scripts and imports, and mirror the structure in the tests.
This commit is contained in:
Claude
2026-09-01 00:59:51 +00:00
parent 3a110ab52c
commit 7a510a926d
66 changed files with 8174 additions and 8039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pilot/feedback_scores.py
+5 -245
View File
@@ -1,247 +1,7 @@
#!/usr/bin/env python3
"""pragent pilot — feedback DB to Langfuse scores.
`feedback.db` already records every reaction, thread resolution and reply a
maintainer leaves on a bot comment. That is the only ground truth pragent has
about whether a finding was any good, and until now it went to a markdown report
nobody reads and nowhere else. This ships it to Langfuse as session-level
scores, so "was the reviewer right" sits on the same axis as "what did it cost".
Session, not trace
------------------
`langfuse_trace` sets `sessionId` to `"{repo}#{pr}"` and lets the trace id be a
fresh uuid per review. Feedback arrives days later against a PR, not against one
particular re-run of the reviewer, and nothing in `feedback.db` records which
trace produced which comment. Scoring the session is therefore both the
available join and the honest granularity: this is feedback on the review of
this PR, not on one invocation.
Two scores, deliberately separated
----------------------------------
* `review_engagement` — the share of a PR's findings that got any human
response at all. This is a signal about the *feedback loop*, not the
reviewer: at the time of writing it is 0.0 across all 113 recorded reviews,
which is exactly the fact that makes an accuracy metric impossible today.
It must be watched first, because every other quality number is vapour
until it moves.
* `review_acceptance` — net verdict over the findings that *did* get a
response: (upvotes + resolved) - (downvotes + negation replies), normalised
to -1..1. Computed only over engaged findings, so an ignored review scores
`None` rather than 0. Zero would read as "humans judged this exactly
neutral"; the truth is nobody looked.
Fail-open and idempotent. Score ids are derived from (repo, pr, name) so a
re-run overwrites rather than duplicates.
"""
from __future__ import annotations
import argparse
import json
import os
import sqlite3
"""Compatibility import for feedback scores."""
import importlib
import sys
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from feedback_harvest import classify_reaction, _is_negation_reply # noqa: E402
REVIEW_ENGAGEMENT = "review_engagement"
REVIEW_ACCEPTANCE = "review_acceptance"
# Stable namespace so the same (repo, pr, score) always produces the same score
# id — Langfuse treats a repeated id as an update, which is what a backfill of a
# still-accumulating PR should do.
_NS = uuid.UUID("6f1d9c2e-4a77-4f2a-9c1a-0d3b5e8a7c41")
def _score_id(repo: str, pr: int, name: str) -> str:
return str(uuid.uuid5(_NS, f"{repo}#{pr}#{name}"))
def collect_pr_feedback(conn: sqlite3.Connection, repo: str, pr: int) -> dict:
"""Tally one PR's findings and the human responses attached to them.
Returns counts only — the scoring maths lives in `score_pr` so it can be
tested without a database.
"""
rows = conn.execute(
"SELECT id, comment_id FROM inline_finding WHERE repo = ? AND pr = ?",
(repo, pr),
).fetchall()
total = len(rows)
engaged = 0
positive = 0
negative = 0
for row in rows:
fid = row["id"] if isinstance(row, sqlite3.Row) else row[0]
cid = row["comment_id"] if isinstance(row, sqlite3.Row) else row[1]
pos = neg = 0
if cid is not None:
for r in conn.execute(
"SELECT content FROM reaction WHERE comment_id = ?", (cid,)
):
kind = classify_reaction(r[0])
if kind == "positive":
pos += 1
elif kind == "negative":
neg += 1
for r in conn.execute(
"SELECT resolved FROM thread_state WHERE finding_id = ?", (fid,)
):
# A resolved thread means the maintainer acted on the finding.
if r[0]:
pos += 1
# A reply counts as engagement either way; only a negation phrase makes
# it a vote against. A neutral reply ("done", "good catch, but…") is
# deliberately not a positive vote — it says someone looked, not that
# they agreed.
replied = 0
for r in conn.execute(
"SELECT body FROM reply WHERE finding_id = ?", (fid,)
):
replied += 1
if _is_negation_reply(r[0]):
neg += 1
if pos or neg or replied:
engaged += 1
positive += pos
negative += neg
return {"total": total, "engaged": engaged, "positive": positive, "negative": negative}
def score_pr(tally: dict) -> dict:
"""Turn one PR's tally into score values.
`review_acceptance` is `None` when nothing was engaged — see the module
docstring on why that is not 0.
"""
total = int(tally.get("total") or 0)
engaged = int(tally.get("engaged") or 0)
pos = int(tally.get("positive") or 0)
neg = int(tally.get("negative") or 0)
engagement = round(engaged / total, 4) if total else None
acceptance = None
if pos or neg:
acceptance = round((pos - neg) / (pos + neg), 4)
return {REVIEW_ENGAGEMENT: engagement, REVIEW_ACCEPTANCE: acceptance}
def build_score_events(
repo: str, pr: int, values: dict, environment: str = "default",
timestamp: str | None = None,
) -> list[dict]:
"""`score-create` events for one PR's feedback.
Every event carries a timestamp: the ingestion endpoint rejects those that
do not, and it reports the rejection as a per-event 400 inside an HTTP 207,
which reads as success to a caller that only checks the status code.
"""
ts = timestamp or datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
events = []
for name, value in values.items():
if value is None:
continue
events.append(
{
"id": str(uuid.uuid4()),
"type": "score-create",
"timestamp": ts,
"body": {
"id": _score_id(repo, pr, name),
"sessionId": f"{repo}#{pr}",
"name": name,
"value": float(value),
"dataType": "NUMERIC",
"environment": environment,
"comment": f"from feedback.db · {repo}#{pr}",
},
}
)
return events
SCORE_CONFIGS = [
{
"name": REVIEW_ENGAGEMENT,
"dataType": "NUMERIC",
"minValue": 0,
"maxValue": 1,
"description": "Share of a PR's findings that drew any human reaction, resolution or reply. 0 = nobody engaged with the review.",
},
{
"name": REVIEW_ACCEPTANCE,
"dataType": "NUMERIC",
"minValue": -1,
"maxValue": 1,
"description": "Net human verdict over engaged findings: +1 all accepted, -1 all rejected. Absent when nothing was engaged.",
},
]
def iter_prs(conn: sqlite3.Connection):
for row in conn.execute(
"SELECT DISTINCT repo, pr FROM inline_finding ORDER BY repo, pr"
):
yield row[0], int(row[1])
def backfill(db_path: str, *, environment: str = "default", dry_run: bool = False) -> dict:
"""Score every PR in the feedback DB. Returns a summary dict."""
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
events: list[dict] = []
scanned = 0
engaged_prs = 0
try:
for repo, pr in iter_prs(conn):
scanned += 1
tally = collect_pr_feedback(conn, repo, pr)
values = score_pr(tally)
if (values.get(REVIEW_ENGAGEMENT) or 0) > 0:
engaged_prs += 1
events.extend(build_score_events(repo, pr, values, environment))
finally:
conn.close()
summary = {"prs_scanned": scanned, "prs_with_engagement": engaged_prs, "scores": len(events)}
if dry_run or not events:
summary["posted"] = False
return summary
import langfuse_trace
conf = langfuse_trace._enabled()
if conf is None:
summary["posted"] = False
summary["error"] = "Langfuse not configured (LANGFUSE_HOST / keys unset)"
return summary
host, pk, sk = conf
status = langfuse_trace._post(host, pk, sk, events, 15.0)
summary["posted"] = status in (200, 201, 207)
summary["http_status"] = status
return summary
def main() -> int:
ap = argparse.ArgumentParser(description="Ship feedback.db verdicts to Langfuse as scores")
ap.add_argument("--db", default=os.environ.get("PRAGENT_FEEDBACK_DB", "/data/feedback.db"))
ap.add_argument("--environment", default="default")
ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args()
summary = backfill(args.db, environment=args.environment, dry_run=args.dry_run)
print(json.dumps(summary, indent=2))
return 0 if summary.get("posted") or args.dry_run else 1
_module = importlib.import_module("feedback.scores")
sys.modules[__name__] = _module
if __name__ == "__main__":
raise SystemExit(main())
raise SystemExit(_module.main())