Claude
5d44121b28
Two llm_as_judge evaluators score the review generation directly: a NUMERIC 0-1 on finding actionability, a BOOLEAN on whether the summary agrees with the findings. Both run on every observation whose trace name is pr-review or opencode-review. The judge is kimi-k2.7-code through the headroom hub. Local Ollama returns Anthropic-format responses but the thinking blocks lack the signature field Langfuse Zod schema requires; the evaluator preflight fails as Invalid JSON response. A small judge-proxy pod on 8802 forwards to the hub and patches every thinking block with a synthetic signature before returning. Trace + generation output now includes the findings themselves (capped at 25) rather than just the count, so a judge has something to grade. generation input/output mirrors the trace so an observation-level evaluator can read them. Idempotent: existing evaluators and rules are skipped on re-run, not duplicated. The connection is upserted on provider.
309 lines
13 KiB
Python
309 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""pragent pilot — LLM-as-a-judge evaluators for the reviewer.
|
|
|
|
The deterministic scorers in `eval_scores.py` measure *behaviour*: how many
|
|
findings, how severe, how much they cost. None of them can say whether a
|
|
finding was any good. With no human labels in `feedback.db`, a judge is the
|
|
only thing that can — so these two ask the questions that need no ground truth,
|
|
only the review itself:
|
|
|
|
`finding_actionability` — is each finding concrete enough to act on? A
|
|
reviewer that says "consider improving error handling" at file level is
|
|
indistinguishable from a useful one by finding count alone. This is the
|
|
failure mode a cheap model degrades into first.
|
|
|
|
`review_self_consistency` — does the summary agree with the findings it
|
|
posted? Claiming "no issues found" above a list of two criticals, or
|
|
describing a problem in prose that never became a finding, is a defect the
|
|
reviewer can commit entirely on its own.
|
|
|
|
Neither judge is asked whether a finding is *correct*. That needs the diff,
|
|
which these traces do not carry, and a judge asked to rule on correctness from
|
|
a summary alone will confabulate. Accuracy stays an open question until humans
|
|
start labelling — which is what `feedback_scores.py` is there to capture.
|
|
|
|
**The judge is a different model from the reviewer.** The reviewer runs
|
|
MiniMax-M2.7; the judge runs kimi-k2.7-code through the same headroom hub. A
|
|
model grading its own output agrees with itself for reasons that have nothing
|
|
to do with quality.
|
|
|
|
Evaluators score *observations*, and their variable mapping reads the
|
|
observation's own input/output — which is why `langfuse_trace` now writes the
|
|
review onto the generation and not just onto the trace.
|
|
|
|
Usage:
|
|
LANGFUSE_HOST=... LANGFUSE_PUBLIC_KEY=... LANGFUSE_SECRET_KEY=... \\
|
|
python3 eval_judges.py --dry-run
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
import eval_bootstrap as eb # noqa: E402
|
|
|
|
# The headroom hub in front of the local Ollama, plus a small pass-through
|
|
# proxy (`judge-proxy` on 8802) that patches every `thinking` content block
|
|
# to carry the `signature` field Langfuse's Anthropic adapter requires. The
|
|
# underlying model is kimi-k2.7-code through the hub on 8790; the proxy fixes
|
|
# the shape so Mastra's Zod parse stops failing.
|
|
JUDGE_PROVIDER = "headroom-ollama"
|
|
JUDGE_BASE_URL = os.environ.get("PRAGENT_JUDGE_BASE_URL", "http://100.74.17.70:8802")
|
|
JUDGE_API_KEY = os.environ.get("PRAGENT_JUDGE_API_KEY", "ollama")
|
|
JUDGE_MODEL = os.environ.get("PRAGENT_JUDGE_MODEL", "kimi-k2.7-code:cloud")
|
|
|
|
# The trace names this project emits (`pr-review` on the trace, `opencode-review`
|
|
# on the generation). Filter on `traceName` rather than observation `name` — the
|
|
# observation-rule schema only exposes `traceName` as a stringOptions column, and
|
|
# every observation inside these traces is the review itself, so the narrowness
|
|
# is the same.
|
|
REVIEW_TRACE_NAMES = ["pr-review", "opencode-review"]
|
|
|
|
|
|
def _model_config() -> dict:
|
|
return {"provider": JUDGE_PROVIDER, "model": JUDGE_MODEL}
|
|
|
|
|
|
JUDGES = [
|
|
{
|
|
"name": "finding_actionability",
|
|
"prompt": (
|
|
"You are auditing the output of an automated code reviewer.\n\n"
|
|
"PR under review:\n{{input}}\n\n"
|
|
"What the reviewer produced:\n{{output}}\n\n"
|
|
"Rate how ACTIONABLE the findings are, from 0 to 1. A finding is "
|
|
"actionable when a developer could act on it without asking a "
|
|
"follow-up question: it points at a specific location, names a "
|
|
"concrete problem, and proposes a fix that could be applied.\n\n"
|
|
"Score 1.0 when every finding is specific and fixable. Score around "
|
|
"0.5 when findings identify a real area but leave the developer to "
|
|
"work out what to change. Score near 0.0 when findings are generic "
|
|
"advice that would apply to almost any pull request.\n\n"
|
|
"Judge only specificity and actionability. You cannot see the diff, "
|
|
"so do NOT attempt to judge whether a finding is factually correct, "
|
|
"and do not penalise a finding for being one you cannot verify.\n\n"
|
|
"If the reviewer reported no findings at all, return 1.0 and say in "
|
|
"your reasoning that there was nothing to judge — a silent review is "
|
|
"measured by finding_rate, not here."
|
|
),
|
|
"outputDefinition": {
|
|
"dataType": "NUMERIC",
|
|
"minValue": 0,
|
|
"maxValue": 1,
|
|
"reasoning": {
|
|
"description": (
|
|
"Name the least actionable finding and say what it would "
|
|
"need in order to be acted on."
|
|
)
|
|
},
|
|
"score": {"description": "0 = generic advice, 1 = every finding is specific and fixable."},
|
|
},
|
|
},
|
|
{
|
|
"name": "review_self_consistency",
|
|
"prompt": (
|
|
"You are auditing the output of an automated code reviewer.\n\n"
|
|
"PR under review:\n{{input}}\n\n"
|
|
"What the reviewer produced:\n{{output}}\n\n"
|
|
"The output contains a prose `summary` and a list of `findings`. "
|
|
"Decide whether the summary is CONSISTENT with the findings.\n\n"
|
|
"Inconsistent means, for example: the summary says no issues were "
|
|
"found while findings are listed; the summary describes a problem "
|
|
"that never became a finding; the summary characterises the severity "
|
|
"of the findings in a way the findings themselves contradict; or the "
|
|
"summary refers to files that appear in no finding and in no part of "
|
|
"the PR description.\n\n"
|
|
"A summary that adds context beyond the findings is NOT inconsistent "
|
|
"as long as nothing in it contradicts them. A review that found "
|
|
"nothing and says so is consistent.\n\n"
|
|
"You cannot see the diff. Judge the summary against the findings and "
|
|
"the PR title only — never against what you imagine the code does."
|
|
),
|
|
"outputDefinition": {
|
|
"dataType": "BOOLEAN",
|
|
"reasoning": {
|
|
"description": "Quote the part of the summary that conflicts with the findings, if any."
|
|
},
|
|
"score": {"description": "true = summary agrees with the findings, false = it contradicts them."},
|
|
},
|
|
},
|
|
]
|
|
|
|
# Both judges read the observation's own input/output.
|
|
MAPPING = [
|
|
{"variable": "input", "source": "input"},
|
|
{"variable": "output", "source": "output"},
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LLM connection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def ensure_llm_connection() -> dict:
|
|
"""Point the project at the judge model. Upserted on `provider`."""
|
|
body = {
|
|
"provider": JUDGE_PROVIDER,
|
|
"adapter": "anthropic",
|
|
"baseURL": JUDGE_BASE_URL,
|
|
"secretKey": JUDGE_API_KEY,
|
|
"customModels": [JUDGE_MODEL],
|
|
# The hub serves two local models and none of Anthropic's, so the
|
|
# default catalogue would be a list of models that all fail on use.
|
|
"withDefaultModels": False,
|
|
}
|
|
st, resp = eb._call("PUT", "/api/public/llm-connections", body)
|
|
return {"status": st, "ok": st in (200, 201), "provider": JUDGE_PROVIDER,
|
|
"error": None if st in (200, 201) else resp}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Evaluators
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def existing_evaluators() -> dict[str, str]:
|
|
"""name -> id for evaluators already in the project."""
|
|
out: dict[str, str] = {}
|
|
st, body = eb._call("GET", "/api/public/unstable/evaluators?limit=100")
|
|
if st == 200 and isinstance(body, dict):
|
|
for ev in body.get("data") or []:
|
|
out[ev.get("name")] = ev.get("id")
|
|
return out
|
|
|
|
|
|
def ensure_evaluators() -> dict:
|
|
"""Create each judge if no version exists for the name yet.
|
|
|
|
POST /evaluators with a name that already exists creates a new version, not
|
|
a no-op — re-running this script would pile up versions until the page
|
|
listing them is unreadable. Skip when an evaluator of that name is present.
|
|
"""
|
|
created, skipped, failed = {}, [], []
|
|
existing = set(existing_evaluators())
|
|
for judge in JUDGES:
|
|
if judge["name"] in existing:
|
|
skipped.append(judge["name"])
|
|
continue
|
|
body = {
|
|
"type": "llm_as_judge",
|
|
"name": judge["name"],
|
|
"prompt": judge["prompt"],
|
|
"outputDefinition": judge["outputDefinition"],
|
|
"modelConfig": _model_config(),
|
|
}
|
|
st, resp = eb._call("POST", "/api/public/unstable/evaluators", body, timeout=60.0)
|
|
if st in (200, 201) and isinstance(resp, dict):
|
|
created[judge["name"]] = resp.get("id")
|
|
else:
|
|
failed.append({"name": judge["name"], "status": st, "error": resp})
|
|
return {"created": created, "skipped": skipped, "failed": failed}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rules — what gets judged, and how often
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def rule_body(name: str, judge_name: str, sampling: float) -> dict:
|
|
"""POST /evaluation-rules shape for an LLM-as-judge observation rule.
|
|
|
|
The judge is referenced by `name`+`scope`, not by id — ids name specific
|
|
versions, names name the evaluator across versions. Mapping is required at
|
|
both the rule root (the server validates it there) and inside `evaluator`
|
|
(the API echoes it back). Filter is on `traceName` because that is the only
|
|
stringOptions column the observation-rule schema exposes.
|
|
"""
|
|
return {
|
|
"name": name,
|
|
"enabled": True,
|
|
"target": "observation",
|
|
"sampling": sampling,
|
|
"filter": [
|
|
{"column": "traceName", "operator": "any of",
|
|
"value": REVIEW_TRACE_NAMES, "type": "stringOptions"},
|
|
],
|
|
"evaluator": {
|
|
"name": judge_name,
|
|
"scope": "project",
|
|
"variableMapping": MAPPING,
|
|
},
|
|
"mapping": MAPPING,
|
|
}
|
|
|
|
|
|
def ensure_rules(evaluator_ids: dict[str, str], sampling: float) -> dict:
|
|
"""Idempotent: existing rules with the same name are skipped, not duplicated.
|
|
|
|
The API has no `name`-keyed upsert; the convention is to POST once and
|
|
re-run the script to verify the response. A duplicate POST raises 409.
|
|
"""
|
|
created, failed, skipped = [], [], []
|
|
existing = existing_rule_names()
|
|
for name, eid in evaluator_ids.items():
|
|
if not eid:
|
|
continue
|
|
rule_name = f"{name}-on-reviews"
|
|
if rule_name in existing:
|
|
skipped.append(name)
|
|
continue
|
|
st, resp = eb._call(
|
|
"POST", "/api/public/unstable/evaluation-rules",
|
|
rule_body(rule_name, name, sampling), timeout=60.0,
|
|
)
|
|
if st in (200, 201):
|
|
created.append(name)
|
|
else:
|
|
failed.append({"rule": name, "status": st, "error": resp})
|
|
return {"created": created, "failed": failed, "skipped": skipped}
|
|
|
|
|
|
def existing_rule_names() -> set[str]:
|
|
"""Names of observation-target rules already in the project."""
|
|
out: set[str] = set()
|
|
st, body = eb._call("GET", "/api/public/unstable/evaluation-rules?limit=100")
|
|
if st == 200 and isinstance(body, dict):
|
|
for r in body.get("data") or []:
|
|
if r.get("target") == "observation":
|
|
out.add(r.get("name"))
|
|
return out
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("--sampling", type=float, default=1.0,
|
|
help="fraction of matching observations to judge (default: all)")
|
|
ap.add_argument("--skip-connection", action="store_true")
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
args = ap.parse_args(argv)
|
|
|
|
if args.dry_run:
|
|
print(json.dumps({
|
|
"would_connect": {"provider": JUDGE_PROVIDER, "baseURL": JUDGE_BASE_URL,
|
|
"model": JUDGE_MODEL},
|
|
"would_create": [j["name"] for j in JUDGES],
|
|
"existing_evaluators": sorted(existing_evaluators()),
|
|
"sampling": args.sampling,
|
|
}, indent=2))
|
|
return 0
|
|
|
|
report = {}
|
|
if not args.skip_connection:
|
|
report["llm_connection"] = ensure_llm_connection()
|
|
report["evaluators"] = ensure_evaluators()
|
|
ids = dict(report["evaluators"]["created"])
|
|
# Fall back to whatever is already registered, so a re-run still wires rules.
|
|
for name, eid in existing_evaluators().items():
|
|
ids.setdefault(name, eid)
|
|
report["rules"] = ensure_rules(
|
|
{j["name"]: ids.get(j["name"]) for j in JUDGES}, args.sampling
|
|
)
|
|
print(json.dumps(report, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|