Claude
5d44121b28
Two llm_as_judge evaluators score the review generation directly: a NUMERIC 0-1 on finding actionability, a BOOLEAN on whether the summary agrees with the findings. Both run on every observation whose trace name is pr-review or opencode-review. The judge is kimi-k2.7-code through the headroom hub. Local Ollama returns Anthropic-format responses but the thinking blocks lack the signature field Langfuse Zod schema requires; the evaluator preflight fails as Invalid JSON response. A small judge-proxy pod on 8802 forwards to the hub and patches every thinking block with a synthetic signature before returning. Trace + generation output now includes the findings themselves (capped at 25) rather than just the count, so a judge has something to grade. generation input/output mirrors the trace so an observation-level evaluator can read them. Idempotent: existing evaluators and rules are skipped on re-run, not duplicated. The connection is upserted on provider.
127 lines
4.6 KiB
Python
127 lines
4.6 KiB
Python
|
|
|
|
"""Tests for the LLM-as-judge evaluator bootstrap."""
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
|
|
|
|
import eval_judges as ej # noqa: E402
|
|
|
|
|
|
# --- rule_body ------------------------------------------------------------
|
|
|
|
def test_rule_body_targets_observations():
|
|
"""Trace-level rules wouldn't see observation input/output."""
|
|
body = ej.rule_body("rule-x", "finding_actionability", 1.0)
|
|
assert body["target"] == "observation"
|
|
assert body["enabled"] is True
|
|
|
|
|
|
def test_rule_body_filters_on_trace_name():
|
|
"""`name` isn't a stringOptions column; only `traceName` is."""
|
|
body = ej.rule_body("rule-x", "finding_actionability", 1.0)
|
|
f = body["filter"][0]
|
|
assert f["column"] == "traceName"
|
|
assert f["operator"] == "any of"
|
|
assert f["type"] == "stringOptions"
|
|
assert "pr-review" in f["value"]
|
|
|
|
|
|
def test_rule_body_references_evaluator_by_name():
|
|
"""Ids are version-specific; rules must name the evaluator across versions."""
|
|
body = ej.rule_body("rule-x", "finding_actionability", 1.0)
|
|
assert body["evaluator"]["name"] == "finding_actionability"
|
|
assert body["evaluator"]["scope"] == "project"
|
|
|
|
|
|
def test_rule_body_maps_input_and_output():
|
|
"""Both judges read the observation's own input/output."""
|
|
body = ej.rule_body("rule-x", "any", 1.0)
|
|
sources = {m["source"] for m in body["mapping"]}
|
|
assert sources == {"input", "output"}
|
|
|
|
|
|
def test_rule_body_carries_mapping_at_both_levels():
|
|
"""The server validates `mapping` at the rule root and echoes it on the evaluator."""
|
|
body = ej.rule_body("rule-x", "any", 1.0)
|
|
assert body["mapping"]
|
|
assert body["evaluator"]["variableMapping"] == body["mapping"]
|
|
|
|
|
|
def test_rule_body_passes_sampling_through():
|
|
assert ej.rule_body("r", "any", 0.25)["sampling"] == 0.25
|
|
|
|
|
|
# --- ensure_evaluators idempotency ---------------------------------------
|
|
|
|
def test_ensure_evaluators_skips_existing(monkeypatch):
|
|
seen = []
|
|
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
seen.append(path)
|
|
return 200, {}
|
|
|
|
monkeypatch.setattr(ej.eb, "_call", fake_call)
|
|
monkeypatch.setattr(ej, "existing_evaluators",
|
|
lambda: {"finding_actionability": "id-1", "review_self_consistency": "id-2"})
|
|
res = ej.ensure_evaluators()
|
|
assert res["created"] == {}
|
|
assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
|
|
assert res["failed"] == []
|
|
assert seen == []
|
|
|
|
|
|
def test_ensure_evaluators_records_failures(monkeypatch):
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
return 422, "boom"
|
|
|
|
monkeypatch.setattr(ej.eb, "_call", fake_call)
|
|
monkeypatch.setattr(ej, "existing_evaluators", lambda: {})
|
|
res = ej.ensure_evaluators()
|
|
assert res["created"] == {}
|
|
assert res["failed"][0]["status"] == 422
|
|
|
|
|
|
# --- ensure_rules idempotency --------------------------------------------
|
|
|
|
def test_ensure_rules_skips_existing(monkeypatch):
|
|
calls = []
|
|
monkeypatch.setattr(ej.eb, "_call",
|
|
lambda *a, **k: calls.append(a) or (200, {}))
|
|
monkeypatch.setattr(ej, "existing_evaluators",
|
|
lambda: {"finding_actionability": "id-1",
|
|
"review_self_consistency": "id-2"})
|
|
monkeypatch.setattr(ej, "existing_rule_names",
|
|
lambda: {"finding_actionability-on-reviews",
|
|
"review_self_consistency-on-reviews"})
|
|
res = ej.ensure_rules({"finding_actionability": "id-1",
|
|
"review_self_consistency": "id-2"}, 1.0)
|
|
assert res["created"] == []
|
|
assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
|
|
assert calls == []
|
|
|
|
|
|
def test_ensure_rules_creates_when_missing(monkeypatch):
|
|
calls = []
|
|
monkeypatch.setattr(ej.eb, "_call",
|
|
lambda *a, **k: calls.append(a) or (201, {}))
|
|
monkeypatch.setattr(ej, "existing_rule_names", lambda: set())
|
|
res = ej.ensure_rules({"finding_actionability": "id-1"}, 1.0)
|
|
assert res["created"] == ["finding_actionability"]
|
|
assert calls[0][0] == "POST"
|
|
assert calls[0][1] == "/api/public/unstable/evaluation-rules"
|
|
|
|
|
|
# --- judge shape ----------------------------------------------------------
|
|
|
|
def test_judges_have_required_keys():
|
|
for j in ej.JUDGES:
|
|
assert j["prompt"]
|
|
assert j["outputDefinition"]["dataType"] in ("NUMERIC", "BOOLEAN", "CATEGORICAL")
|
|
|
|
|
|
def test_default_base_url_points_at_the_thinking_patch_proxy():
|
|
"""`8802` is the judge-proxy that adds a `signature` to thinking blocks."""
|
|
assert "8802" in ej.JUDGE_BASE_URL
|