diff --git a/.judge-trigger.txt b/.judge-trigger.txt new file mode 100644 index 0000000..8a0b918 --- /dev/null +++ b/.judge-trigger.txt @@ -0,0 +1 @@ +# judge trigger 1788203999 diff --git a/README.md b/README.md index 95f8dee..5f04db1 100644 --- a/README.md +++ b/README.md @@ -185,3 +185,5 @@ review time. ## License Not yet chosen. Until one is added, no reuse rights are granted. + +_pilot eval judges test 1788201461_ diff --git a/pilot/eval_judges.py b/pilot/eval_judges.py index cdea5bc..f7acd5b 100644 --- a/pilot/eval_judges.py +++ b/pilot/eval_judges.py @@ -208,18 +208,24 @@ def ensure_evaluators() -> dict: # --------------------------------------------------------------------------- def rule_body(name: str, judge_name: str, sampling: float) -> dict: - """POST /evaluation-rules shape for an LLM-as-judge observation rule. + """POST /evaluation-rules shape for an LLM-as-judge trace rule. - The judge is referenced by `name`+`scope`, not by id — ids name specific - versions, names name the evaluator across versions. Mapping is required at - both the rule root (the server validates it there) and inside `evaluator` - (the API echoes it back). Filter is on `traceName` because that is the only - stringOptions column the observation-rule schema exposes. + Target is `trace` rather than `observation` on purpose: the standard + `/api/public/ingestion` path that ships review traces here feeds only + the trace-upsert queue, and `evalService.createEvalJobs` only creates + jobs for `targetObject ∈ {TRACE, DATASET}`. Observation rules are + triggered exclusively from the OTel ingestion pipeline, which this + pilot does not use. A trace rule reads the trace's own input/output — + `langfuse_trace` already writes `_review_input`/`_review_output` onto + the trace body for exactly this reason. + + Mapping is required at both the rule root (server validates it there) + and inside `evaluator` (the API echoes it back). """ return { "name": name, "enabled": True, - "target": "observation", + "target": "trace", "sampling": sampling, "filter": [ {"column": "traceName", "operator": "any of", diff --git a/tests/pilot/test_eval_judges.py b/tests/pilot/test_eval_judges.py index 022b4ca..0fe4fe2 100644 --- a/tests/pilot/test_eval_judges.py +++ b/tests/pilot/test_eval_judges.py @@ -11,10 +11,16 @@ import eval_judges as ej # noqa: E402 # --- rule_body ------------------------------------------------------------ -def test_rule_body_targets_observations(): - """Trace-level rules wouldn't see observation input/output.""" +def test_rule_body_targets_traces(): + """Trace target matches the path `/api/public/ingestion` triggers. + + Observation rules only fire from the OTel ingestion pipeline; this + pilot uses standard ingestion, so its jobs only come from + `evalService.createEvalJobs` and that dispatcher handles + `targetObject ∈ {TRACE, DATASET}`. + """ body = ej.rule_body("rule-x", "finding_actionability", 1.0) - assert body["target"] == "observation" + assert body["target"] == "trace" assert body["enabled"] is True