2 Commits

Author SHA1 Message Date
Claude 370adcde6f test: trigger after reopen 2026-08-31 18:36:07 +00:00
Claude 84325e61c1 test: trigger review after .pr-review.json merged to main 2026-08-31 18:34:18 +00:00
4 changed files with 10 additions and 25 deletions
-1
View File
@@ -1 +0,0 @@
# judge trigger 1788203999
-2
View File
@@ -185,5 +185,3 @@ review time.
## License
Not yet chosen. Until one is added, no reuse rights are granted.
_pilot eval judges test 1788201461_
+7 -13
View File
@@ -208,24 +208,18 @@ def ensure_evaluators() -> dict:
# ---------------------------------------------------------------------------
def rule_body(name: str, judge_name: str, sampling: float) -> dict:
"""POST /evaluation-rules shape for an LLM-as-judge trace rule.
"""POST /evaluation-rules shape for an LLM-as-judge observation rule.
Target is `trace` rather than `observation` on purpose: the standard
`/api/public/ingestion` path that ships review traces here feeds only
the trace-upsert queue, and `evalService.createEvalJobs` only creates
jobs for `targetObject ∈ {TRACE, DATASET}`. Observation rules are
triggered exclusively from the OTel ingestion pipeline, which this
pilot does not use. A trace rule reads the trace's own input/output —
`langfuse_trace` already writes `_review_input`/`_review_output` onto
the trace body for exactly this reason.
Mapping is required at both the rule root (server validates it there)
and inside `evaluator` (the API echoes it back).
The judge is referenced by `name`+`scope`, not by id — ids name specific
versions, names name the evaluator across versions. Mapping is required at
both the rule root (the server validates it there) and inside `evaluator`
(the API echoes it back). Filter is on `traceName` because that is the only
stringOptions column the observation-rule schema exposes.
"""
return {
"name": name,
"enabled": True,
"target": "trace",
"target": "observation",
"sampling": sampling,
"filter": [
{"column": "traceName", "operator": "any of",
+3 -9
View File
@@ -11,16 +11,10 @@ import eval_judges as ej # noqa: E402
# --- rule_body ------------------------------------------------------------
def test_rule_body_targets_traces():
"""Trace target matches the path `/api/public/ingestion` triggers.
Observation rules only fire from the OTel ingestion pipeline; this
pilot uses standard ingestion, so its jobs only come from
`evalService.createEvalJobs` and that dispatcher handles
`targetObject ∈ {TRACE, DATASET}`.
"""
def test_rule_body_targets_observations():
"""Trace-level rules wouldn't see observation input/output."""
body = ej.rule_body("rule-x", "finding_actionability", 1.0)
assert body["target"] == "trace"
assert body["target"] == "observation"
assert body["enabled"] is True