refactor: organize pilot packages

Group review, feedback, evaluation, observability, and entrypoint code into packages. Keep thin top-level compatibility shims for existing scripts and imports, and mirror the structure in the tests.

tests/pilot/evaluation_tests/judges_test.py

@@ -0,0 +1,132 @@
+
+
+"""Tests for the LLM-as-judge evaluator bootstrap."""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
+
+import eval_judges as ej  # noqa: E402
+
+
+# --- rule_body ------------------------------------------------------------
+
+def test_rule_body_targets_traces():
+    """Trace target matches the path `/api/public/ingestion` triggers.
+
+    Observation rules only fire from the OTel ingestion pipeline; this
+    pilot uses standard ingestion, so its jobs only come from
+    `evalService.createEvalJobs` and that dispatcher handles
+    `targetObject ∈ {TRACE, DATASET}`.
+    """
+    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
+    assert body["target"] == "trace"
+    assert body["enabled"] is True
+
+
+def test_rule_body_filters_on_trace_name():
+    """`name` isn't a stringOptions column; only `traceName` is."""
+    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
+    f = body["filter"][0]
+    assert f["column"] == "traceName"
+    assert f["operator"] == "any of"
+    assert f["type"] == "stringOptions"
+    assert "pr-review" in f["value"]
+
+
+def test_rule_body_references_evaluator_by_name():
+    """Ids are version-specific; rules must name the evaluator across versions."""
+    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
+    assert body["evaluator"]["name"] == "finding_actionability"
+    assert body["evaluator"]["scope"] == "project"
+
+
+def test_rule_body_maps_input_and_output():
+    """Both judges read the observation's own input/output."""
+    body = ej.rule_body("rule-x", "any", 1.0)
+    sources = {m["source"] for m in body["mapping"]}
+    assert sources == {"input", "output"}
+
+
+def test_rule_body_carries_mapping_at_both_levels():
+    """The server validates `mapping` at the rule root and echoes it on the evaluator."""
+    body = ej.rule_body("rule-x", "any", 1.0)
+    assert body["mapping"]
+    assert body["evaluator"]["variableMapping"] == body["mapping"]
+
+
+def test_rule_body_passes_sampling_through():
+    assert ej.rule_body("r", "any", 0.25)["sampling"] == 0.25
+
+
+# --- ensure_evaluators idempotency ---------------------------------------
+
+def test_ensure_evaluators_skips_existing(monkeypatch):
+    seen = []
+
+    def fake_call(method, path, body=None, timeout=20.0):
+        seen.append(path)
+        return 200, {}
+
+    monkeypatch.setattr(ej.eb, "_call", fake_call)
+    monkeypatch.setattr(ej, "existing_evaluators",
+                        lambda: {"finding_actionability": "id-1", "review_self_consistency": "id-2"})
+    res = ej.ensure_evaluators()
+    assert res["created"] == {}
+    assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
+    assert res["failed"] == []
+    assert seen == []
+
+
+def test_ensure_evaluators_records_failures(monkeypatch):
+    def fake_call(method, path, body=None, timeout=20.0):
+        return 422, "boom"
+
+    monkeypatch.setattr(ej.eb, "_call", fake_call)
+    monkeypatch.setattr(ej, "existing_evaluators", lambda: {})
+    res = ej.ensure_evaluators()
+    assert res["created"] == {}
+    assert res["failed"][0]["status"] == 422
+
+
+# --- ensure_rules idempotency --------------------------------------------
+
+def test_ensure_rules_skips_existing(monkeypatch):
+    calls = []
+    monkeypatch.setattr(ej.eb, "_call",
+                        lambda *a, **k: calls.append(a) or (200, {}))
+    monkeypatch.setattr(ej, "existing_evaluators",
+                        lambda: {"finding_actionability": "id-1",
+                                 "review_self_consistency": "id-2"})
+    monkeypatch.setattr(ej, "existing_rule_names",
+                        lambda: {"finding_actionability-on-reviews",
+                                 "review_self_consistency-on-reviews"})
+    res = ej.ensure_rules({"finding_actionability": "id-1",
+                           "review_self_consistency": "id-2"}, 1.0)
+    assert res["created"] == []
+    assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
+    assert calls == []
+
+
+def test_ensure_rules_creates_when_missing(monkeypatch):
+    calls = []
+    monkeypatch.setattr(ej.eb, "_call",
+                        lambda *a, **k: calls.append(a) or (201, {}))
+    monkeypatch.setattr(ej, "existing_rule_names", lambda: set())
+    res = ej.ensure_rules({"finding_actionability": "id-1"}, 1.0)
+    assert res["created"] == ["finding_actionability"]
+    assert calls[0][0] == "POST"
+    assert calls[0][1] == "/api/public/unstable/evaluation-rules"
+
+
+# --- judge shape ----------------------------------------------------------
+
+def test_judges_have_required_keys():
+    for j in ej.JUDGES:
+        assert j["prompt"]
+        assert j["outputDefinition"]["dataType"] in ("NUMERIC", "BOOLEAN", "CATEGORICAL")
+
+
+def test_default_base_url_points_at_the_thinking_patch_proxy():
+    """`8802` is the judge-proxy that adds a `signature` to thinking blocks."""
+    assert "8802" in ej.JUDGE_BASE_URL