"""Tests for the LLM-as-judge evaluator bootstrap."""
import os
import sys

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))

import eval_judges as ej  # noqa: E402


# --- rule_body ------------------------------------------------------------

def test_rule_body_targets_observations():
    """Trace-level rules wouldn't see observation input/output."""
    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
    assert body["target"] == "observation"
    assert body["enabled"] is True


def test_rule_body_filters_on_trace_name():
    """`name` isn't a stringOptions column; only `traceName` is."""
    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
    f = body["filter"][0]
    assert f["column"] == "traceName"
    assert f["operator"] == "any of"
    assert f["type"] == "stringOptions"
    assert "pr-review" in f["value"]


def test_rule_body_references_evaluator_by_name():
    """Ids are version-specific; rules must name the evaluator across versions."""
    body = ej.rule_body("rule-x", "finding_actionability", 1.0)
    assert body["evaluator"]["name"] == "finding_actionability"
    assert body["evaluator"]["scope"] == "project"


def test_rule_body_maps_input_and_output():
    """Both judges read the observation's own input/output."""
    body = ej.rule_body("rule-x", "any", 1.0)
    sources = {m["source"] for m in body["mapping"]}
    assert sources == {"input", "output"}


def test_rule_body_carries_mapping_at_both_levels():
    """The server validates `mapping` at the rule root and echoes it on the evaluator."""
    body = ej.rule_body("rule-x", "any", 1.0)
    assert body["mapping"]
    assert body["evaluator"]["variableMapping"] == body["mapping"]


def test_rule_body_passes_sampling_through():
    assert ej.rule_body("r", "any", 0.25)["sampling"] == 0.25


# --- ensure_evaluators idempotency ---------------------------------------

def test_ensure_evaluators_skips_existing(monkeypatch):
    seen = []

    def fake_call(method, path, body=None, timeout=20.0):
        seen.append(path)
        return 200, {}

    monkeypatch.setattr(ej.eb, "_call", fake_call)
    monkeypatch.setattr(ej, "existing_evaluators",
                        lambda: {"finding_actionability": "id-1", "review_self_consistency": "id-2"})
    res = ej.ensure_evaluators()
    assert res["created"] == {}
    assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
    assert res["failed"] == []
    assert seen == []


def test_ensure_evaluators_records_failures(monkeypatch):
    def fake_call(method, path, body=None, timeout=20.0):
        return 422, "boom"

    monkeypatch.setattr(ej.eb, "_call", fake_call)
    monkeypatch.setattr(ej, "existing_evaluators", lambda: {})
    res = ej.ensure_evaluators()
    assert res["created"] == {}
    assert res["failed"][0]["status"] == 422


# --- ensure_rules idempotency --------------------------------------------

def test_ensure_rules_skips_existing(monkeypatch):
    calls = []
    monkeypatch.setattr(ej.eb, "_call",
                        lambda *a, **k: calls.append(a) or (200, {}))
    monkeypatch.setattr(ej, "existing_evaluators",
                        lambda: {"finding_actionability": "id-1",
                                 "review_self_consistency": "id-2"})
    monkeypatch.setattr(ej, "existing_rule_names",
                        lambda: {"finding_actionability-on-reviews",
                                 "review_self_consistency-on-reviews"})
    res = ej.ensure_rules({"finding_actionability": "id-1",
                           "review_self_consistency": "id-2"}, 1.0)
    assert res["created"] == []
    assert sorted(res["skipped"]) == ["finding_actionability", "review_self_consistency"]
    assert calls == []


def test_ensure_rules_creates_when_missing(monkeypatch):
    calls = []
    monkeypatch.setattr(ej.eb, "_call",
                        lambda *a, **k: calls.append(a) or (201, {}))
    monkeypatch.setattr(ej, "existing_rule_names", lambda: set())
    res = ej.ensure_rules({"finding_actionability": "id-1"}, 1.0)
    assert res["created"] == ["finding_actionability"]
    assert calls[0][0] == "POST"
    assert calls[0][1] == "/api/public/unstable/evaluation-rules"


# --- judge shape ----------------------------------------------------------

def test_judges_have_required_keys():
    for j in ej.JUDGES:
        assert j["prompt"]
        assert j["outputDefinition"]["dataType"] in ("NUMERIC", "BOOLEAN", "CATEGORICAL")


def test_default_base_url_points_at_the_thinking_patch_proxy():
    """`8802` is the judge-proxy that adds a `signature` to thinking blocks."""
    assert "8802" in ej.JUDGE_BASE_URL