merge: resolve eval judge rule conflict

2026-09-01 00:26:25 +00:00
parent 5b8be61e2f 193a90e63e
commit 3f30bdd450
8 changed files with 824 additions and 11 deletions
@@ -0,0 +1,158 @@
+"""Tests for the eval bootstrap's dataset-item construction."""
+import os
+import sqlite3
+import sys
+import urllib.parse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
+
+import eval_bootstrap as eb  # noqa: E402
+
+
+# --- item_id --------------------------------------------------------------
+
+def test_item_id_has_no_path_separator():
+    """A `/` would split the UI's item route into extra path segments."""
+    assert "/" not in eb.item_id("netcracker/interview", 29)
+
+
+def test_item_id_has_no_fragment_marker():
+    """Everything after a `#` is a fragment the browser never sends."""
+    assert "#" not in eb.item_id("netcracker/interview", 29)
+
+
+def test_item_id_survives_a_url_round_trip():
+    """The id must appear verbatim in a path, needing no percent-encoding."""
+    ident = eb.item_id("netcracker/interview", 29)
+    assert urllib.parse.quote(ident, safe="") == ident
+
+
+def test_item_id_keeps_repo_and_pr_readable():
+    assert eb.item_id("netcracker/interview", 29) == "netcracker__interview__pr29"
+
+
+def test_item_id_is_unique_per_pr():
+    assert eb.item_id("o/r", 1) != eb.item_id("o/r", 2)
+
+
+def test_item_id_is_unique_per_repo():
+    assert eb.item_id("o/one", 1) != eb.item_id("o/two", 1)
+
+
+def test_item_id_accepts_a_string_pr():
+    assert eb.item_id("o/r", "29") == eb.item_id("o/r", 29)
+
+
+# --- read_review_items ----------------------------------------------------
+
+def _db(tmp_path, rows, findings=()):
+    path = str(tmp_path / "feedback.db")
+    conn = sqlite3.connect(path)
+    conn.execute(
+        "CREATE TABLE review (repo TEXT, pr INTEGER, posted_at INTEGER, head_sha TEXT)"
+    )
+    conn.execute(
+        "CREATE TABLE inline_finding (repo TEXT, pr INTEGER, path TEXT, line INTEGER,"
+        " severity TEXT, problem TEXT, fix TEXT)"
+    )
+    conn.executemany("INSERT INTO review VALUES (?,?,?,?)", rows)
+    conn.executemany("INSERT INTO inline_finding VALUES (?,?,?,?,?,?,?)", findings)
+    conn.commit()
+    conn.close()
+    return path
+
+
+def test_items_use_url_safe_ids(tmp_path):
+    path = _db(tmp_path, [("netcracker/interview", 29, 100, "abc")])
+    items = eb.read_review_items(path)
+    assert [i["id"] for i in items] == ["netcracker__interview__pr29"]
+
+
+def test_item_input_keeps_the_real_repo_name(tmp_path):
+    """The id is mangled for the URL; the payload must stay faithful."""
+    path = _db(tmp_path, [("netcracker/interview", 29, 100, "abc")])
+    item = eb.read_review_items(path)[0]
+    assert item["input"]["repo"] == "netcracker/interview"
+    assert item["input"]["pr"] == 29
+
+
+def test_one_item_per_pr_not_per_review(tmp_path):
+    path = _db(
+        tmp_path,
+        [
+            ("o/r", 1, 100, "a"),
+            ("o/r", 1, 200, "b"),
+            ("o/r", 2, 300, "c"),
+        ],
+    )
+    items = eb.read_review_items(path)
+    assert [i["id"] for i in items] == ["o__r__pr1", "o__r__pr2"]
+    assert items[0]["metadata"]["reviews_run"] == 2
+
+
+def test_items_are_not_flagged_as_human_labelled(tmp_path):
+    path = _db(tmp_path, [("o/r", 1, 100, "a")])
+    assert eb.read_review_items(path)[0]["metadata"]["labelled_by_human"] is False
+
+
+# --- metadata facets ------------------------------------------------------
+
+def _md(findings=(), repo="netcracker/interview", pr=29):
+    return eb._item_metadata(
+        repo=repo, pr=pr, head_sha="abc", reviews_run=2, last_seen=1788189422,
+        findings=[{"severity": s} for s in findings],
+    )
+
+
+def test_metadata_carries_the_repo_for_filtering():
+    assert _md()["repo"] == "netcracker/interview"
+
+
+def test_metadata_splits_owner_from_repo_name():
+    """A filter on the joined repo can match one repo; owner matches an org."""
+    md = _md()
+    assert md["owner"] == "netcracker"
+    assert md["repo_name"] == "interview"
+
+
+def test_owner_falls_back_when_the_repo_is_unqualified():
+    md = _md(repo="standalone")
+    assert md["owner"] == "standalone"
+    assert md["repo_name"] == "standalone"
+
+
+def test_metadata_values_are_filterable_primitives():
+    """Nested objects and lists are not reachable from the filter bar."""
+    for key, value in _md(["high"]).items():
+        assert isinstance(value, (str, int, float, bool)), key
+
+
+def test_max_severity_is_the_worst_finding():
+    assert _md(["low", "critical", "medium"])["max_severity"] == "critical"
+
+
+def test_max_severity_is_none_not_absent_for_a_silent_review():
+    md = _md([])
+    assert md["max_severity"] == "none"
+    assert md["has_findings"] is False
+
+
+def test_unknown_severity_does_not_win_the_max():
+    assert _md(["banana", "low"])["max_severity"] == "low"
+
+
+def test_severity_comparison_ignores_case():
+    assert _md(["HIGH"])["max_severity"] == "high"
+
+
+def test_finding_count_matches_the_findings():
+    md = _md(["low", "low"])
+    assert md["finding_count"] == 2
+    assert md["has_findings"] is True
+
+
+def test_last_reviewed_is_exposed_both_ways():
+    """The epoch sorts; the ISO string is what a human reads in a filter."""
+    md = _md()
+    assert md["last_reviewed_at"] == 1788189422
+    assert md["last_reviewed_iso"].startswith("2026-08-31T")
@@ -0,0 +1,159 @@
+"""Tests for linking existing review traces into dataset runs."""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
+
+import eval_experiment as ex  # noqa: E402
+
+
+def trace(tid, repo="o/r", pr=1, model="M2", ts="2026-08-01T00:00:00Z", **md):
+    meta = {"repo": repo, "pr": pr}
+    meta.update(md)
+    return {
+        "id": tid,
+        "timestamp": ts,
+        "tags": [f"model:{model}", "engine:opencode"],
+        "metadata": meta,
+    }
+
+
+# --- trace_model ----------------------------------------------------------
+
+def test_model_read_from_tag():
+    assert ex.trace_model(trace("t1", model="MiniMax-M2.7")) == "MiniMax-M2.7"
+
+
+def test_model_falls_back_when_untagged():
+    assert ex.trace_model({"tags": ["engine:opencode"]}) == "unknown"
+
+
+def test_model_falls_back_when_tags_absent():
+    assert ex.trace_model({}) == "unknown"
+
+
+# --- trace_item_id --------------------------------------------------------
+
+def test_item_id_matches_the_bootstrap_scheme():
+    assert ex.trace_item_id(trace("t1", repo="netcracker/interview", pr=29)) == \
+        "netcracker__interview__pr29"
+
+
+def test_trace_without_repo_is_not_an_item():
+    assert ex.trace_item_id({"metadata": {"pr": 1}}) is None
+
+
+def test_trace_without_pr_is_not_an_item():
+    assert ex.trace_item_id({"metadata": {"repo": "o/r"}}) is None
+
+
+def test_trace_without_metadata_is_not_an_item():
+    assert ex.trace_item_id({}) is None
+
+
+# --- plan_runs ------------------------------------------------------------
+
+ITEMS = {"o__r__pr1", "o__r__pr2"}
+
+
+def test_traces_group_by_model():
+    plan = ex.plan_runs(
+        [trace("a", pr=1, model="x"), trace("b", pr=2, model="y")], ITEMS
+    )
+    assert set(plan["runs"]) == {"x", "y"}
+
+
+def test_group_by_none_collapses_to_one_run():
+    plan = ex.plan_runs(
+        [trace("a", pr=1, model="x"), trace("b", pr=2, model="y")],
+        ITEMS,
+        group_by="none",
+    )
+    assert list(plan["runs"]) == ["all-traces"]
+
+
+def test_only_the_newest_trace_per_item_is_kept():
+    """A re-reviewed PR has many traces; a run takes one output per input."""
+    plan = ex.plan_runs(
+        [
+            trace("old", pr=1, ts="2026-08-01T00:00:00Z"),
+            trace("new", pr=1, ts="2026-08-09T00:00:00Z"),
+        ],
+        ITEMS,
+    )
+    assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
+
+
+def test_newest_wins_regardless_of_input_order():
+    older = trace("old", pr=1, ts="2026-08-01T00:00:00Z")
+    newer = trace("new", pr=1, ts="2026-08-09T00:00:00Z")
+    for order in ([older, newer], [newer, older]):
+        plan = ex.plan_runs(order, ITEMS)
+        assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
+
+
+def test_trace_for_a_pr_outside_the_dataset_is_skipped():
+    plan = ex.plan_runs([trace("a", pr=99)], ITEMS)
+    assert plan["runs"] == {}
+    assert plan["skipped_not_in_dataset"] == 1
+
+
+def test_non_review_trace_is_counted_separately():
+    plan = ex.plan_runs([{"id": "x", "metadata": {}}], ITEMS)
+    assert plan["skipped_not_a_review"] == 1
+    assert plan["skipped_not_in_dataset"] == 0
+
+
+def test_same_pr_different_models_lands_in_both_runs():
+    plan = ex.plan_runs([trace("a", pr=1, model="x"), trace("b", pr=1, model="y")], ITEMS)
+    assert plan["runs"]["x"]["o__r__pr1"]["id"] == "a"
+    assert plan["runs"]["y"]["o__r__pr1"]["id"] == "b"
+
+
+# --- run_name -------------------------------------------------------------
+
+def test_run_name_prefixed():
+    assert ex.run_name("baseline", "MiniMax-M2.7") == "baseline-MiniMax-M2.7"
+
+
+def test_empty_prefix_leaves_the_key_bare():
+    assert ex.run_name("", "MiniMax-M2.7") == "MiniMax-M2.7"
+
+
+# --- create_run -----------------------------------------------------------
+
+def test_create_run_posts_one_item_per_pair(monkeypatch):
+    calls = []
+
+    def fake_call(method, path, body=None, timeout=20.0):
+        calls.append((method, path, body))
+        return 201, {}
+
+    monkeypatch.setattr(ex.eb, "_call", fake_call)
+    res = ex.create_run("run-1", {"o__r__pr1": trace("t1"), "o__r__pr2": trace("t2", pr=2)})
+    assert res["items_linked"] == 2
+    assert res["failed"] == []
+    assert {c[1] for c in calls} == {"/api/public/dataset-run-items"}
+    assert {c[2]["runName"] for c in calls} == {"run-1"}
+
+
+def test_create_run_links_the_trace_to_the_item(monkeypatch):
+    seen = {}
+
+    def fake_call(method, path, body=None, timeout=20.0):
+        seen.update(body)
+        return 201, {}
+
+    monkeypatch.setattr(ex.eb, "_call", fake_call)
+    ex.create_run("run-1", {"o__r__pr1": trace("t1")})
+    assert seen["datasetItemId"] == "o__r__pr1"
+    assert seen["traceId"] == "t1"
+    assert seen["metadata"]["model"] == "M2"
+
+
+def test_create_run_reports_rejected_items(monkeypatch):
+    monkeypatch.setattr(ex.eb, "_call", lambda *a, **k: (400, "nope"))
+    res = ex.create_run("run-1", {"o__r__pr1": trace("t1")})
+    assert res["items_linked"] == 0
+    assert res["failed"][0]["item"] == "o__r__pr1"
+    assert res["failed"][0]["status"] == 400