Claude
7a510a926d
Group review, feedback, evaluation, observability, and entrypoint code into packages. Keep thin top-level compatibility shims for existing scripts and imports, and mirror the structure in the tests.
160 lines
4.9 KiB
Python
160 lines
4.9 KiB
Python
"""Tests for linking existing review traces into dataset runs."""
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "pilot"))
|
|
|
|
import eval_experiment as ex # noqa: E402
|
|
|
|
|
|
def trace(tid, repo="o/r", pr=1, model="M2", ts="2026-08-01T00:00:00Z", **md):
|
|
meta = {"repo": repo, "pr": pr}
|
|
meta.update(md)
|
|
return {
|
|
"id": tid,
|
|
"timestamp": ts,
|
|
"tags": [f"model:{model}", "engine:opencode"],
|
|
"metadata": meta,
|
|
}
|
|
|
|
|
|
# --- trace_model ----------------------------------------------------------
|
|
|
|
def test_model_read_from_tag():
|
|
assert ex.trace_model(trace("t1", model="MiniMax-M2.7")) == "MiniMax-M2.7"
|
|
|
|
|
|
def test_model_falls_back_when_untagged():
|
|
assert ex.trace_model({"tags": ["engine:opencode"]}) == "unknown"
|
|
|
|
|
|
def test_model_falls_back_when_tags_absent():
|
|
assert ex.trace_model({}) == "unknown"
|
|
|
|
|
|
# --- trace_item_id --------------------------------------------------------
|
|
|
|
def test_item_id_matches_the_bootstrap_scheme():
|
|
assert ex.trace_item_id(trace("t1", repo="netcracker/interview", pr=29)) == \
|
|
"netcracker__interview__pr29"
|
|
|
|
|
|
def test_trace_without_repo_is_not_an_item():
|
|
assert ex.trace_item_id({"metadata": {"pr": 1}}) is None
|
|
|
|
|
|
def test_trace_without_pr_is_not_an_item():
|
|
assert ex.trace_item_id({"metadata": {"repo": "o/r"}}) is None
|
|
|
|
|
|
def test_trace_without_metadata_is_not_an_item():
|
|
assert ex.trace_item_id({}) is None
|
|
|
|
|
|
# --- plan_runs ------------------------------------------------------------
|
|
|
|
ITEMS = {"o__r__pr1", "o__r__pr2"}
|
|
|
|
|
|
def test_traces_group_by_model():
|
|
plan = ex.plan_runs(
|
|
[trace("a", pr=1, model="x"), trace("b", pr=2, model="y")], ITEMS
|
|
)
|
|
assert set(plan["runs"]) == {"x", "y"}
|
|
|
|
|
|
def test_group_by_none_collapses_to_one_run():
|
|
plan = ex.plan_runs(
|
|
[trace("a", pr=1, model="x"), trace("b", pr=2, model="y")],
|
|
ITEMS,
|
|
group_by="none",
|
|
)
|
|
assert list(plan["runs"]) == ["all-traces"]
|
|
|
|
|
|
def test_only_the_newest_trace_per_item_is_kept():
|
|
"""A re-reviewed PR has many traces; a run takes one output per input."""
|
|
plan = ex.plan_runs(
|
|
[
|
|
trace("old", pr=1, ts="2026-08-01T00:00:00Z"),
|
|
trace("new", pr=1, ts="2026-08-09T00:00:00Z"),
|
|
],
|
|
ITEMS,
|
|
)
|
|
assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
|
|
|
|
|
|
def test_newest_wins_regardless_of_input_order():
|
|
older = trace("old", pr=1, ts="2026-08-01T00:00:00Z")
|
|
newer = trace("new", pr=1, ts="2026-08-09T00:00:00Z")
|
|
for order in ([older, newer], [newer, older]):
|
|
plan = ex.plan_runs(order, ITEMS)
|
|
assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
|
|
|
|
|
|
def test_trace_for_a_pr_outside_the_dataset_is_skipped():
|
|
plan = ex.plan_runs([trace("a", pr=99)], ITEMS)
|
|
assert plan["runs"] == {}
|
|
assert plan["skipped_not_in_dataset"] == 1
|
|
|
|
|
|
def test_non_review_trace_is_counted_separately():
|
|
plan = ex.plan_runs([{"id": "x", "metadata": {}}], ITEMS)
|
|
assert plan["skipped_not_a_review"] == 1
|
|
assert plan["skipped_not_in_dataset"] == 0
|
|
|
|
|
|
def test_same_pr_different_models_lands_in_both_runs():
|
|
plan = ex.plan_runs([trace("a", pr=1, model="x"), trace("b", pr=1, model="y")], ITEMS)
|
|
assert plan["runs"]["x"]["o__r__pr1"]["id"] == "a"
|
|
assert plan["runs"]["y"]["o__r__pr1"]["id"] == "b"
|
|
|
|
|
|
# --- run_name -------------------------------------------------------------
|
|
|
|
def test_run_name_prefixed():
|
|
assert ex.run_name("baseline", "MiniMax-M2.7") == "baseline-MiniMax-M2.7"
|
|
|
|
|
|
def test_empty_prefix_leaves_the_key_bare():
|
|
assert ex.run_name("", "MiniMax-M2.7") == "MiniMax-M2.7"
|
|
|
|
|
|
# --- create_run -----------------------------------------------------------
|
|
|
|
def test_create_run_posts_one_item_per_pair(monkeypatch):
|
|
calls = []
|
|
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
calls.append((method, path, body))
|
|
return 201, {}
|
|
|
|
monkeypatch.setattr(ex.eb, "_call", fake_call)
|
|
res = ex.create_run("run-1", {"o__r__pr1": trace("t1"), "o__r__pr2": trace("t2", pr=2)})
|
|
assert res["items_linked"] == 2
|
|
assert res["failed"] == []
|
|
assert {c[1] for c in calls} == {"/api/public/dataset-run-items"}
|
|
assert {c[2]["runName"] for c in calls} == {"run-1"}
|
|
|
|
|
|
def test_create_run_links_the_trace_to_the_item(monkeypatch):
|
|
seen = {}
|
|
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
seen.update(body)
|
|
return 201, {}
|
|
|
|
monkeypatch.setattr(ex.eb, "_call", fake_call)
|
|
ex.create_run("run-1", {"o__r__pr1": trace("t1")})
|
|
assert seen["datasetItemId"] == "o__r__pr1"
|
|
assert seen["traceId"] == "t1"
|
|
assert seen["metadata"]["model"] == "M2"
|
|
|
|
|
|
def test_create_run_reports_rejected_items(monkeypatch):
|
|
monkeypatch.setattr(ex.eb, "_call", lambda *a, **k: (400, "nope"))
|
|
res = ex.create_run("run-1", {"o__r__pr1": trace("t1")})
|
|
assert res["items_linked"] == 0
|
|
assert res["failed"][0]["item"] == "o__r__pr1"
|
|
assert res["failed"][0]["status"] == 400
|