Marcos
2e1ad817e7
The filter bar matches on metadata only — not on input and not on the item id — so a dataset seeded with repo/pr in `input` alone could not be sliced by repo at all. Every facet worth filtering on is now a flat primitive in `metadata`: repo, owner, repo_name, pr, head_sha, finding_count, has_findings, max_severity, reviews_run and the review timestamp both ways. `owner` is split out because a filter on the joined repo matches one repo, never a whole org, and `max_severity` is "none" rather than absent because an absent key matches no filter. `eval_experiment.py` links reviews that already ran into a dataset run, one run per model, so the Experiments tab is populated without re-running the reviewer. One trace per (run, item), the most recent: a PR re-reviewed on every push has many traces and a run is one output per input. It posts to the deprecated /api/public/dataset-run-items — the notice exempts self-hosted v3 from the cutoff date and the pilot is stdlib-only by design. Revisit at v4. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
160 lines
4.9 KiB
Python
160 lines
4.9 KiB
Python
"""Tests for linking existing review traces into dataset runs."""
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
|
|
|
|
import eval_experiment as ex # noqa: E402
|
|
|
|
|
|
def trace(tid, repo="o/r", pr=1, model="M2", ts="2026-08-01T00:00:00Z", **md):
|
|
meta = {"repo": repo, "pr": pr}
|
|
meta.update(md)
|
|
return {
|
|
"id": tid,
|
|
"timestamp": ts,
|
|
"tags": [f"model:{model}", "engine:opencode"],
|
|
"metadata": meta,
|
|
}
|
|
|
|
|
|
# --- trace_model ----------------------------------------------------------
|
|
|
|
def test_model_read_from_tag():
|
|
assert ex.trace_model(trace("t1", model="MiniMax-M2.7")) == "MiniMax-M2.7"
|
|
|
|
|
|
def test_model_falls_back_when_untagged():
|
|
assert ex.trace_model({"tags": ["engine:opencode"]}) == "unknown"
|
|
|
|
|
|
def test_model_falls_back_when_tags_absent():
|
|
assert ex.trace_model({}) == "unknown"
|
|
|
|
|
|
# --- trace_item_id --------------------------------------------------------
|
|
|
|
def test_item_id_matches_the_bootstrap_scheme():
|
|
assert ex.trace_item_id(trace("t1", repo="netcracker/interview", pr=29)) == \
|
|
"netcracker__interview__pr29"
|
|
|
|
|
|
def test_trace_without_repo_is_not_an_item():
|
|
assert ex.trace_item_id({"metadata": {"pr": 1}}) is None
|
|
|
|
|
|
def test_trace_without_pr_is_not_an_item():
|
|
assert ex.trace_item_id({"metadata": {"repo": "o/r"}}) is None
|
|
|
|
|
|
def test_trace_without_metadata_is_not_an_item():
|
|
assert ex.trace_item_id({}) is None
|
|
|
|
|
|
# --- plan_runs ------------------------------------------------------------
|
|
|
|
ITEMS = {"o__r__pr1", "o__r__pr2"}
|
|
|
|
|
|
def test_traces_group_by_model():
|
|
plan = ex.plan_runs(
|
|
[trace("a", pr=1, model="x"), trace("b", pr=2, model="y")], ITEMS
|
|
)
|
|
assert set(plan["runs"]) == {"x", "y"}
|
|
|
|
|
|
def test_group_by_none_collapses_to_one_run():
|
|
plan = ex.plan_runs(
|
|
[trace("a", pr=1, model="x"), trace("b", pr=2, model="y")],
|
|
ITEMS,
|
|
group_by="none",
|
|
)
|
|
assert list(plan["runs"]) == ["all-traces"]
|
|
|
|
|
|
def test_only_the_newest_trace_per_item_is_kept():
|
|
"""A re-reviewed PR has many traces; a run takes one output per input."""
|
|
plan = ex.plan_runs(
|
|
[
|
|
trace("old", pr=1, ts="2026-08-01T00:00:00Z"),
|
|
trace("new", pr=1, ts="2026-08-09T00:00:00Z"),
|
|
],
|
|
ITEMS,
|
|
)
|
|
assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
|
|
|
|
|
|
def test_newest_wins_regardless_of_input_order():
|
|
older = trace("old", pr=1, ts="2026-08-01T00:00:00Z")
|
|
newer = trace("new", pr=1, ts="2026-08-09T00:00:00Z")
|
|
for order in ([older, newer], [newer, older]):
|
|
plan = ex.plan_runs(order, ITEMS)
|
|
assert plan["runs"]["M2"]["o__r__pr1"]["id"] == "new"
|
|
|
|
|
|
def test_trace_for_a_pr_outside_the_dataset_is_skipped():
|
|
plan = ex.plan_runs([trace("a", pr=99)], ITEMS)
|
|
assert plan["runs"] == {}
|
|
assert plan["skipped_not_in_dataset"] == 1
|
|
|
|
|
|
def test_non_review_trace_is_counted_separately():
|
|
plan = ex.plan_runs([{"id": "x", "metadata": {}}], ITEMS)
|
|
assert plan["skipped_not_a_review"] == 1
|
|
assert plan["skipped_not_in_dataset"] == 0
|
|
|
|
|
|
def test_same_pr_different_models_lands_in_both_runs():
|
|
plan = ex.plan_runs([trace("a", pr=1, model="x"), trace("b", pr=1, model="y")], ITEMS)
|
|
assert plan["runs"]["x"]["o__r__pr1"]["id"] == "a"
|
|
assert plan["runs"]["y"]["o__r__pr1"]["id"] == "b"
|
|
|
|
|
|
# --- run_name -------------------------------------------------------------
|
|
|
|
def test_run_name_prefixed():
|
|
assert ex.run_name("baseline", "MiniMax-M2.7") == "baseline-MiniMax-M2.7"
|
|
|
|
|
|
def test_empty_prefix_leaves_the_key_bare():
|
|
assert ex.run_name("", "MiniMax-M2.7") == "MiniMax-M2.7"
|
|
|
|
|
|
# --- create_run -----------------------------------------------------------
|
|
|
|
def test_create_run_posts_one_item_per_pair(monkeypatch):
|
|
calls = []
|
|
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
calls.append((method, path, body))
|
|
return 201, {}
|
|
|
|
monkeypatch.setattr(ex.eb, "_call", fake_call)
|
|
res = ex.create_run("run-1", {"o__r__pr1": trace("t1"), "o__r__pr2": trace("t2", pr=2)})
|
|
assert res["items_linked"] == 2
|
|
assert res["failed"] == []
|
|
assert {c[1] for c in calls} == {"/api/public/dataset-run-items"}
|
|
assert {c[2]["runName"] for c in calls} == {"run-1"}
|
|
|
|
|
|
def test_create_run_links_the_trace_to_the_item(monkeypatch):
|
|
seen = {}
|
|
|
|
def fake_call(method, path, body=None, timeout=20.0):
|
|
seen.update(body)
|
|
return 201, {}
|
|
|
|
monkeypatch.setattr(ex.eb, "_call", fake_call)
|
|
ex.create_run("run-1", {"o__r__pr1": trace("t1")})
|
|
assert seen["datasetItemId"] == "o__r__pr1"
|
|
assert seen["traceId"] == "t1"
|
|
assert seen["metadata"]["model"] == "M2"
|
|
|
|
|
|
def test_create_run_reports_rejected_items(monkeypatch):
|
|
monkeypatch.setattr(ex.eb, "_call", lambda *a, **k: (400, "nope"))
|
|
res = ex.create_run("run-1", {"o__r__pr1": trace("t1")})
|
|
assert res["items_linked"] == 0
|
|
assert res["failed"][0]["item"] == "o__r__pr1"
|
|
assert res["failed"][0]["status"] == 400
|