feat(eval): filterable item metadata and dataset runs for the Experiments tab

The filter bar matches on metadata only — not on input and not on the item id —
so a dataset seeded with repo/pr in `input` alone could not be sliced by repo
at all. Every facet worth filtering on is now a flat primitive in `metadata`:
repo, owner, repo_name, pr, head_sha, finding_count, has_findings,
max_severity, reviews_run and the review timestamp both ways. `owner` is split
out because a filter on the joined repo matches one repo, never a whole org,
and `max_severity` is "none" rather than absent because an absent key matches
no filter.

`eval_experiment.py` links reviews that already ran into a dataset run, one run
per model, so the Experiments tab is populated without re-running the reviewer.
One trace per (run, item), the most recent: a PR re-reviewed on every push has
many traces and a run is one output per input.

It posts to the deprecated /api/public/dataset-run-items — the notice exempts
self-hosted v3 from the cutoff date and the pilot is stdlib-only by design.
Revisit at v4.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Marcos
2026-08-31 15:53:35 +00:00
parent 1534a99630
commit 2e1ad817e7
5 changed files with 535 additions and 8 deletions
+63
View File
@@ -93,3 +93,66 @@ def test_one_item_per_pr_not_per_review(tmp_path):
def test_items_are_not_flagged_as_human_labelled(tmp_path):
path = _db(tmp_path, [("o/r", 1, 100, "a")])
assert eb.read_review_items(path)[0]["metadata"]["labelled_by_human"] is False
# --- metadata facets ------------------------------------------------------
def _md(findings=(), repo="netcracker/interview", pr=29):
return eb._item_metadata(
repo=repo, pr=pr, head_sha="abc", reviews_run=2, last_seen=1788189422,
findings=[{"severity": s} for s in findings],
)
def test_metadata_carries_the_repo_for_filtering():
assert _md()["repo"] == "netcracker/interview"
def test_metadata_splits_owner_from_repo_name():
"""A filter on the joined repo can match one repo; owner matches an org."""
md = _md()
assert md["owner"] == "netcracker"
assert md["repo_name"] == "interview"
def test_owner_falls_back_when_the_repo_is_unqualified():
md = _md(repo="standalone")
assert md["owner"] == "standalone"
assert md["repo_name"] == "standalone"
def test_metadata_values_are_filterable_primitives():
"""Nested objects and lists are not reachable from the filter bar."""
for key, value in _md(["high"]).items():
assert isinstance(value, (str, int, float, bool)), key
def test_max_severity_is_the_worst_finding():
assert _md(["low", "critical", "medium"])["max_severity"] == "critical"
def test_max_severity_is_none_not_absent_for_a_silent_review():
md = _md([])
assert md["max_severity"] == "none"
assert md["has_findings"] is False
def test_unknown_severity_does_not_win_the_max():
assert _md(["banana", "low"])["max_severity"] == "low"
def test_severity_comparison_ignores_case():
assert _md(["HIGH"])["max_severity"] == "high"
def test_finding_count_matches_the_findings():
md = _md(["low", "low"])
assert md["finding_count"] == 2
assert md["has_findings"] is True
def test_last_reviewed_is_exposed_both_ways():
"""The epoch sorts; the ISO string is what a human reads in a filter."""
md = _md()
assert md["last_reviewed_at"] == 1788189422
assert md["last_reviewed_iso"].startswith("2026-08-31T")