feat(eval): filterable item metadata and dataset runs for the Experiments tab
The filter bar matches on metadata only — not on input and not on the item id — so a dataset seeded with repo/pr in `input` alone could not be sliced by repo at all. Every facet worth filtering on is now a flat primitive in `metadata`: repo, owner, repo_name, pr, head_sha, finding_count, has_findings, max_severity, reviews_run and the review timestamp both ways. `owner` is split out because a filter on the joined repo matches one repo, never a whole org, and `max_severity` is "none" rather than absent because an absent key matches no filter. `eval_experiment.py` links reviews that already ran into a dataset run, one run per model, so the Experiments tab is populated without re-running the reviewer. One trace per (run, item), the most recent: a PR re-reviewed on every push has many traces and a run is one output per input. It posts to the deprecated /api/public/dataset-run-items — the notice exempts self-hosted v3 from the cutoff date and the pilot is stdlib-only by design. Revisit at v4. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -93,3 +93,66 @@ def test_one_item_per_pr_not_per_review(tmp_path):
|
||||
def test_items_are_not_flagged_as_human_labelled(tmp_path):
|
||||
path = _db(tmp_path, [("o/r", 1, 100, "a")])
|
||||
assert eb.read_review_items(path)[0]["metadata"]["labelled_by_human"] is False
|
||||
|
||||
|
||||
# --- metadata facets ------------------------------------------------------
|
||||
|
||||
def _md(findings=(), repo="netcracker/interview", pr=29):
|
||||
return eb._item_metadata(
|
||||
repo=repo, pr=pr, head_sha="abc", reviews_run=2, last_seen=1788189422,
|
||||
findings=[{"severity": s} for s in findings],
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_carries_the_repo_for_filtering():
|
||||
assert _md()["repo"] == "netcracker/interview"
|
||||
|
||||
|
||||
def test_metadata_splits_owner_from_repo_name():
|
||||
"""A filter on the joined repo can match one repo; owner matches an org."""
|
||||
md = _md()
|
||||
assert md["owner"] == "netcracker"
|
||||
assert md["repo_name"] == "interview"
|
||||
|
||||
|
||||
def test_owner_falls_back_when_the_repo_is_unqualified():
|
||||
md = _md(repo="standalone")
|
||||
assert md["owner"] == "standalone"
|
||||
assert md["repo_name"] == "standalone"
|
||||
|
||||
|
||||
def test_metadata_values_are_filterable_primitives():
|
||||
"""Nested objects and lists are not reachable from the filter bar."""
|
||||
for key, value in _md(["high"]).items():
|
||||
assert isinstance(value, (str, int, float, bool)), key
|
||||
|
||||
|
||||
def test_max_severity_is_the_worst_finding():
|
||||
assert _md(["low", "critical", "medium"])["max_severity"] == "critical"
|
||||
|
||||
|
||||
def test_max_severity_is_none_not_absent_for_a_silent_review():
|
||||
md = _md([])
|
||||
assert md["max_severity"] == "none"
|
||||
assert md["has_findings"] is False
|
||||
|
||||
|
||||
def test_unknown_severity_does_not_win_the_max():
|
||||
assert _md(["banana", "low"])["max_severity"] == "low"
|
||||
|
||||
|
||||
def test_severity_comparison_ignores_case():
|
||||
assert _md(["HIGH"])["max_severity"] == "high"
|
||||
|
||||
|
||||
def test_finding_count_matches_the_findings():
|
||||
md = _md(["low", "low"])
|
||||
assert md["finding_count"] == 2
|
||||
assert md["has_findings"] is True
|
||||
|
||||
|
||||
def test_last_reviewed_is_exposed_both_ways():
|
||||
"""The epoch sorts; the ISO string is what a human reads in a filter."""
|
||||
md = _md()
|
||||
assert md["last_reviewed_at"] == 1788189422
|
||||
assert md["last_reviewed_iso"].startswith("2026-08-31T")
|
||||
|
||||
Reference in New Issue
Block a user