feat(pilot): behavioural scorers, feedback ground truth, and an eval dataset

Adds the evaluation layer on top of the review traces: five deterministic
scores describing how the reviewer behaved, a bridge that turns human reactions
into ground truth, and a dataset seeded from the reviews already run.

The two are kept apart on purpose. feedback.db has recorded 113 reviews and
zero reactions, resolutions or replies — nobody has ever responded to a bot
comment — so an accuracy metric cannot be built yet. The scorers therefore
measure behaviour, which is computable from data in hand, and feedback_scores
turns verdicts into scores the moment any arrive.

eval_scores.py emits finding_rate, severity_info_ratio, severity_max,
dropped_findings and cost_per_finding into the same ingestion batch as the
trace. Undefined values are omitted rather than reported as zero: an info ratio
over a silent review is undefined, and charting it as 0 would read as perfect
calibration.

dropped_findings needed a parser change. Both parsers silently discard findings
with an unusable path/line, which made a model emitting garbage locations
indistinguishable from one that found nothing. last_parse_dropped() exposes the
delta, read at parse time — after apply_repo_config the drops are the config
working as intended, not the model misbehaving.

feedback_scores.py scores the session ("{repo}#{pr}"), because feedback arrives
days later against a PR and nothing records which re-run produced which
comment. review_acceptance is absent rather than 0 when nothing was engaged.

eval_bootstrap.py registers the score configs, seeds the pragent-reviews
dataset, and can backfill scores onto traces that predate the scorers.
expectedOutput is the reviewer's own prior output, flagged
labelled_by_human: false — a regression baseline, not verified truth.

Also fixes a silent telemetry failure: the ingestion endpoint answers 207 when
only some events succeed, so a batch with every event rejected still looked
like success. Score events were missing the required per-event timestamp and
ingested nothing while reporting 207. _warn_on_rejected_events now logs the
per-event errors under LANGFUSE_DEBUG.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/pilot/test_eval_scores.py

@@ -0,0 +1,200 @@
+"""Tests for the deterministic review scorers."""
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
+
+import eval_scores as es  # noqa: E402
+
+
+def f(sev, path="a.py", line=1):
+    return {"severity": sev, "path": path, "line": line, "problem": "p", "fix": ""}
+
+
+# --- finding_rate ---------------------------------------------------------
+
+def test_finding_rate_counts_findings():
+    assert es.finding_rate([f("high"), f("low")]) == 2.0
+
+
+def test_finding_rate_zero_for_silent_review():
+    assert es.finding_rate([]) == 0.0
+    assert es.finding_rate(None) == 0.0
+
+
+# --- severity_info_ratio --------------------------------------------------
+
+def test_info_ratio_all_advisory():
+    assert es.severity_info_ratio([f("info"), f("trivial")]) == 1.0
+
+
+def test_info_ratio_mixed():
+    assert es.severity_info_ratio([f("info"), f("high")]) == 0.5
+
+
+def test_info_ratio_none_when_no_findings():
+    # Undefined, not zero — zero would read as perfectly calibrated.
+    assert es.severity_info_ratio([]) is None
+
+
+def test_info_ratio_unknown_severity_treated_as_medium():
+    # Matches _normalize_finding's fallback, so an odd severity is not
+    # silently counted as advisory.
+    assert es.severity_info_ratio([f("bogus")]) == 0.0
+
+
+# --- severity_max ---------------------------------------------------------
+
+def test_severity_max_picks_highest():
+    assert es.severity_max([f("info"), f("critical"), f("low")]) == "critical"
+
+
+def test_severity_max_none_when_silent():
+    assert es.severity_max([]) == "none"
+
+
+def test_severity_max_case_insensitive():
+    assert es.severity_max([f("HIGH")]) == "high"
+
+
+# --- dropped_findings -----------------------------------------------------
+
+def test_dropped_findings_delta():
+    assert es.dropped_findings(5, 2) == 3.0
+
+
+def test_dropped_findings_never_negative():
+    assert es.dropped_findings(1, 3) == 0.0
+
+
+def test_dropped_findings_none_when_unknown():
+    assert es.dropped_findings(None, 2) is None
+
+
+# --- cost_per_finding -----------------------------------------------------
+
+def test_cost_per_finding_divides():
+    assert es.cost_per_finding(1.0, [f("high"), f("low")]) == 0.5
+
+
+def test_cost_per_finding_silent_review_divides_by_one():
+    # The run still cost money; attributing all of it to "found nothing" is
+    # the honest reading, and it avoids a division by zero.
+    assert es.cost_per_finding(0.25, []) == 0.25
+
+
+def test_cost_per_finding_none_when_unpriced():
+    assert es.cost_per_finding(None, [f("high")]) is None
+
+
+def test_cost_per_finding_none_on_garbage():
+    assert es.cost_per_finding("abc", [f("high")]) is None
+
+
+# --- build_scores ---------------------------------------------------------
+
+def _by_name(events):
+    return {e["body"]["name"]: e["body"] for e in events}
+
+
+def test_build_scores_emits_expected_set():
+    events = es.build_scores(
+        trace_id="t1", findings=[f("high"), f("info")], environment="claude",
+        cost_usd=0.5, dropped_count=2, timestamp="2026-01-01T00:00:00Z",
+    )
+    names = _by_name(events)
+    assert set(names) == {
+        es.FINDING_RATE, es.SEVERITY_INFO_RATIO, es.SEVERITY_MAX,
+        es.DROPPED_FINDINGS, es.COST_PER_FINDING,
+    }
+    assert names[es.FINDING_RATE]["value"] == 2.0
+    assert names[es.SEVERITY_MAX]["value"] == "high"
+    assert names[es.DROPPED_FINDINGS]["value"] == 2.0
+    assert names[es.COST_PER_FINDING]["value"] == 0.25
+
+
+def test_build_scores_all_events_are_score_create_on_the_trace():
+    events = es.build_scores(
+        trace_id="t9", findings=[f("low")], environment="ollama", cost_usd=1.0,
+    )
+    assert all(e["type"] == "score-create" for e in events)
+    assert all(e["body"]["traceId"] == "t9" for e in events)
+    assert all(e["body"]["environment"] == "ollama" for e in events)
+
+
+def test_build_scores_omits_undefined_scores():
+    # No cost and no drop count measured -> those scores are absent, not zero.
+    events = es.build_scores(trace_id="t2", findings=[], environment="ollama")
+    names = set(_by_name(events))
+    assert es.COST_PER_FINDING not in names
+    assert es.DROPPED_FINDINGS not in names
+    assert es.SEVERITY_INFO_RATIO not in names
+    assert names == {es.FINDING_RATE, es.SEVERITY_MAX}
+
+
+def test_build_scores_categorical_value_is_string():
+    events = es.build_scores(trace_id="t3", findings=[f("high")], environment="claude")
+    sev = _by_name(events)[es.SEVERITY_MAX]
+    assert sev["dataType"] == "CATEGORICAL"
+    assert isinstance(sev["value"], str)
+
+
+def test_build_scores_numeric_values_are_floats():
+    events = es.build_scores(
+        trace_id="t4", findings=[f("high")], environment="claude", cost_usd=1,
+    )
+    for name, body in _by_name(events).items():
+        if body["dataType"] == "NUMERIC":
+            assert isinstance(body["value"], float), name
+
+
+def test_build_scores_comment_propagates():
+    events = es.build_scores(
+        trace_id="t5", findings=[f("high")], environment="claude",
+        cost_usd=1.0, comment="cost basis: equivalent:claude-sonnet-5",
+    )
+    assert all("equivalent" in e["body"]["comment"] for e in events)
+
+
+# --- score configs --------------------------------------------------------
+
+def test_every_emitted_score_has_a_config():
+    configured = {c["name"] for c in es.SCORE_CONFIGS}
+    events = es.build_scores(
+        trace_id="t6", findings=[f("high")], environment="claude",
+        cost_usd=1.0, dropped_count=0,
+    )
+    assert set(_by_name(events)) <= configured
+
+
+def test_severity_max_config_covers_every_severity_it_can_emit():
+    labels = {c["label"] for c in
+              next(c for c in es.SCORE_CONFIGS if c["name"] == es.SEVERITY_MAX)["categories"]}
+    assert set(es.SEVERITY_RANK) | {"none"} == labels
+
+
+# --- ingestion envelope ---------------------------------------------------
+
+def test_every_event_carries_a_timestamp():
+    # Ingestion rejects events without one, and reports the rejection as a
+    # per-event 400 inside an HTTP 207 that reads as success.
+    events = es.build_scores(
+        trace_id="t7", findings=[f("high")], environment="claude", cost_usd=1.0,
+    )
+    assert events
+    assert all(e.get("timestamp") for e in events)
+
+
+def test_timestamp_defaults_when_caller_omits_it():
+    events = es.build_scores(trace_id="t8", findings=[f("low")], environment="claude")
+    assert all(isinstance(e["timestamp"], str) and e["timestamp"].endswith("Z") for e in events)
+
+
+def test_explicit_timestamp_is_used():
+    events = es.build_scores(
+        trace_id="t9", findings=[f("low")], environment="claude",
+        timestamp="2026-01-02T03:04:05Z",
+    )
+    assert all(e["timestamp"] == "2026-01-02T03:04:05Z" for e in events)

tests/pilot/test_feedback_scores.py

@@ -0,0 +1,207 @@
+"""Tests for the feedback.db -> Langfuse score bridge."""
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "pilot"))
+
+import feedback  # noqa: E402
+import feedback_scores as fs  # noqa: E402
+
+
+@pytest.fixture
+def db(tmp_path):
+    conn = feedback.init(str(tmp_path / "fb.db"))
+    yield conn
+    conn.close()
+
+
+def _seed_finding(conn, repo="o/r", pr=1, comment_id=100, path="a.py", line=1):
+    cur = conn.execute(
+        "INSERT INTO review (repo, pr, head_sha, posted_at) VALUES (?,?,?,?)",
+        (repo, pr, "deadbeef", 1000),
+    )
+    review_id = cur.lastrowid
+    cur = conn.execute(
+        """INSERT INTO inline_finding
+           (review_id, repo, pr, path, line, severity, problem, comment_id, posthash, posted_at)
+           VALUES (?,?,?,?,?,?,?,?,?,?)""",
+        (review_id, repo, pr, path, line, "HIGH", "problem", comment_id, f"h{comment_id}", 1000),
+    )
+    conn.commit()
+    return cur.lastrowid
+
+
+# --- score_pr maths -------------------------------------------------------
+
+def test_engagement_zero_when_nobody_responded():
+    v = fs.score_pr({"total": 4, "engaged": 0, "positive": 0, "negative": 0})
+    assert v[fs.REVIEW_ENGAGEMENT] == 0.0
+
+
+def test_acceptance_absent_when_nobody_engaged():
+    # Not 0.0 — zero would claim humans judged it neutral.
+    v = fs.score_pr({"total": 4, "engaged": 0, "positive": 0, "negative": 0})
+    assert v[fs.REVIEW_ACCEPTANCE] is None
+
+
+def test_engagement_is_a_share_of_findings():
+    v = fs.score_pr({"total": 4, "engaged": 1, "positive": 1, "negative": 0})
+    assert v[fs.REVIEW_ENGAGEMENT] == 0.25
+
+
+def test_acceptance_all_positive():
+    v = fs.score_pr({"total": 2, "engaged": 2, "positive": 3, "negative": 0})
+    assert v[fs.REVIEW_ACCEPTANCE] == 1.0
+
+
+def test_acceptance_all_negative():
+    v = fs.score_pr({"total": 2, "engaged": 2, "positive": 0, "negative": 2})
+    assert v[fs.REVIEW_ACCEPTANCE] == -1.0
+
+
+def test_acceptance_mixed_is_normalised():
+    v = fs.score_pr({"total": 4, "engaged": 4, "positive": 3, "negative": 1})
+    assert v[fs.REVIEW_ACCEPTANCE] == 0.5
+
+
+def test_engagement_absent_when_no_findings_at_all():
+    v = fs.score_pr({"total": 0, "engaged": 0, "positive": 0, "negative": 0})
+    assert v[fs.REVIEW_ENGAGEMENT] is None
+
+
+# --- collect_pr_feedback over a real sqlite ------------------------------
+
+def test_collect_counts_nothing_on_untouched_findings(db):
+    _seed_finding(db)
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally == {"total": 1, "engaged": 0, "positive": 0, "negative": 0}
+
+
+def test_collect_counts_positive_reaction(db):
+    _seed_finding(db, comment_id=101)
+    db.execute(
+        "INSERT INTO reaction (comment_id, user, content, created_at) VALUES (?,?,?,?)",
+        (101, "alice", "+1", 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally["positive"] == 1 and tally["engaged"] == 1
+
+
+def test_collect_counts_negative_reaction(db):
+    _seed_finding(db, comment_id=102)
+    db.execute(
+        "INSERT INTO reaction (comment_id, user, content, created_at) VALUES (?,?,?,?)",
+        (102, "bob", "-1", 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally["negative"] == 1 and tally["engaged"] == 1
+
+
+def test_resolved_thread_counts_positive(db):
+    fid = _seed_finding(db, comment_id=103)
+    db.execute(
+        "INSERT INTO thread_state (finding_id, resolved, checked_at) VALUES (?,?,?)",
+        (fid, 1, 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally["positive"] == 1 and tally["engaged"] == 1
+
+
+def test_unresolved_thread_is_not_a_vote(db):
+    fid = _seed_finding(db, comment_id=104)
+    db.execute(
+        "INSERT INTO thread_state (finding_id, resolved, checked_at) VALUES (?,?,?)",
+        (fid, 0, 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally == {"total": 1, "engaged": 0, "positive": 0, "negative": 0}
+
+
+def test_negation_reply_counts_negative(db):
+    fid = _seed_finding(db, comment_id=105)
+    db.execute(
+        "INSERT INTO reply (finding_id, author, body, created_at) VALUES (?,?,?,?)",
+        (fid, "carol", "this is a false positive", 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally["negative"] == 1 and tally["engaged"] == 1
+
+
+def test_neutral_reply_is_engagement_but_not_a_vote(db):
+    fid = _seed_finding(db, comment_id=106)
+    db.execute(
+        "INSERT INTO reply (finding_id, author, body, created_at) VALUES (?,?,?,?)",
+        (fid, "dave", "done", 1),
+    )
+    db.commit()
+    tally = fs.collect_pr_feedback(db, "o/r", 1)
+    assert tally["engaged"] == 1
+    assert tally["positive"] == 0 and tally["negative"] == 0
+
+
+# --- event shape ----------------------------------------------------------
+
+def test_build_score_events_shape():
+    events = fs.build_score_events("o/r", 7, {fs.REVIEW_ENGAGEMENT: 0.5}, "claude")
+    assert len(events) == 1
+    body = events[0]["body"]
+    assert events[0]["type"] == "score-create"
+    assert body["sessionId"] == "o/r#7"
+    assert body["value"] == 0.5
+    assert body["environment"] == "claude"
+
+
+def test_build_score_events_skips_none():
+    events = fs.build_score_events("o/r", 7, {fs.REVIEW_ACCEPTANCE: None})
+    assert events == []
+
+
+def test_score_ids_are_stable_across_runs():
+    # A backfill re-run must update, not duplicate.
+    a = fs.build_score_events("o/r", 7, {fs.REVIEW_ENGAGEMENT: 0.5})[0]["body"]["id"]
+    b = fs.build_score_events("o/r", 7, {fs.REVIEW_ENGAGEMENT: 0.9})[0]["body"]["id"]
+    assert a == b
+
+
+def test_score_ids_differ_per_pr_and_name():
+    e1 = fs.build_score_events("o/r", 7, {fs.REVIEW_ENGAGEMENT: 1})[0]["body"]["id"]
+    e2 = fs.build_score_events("o/r", 8, {fs.REVIEW_ENGAGEMENT: 1})[0]["body"]["id"]
+    e3 = fs.build_score_events("o/r", 7, {fs.REVIEW_ACCEPTANCE: 1})[0]["body"]["id"]
+    assert len({e1, e2, e3}) == 3
+
+
+def test_backfill_dry_run_reports_without_posting(db, tmp_path):
+    _seed_finding(db, comment_id=107)
+    db.commit()
+    path = db.execute("PRAGMA database_list").fetchone()[2]
+    summary = fs.backfill(path, dry_run=True)
+    assert summary["prs_scanned"] == 1
+    assert summary["prs_with_engagement"] == 0
+    assert summary["posted"] is False
+
+
+def test_every_emitted_score_has_a_config():
+    configured = {c["name"] for c in fs.SCORE_CONFIGS}
+    assert {fs.REVIEW_ENGAGEMENT, fs.REVIEW_ACCEPTANCE} == configured
+
+
+def test_every_event_carries_a_timestamp():
+    # Without one the ingestion endpoint 400s the event inside a 207 that the
+    # caller reads as success.
+    events = fs.build_score_events("o/r", 1, {fs.REVIEW_ENGAGEMENT: 0.0})
+    assert events
+    assert all(e.get("timestamp") for e in events)
+
+
+def test_explicit_timestamp_is_used():
+    events = fs.build_score_events(
+        "o/r", 1, {fs.REVIEW_ENGAGEMENT: 0.0}, timestamp="2026-01-02T03:04:05Z"
+    )
+    assert events[0]["timestamp"] == "2026-01-02T03:04:05Z"

tests/pilot/test_langfuse_trace.py

@@ -145,15 +145,18 @@ def test_unknown_comparison_target_yields_no_cost_block_rather_than_a_wrong_one(
 def test_batch_has_a_trace_and_a_generation_linked_by_trace_id():
     batch = lt.build_batch(model="headroom/claude-sonnet-5", **BASE)
     types = [e["type"] for e in batch]
-    assert types == ["trace-create", "generation-create"]
-    trace, gen = batch
+    # Scores ride in the same batch; the trace and generation lead it.
+    assert types[:2] == ["trace-create", "generation-create"]
+    trace, gen = batch[0], batch[1]
     assert gen["body"]["traceId"] == trace["body"]["id"]
     assert trace["body"]["environment"] == gen["body"]["environment"] == "claude"
 
 
-def test_batch_without_usage_is_trace_only():
+def test_batch_without_usage_has_no_generation():
     batch = lt.build_batch(model="headroom/glm-5.2:cloud", **{**BASE, "usage": None})
-    assert [e["type"] for e in batch] == ["trace-create"]
+    types = [e["type"] for e in batch]
+    assert "generation-create" not in types
+    assert types[0] == "trace-create"
 
 
 def test_trace_carries_repo_pr_session_and_severity_counts():
@@ -226,7 +229,9 @@ def test_configured_emit_posts_to_the_ingestion_endpoint(monkeypatch):
     assert lt.emit_review_trace(model="headroom/claude-sonnet-5", **BASE) is True
     # Trailing slash stripped so the path is not doubled.
     assert seen["host"] == "http://langfuse.test:3000"
-    assert len(seen["batch"]) == 2
+    kinds = [e["type"] for e in seen["batch"]]
+    assert kinds[:2] == ["trace-create", "generation-create"]
+    assert "score-create" in kinds
 
 
 def test_transport_failure_is_swallowed(monkeypatch):
@@ -243,3 +248,79 @@ def test_non_success_status_reports_failure_without_raising(monkeypatch):
     _configure(monkeypatch)
     monkeypatch.setattr(lt, "_post", lambda *a, **k: 401)
     assert lt.emit_review_trace(model="headroom/glm-5.2:cloud", **BASE) is False
+
+
+# ---------------------------------------------------------------------------
+# Scores folded into the review batch (added with eval_scores)
+# ---------------------------------------------------------------------------
+
+def _scores(events):
+    return {e["body"]["name"]: e["body"] for e in events if e["type"] == "score-create"}
+
+
+def test_build_batch_appends_scores():
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t",
+        model="headroom/claude-sonnet-5",
+        usage={"input": 100, "output": 10},
+        findings=[{"severity": "high", "path": "a.py", "line": 1}],
+    )
+    names = set(_scores(events))
+    assert "finding_rate" in names
+    assert "severity_max" in names
+
+
+def test_scores_attach_to_the_same_trace():
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t", model="m",
+        usage={"input": 1, "output": 1}, findings=[], trace_id="fixed-id",
+    )
+    for body in _scores(events).values():
+        assert body["traceId"] == "fixed-id"
+
+
+def test_scores_inherit_the_trace_environment():
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t",
+        model="headroom/glm-5.2:cloud",
+        usage={"input": 1, "output": 1}, findings=[],
+    )
+    for body in _scores(events).values():
+        assert body["environment"] == "ollama"
+
+
+def test_dropped_findings_scored_when_provided():
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t", model="m",
+        usage={"input": 1, "output": 1}, findings=[], dropped_count=3,
+    )
+    assert _scores(events)["dropped_findings"]["value"] == 3.0
+
+
+def test_dropped_findings_absent_when_not_measured():
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t", model="m",
+        usage={"input": 1, "output": 1}, findings=[],
+    )
+    assert "dropped_findings" not in _scores(events)
+
+
+def test_cost_score_carries_its_basis_in_the_comment():
+    # An equivalent-cost $/finding must never be read as money spent.
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t",
+        model="headroom/glm-5.2:cloud",
+        usage={"input": 1000, "output": 100}, findings=[{"severity": "low", "path": "a", "line": 1}],
+    )
+    cpf = _scores(events).get("cost_per_finding")
+    if cpf is not None:  # only when cost_model could price the comparison target
+        assert "equivalent" in cpf["comment"]
+
+
+def test_batch_without_usage_still_scores_findings():
+    # A run with no usage report still produced findings worth scoring.
+    events = lt.build_batch(
+        repo="o/r", index="1", sha="abc", title="t", model="m",
+        usage=None, findings=[{"severity": "critical", "path": "a", "line": 2}],
+    )
+    assert _scores(events)["severity_max"]["value"] == "critical"

tests/pilot/test_parse_dropped.py

@@ -0,0 +1,90 @@
+"""The parse-time drop counter feeding the `dropped_findings` score.
+
+A model that emits findings at unusable locations produces an empty findings
+list, exactly like a model that found nothing. These tests pin the signal that
+tells the two apart.
+"""
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.abspath(os.path.join(HERE, "..", "..", "pilot")))
+
+import ai_review  # noqa: E402
+
+
+def _payload(findings):
+    return "```json\n" + json.dumps({"summary": "s", "findings": findings}) + "\n```"
+
+
+GOOD = {"severity": "high", "path": "a.py", "line": 3, "problem": "p", "fix": "f"}
+NO_PATH = {"severity": "high", "line": 3, "problem": "p"}
+NO_LINE = {"severity": "high", "path": "a.py", "problem": "p"}
+BAD_LINE = {"severity": "high", "path": "a.py", "line": 0, "problem": "p"}
+
+
+def test_no_drops_on_clean_output():
+    _, findings, *_ = ai_review.parse_review_output(_payload([GOOD, GOOD]))
+    assert len(findings) == 2
+    assert ai_review.last_parse_dropped() == 0
+
+
+def test_counts_findings_missing_path():
+    _, findings, *_ = ai_review.parse_review_output(_payload([GOOD, NO_PATH]))
+    assert len(findings) == 1
+    assert ai_review.last_parse_dropped() == 1
+
+
+def test_counts_findings_missing_line():
+    _, findings, *_ = ai_review.parse_review_output(_payload([NO_LINE, NO_LINE]))
+    assert findings == []
+    assert ai_review.last_parse_dropped() == 2
+
+
+def test_counts_findings_with_unusable_line():
+    _, findings, *_ = ai_review.parse_review_output(_payload([BAD_LINE]))
+    assert findings == []
+    assert ai_review.last_parse_dropped() == 1
+
+
+def test_all_dropped_is_distinguishable_from_found_nothing():
+    ai_review.parse_review_output(_payload([NO_PATH, NO_PATH, NO_PATH]))
+    all_dropped = ai_review.last_parse_dropped()
+    ai_review.parse_review_output(_payload([]))
+    found_nothing = ai_review.last_parse_dropped()
+    assert all_dropped == 3 and found_nothing == 0
+
+
+def test_counter_resets_on_unparseable_output():
+    # Otherwise a salvage-path review inherits the previous review's count.
+    ai_review.parse_review_output(_payload([NO_PATH, NO_PATH]))
+    assert ai_review.last_parse_dropped() == 2
+    ai_review.parse_review_output("no json here at all")
+    assert ai_review.last_parse_dropped() == 0
+
+
+def test_counter_resets_on_malformed_json():
+    ai_review.parse_review_output(_payload([NO_PATH]))
+    ai_review.parse_review_output("```json\n{not valid json,,,}\n```")
+    assert ai_review.last_parse_dropped() == 0
+
+
+def test_parse_findings_tracks_drops_too():
+    # The non-opencode path must be scored on the same basis.
+    findings = ai_review.parse_findings(json.dumps({"findings": [GOOD, NO_PATH]}))
+    assert len(findings) == 1
+    assert ai_review.last_parse_dropped() == 1
+
+
+def test_parse_findings_resets_on_garbage():
+    ai_review.parse_findings(json.dumps({"findings": [NO_PATH]}))
+    assert ai_review.last_parse_dropped() == 1
+    ai_review.parse_findings("not json")
+    assert ai_review.last_parse_dropped() == 0
+
+
+def test_bare_array_output_is_counted():
+    _, findings, *_ = ai_review.parse_review_output("```json\n" + json.dumps([GOOD, NO_PATH]) + "\n```")
+    assert len(findings) == 1
+    assert ai_review.last_parse_dropped() == 1