Claude
7a510a926d
Group review, feedback, evaluation, observability, and entrypoint code into packages. Keep thin top-level compatibility shims for existing scripts and imports, and mirror the structure in the tests.
91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
"""The parse-time drop counter feeding the `dropped_findings` score.
|
|
|
|
A model that emits findings at unusable locations produces an empty findings
|
|
list, exactly like a model that found nothing. These tests pin the signal that
|
|
tells the two apart.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, os.path.abspath(os.path.join(HERE, "..", "..", "..", "pilot")))
|
|
|
|
import ai_review # noqa: E402
|
|
|
|
|
|
def _payload(findings):
|
|
return "```json\n" + json.dumps({"summary": "s", "findings": findings}) + "\n```"
|
|
|
|
|
|
GOOD = {"severity": "high", "path": "a.py", "line": 3, "problem": "p", "fix": "f"}
|
|
NO_PATH = {"severity": "high", "line": 3, "problem": "p"}
|
|
NO_LINE = {"severity": "high", "path": "a.py", "problem": "p"}
|
|
BAD_LINE = {"severity": "high", "path": "a.py", "line": 0, "problem": "p"}
|
|
|
|
|
|
def test_no_drops_on_clean_output():
|
|
_, findings, *_ = ai_review.parse_review_output(_payload([GOOD, GOOD]))
|
|
assert len(findings) == 2
|
|
assert ai_review.last_parse_dropped() == 0
|
|
|
|
|
|
def test_counts_findings_missing_path():
|
|
_, findings, *_ = ai_review.parse_review_output(_payload([GOOD, NO_PATH]))
|
|
assert len(findings) == 1
|
|
assert ai_review.last_parse_dropped() == 1
|
|
|
|
|
|
def test_counts_findings_missing_line():
|
|
_, findings, *_ = ai_review.parse_review_output(_payload([NO_LINE, NO_LINE]))
|
|
assert findings == []
|
|
assert ai_review.last_parse_dropped() == 2
|
|
|
|
|
|
def test_counts_findings_with_unusable_line():
|
|
_, findings, *_ = ai_review.parse_review_output(_payload([BAD_LINE]))
|
|
assert findings == []
|
|
assert ai_review.last_parse_dropped() == 1
|
|
|
|
|
|
def test_all_dropped_is_distinguishable_from_found_nothing():
|
|
ai_review.parse_review_output(_payload([NO_PATH, NO_PATH, NO_PATH]))
|
|
all_dropped = ai_review.last_parse_dropped()
|
|
ai_review.parse_review_output(_payload([]))
|
|
found_nothing = ai_review.last_parse_dropped()
|
|
assert all_dropped == 3 and found_nothing == 0
|
|
|
|
|
|
def test_counter_resets_on_unparseable_output():
|
|
# Otherwise a salvage-path review inherits the previous review's count.
|
|
ai_review.parse_review_output(_payload([NO_PATH, NO_PATH]))
|
|
assert ai_review.last_parse_dropped() == 2
|
|
ai_review.parse_review_output("no json here at all")
|
|
assert ai_review.last_parse_dropped() == 0
|
|
|
|
|
|
def test_counter_resets_on_malformed_json():
|
|
ai_review.parse_review_output(_payload([NO_PATH]))
|
|
ai_review.parse_review_output("```json\n{not valid json,,,}\n```")
|
|
assert ai_review.last_parse_dropped() == 0
|
|
|
|
|
|
def test_parse_findings_tracks_drops_too():
|
|
# The non-opencode path must be scored on the same basis.
|
|
findings = ai_review.parse_findings(json.dumps({"findings": [GOOD, NO_PATH]}))
|
|
assert len(findings) == 1
|
|
assert ai_review.last_parse_dropped() == 1
|
|
|
|
|
|
def test_parse_findings_resets_on_garbage():
|
|
ai_review.parse_findings(json.dumps({"findings": [NO_PATH]}))
|
|
assert ai_review.last_parse_dropped() == 1
|
|
ai_review.parse_findings("not json")
|
|
assert ai_review.last_parse_dropped() == 0
|
|
|
|
|
|
def test_bare_array_output_is_counted():
|
|
_, findings, *_ = ai_review.parse_review_output("```json\n" + json.dumps([GOOD, NO_PATH]) + "\n```")
|
|
assert len(findings) == 1
|
|
assert ai_review.last_parse_dropped() == 1
|