Claude
7a510a926d
Group review, feedback, evaluation, observability, and entrypoint code into packages. Keep thin top-level compatibility shims for existing scripts and imports, and mirror the structure in the tests.
327 lines
12 KiB
Python
327 lines
12 KiB
Python
"""Unit tests for Langfuse trace emission. No network.
|
|
|
|
`_post` is monkeypatched everywhere a POST would happen; a test that reaches
|
|
the real network is a bug in the test, not a slow test.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
|
|
sys.path.insert(0, os.path.join(ROOT, "pilot"))
|
|
|
|
import langfuse_trace as lt # noqa: E402
|
|
|
|
|
|
USAGE = {
|
|
"input": 2_000_000,
|
|
"output": 17_000,
|
|
"reasoning": 500,
|
|
"cache_read": 400_000,
|
|
"cache_write": 50_000,
|
|
"total": 2_017_000,
|
|
"cost": 0.0,
|
|
"steps": 28,
|
|
"duration_s": 348.3,
|
|
}
|
|
|
|
BASE = dict(
|
|
repo="techspark/pragent",
|
|
index="42",
|
|
sha="2613b3e1122334455",
|
|
title="Harden the review path",
|
|
usage=USAGE,
|
|
findings=[
|
|
{"severity": "critical", "path": "a.py"},
|
|
{"severity": "minor", "path": "b.py"},
|
|
{"severity": "minor", "path": "c.py"},
|
|
],
|
|
summary="Three findings.",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# model -> environment split (the whole point of the integration)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_claude_models_land_in_the_claude_environment():
|
|
assert lt.resolve_environment("headroom/claude-sonnet-5") == "claude"
|
|
assert lt.resolve_environment("claude-opus-5") == "claude"
|
|
|
|
|
|
def test_everything_else_lands_in_the_ollama_environment():
|
|
for m in (
|
|
"headroom/glm-5.2:cloud",
|
|
"headroom/MiniMax-M2.7",
|
|
"vllm-qwen38/qwen3.8-27b",
|
|
"gpt-5",
|
|
):
|
|
assert lt.resolve_environment(m) == "ollama", m
|
|
|
|
|
|
def test_provider_and_bare_model_are_split_on_the_first_slash_only():
|
|
assert lt.provider_of("vllm-qwen38/qwen3.8-27b") == "vllm-qwen38"
|
|
assert lt.strip_provider("headroom/glm-5.2:cloud") == "glm-5.2:cloud"
|
|
# A bare name has no provider prefix; default to the pilot's proxy.
|
|
assert lt.provider_of("glm-5.2:cloud") == "headroom"
|
|
assert lt.strip_provider("glm-5.2:cloud") == "glm-5.2:cloud"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# usage accounting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_cache_reads_are_subtracted_from_input_not_added():
|
|
# Langfuse sums usageDetails keys; opencode reports cache_read *inside*
|
|
# input, so reporting both raw would bill the prefix twice.
|
|
d = lt._usage_details(USAGE)
|
|
assert d["input"] == 2_000_000 - 400_000
|
|
assert d["cache_read_input_tokens"] == 400_000
|
|
assert d["cache_write_input_tokens"] == 50_000
|
|
assert d["output"] == 17_000
|
|
assert d["reasoning"] == 500
|
|
|
|
|
|
def test_zero_cache_fields_are_omitted_rather_than_sent_as_zero():
|
|
d = lt._usage_details({"input": 100, "output": 10})
|
|
assert d == {"input": 100, "output": 10}
|
|
|
|
|
|
def test_a_paid_model_is_priced_as_itself():
|
|
costs, basis = lt._cost_details(USAGE, "headroom/claude-sonnet-5")
|
|
assert costs["total"] > 0
|
|
assert basis == "actual"
|
|
|
|
|
|
def test_minimax_is_priced_against_the_comparison_target_not_zero():
|
|
# MiniMax-M2.7 is the model the webhook actually runs and it is absent from
|
|
# PRICES; charting it at $0 would make the whole dashboard a flat line.
|
|
costs, basis = lt._cost_details(USAGE, "headroom/MiniMax-M2.7")
|
|
assert costs["total"] > 0
|
|
assert basis == "equivalent:claude-sonnet-5"
|
|
|
|
|
|
def test_glm_is_priced_against_the_comparison_target():
|
|
costs, basis = lt._cost_details(USAGE, "headroom/glm-5.2:cloud")
|
|
assert costs["total"] > 0
|
|
assert basis.startswith("equivalent:")
|
|
|
|
|
|
def test_an_all_zero_price_entry_counts_as_free_not_as_priced():
|
|
# The self-hosted vLLM qwen IS in PRICES, at 0.00 across the board.
|
|
costs, basis = lt._cost_details(USAGE, "vllm-qwen38/qwen3.8-27b")
|
|
assert costs["total"] > 0
|
|
assert basis.startswith("equivalent:")
|
|
|
|
|
|
def test_explicit_price_target_wins_over_the_default():
|
|
costs, basis = lt._cost_details(USAGE, "headroom/MiniMax-M2.7", "claude-opus-5")
|
|
assert basis == "equivalent:claude-opus-5"
|
|
sonnet, _ = lt._cost_details(USAGE, "headroom/MiniMax-M2.7", "claude-sonnet-5")
|
|
assert costs["total"] > sonnet["total"]
|
|
|
|
|
|
def test_env_overrides_the_default_target(monkeypatch):
|
|
monkeypatch.setenv("PRAGENT_PRICE_TARGET", "claude-haiku-4-5")
|
|
assert lt.resolve_price_target() == "claude-haiku-4-5"
|
|
# An explicit argument still beats the env.
|
|
assert lt.resolve_price_target("gpt-5") == "gpt-5"
|
|
|
|
|
|
def test_unknown_comparison_target_yields_no_cost_block_rather_than_a_wrong_one():
|
|
costs, basis = lt._cost_details(USAGE, "headroom/MiniMax-M2.7", "not-a-real-model")
|
|
assert costs == {}
|
|
assert basis == ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# batch shape
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_batch_has_a_trace_and_a_generation_linked_by_trace_id():
|
|
batch = lt.build_batch(model="headroom/claude-sonnet-5", **BASE)
|
|
types = [e["type"] for e in batch]
|
|
# Scores ride in the same batch; the trace and generation lead it.
|
|
assert types[:2] == ["trace-create", "generation-create"]
|
|
trace, gen = batch[0], batch[1]
|
|
assert gen["body"]["traceId"] == trace["body"]["id"]
|
|
assert trace["body"]["environment"] == gen["body"]["environment"] == "claude"
|
|
|
|
|
|
def test_batch_without_usage_has_no_generation():
|
|
batch = lt.build_batch(model="headroom/glm-5.2:cloud", **{**BASE, "usage": None})
|
|
types = [e["type"] for e in batch]
|
|
assert "generation-create" not in types
|
|
assert types[0] == "trace-create"
|
|
|
|
|
|
def test_trace_carries_repo_pr_session_and_severity_counts():
|
|
batch = lt.build_batch(model="headroom/glm-5.2:cloud", **BASE)
|
|
body = batch[0]["body"]
|
|
assert body["sessionId"] == "techspark/pragent#42"
|
|
assert body["metadata"]["severities"] == {"critical": 1, "minor": 2}
|
|
assert body["metadata"]["findings"] == 3
|
|
assert "provider:headroom" in body["tags"]
|
|
assert "model:glm-5.2:cloud" in body["tags"]
|
|
|
|
|
|
def test_lens_names_become_tags():
|
|
batch = lt.build_batch(
|
|
model="headroom/glm-5.2:cloud", lenses=["security", "tests"], **BASE
|
|
)
|
|
assert "lens:security" in batch[0]["body"]["tags"]
|
|
assert "lens:tests" in batch[0]["body"]["tags"]
|
|
|
|
|
|
def test_cost_basis_is_tagged_so_equivalent_is_never_read_as_spend():
|
|
batch = lt.build_batch(model="headroom/MiniMax-M2.7", **BASE)
|
|
trace = batch[0]["body"]
|
|
assert "cost:equivalent:claude-sonnet-5" in trace["tags"]
|
|
assert trace["metadata"]["cost_basis"] == "equivalent:claude-sonnet-5"
|
|
|
|
paid = lt.build_batch(model="headroom/claude-sonnet-5", **BASE)
|
|
assert "cost:actual" in paid[0]["body"]["tags"]
|
|
|
|
|
|
def test_minimax_generation_carries_a_nonzero_cost():
|
|
batch = lt.build_batch(model="headroom/MiniMax-M2.7", **BASE)
|
|
assert batch[1]["body"]["costDetails"]["total"] > 0
|
|
|
|
|
|
def test_batch_is_json_serializable():
|
|
batch = lt.build_batch(model="headroom/claude-sonnet-5", **BASE)
|
|
json.dumps({"batch": batch})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# emit_review_trace — config gate and fail-open
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _configure(monkeypatch):
|
|
monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.test:3000/")
|
|
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
|
|
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")
|
|
|
|
|
|
def test_no_config_means_no_post_and_no_error(monkeypatch):
|
|
for k in ("LANGFUSE_HOST", "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"):
|
|
monkeypatch.delenv(k, raising=False)
|
|
calls = []
|
|
monkeypatch.setattr(lt, "_post", lambda *a, **k: calls.append(a) or 200)
|
|
assert lt.emit_review_trace(model="headroom/glm-5.2:cloud", **BASE) is False
|
|
assert calls == []
|
|
|
|
|
|
def test_configured_emit_posts_to_the_ingestion_endpoint(monkeypatch):
|
|
_configure(monkeypatch)
|
|
seen = {}
|
|
|
|
def fake_post(host, pk, sk, batch, timeout):
|
|
seen.update(host=host, pk=pk, sk=sk, batch=batch, timeout=timeout)
|
|
return 207
|
|
|
|
monkeypatch.setattr(lt, "_post", fake_post)
|
|
assert lt.emit_review_trace(model="headroom/claude-sonnet-5", **BASE) is True
|
|
# Trailing slash stripped so the path is not doubled.
|
|
assert seen["host"] == "http://langfuse.test:3000"
|
|
kinds = [e["type"] for e in seen["batch"]]
|
|
assert kinds[:2] == ["trace-create", "generation-create"]
|
|
assert "score-create" in kinds
|
|
|
|
|
|
def test_transport_failure_is_swallowed(monkeypatch):
|
|
_configure(monkeypatch)
|
|
|
|
def boom(*a, **k):
|
|
raise OSError("connection refused")
|
|
|
|
monkeypatch.setattr(lt, "_post", boom)
|
|
assert lt.emit_review_trace(model="headroom/glm-5.2:cloud", **BASE) is False
|
|
|
|
|
|
def test_non_success_status_reports_failure_without_raising(monkeypatch):
|
|
_configure(monkeypatch)
|
|
monkeypatch.setattr(lt, "_post", lambda *a, **k: 401)
|
|
assert lt.emit_review_trace(model="headroom/glm-5.2:cloud", **BASE) is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scores folded into the review batch (added with eval_scores)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _scores(events):
|
|
return {e["body"]["name"]: e["body"] for e in events if e["type"] == "score-create"}
|
|
|
|
|
|
def test_build_batch_appends_scores():
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t",
|
|
model="headroom/claude-sonnet-5",
|
|
usage={"input": 100, "output": 10},
|
|
findings=[{"severity": "high", "path": "a.py", "line": 1}],
|
|
)
|
|
names = set(_scores(events))
|
|
assert "finding_rate" in names
|
|
assert "severity_max" in names
|
|
|
|
|
|
def test_scores_attach_to_the_same_trace():
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t", model="m",
|
|
usage={"input": 1, "output": 1}, findings=[], trace_id="fixed-id",
|
|
)
|
|
for body in _scores(events).values():
|
|
assert body["traceId"] == "fixed-id"
|
|
|
|
|
|
def test_scores_inherit_the_trace_environment():
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t",
|
|
model="headroom/glm-5.2:cloud",
|
|
usage={"input": 1, "output": 1}, findings=[],
|
|
)
|
|
for body in _scores(events).values():
|
|
assert body["environment"] == "ollama"
|
|
|
|
|
|
def test_dropped_findings_scored_when_provided():
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t", model="m",
|
|
usage={"input": 1, "output": 1}, findings=[], dropped_count=3,
|
|
)
|
|
assert _scores(events)["dropped_findings"]["value"] == 3.0
|
|
|
|
|
|
def test_dropped_findings_absent_when_not_measured():
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t", model="m",
|
|
usage={"input": 1, "output": 1}, findings=[],
|
|
)
|
|
assert "dropped_findings" not in _scores(events)
|
|
|
|
|
|
def test_cost_score_carries_its_basis_in_the_comment():
|
|
# An equivalent-cost $/finding must never be read as money spent.
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t",
|
|
model="headroom/glm-5.2:cloud",
|
|
usage={"input": 1000, "output": 100}, findings=[{"severity": "low", "path": "a", "line": 1}],
|
|
)
|
|
cpf = _scores(events).get("cost_per_finding")
|
|
if cpf is not None: # only when cost_model could price the comparison target
|
|
assert "equivalent" in cpf["comment"]
|
|
|
|
|
|
def test_batch_without_usage_still_scores_findings():
|
|
# A run with no usage report still produced findings worth scoring.
|
|
events = lt.build_batch(
|
|
repo="o/r", index="1", sha="abc", title="t", model="m",
|
|
usage=None, findings=[{"severity": "critical", "path": "a", "line": 2}],
|
|
)
|
|
assert _scores(events)["severity_max"]["value"] == "critical"
|