132 lines
5.2 KiB
Python
132 lines
5.2 KiB
Python
"""质量门控测试（第三道防线）."""
|
|
|
|
import json
|
|
|
|
from src.ai.services.evaluation.llm_judge import JudgeResult, LLMJudge
|
|
from src.ai.services.evaluation.quality_gate import QualityGate
|
|
from src.ai.services.evaluation.rule_validator import RuleValidator
|
|
|
|
from .conftest import MockProvider
|
|
|
|
|
|
class TestQualityGate:
|
|
"""QualityGate 测试."""
|
|
|
|
async def test_rule_fail_rejects(self) -> None:
|
|
"""规则校验失败 → 拒绝."""
|
|
gate = QualityGate(rule_validator=RuleValidator())
|
|
result = await gate.evaluate(llm_output="invalid json")
|
|
assert result.passed is False
|
|
assert result.degraded is True
|
|
assert result.score == 0.0
|
|
|
|
async def test_rule_pass_no_judge(self) -> None:
|
|
"""规则通过 + 无 LLM Judge → 放行."""
|
|
gate = QualityGate(rule_validator=RuleValidator(), llm_judge=None)
|
|
output = json.dumps({"question": "完整的题目", "answer": "完整的答案"})
|
|
result = await gate.evaluate(output)
|
|
assert result.passed is True
|
|
assert result.score == 1.0
|
|
|
|
async def test_rule_pass_with_warnings_degraded(self) -> None:
|
|
"""规则通过但有 warning + 无 Judge → degraded."""
|
|
gate = QualityGate(rule_validator=RuleValidator(), llm_judge=None)
|
|
output = json.dumps({"question": "ab", "answer": "答"})
|
|
result = await gate.evaluate(output)
|
|
assert result.passed is True
|
|
assert result.degraded is True
|
|
|
|
async def test_with_llm_judge_pass(self) -> None:
|
|
"""规则 + LLM Judge 均通过."""
|
|
judge = LLMJudge(provider=MockProvider(
|
|
response_content=json.dumps({
|
|
"overall": 0.9,
|
|
"accuracy": 0.9,
|
|
"clarity": 0.9,
|
|
"correctness": 0.9,
|
|
"completeness": 0.9,
|
|
"difficulty_match": 0.9,
|
|
"issues": [],
|
|
}),
|
|
))
|
|
gate = QualityGate(rule_validator=RuleValidator(), llm_judge=judge)
|
|
output = json.dumps({"question": "完整题目", "answer": "完整答案"})
|
|
result = await gate.evaluate(output)
|
|
assert result.passed is True
|
|
assert result.judge_result is not None
|
|
|
|
async def test_combine_scores(self) -> None:
|
|
"""综合评分权重 0.4 + 0.6."""
|
|
gate = QualityGate(rule_validator=RuleValidator())
|
|
combined = gate._combine_scores(1.0, 0.5)
|
|
assert combined == 0.7 # 1.0*0.4 + 0.5*0.6
|
|
|
|
async def test_judge_unavailable_degrades(self) -> None:
|
|
"""LLM Judge 不可用 → 仅规则校验."""
|
|
judge = LLMJudge(provider=None) # provider=None
|
|
gate = QualityGate(rule_validator=RuleValidator(), llm_judge=judge)
|
|
output = json.dumps({"question": "完整题目", "answer": "完整答案"})
|
|
result = await gate.evaluate(output)
|
|
assert result.judge_result is not None
|
|
assert result.judge_result.available is False
|
|
|
|
|
|
class TestLLMJudge:
|
|
"""LLMJudge 测试."""
|
|
|
|
async def test_no_provider_degrades(self) -> None:
|
|
"""无 provider → available=False."""
|
|
judge = LLMJudge(provider=None)
|
|
result = await judge.judge("题", "答")
|
|
assert result.available is False
|
|
assert result.score == 0.7
|
|
|
|
async def test_provider_unavailable_degrades(self) -> None:
|
|
"""provider 未配置 → available=False."""
|
|
judge = LLMJudge(provider=MockProvider(available=False))
|
|
result = await judge.judge("题", "答")
|
|
assert result.available is False
|
|
|
|
async def test_parse_valid_response(self) -> None:
|
|
"""解析合法 JSON 评审响应."""
|
|
judge = LLMJudge(provider=MockProvider(
|
|
response_content=json.dumps({
|
|
"overall": 0.85,
|
|
"accuracy": 0.9,
|
|
"clarity": 0.8,
|
|
"correctness": 0.9,
|
|
"completeness": 0.8,
|
|
"difficulty_match": 0.85,
|
|
"issues": ["解析不够详细"],
|
|
}),
|
|
))
|
|
result = await judge.judge("题", "答", difficulty="easy")
|
|
assert result.available is True
|
|
assert result.score == 0.85
|
|
assert result.issues == ["解析不够详细"]
|
|
|
|
async def test_parse_invalid_json(self) -> None:
|
|
"""非法 JSON → available=False, score=0.5."""
|
|
judge = LLMJudge(provider=MockProvider(response_content="not json"))
|
|
result = await judge.judge("题", "答")
|
|
assert result.available is False
|
|
assert result.score == 0.5
|
|
|
|
async def test_judge_result_passed(self) -> None:
|
|
"""JudgeResult.passed 属性."""
|
|
passed = JudgeResult(score=0.8, available=True)
|
|
assert passed.passed is True
|
|
failed = JudgeResult(score=0.3, available=True)
|
|
assert failed.passed is False
|
|
unavailable = JudgeResult(score=0.8, available=False)
|
|
assert unavailable.passed is False
|
|
|
|
async def test_parse_markdown_wrapped(self) -> None:
|
|
"""markdown 包裹的 JSON 能解析."""
|
|
judge = LLMJudge(provider=MockProvider(
|
|
response_content='```json\n{"overall": 0.7}\n```',
|
|
))
|
|
result = await judge.judge("题", "答")
|
|
assert result.available is True
|
|
assert result.score == 0.7
|