Import Human Voice Rewrite Demo (PRD v1.1) + Dockerfile for Coolify

This commit is contained in:
Jeamee
2026-08-24 16:49:57 +08:00
commit d6f70b02f4
16 changed files with 3580 additions and 0 deletions
+551
View File
@@ -0,0 +1,551 @@
"""Hermetic tests for Human Voice Rewrite Demo (PRD v1.1).
No real model calls — the LLM seam is stubbed.
"""
import json
import pytest
from fastapi.testclient import TestClient
from evidence import sanitize_diagnosis, sanitize_patterns
from llm import LLMError, LlmClient, extract_json, message_text, reasoning_config
from main import app
from prompts import normalize_diagnose_payload
from schemas import Annotation, DiagnoseResponse, ParagraphBrief, Pattern
SAMPLE_PARAGRAPHS = [
"The pencil in my hand was still. I wanted to find the answer.",
"I met a problem that I could not solve right away.",
"I learned to be patient.",
"Now, I sit at my desk again.",
]
class FakeCompleter:
def __init__(self, payload):
self.payload = payload
def complete_json(self, system, user, validate=None):
return self.payload
# ---------------------------------------------------------------- evidence
def test_evidence_keeps_verbatim_hits_drops_misses():
pats = [
Pattern(
pattern_id="P05",
name="Metaphor Stack",
affected_paragraphs=["p1"],
evidence=["the pencil in my hand", "not in the text"],
why_ai_like="w",
human_impact="h",
transformation_rule="Keep One Motif",
)
]
out = sanitize_patterns(pats, SAMPLE_PARAGRAPHS)
assert out[0].evidence == ["the pencil in my hand"]
assert out[0].category == "rhetoric"
def test_evidence_unmatched_cleared_but_pattern_kept():
pats = [
Pattern(
pattern_id="P02",
name="Explicit Lesson",
affected_paragraphs=["p3"],
evidence=["somewhere else entirely"],
why_ai_like="w",
human_impact="h",
transformation_rule="Lesson -> Change in Judgment",
)
]
out = sanitize_patterns(pats, SAMPLE_PARAGRAPHS)
assert len(out) == 1 and out[0].pattern_id == "P02"
assert out[0].evidence == []
assert out[0].category == "growth"
def test_fictional_annotation_is_dropped():
resp = DiagnoseResponse(
overall_diagnosis="x",
paragraph_briefs=[
ParagraphBrief(
paragraph_id="p1",
annotations=[
Annotation(
pattern_id="P05",
category="rhetoric",
title="虚构",
evidence=["this phrase is not in the paragraph"],
observation="o",
rewrite_action="a",
),
Annotation(
pattern_id="P05",
category="rhetoric",
title="真实",
evidence=["the pencil in my hand"],
observation="o",
rewrite_action="a",
),
],
)
],
)
cleaned = sanitize_diagnosis(resp, SAMPLE_PARAGRAPHS)
assert len(cleaned.paragraph_briefs[0].annotations) == 1
assert cleaned.paragraph_briefs[0].annotations[0].title == "真实"
# ---------------------------------------------------------------- extract_json / retry
def test_extract_json_strips_fences_and_wraps():
assert extract_json('```json\n{"a": 1}\n```') == {"a": 1}
assert extract_json('here you go: {"a": 1} thanks') == {"a": 1}
def test_message_text_joins_content_parts():
assert message_text({"choices": [{"message": {"content": [{"type": "text", "text": '{"ok":1}'}]}}]}) == '{"ok":1}'
def test_reasoning_config_does_not_set_effort_and_max_tokens():
first = reasoning_config(False)
retry = reasoning_config(True)
assert not ("effort" in first and "max_tokens" in first)
assert not ("effort" in retry and "max_tokens" in retry)
assert first == {"max_tokens": 2048}
assert retry == {"effort": "none", "exclude": True}
def test_message_text_empty_mentions_finish_reason():
with pytest.raises(LLMError, match="finish_reason=length"):
message_text(
{
"choices": [
{
"finish_reason": "length",
"message": {"content": "", "reasoning_content": "thinking..."},
}
]
}
)
class SeqClient(LlmClient):
def __init__(self, responses):
super().__init__(api_key="sk-test", base_url="http://stub", model="stub")
self.responses = list(responses)
self.calls = 0
def _complete(self, system, user, json_reminder=False, **kwargs):
self.calls += 1
kind, val = self.responses.pop(0)
if kind == "error":
raise LLMError(val)
return val
def test_parse_failure_retries_once():
c = SeqClient([("content", "not json at all"), ("content", '{"ok": 1}')])
assert c.complete_json("s", "u") == {"ok": 1}
assert c.calls == 2
def test_transient_5xx_retries_once():
c = SeqClient([("error", "HTTP 502 bad gateway"), ("content", '{"ok": 2}')])
assert c.complete_json("s", "u") == {"ok": 2}
assert c.calls == 2
def test_parse_failure_twice_raises():
c = SeqClient([("content", "nope"), ("content", "still not json")])
with pytest.raises(LLMError):
c.complete_json("s", "u")
assert c.calls == 2
def test_http_402_error_shows_only_error_message():
"""HTTP 非 200(如 OpenRouter 402)→ LLMError 只带 error.message,不把完整 JSON body 甩给页面(PRD §11 失败兜底)。"""
import httpx
resp = httpx.Response(
402,
json={
"error": {
"message": "Insufficient credits. Add more using https://openrouter.ai/settings/credits",
"code": 402,
"type": "insufficient_quota",
}
},
request=httpx.Request("POST", "http://stub/chat/completions"),
)
class HttpErrorClient(LlmClient):
def __init__(self):
super().__init__(api_key="sk-test", base_url="http://stub", model="stub")
def _post(self, payload):
return resp
with pytest.raises(LLMError) as ei:
HttpErrorClient().complete_json("s", "u")
msg = str(ei.value)
assert "Insufficient credits" in msg # error.message 对用户可见
assert '"error"' not in msg # 不是原始 JSON body
assert "insufficient_quota" not in msg # 不在 message 里的字段不外泄
def test_no_key_fails_closed(monkeypatch):
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
monkeypatch.delenv("PRODREAM_BACKEND_OPENROUTER_API_KEY", raising=False)
with pytest.raises(LLMError):
LlmClient(api_key="")
def test_normalize_diagnose_payload_maps_legacy_fields():
data = normalize_diagnose_payload(
{
"paragraph_briefs": [
{
"paragraph_id": "p1",
"primary_goal": "收修辞",
"rewrite_guidance": "不要连续比喻",
"context_before": "上一段已建立确定感",
"context_after": "这一段只需写挫败",
"scaffold": ["What I liked was ______."],
}
],
"patterns": [{"pattern_id": "P05", "name": "Metaphor Stack"}],
}
)
brief = data["paragraph_briefs"][0]
assert brief["rewrite_goal"] == "收修辞"
assert brief["ai_focus"] == "不要连续比喻"
assert "确定感" in brief["context_hint"]
assert brief["scaffold"] == "What I liked was ______."
assert data["patterns"][0]["category"] == "rhetoric"
def test_normalize_diagnose_payload_coerces_list_fields():
"""模型偶发把字符串字段返回成数组(2026-08-21 用户实测 ai_focus 4 段全中)——
normalize 必须规整为字符串/数组,且规整后能通过 schema(不再 502)。"""
data = normalize_diagnose_payload(
{
"overall_diagnosis": ["开头不错", "结尾乏力"],
"paragraph_briefs": [
{
"paragraph_id": "p1",
"confirmed_meaning": ["第一段写犹豫"],
"rewrite_goal": "收修辞",
"ai_focus": ["重复修辞", "抽象总结"],
"must_preserve": "secret code",
"annotations": [{"title": ["短标题"], "observation": "连续两个比喻", "evidence": "原文句"}],
}
],
"patterns": [{"pattern_id": "P05", "name": ["Metaphor", "Stack"]}],
}
)
brief = data["paragraph_briefs"][0]
assert brief["ai_focus"] == "重复修辞;抽象总结"
assert brief["confirmed_meaning"] == "第一段写犹豫"
assert brief["must_preserve"] == ["secret code"]
assert brief["annotations"][0]["title"] == "短标题"
assert brief["annotations"][0]["evidence"] == ["原文句"]
assert data["overall_diagnosis"] == "开头不错;结尾乏力"
assert data["patterns"][0]["name"] == "MetaphorStack"
# 规整后必须能过 schema——复现用户实测 502 场景不再发生
resp = DiagnoseResponse.model_validate(data)
assert resp.paragraph_briefs[0].ai_focus == "重复修辞;抽象总结"
# ---------------------------------------------------------------- API shape
def api_client(payload):
import main as main_mod
main_mod.get_client = lambda: FakeCompleter(payload)
return TestClient(app)
class CaptureCompleter:
def __init__(self, payload):
self.payload = payload
self.calls = []
def complete_json(self, system, user, validate=None):
self.calls.append({"system": system, "user": user})
return self.payload
def test_api_diagnose_passes_constraints_to_llm():
"""补充改写要求/约束必须真实传给 LLM(诊断重生成链路)。"""
import main as main_mod
cap = CaptureCompleter(
{
"overall_diagnosis": "d",
"patterns": [],
"paragraph_briefs": [
{
"paragraph_id": "p1",
"confirmed_meaning": "m",
"rewrite_goal": "g",
"ai_focus": "a",
"annotations": [],
}
],
"optional_suggestion": "",
}
)
main_mod.get_client = lambda: cap
r = TestClient(app).post(
"/api/diagnose",
json={
"paragraphs": SAMPLE_PARAGRAPHS,
"confirmed_anchors": ["a"],
"global_constraints": ["整体更直接、克制"],
"paragraph_constraints": {"p1": ["保留 secret code"]},
},
)
assert r.status_code == 200
assert "整体更直接、克制" in cap.calls[0]["user"]
assert "保留 secret code" in cap.calls[0]["user"]
def test_api_diagnose_coerces_ai_focus_list():
"""复现 2026-08-21 用户实测:模型把 ai_focus 返回成数组——
路由必须规整为字符串并返回 200,而不是 502。"""
import main as main_mod
cap = CaptureCompleter(
{
"overall_diagnosis": ["开头不错", "结尾乏力"],
"patterns": [],
"paragraph_briefs": [
{
"paragraph_id": f"p{i + 1}",
"confirmed_meaning": "m",
"rewrite_goal": "g",
"ai_focus": ["重复修辞", "抽象总结"],
"annotations": [],
}
for i in range(4)
],
"optional_suggestion": "",
}
)
main_mod.get_client = lambda: cap
r = TestClient(app).post(
"/api/diagnose",
json={"paragraphs": SAMPLE_PARAGRAPHS, "confirmed_anchors": [], "global_constraints": []},
)
assert r.status_code == 200
briefs = r.json()["paragraph_briefs"]
assert len(briefs) == 4
assert briefs[0]["ai_focus"] == "重复修辞;抽象总结"
def test_api_analyze_shape():
payload = {
"essay_summary": "学生和数学关系的变化。",
"prompt_alignment": {"prompt_intent": "", "current_alignment": "", "optional_opportunity": ""},
"paragraphs": [
{
"id": "p1",
"original_text": SAMPLE_PARAGRAPHS[0],
"natural_meaning_zh": "喜欢数学带来的确定感。",
"semantic_anchor": "数学给我确定感。",
"optional_content_opportunity": "",
}
],
}
r = api_client(payload).post(
"/api/analyze", json={"prompt": "P1", "word_limit": 650, "paragraphs": [SAMPLE_PARAGRAPHS[0]]}
)
assert r.status_code == 200
assert r.json()["paragraphs"][0]["id"] == "p1"
def test_api_analyze_empty_text_422():
r = api_client({}).post("/api/analyze", json={"paragraphs": [" "]})
assert r.status_code == 422
def test_api_analyze_bad_payload_returns_one_line_detail():
"""LLM 输出不符合 schema → 502 detail 是一行用户可读摘要,不把 ValidationError 堆栈甩给页面(PRD §11 失败兜底)。"""
r = api_client({"essay_summary": "s", "paragraphs": [123]}).post( # 类型错误触发 ValidationError
"/api/analyze", json={"prompt": "P1", "word_limit": 650, "paragraphs": [SAMPLE_PARAGRAPHS[0]]}
)
assert r.status_code == 502
detail = r.json()["detail"]
assert "结果格式不符合预期" in detail
assert "validation errors for" not in detail # 没有完整校验堆栈
assert detail.count("\n") == 0 # 单行
assert len(detail) < 200 # 摘要级长度
def test_api_diagnose_drops_fictional_evidence():
payload = {
"overall_diagnosis": "修辞偏密。",
"patterns": [
{
"pattern_id": "P05",
"name": "Metaphor Stack",
"category": "rhetoric",
"affected_paragraphs": ["p1"],
"evidence": ["不存在的证据"],
"why_ai_like": "w",
"human_impact": "h",
"transformation_rule": "Keep One Motif",
}
],
"paragraph_briefs": [
{
"paragraph_id": "p1",
"confirmed_meaning": "确定感",
"rewrite_goal": "收修辞",
"ai_focus": "连续比喻",
"annotations": [
{
"pattern_id": "P05",
"category": "rhetoric",
"kind_label": "修辞包装",
"title": "虚构",
"evidence": ["不存在的证据"],
"observation": "o",
"rewrite_action": "a",
}
],
"context_hint": "",
"scaffold": "What I liked was ______.",
"reference_snippet": "I liked the certainty.",
}
],
"optional_suggestion": "",
}
r = api_client(payload).post(
"/api/diagnose",
json={"paragraphs": SAMPLE_PARAGRAPHS, "confirmed_anchors": ["a"], "global_constraints": [], "paragraph_constraints": {}},
)
assert r.status_code == 200
body = r.json()
assert body["patterns"][0]["pattern_id"] == "P05"
assert body["patterns"][0]["evidence"] == []
assert body["paragraph_briefs"][0]["annotations"] == []
def test_api_recheck_revision_single_target():
payload = {
"status": "revision_required",
"checked_rewrite_version": "rv_12",
"global_checks": {
k: "pass"
for k in [
"semantic_preservation",
"voice_consistency",
"pattern_reduction",
"new_pattern",
"coherence",
"reference_copying",
"word_limit",
]
},
"revision_targets": [
{
"paragraph_id": "p3",
"blocking_issue": "新版本仍是 Explicit Lesson",
"evidence": ["gradually came to realize"],
"single_revision_goal": "不要总结我学到了什么,写判断怎么变化。",
}
],
}
r = api_client(payload).post(
"/api/recheck",
json={
"original_paragraphs": SAMPLE_PARAGRAPHS,
"rewrite_paragraphs": ["a" * 60] * 4,
"rewrite_version": "rv_12",
},
)
assert r.status_code == 200
body = r.json()
assert body["status"] == "revision_required"
assert body["checked_rewrite_version"] == "rv_12"
assert len(body["revision_targets"]) == 1
def test_api_recheck_empty_paragraph_422():
r = api_client({}).post(
"/api/recheck",
json={"original_paragraphs": SAMPLE_PARAGRAPHS, "rewrite_paragraphs": ["ok", "", "ok", "ok"]},
)
assert r.status_code == 422
def test_api_reference_shape():
payload = {"paragraph_id": "p1", "starter": "What I liked about math was ______."}
r = api_client(payload).post(
"/api/reference",
json={"paragraph_id": "p1", "original_text": SAMPLE_PARAGRAPHS[0], "semantic_anchor": "a", "rewrite_goal": "g"},
)
assert r.status_code == 200
assert r.json()["starter"].startswith("What I liked")
assert r.json()["reference_snippet"].startswith("What I liked")
def test_api_scaffold_shape():
payload = {"paragraph_id": "p1", "scaffold": "What I liked about math was ______."}
r = api_client(payload).post(
"/api/scaffold",
json={"paragraph_id": "p1", "original_text": SAMPLE_PARAGRAPHS[0], "semantic_anchor": "a", "rewrite_goal": "g"},
)
assert r.status_code == 200
assert "liked" in r.json()["scaffold"]
def test_complete_json_paragraph_count_mismatch_retries_once():
"""语义校验:段落数不匹配视为一次失败并自动重试,重试成功返回完整结果。
真实 LLM 偶发返回合法 JSON 但段落条目不全(AI 初审"4 段只显示 1 段"),
此前无校验被静默接受;现在按 complete_json 既有重试约定自动再试一次。"""
one = {"essay_summary": "s", "paragraphs": [{"id": "p1", "natural_meaning_zh": "第一段"}]}
four = {"essay_summary": "s", "paragraphs": [{"id": f"p{i}", "natural_meaning_zh": f"{i}"} for i in range(1, 5)]}
c = SeqClient([("content", json.dumps(one)), ("content", json.dumps(four))])
out = c.complete_json(
"s", "u", validate=lambda d: "结果段落数与输入不一致" if len(d.get("paragraphs") or []) != 4 else None
)
assert len(out["paragraphs"]) == 4
assert c.calls == 2 # 第一次因数量不匹配被重试
def test_complete_json_paragraph_count_mismatch_twice_raises():
"""两次都数量不匹配 → LLMError 带一行可读说明(不把原始 JSON 甩给页面)。"""
one = {"essay_summary": "s", "paragraphs": [{"id": "p1", "natural_meaning_zh": "第一段"}]}
c = SeqClient([("content", json.dumps(one)), ("content", json.dumps(one))])
with pytest.raises(LLMError) as ei:
c.complete_json(
"s", "u",
validate=lambda d: "结果段落数与输入不一致(输入 4 段,返回 1 段)" if len(d.get("paragraphs") or []) != 4 else None,
)
assert "输入 4 段" in str(ei.value) and "返回 1 段" in str(ei.value)
assert c.calls == 2
def test_api_analyze_wires_paragraph_count_check():
"""analyze 路由必须把段落数量校验传给 complete_json(否则 4 段输入只返回 1 段会被静默接受)。"""
import main as main_mod
class SpyCompleter(CaptureCompleter):
def complete_json(self, system, user, validate=None):
self.validate = validate
return super().complete_json(system, user, validate)
payload = {"essay_summary": "s", "paragraphs": [{"id": "p1", "natural_meaning_zh": "第一段"}]}
spy = SpyCompleter(payload)
main_mod.get_client = lambda: spy
r = TestClient(app).post(
"/api/analyze", json={"prompt": "P1", "word_limit": 650, "paragraphs": SAMPLE_PARAGRAPHS}
)
assert r.status_code == 200
assert spy.validate is not None
bad = spy.validate({"essay_summary": "s", "paragraphs": [{"id": "p1"}]})
assert bad is not None and "段落数" in bad
assert spy.validate({"essay_summary": "s", "paragraphs": [{"id": f"p{i}"} for i in range(1, 5)]}) is None