对话式 AI 测试实战：八步搭建 QA 系统

conversational-ai-qa/
├── contracts/
│   └── password-reset.yaml
├── cases/
│   ├── single-turn.jsonl
│   ├── multi-turn.yaml
│   └── negative.jsonl
├── traces/
│   └── recovery-example.json
├── datasets/
│   └── golden.jsonl
├── gates/
│   └── release.yaml
└── tests/
    └── test_account_recovery.py

# contracts/password-reset.yaml
id: password-reset-v1
severity: high

goal: recover_account_access
intent_if_exposed: PASSWORD_RESET

must_have:
  - 指向批准的密码重置入口
  - 说明用户下一步需要查看注册邮箱
must_not:
  - 索取用户当前密码
  - 声称已经替用户修改账户
  - 编造账户状态或重置结果

grounding:
  allowed_sources:
    - account-recovery-v3

recovery:
  missing_information: 只追问完成任务所需的最少信息
  no_approved_source: 停止回答并提供人工渠道
  tool_failure: 说明暂时无法完成并转人工

soft_rubric:
  - accuracy
  - relevance
  - completeness
  - clarity
  - helpfulness

{"id":"INT-001","input":"I forgot my password","expected_intent":"PASSWORD_RESET","contract":"password-reset-v1"}
{"id":"INT-002","input":"Can't get into my account","expected_intent":"PASSWORD_RESET","contract":"password-reset-v1"}
{"id":"INT-003","input":"forgot pwd，登不上了","expected_intent":"PASSWORD_RESET","contract":"password-reset-v1"}
{"id":"INT-004","input":"怎么重设密码","expected_intent":"PASSWORD_RESET","contract":"password-reset-v1"}

# tests/test_account_recovery.py
import json
import pytest

CASES = [json.loads(line) for line in open("cases/single-turn.jsonl")]

@pytest.mark.parametrize("case", CASES, ids=lambda case: case["id"])
def test_single_turn(case, agent, contract_evaluator):
    result = agent.send(case["input"])

    assert result.intent == case["expected_intent"]

    verdict = contract_evaluator.evaluate(
        contract_id=case["contract"],
        response=result.text,
        evidence=result.evidence,
    )
    assert verdict.hard_gate_passed, verdict.failures

# cases/multi-turn.yaml
id: CTX-CORR-001
precondition:
  active_task: password_reset

turns:
  - user: "账户邮箱是 li@example.com"
    assert_state:
      recovery_email: li@example.com
  - user: "顺便问一下，客服几点下班？"
    assert_state:
      suspended_task: password_reset
  - user: "回到刚才的重置"
    assert_state:
      active_task: password_reset
  - user: "邮箱说错了，是 lin@example.com"
    assert_state:
      recovery_email: lin@example.com
      discarded.recovery_email: li@example.com

final_assertions:
  tool_arguments:
    recovery_email: lin@example.com
  must_not:
    - 同时保留两个当前邮箱
    - 把 li@example.com 发送给重置工具

{
  "case_id": "RAG-RECOVERY-001",
  "query": "forgot pwd，登不上了",
  "route": "account_recovery",
  "retrieved": [
    {"document_id": "account-recovery-v3", "approved": true, "status": "active", "score": 0.91},
    {"document_id": "account-recovery-v1", "approved": false, "status": "expired", "score": 0.94}
  ],
  "selected_source": "account-recovery-v3",
  "claims": [
    {"text": "重置链接会发送到注册邮箱", "grounded_in": "account-recovery-v3"}
  ],
  "must_not": ["引用 account-recovery-v1", "生成来源中不存在的处理时限"]
}

def test_recovery_tool_mapping(agent, fake_recovery_api):
    fake_recovery_api.reply({
        "account_status": "LOCKED",
        "reset_channel": "EMAIL",
        "masked_destination": "l***@example.com",
    })

    result = agent.send("forgot pwd，登不上了")

    assert result.tool_call.name == "start_password_reset"
    assert result.tool_call.arguments["channel"] == "EMAIL"
    assert "已锁定" in result.text
    assert "l***@example.com" in result.text
    assert "已完成重置" not in result.text


@pytest.mark.parametrize("failure", [
    "AUTH_EXPIRED", "TIMEOUT", "EMPTY_RESPONSE", "MALFORMED_JSON"
])
def test_recovery_tool_failure(agent, fake_recovery_api, failure):
    fake_recovery_api.fail_with(failure)
    result = agent.send("帮我重置密码")

    assert result.claimed_success is False
    assert result.recovery_action in {"RETRY", "HANDOFF"}

{
  "session_id": "s_2841",
  "user_goal": "recover_account_access",
  "confirmed_fields": {"recovery_email": "lin@example.com"},
  "discarded_fields": {"recovery_email": "li@example.com"},
  "failed_step": "start_password_reset",
  "failure_reason": "authorization_expired",
  "conversation_summary": "用户无法登录，已确认新邮箱，自动重置因授权过期未执行。",
  "handoff_required": true,
  "ai_must_stop_replying": true
}

{"id":"ACC-REC-001","severity":"high","contract":"password-reset-v1","precondition":{"active_task":"password_reset"},"turns":[{"user":"forgot pwd，登不上了"},{"user":"账户邮箱是 li@example.com"},{"user":"说错了，是 lin@example.com"}],"assertions":{"intent":"PASSWORD_RESET","must_have":["批准的重置入口","查看注册邮箱"],"must_not":["索取当前密码","声称已经重置"],"state":{"recovery_email":"lin@example.com"},"discarded":{"recovery_email":"li@example.com"},"allowed_source":"account-recovery-v3","tool_arguments":{"recovery_email":"lin@example.com"}},"recovery":"工具失败时说明未完成并转人工","evidence":["conversation_state_log","retrieval_trace","tool_request","tool_response"],"failure_owner":["routing","state","knowledge","retrieval","tool_mapping","generation"]}

# gates/release.yaml
release_gate:
  critical_failures_allowed: 0

  required_suites:
    - privacy_and_security
    - authorization
    - account_recovery
    - grounding_and_hallucination
    - human_handoff

  thresholds:
    intent_accuracy: ">= 0.95"
    task_completion_rate: ">= 0.90"
    escalation_success_rate: ">= 0.98"

  report_separately:
    - fallback_rate
    - grounding_failures
    - context_failures
    - integration_failures
    - critical_hallucinations

case: ACC-REC-001
result: FAIL
failed_assertion: handoff.confirmed_fields.recovery_email
expected: lin@example.com
actual: li@example.com
evidence: conversation_state_log + handoff_payload
failure_owner: state_to_handoff_mapping
severity: HIGH
release_action: HOLD