meida_backend/tests/golden/runner.py

"""黄金集回归 runner：JSON 用例加载 → 执行 → 指标计算 → 逐用例明细（T-P2-11）.

黄金集是"指标退化直接阻断合并"的防线（FR-KB-11 / NFR-MAINT-04）。本模块把
用例（数据）、阈值（配置）与判定（代码）彻底分开——加用例 = 加 JSON 行，
规模扩充（知识库文档 §7 的四子集 330 例口径，由内容轨 T-P2-16 达成）不需要
动框架。

设计约束：

1. **不依赖 pytest**：``test_golden.py`` 只是薄壳。将来本地脚本或 admin
   后台跑同一套黄金集，直接 import 本模块即可。
2. **失败必须可解释**：每条用例的判定携带"期望 vs 实际"的差异摘要，
   CI 日志里要能直接看出哪条用例在哪个断言上退化，而不是只有一个通过率。
3. **种子实体自管理**：检索用例的实体以 ``golden:`` 前缀命名；跑前先删
   （上次运行可能中断在判定之前，保证幂等重跑）、跑后必删（集成库长活，
   留下假实体会破坏其它用例对结果列表的精确断言）。
4. **参数钉死**：黄金集断言确定性结果，检索的维度/top_k 固定在框架里而
   不读 env——env 漂移会让同一份用例集在不同机器上口径不一。
5. **封闭词汇表**：用例/expect 的未知键一律判为用例失败而非静默忽略——
   ``must_recomend`` 这类 typo 若被忽略，意味着断言悄悄失效（与规则 DSL
   同一哲学，FR-KB-02）。
"""

from __future__ import annotations

import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession

from packages.domain.ids import next_id
from packages.retrieval.embedding import DeterministicEmbedder
from packages.retrieval.repo import ChunkInput, hybrid_search, upsert_chunks
from packages.retrieval.rerank import ScoredChunk
from packages.rules import dsl, engine
from packages.shared.db import session_scope
from packages.shared.models import KbChunk, KbEntity

GOLDEN_DIR = Path(__file__).resolve().parent

#: 种子实体命名前缀：runner 只清理带此前缀的实体——既不误伤业务数据，
#: 也让"哪些行是黄金集造的"在库内一眼可辨（幂等重跑契约，任务文档 §3.3）。
SEED_NAME_PREFIX = "golden:"

#: 检索执行的固定参数（与 Settings 默认值一致但刻意不读 Settings）。
_VECTOR_DIMENSIONS = 1024  # 与 kb_chunk.embedding 的 Vector(1024) 列宽一致
_RETRIEVAL_TOP_K = 50  # 与 Settings.retrieval_top_k 默认值一致

_RULES_CASE_KEYS = frozenset({"name", "rules", "context", "expect"})
_RULES_EXPECT_KEYS = frozenset(
    {"must_recommend", "must_avoid", "forbidden_recommend", "hard_blocks_nonempty"}
)
_RETRIEVAL_CASE_KEYS = frozenset({"name", "seed_entities", "query", "user_ctx", "expect"})
_RETRIEVAL_EXPECT_KEYS = frozenset({"must_hit_names", "must_miss_names", "first_hit_in"})
_USER_CTX_KEYS = frozenset({"body_shape", "occasion", "entity_types"})
_FIRST_HIT_IN_CHOICES = ("top1", "top3")


# ── 结果模型 ────────────────────────────────────────────────────────────────


@dataclass(frozen=True, slots=True)
class CaseResult:
    """一条用例的判定结论；problems 是"期望 vs 实际"的差异摘要列表."""

    name: str
    passed: bool
    problems: tuple[str, ...] = ()
    #: 检索子集专用：首个 must_hit 落位是否在 top3（未声明 first_hit_in 为 None）。
    first_hit_in_top3: bool | None = None


@dataclass(frozen=True, slots=True)
class SubsetReport:
    """一个子集的指标汇总。阈值随报告携带，调用方只需对齐指标与阈值."""

    subset: str
    cases: tuple[CaseResult, ...]
    pass_rate_min: float
    first_hit_in_top3_min: float | None = None
    first_hit_in_top3_total: int = 0
    first_hit_in_top3_passed: int = 0

    @property
    def total(self) -> int:
        return len(self.cases)

    @property
    def passed(self) -> int:
        return sum(1 for case in self.cases if case.passed)

    @property
    def pass_rate(self) -> float:
        return self.passed / self.total if self.total else 0.0

    @property
    def first_hit_in_top3_rate(self) -> float | None:
        """分母 = 声明了 first_hit_in 的用例数（规则子集恒为 None）."""
        if not self.first_hit_in_top3_total:
            return None
        return self.first_hit_in_top3_passed / self.first_hit_in_top3_total

    def summary(self) -> str:
        """CI 日志直读的明细：头部指标行 + 每条失败用例的差异摘要."""
        head = (
            f"[golden:{self.subset}] 通过 {self.passed}/{self.total}，"
            f"pass_rate={self.pass_rate:.4f}（阈值 {self.pass_rate_min:.2f}）"
        )
        if self.first_hit_in_top3_min is not None:
            rate = self.first_hit_in_top3_rate
            rate_text = "n/a（无用例声明该指标）" if rate is None else f"{rate:.4f}"
            head += (
                f"；first_hit_in_top3={rate_text}"
                f"（阈值 {self.first_hit_in_top3_min:.2f}，"
                f"{self.first_hit_in_top3_passed}/{self.first_hit_in_top3_total} 例参与）"
            )
        lines = [head]
        for case in self.cases:
            if case.passed:
                continue
            lines.append(f"  [FAIL] {case.name}")
            lines.extend(f"    - {problem}" for problem in case.problems)
        return "\n".join(lines)


# ── 配置与用例加载 ──────────────────────────────────────────────────────────


@dataclass(frozen=True, slots=True)
class SubsetThresholds:
    """thresholds.json 里一个子集的阈值."""

    pass_rate_min: float
    first_hit_in_top3_min: float | None = None


def load_thresholds() -> dict[str, SubsetThresholds]:
    raw = json.loads((GOLDEN_DIR / "thresholds.json").read_text(encoding="utf-8"))
    thresholds: dict[str, SubsetThresholds] = {}
    for subset, cfg in raw.items():
        if not isinstance(cfg, dict) or "pass_rate_min" not in cfg:
            raise ValueError(f"thresholds.json：子集 {subset} 缺少 pass_rate_min")
        thresholds[subset] = SubsetThresholds(
            pass_rate_min=float(cfg["pass_rate_min"]),
            first_hit_in_top3_min=(
                float(cfg["first_hit_in_top3_min"]) if "first_hit_in_top3_min" in cfg else None
            ),
        )
    return thresholds


def _thresholds_for(subset: str) -> SubsetThresholds:
    thresholds = load_thresholds()
    if subset not in thresholds:
        raise ValueError(f"thresholds.json 缺少子集 {subset} 的阈值")
    return thresholds[subset]


def _load_cases(filename: str) -> list[dict[str, Any]]:
    path = GOLDEN_DIR / "cases" / filename
    raw = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(raw, list):
        raise ValueError(f"{path} 应为 JSON 数组（每个元素一个用例对象）")
    if not all(isinstance(case, dict) for case in raw):
        raise ValueError(f"{path} 存在非对象的数组元素（用例必须是 JSON 对象）")
    return raw


def _unknown_keys(actual: Mapping[str, Any], allowed: frozenset[str]) -> list[str]:
    return sorted(set(actual) - allowed)


def _empty_marker(subset: str) -> tuple[CaseResult, ...]:
    """空子集的哨兵失败用例：vacuous green 是回归防线最危险的静默失效."""
    return (
        CaseResult(
            f"<{subset}-empty>",
            False,
            (f"子集 {subset} 没有任何用例：请检查 cases/*.json 是否被清空",),
        ),
    )


# ── 规则子集（packages.rules DSL + 引擎，纯逻辑无 I/O）─────────────────────


def _build_context(raw: Mapping[str, Any]) -> dsl.RuleContext:
    """JSON context → RuleContext。集合字段（数组）收成 tuple；未知字段 TypeError."""
    kwargs: dict[str, Any] = {}
    for key, value in raw.items():
        if key in ("features", "goals", "user_taboos"):
            kwargs[key] = tuple(value)
        else:
            kwargs[key] = value
    return dsl.RuleContext(**kwargs)


def _expected_item(item: Any) -> tuple[str, dict[str, tuple[str, ...]]]:
    """expect 条目 {slot, attrs_include} → (slot, {维度: 取值元组})；attrs 缺省=整 slot."""
    if not isinstance(item, dict) or not item.get("slot"):
        raise ValueError(f"expect 条目缺少 slot：{item!r}")
    attrs = item.get("attrs_include") or {}
    return str(item["slot"]), {str(dim): tuple(values) for dim, values in attrs.items()}


def _covers(actual: dsl.SlotAttrs, slot: str, attrs: dict[str, tuple[str, ...]]) -> bool:
    """子集语义（must_*）：同 slot，且期望每个维度的取值都被 actual 包含.

    attrs 为空 = 对整个 slot 的期望（与 DSL"空 attrs 表示对品类推荐/回避"对齐）。
    """
    if actual.slot != slot:
        return False
    return all(
        dim in actual.attrs and set(values) <= set(actual.attrs[dim])
        for dim, values in attrs.items()
    )


def _overlaps(actual: dsl.SlotAttrs, slot: str, attrs: dict[str, tuple[str, ...]]) -> bool:
    """红线语义（forbidden_*）：同 slot 且任一期望维度与 actual 取值相交即命中."""
    if actual.slot != slot:
        return False
    return any(
        dim in actual.attrs and set(values) & set(actual.attrs[dim])
        for dim, values in attrs.items()
    )


def _fmt_slot_attrs(item: dsl.SlotAttrs) -> str:
    if not item.attrs:
        return item.slot
    dims = ", ".join(f"{dim}={list(values)}" for dim, values in item.attrs.items())
    return f"{item.slot}[{dims}]"


def _fmt_items(items: Sequence[dsl.SlotAttrs]) -> str:
    if not items:
        return "（空）"
    return "[" + ", ".join(_fmt_slot_attrs(item) for item in items) + "]"


def _judge_rules_case(case: Mapping[str, Any]) -> CaseResult:
    name = str(case.get("name", "<缺少 name>"))
    problems: list[str] = []
    unknown = _unknown_keys(case, _RULES_CASE_KEYS)
    if unknown:
        problems.append(f"用例含未识别字段 {unknown}（可用：{sorted(_RULES_CASE_KEYS)}）")
    if "rules" not in case or "expect" not in case:
        problems.append("用例缺少必填字段 rules / expect")
        return CaseResult(name, False, tuple(problems))
    expect: Mapping[str, Any] = case["expect"]
    if not isinstance(expect, Mapping):
        problems.append("expect 应为 JSON 对象")
        return CaseResult(name, False, tuple(problems))

    try:
        rules = [dsl.Rule.model_validate(rule) for rule in case["rules"]]
        ctx = _build_context(case.get("context") or {})
        must_recommend = [_expected_item(item) for item in expect.get("must_recommend", [])]
        must_avoid = [_expected_item(item) for item in expect.get("must_avoid", [])]
        forbidden = [_expected_item(item) for item in expect.get("forbidden_recommend", [])]
    except Exception as exc:
        problems.append(f"用例无法解析：{exc}")
        return CaseResult(name, False, tuple(problems))

    unknown_expect = _unknown_keys(expect, _RULES_EXPECT_KEYS)
    if unknown_expect:
        problems.append(
            f"expect 含未识别字段 {unknown_expect}（可用：{sorted(_RULES_EXPECT_KEYS)}）——"
            "静默忽略会让断言悄悄失效"
        )

    verdict = engine.evaluate(rules, ctx)

    for slot, attrs in must_recommend:
        if not any(_covers(rec, slot, attrs) for rec in verdict.recommend):
            problems.append(
                f"must_recommend 未命中：期望 {slot}[{attrs}]，"
                f"实际 recommend={_fmt_items(verdict.recommend)}"
            )
    for slot, attrs in must_avoid:
        if not any(_covers(av, slot, attrs) for av in verdict.avoid):
            problems.append(
                f"must_avoid 未命中：期望 {slot}[{attrs}]，实际 avoid={_fmt_items(verdict.avoid)}"
            )
    for slot, attrs in forbidden:
        violation = next((rec for rec in verdict.recommend if _overlaps(rec, slot, attrs)), None)
        if violation is not None:
            problems.append(
                f"forbidden_recommend 违规：{_fmt_slot_attrs(violation)} "
                f"与红线 {slot}[{attrs}] 相交"
            )
    if "hard_blocks_nonempty" in expect:
        want = bool(expect["hard_blocks_nonempty"])
        if want != verdict.blocked:
            hit_keys = [block.rule_key for block in verdict.hard_blocks]
            problems.append(
                f"hard_blocks_nonempty 期望 {want}，实际 {verdict.blocked}"
                f"（硬拦截规则：{hit_keys or '无'}）"
            )
    return CaseResult(name, not problems, tuple(problems))


def run_rules_subset() -> SubsetReport:
    """规则子集：纯逻辑求值，同步执行即可（无 DB / 网络）."""
    cases = _load_cases("rules_cases.json")
    results = tuple(_judge_rules_case(case) for case in cases) or _empty_marker("rules")
    thresholds = _thresholds_for("rules")
    return SubsetReport(subset="rules", cases=results, pass_rate_min=thresholds.pass_rate_min)


# ── 检索子集（真实 PG + DeterministicEmbedder + hybrid_search）─────────────


@dataclass(frozen=True, slots=True)
class SeedEntity:
    """一条检索用例自带的种子实体（跑前清理、跑后删除，见模块 docstring）."""

    entity_type: str
    name: str
    attrs: dict[str, Any]
    status: str
    chunks: tuple[str, ...]


def _parse_seeds(raw_seeds: Any) -> list[SeedEntity]:
    if not isinstance(raw_seeds, list) or not raw_seeds:
        raise ValueError("seed_entities 必须是非空数组")
    seeds: list[SeedEntity] = []
    seen: set[str] = set()
    for raw in raw_seeds:
        if not isinstance(raw, dict):
            raise ValueError(f"seed 实体应为 JSON 对象：{raw!r}")
        name = str(raw.get("name", ""))
        if not name.startswith(SEED_NAME_PREFIX):
            raise ValueError(f"种子实体 name 必须以 {SEED_NAME_PREFIX!r} 开头：{name!r}")
        if name in seen:
            raise ValueError(f"种子实体 name 在用例内重复：{name!r}")
        seen.add(name)
        chunks = tuple(str(text) for text in raw.get("chunks", []))
        if not chunks or not all(text.strip() for text in chunks):
            raise ValueError(f"种子实体 {name!r} 至少要有一个非空 chunk 文本")
        seeds.append(
            SeedEntity(
                entity_type=str(raw.get("entity_type", "item")),
                name=name,
                attrs=dict(raw.get("attrs") or {}),
                status=str(raw.get("status", "published")),
                chunks=chunks,
            )
        )
    return seeds


def _build_user_ctx(raw: Mapping[str, Any]) -> dict[str, Any]:
    unknown = _unknown_keys(raw, _USER_CTX_KEYS)
    if unknown:
        raise ValueError(f"user_ctx 含未识别字段 {unknown}（可用：{sorted(_USER_CTX_KEYS)}）")
    ctx: dict[str, Any] = dict(raw)
    if ctx.get("entity_types"):
        ctx["entity_types"] = tuple(ctx["entity_types"])
    else:
        ctx.pop("entity_types", None)  # 空数组视为不限类型（与 hybrid_search 语义一致）
    return ctx


async def _delete_seeded(session: AsyncSession, seeds: Sequence[SeedEntity]) -> int:
    """按 golden: 前缀 name 删除本用例 seed 过的实体及其全部向量（幂等）."""
    names = [seed.name for seed in seeds]
    entity_ids = list(
        (await session.execute(select(KbEntity.entity_id).where(KbEntity.name.in_(names))))
        .scalars()
        .all()
    )
    if not entity_ids:
        return 0
    await session.execute(delete(KbChunk).where(KbChunk.entity_id.in_(entity_ids)))
    await session.execute(delete(KbEntity).where(KbEntity.entity_id.in_(entity_ids)))
    return len(entity_ids)


def _judge_retrieval_expect(
    expect: Mapping[str, Any],
    results: Sequence[ScoredChunk],
    name_by_entity: Mapping[int, str],
) -> tuple[list[str], bool | None]:
    """检索用例判定。返回 (差异摘要列表, 首命中是否在 top3 或 None)."""
    problems: list[str] = []
    unknown = _unknown_keys(expect, _RETRIEVAL_EXPECT_KEYS)
    if unknown:
        problems.append(f"expect 含未识别字段 {unknown}（可用：{sorted(_RETRIEVAL_EXPECT_KEYS)}）")

    position_by_name: dict[str, int] = {}
    ranked_names: list[str] = []
    for index, chunk in enumerate(results, start=1):
        name = name_by_entity.get(chunk.entity_id, f"#实体{chunk.entity_id}")
        ranked_names.append(name)
        if name not in position_by_name:
            position_by_name[name] = index

    must_hit = [str(item) for item in expect.get("must_hit_names", [])]
    must_miss = [str(item) for item in expect.get("must_miss_names", [])]
    for name in must_hit:
        if name not in position_by_name:
            problems.append(f"must_hit 未出现：{name!r} 不在结果中（结果实体序列：{ranked_names}）")
    for name in must_miss:
        position = position_by_name.get(name)
        if position is not None:
            problems.append(
                f"must_miss 红线违规：{name!r} 出现在第 {position} 位——"
                "硬过滤零漏（draft/禁忌/场景外实体绝不出现）被破坏"
            )

    first_hit_in_top3: bool | None = None
    if "first_hit_in" in expect:
        mode = expect["first_hit_in"]
        if mode not in _FIRST_HIT_IN_CHOICES:
            problems.append(f"first_hit_in 只能是 {_FIRST_HIT_IN_CHOICES}，得到 {mode!r}")
        else:
            positions = [position_by_name[name] for name in must_hit if name in position_by_name]
            if not positions:
                problems.append("无法评价 first_hit_in：没有任何 must_hit 实体出现在结果中")
                first_hit_in_top3 = False
            else:
                first = min(positions)
                limit = 1 if mode == "top1" else 3
                if first > limit:
                    problems.append(
                        f"first_hit_in={mode} 未满足：首个命中在第 {first} 位，"
                        f"前 {limit} 位序列：{ranked_names[:limit]}"
                    )
                first_hit_in_top3 = first <= 3
    return problems, first_hit_in_top3


async def run_retrieval_case(
    case: Mapping[str, Any], embedder: DeterministicEmbedder
) -> CaseResult:
    name = str(case.get("name", "<缺少 name>"))
    problems: list[str] = []
    unknown = _unknown_keys(case, _RETRIEVAL_CASE_KEYS)
    if unknown:
        problems.append(f"用例含未识别字段 {unknown}（可用：{sorted(_RETRIEVAL_CASE_KEYS)}）")
    try:
        for required in ("seed_entities", "query", "expect"):
            if required not in case:
                raise ValueError(f"用例缺少必填字段 {required}")
        seeds = _parse_seeds(case["seed_entities"])
        query = str(case["query"])
        user_ctx = _build_user_ctx(case.get("user_ctx") or {})
        expect = case["expect"]
        if not isinstance(expect, Mapping):
            raise ValueError("expect 应为 JSON 对象")
    except Exception as exc:
        problems.append(f"用例无法解析：{exc}")
        return CaseResult(name, False, tuple(problems))

    first_hit_in_top3: bool | None = None
    try:
        # 1) 跑前先清自己 seed 的行：上一次运行可能中断在判定之前（幂等重跑）。
        async with session_scope() as session:
            await _delete_seeded(session, seeds)
        # 2) 重建种子实体并写入向量（DeterministicEmbedder：同文本同向量）。
        entity_ids: dict[str, int] = {}
        async with session_scope() as session:
            for seed in seeds:
                entity_id = next_id()
                session.add(
                    KbEntity(
                        entity_id=entity_id,
                        entity_type=seed.entity_type,
                        name=seed.name,
                        attrs=seed.attrs,
                        status=seed.status,
                    )
                )
                # 先落实体行再写 chunk：kb_chunk 有 entity 外键，而两个模型之间
                # 没有 relationship()，session 又是 autoflush=False——不显式
                # flush 的话 UOW 不保证实体先于 chunk 插入（实测会 FK 违约）。
                await session.flush()
                await upsert_chunks(
                    session,
                    entity_id,
                    [ChunkInput(text=text) for text in seed.chunks],
                    embedder,
                )
                entity_ids[seed.name] = entity_id
        # 3) 混合检索：硬过滤在 SQL WHERE 里，"零漏"是结构保证而非概率保证。
        query_vec = (await embedder.embed([query]))[0]
        async with session_scope() as session:
            results = await hybrid_search(
                session,
                query_vec,
                query,
                entity_types=user_ctx.get("entity_types"),
                occasion=user_ctx.get("occasion"),
                body_shape=user_ctx.get("body_shape"),
                top_k=_RETRIEVAL_TOP_K,
            )
        name_by_entity = {entity_id: seed_name for seed_name, entity_id in entity_ids.items()}
        case_problems, first_hit_in_top3 = _judge_retrieval_expect(expect, results, name_by_entity)
        problems.extend(case_problems)
    finally:
        # 4) 跑后必删：集成库长活，留下的假实体会破坏其它用例对结果列表的精确断言。
        try:
            async with session_scope() as session:
                await _delete_seeded(session, seeds)
        except Exception as exc:
            problems.append(f"种子清理失败（会污染共享集成库）：{exc}")
    return CaseResult(name, not problems, tuple(problems), first_hit_in_top3)


async def run_retrieval_subset() -> SubsetReport:
    """检索子集：真实 PG + pgvector 上执行（种子自管理，幂等可重跑）."""
    cases = _load_cases("retrieval_cases.json")
    embedder = DeterministicEmbedder(dimensions=_VECTOR_DIMENSIONS)
    results = tuple([await run_retrieval_case(case, embedder) for case in cases])
    if not results:
        results = _empty_marker("retrieval")
    thresholds = _thresholds_for("retrieval")
    declaring = [case for case in results if case.first_hit_in_top3 is not None]
    return SubsetReport(
        subset="retrieval",
        cases=results,
        pass_rate_min=thresholds.pass_rate_min,
        first_hit_in_top3_min=thresholds.first_hit_in_top3_min,
        first_hit_in_top3_total=len(declaring),
        first_hit_in_top3_passed=sum(1 for case in declaring if case.first_hit_in_top3),
    )