From 67d9754823ec5212c88fa42f8464fa6d7a289c80 Mon Sep 17 00:00:00 2001 From: xsl Date: Fri, 14 Aug 2026 21:19:57 +0800 Subject: [PATCH] test(golden): data-driven golden set framework wired into CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/golden：JSON 用例 + 阈值配置 + runner（规则 pass_rate=1.0、检索 0.8 起步）+ pytest 壳；规则/检索两子集种子，Agent/报告子集占位（T-P2-11）。 实现： - runner.py 为普通模块（不依赖 pytest）：加载/执行/指标（pass_rate、 first_hit_in_top3）/逐用例差异明细；test_golden.py 只是薄壳（marker=golden， 无 MEIDA_RUN_INTEGRATION 时整体 skip）。用例与 expect 的未知键一律判 fail（封闭词汇表，防断言静默失效）。 - 规则子集 13 例（rules_cases.json，走 packages.rules DSL/evaluate）： 含 3 个硬禁忌场景（无条件合规硬拦、正式场合硬拦、复合优先级链 100硬>95禁忌>90体型>50趋势）+ 禁忌压制/维度隔离/多规则叠加/温度边界/ 缺字段静默/include 子集语义等。 - 检索子集 13 例（retrieval_cases.json，DeterministicEmbedder+hybrid_search）： 含 7 个 must_miss 红线场景（禁忌体型/draft/archived/场景外/类型外/组合红线）。 种子实体以 golden: 前缀命名，跑前先删（幂等重跑）；跑后也删——CI 防线 5c 与集成测试共用长活库，留下假实体会破坏其它用例对结果列表的精确断言。 确定性策略：query=目标 chunk 原文（余弦距离 0 必排第 1）+ 每例独立 marker token（gqNN）隔离关键词路；检索参数（1024 维/top_k 50/默认 RRF 权重）钉死在 runner，不读 env。 - thresholds.json：规则 pass_rate_min=1.0（硬结论子集一条不许退）； 检索 pass_rate_min=0.8 + first_hit_in_top3_min=0.8 起步。 - README.md：四子集 330 例规划、两格式 spec、加用例流程、阈值口径与 “为什么规则必须 1.0”。 测试： - MEIDA_RUN_INTEGRATION=1 pytest -m golden --no-cov：2 passed（规则 13/13=1.0，检索 13/13=1.0、first_hit_in_top3=13/13）；连续重跑幂等， 跑后库内 golden:% 残留为 0；不设 env 时 2 skipped；基线单测 473 passed、 集成 38 passed 不受影响；ruff check/format、lint-imports 全绿。 - 阻断语义自证（已改回，提交里只有正确版本）：临时把规则用例 user-taboo-overrides-trend-recommendation 的 must_avoid 改成不存在的 取值并把 hard_blocks_nonempty 翻转为 true → 规则子集 12/13=0.9231 <1.0 fail，明细输出“must_avoid 未命中：期望 top[length=九分裤脚]，实际 avoid=[top[length=[短款, 露脐]]]”与“hard_blocks_nonempty 期望 True， 实际 False（硬拦截规则：无）”；再临时破坏 3 条检索用例（2 条 must_hit 换成不存在的实体名、1 条 must_miss 加入本应命中的实体）→ 检索子集 10/13=0.7692<0.8 fail，明细含“must_miss 红线违规：golden:r12-good 出现在第 1 位——硬过滤零漏被破坏”。验证完三条全部改回，复跑全绿。 （注：检索阈值 0.8 起步，单条退化 0.92 尚不拦截、≥3 条即拦截，符合 种子期口径；规则阈值 1.0 单条即拦截。） 偏差说明：无 --- tests/golden/README.md | 144 +++++++ tests/golden/__init__.py | 0 tests/golden/cases/retrieval_cases.json | 348 ++++++++++++++++ tests/golden/cases/rules_cases.json | 514 +++++++++++++++++++++++ tests/golden/runner.py | 533 ++++++++++++++++++++++++ tests/golden/test_golden.py | 52 +++ tests/golden/thresholds.json | 9 + 7 files changed, 1600 insertions(+) create mode 100644 tests/golden/README.md create mode 100644 tests/golden/__init__.py create mode 100644 tests/golden/cases/retrieval_cases.json create mode 100644 tests/golden/cases/rules_cases.json create mode 100644 tests/golden/runner.py create mode 100644 tests/golden/test_golden.py create mode 100644 tests/golden/thresholds.json diff --git a/tests/golden/README.md b/tests/golden/README.md new file mode 100644 index 0000000..af5d80b --- /dev/null +++ b/tests/golden/README.md @@ -0,0 +1,144 @@ +# 黄金集回归框架（Golden Set） + +> 任务 T-P2-11 ｜ 需求 FR-KB-11（检索质量可回归）、NFR-MAINT-04（指标退化阻断合并） +> 架构 §17「黄金集是最重要的一道防线」｜ CI 防线 5c（`.github/workflows/ci.yml`） + +黄金集 = **数据驱动的回归防线**：用例存 JSON、阈值存配置、指标低于阈值直接 fail。 +与"散在单测里的固定样本"的本质区别是——**加用例 = 加 JSON 行，零代码改动**； +顾问/运营扩充规模不需要工程师改断言。 + +## 目录 + +``` +tests/golden/ + README.md # 本文档：用例格式 + 加用例流程 + 阈值口径 + thresholds.json # 各子集指标阈值（pass_rate 等） + cases/ + rules_cases.json # 规则黄金集（当前 13 例种子） + retrieval_cases.json # 检索黄金集（当前 13 例种子） + runner.py # 用例加载 + 执行 + 指标计算 + 逐用例明细（不依赖 pytest） + test_golden.py # pytest 薄壳（marker=golden，无 MEIDA_RUN_INTEGRATION 时 skip） +``` + +## 四子集规划（知识库文档 §7，唯一事实源） + +| 子集 | 目标规模 | 当前 | 状态 | +| --- | --- | --- | --- | +| 检索 | 100 | 13 | ✅ 种子已就绪（内容轨 T-P2-16 扩充） | +| 规则 | —（决策表回归，随单测+本子集维护） | 13 | ✅ 种子已就绪 | +| Agent 问答 | 100 | 0 | ⏳ Sprint 3 占位 | +| 诊断报告结构 | 30 | 0 | ⏳ Sprint 4 占位 | + +> 规模达标是内容轨 T-P2-16 的事；本框架的职责是让"加用例 = 加 JSON 行"。 + +## 怎么跑 + +```bash +make golden # = MEIDA_RUN_INTEGRATION=1 uv run pytest -m golden --no-cov +# 或直接： +MEIDA_RUN_INTEGRATION=1 .venv/bin/pytest -m golden --no-cov +``` + +- 默认 `pytest`（`make test`）**不跑** golden（pyproject addopts 已排除），本地无 PG 不受影响； + 未设 `MEIDA_RUN_INTEGRATION` 时本目录用例整体 skip，不拖累本地开发。 +- 需要 PostgreSQL(pgvector) 已迁移到最新 schema（`make up && make migrate`）。 +- CI 在集成测试之后强制跑（防线 5c），**指标低于阈值 = job 失败 = 阻断合并**。 + +## 用例格式 + +### 规则子集（cases/rules_cases.json） + +每条用例是**自包含**的：规则数组 + 画像 + 期望。执行走 +`packages.rules` 的 `dsl.Rule.model_validate` → `dsl.RuleContext` → `engine.evaluate`（纯逻辑）。 + +```json +{ + "name": "user-taboo-overrides-trend-recommendation", + "rules": [ { "rule_key": "...", "category": "...", "priority": 95, + "conditions": {"user_taboos": {"include": ["露脐"]}}, + "effects": {"avoid": [{"slot": "top", "attrs": {"length": ["短款"]}}]} } ], + "context": {"style": "甜酷", "user_taboos": ["露脐"]}, + "expect": { + "must_recommend": [{"slot": "top", "attrs_include": {"shoulder": ["垫肩"]}}], + "must_avoid": [{"slot": "top", "attrs_include": {"length": ["短款"]}}], + "forbidden_recommend": [{"slot": "top", "attrs_include": {"length": ["短款"]}}], + "hard_blocks_nonempty": false + } +} +``` + +expect 判定语义（一条不符 = 该用例 fail）： + +| 键 | 语义 | +| --- | --- | +| `must_recommend` | **子集语义**：实际 recommend 中存在同 slot 条目，且期望每个维度的取值都被其包含。`attrs_include` 为空 `{}` = 对整个 slot 的期望 | +| `must_avoid` | 同上，但核对对象是实际 avoid 列表 | +| `forbidden_recommend` | **红线语义**：实际 recommend 中任何与期望维度取值**相交**的条目 = 违规（用于验证禁忌/合规压制：被压掉的推荐绝不允许复活） | +| `hard_blocks_nonempty` | `engine.evaluate` 的 `hard_blocks` 是否非空（is_hard 规则的 avoid 命中） | + +### 检索子集（cases/retrieval_cases.json) + +```json +{ + "name": "taboo-body-shape-entity-never-leaks", + "seed_entities": [ + {"entity_type": "item", "name": "golden:r02-taboo", + "attrs": {"occasions": ["wedding"], "avoid_body_shapes": ["apple"]}, + "status": "published", "chunks": ["gq02 pear body dress styling guide"]} + ], + "query": "gq02 pear body dress styling guide for apple shapes", + "user_ctx": {"body_shape": "apple", "occasion": "wedding", "entity_types": ["item"]}, + "expect": { + "must_hit_names": ["golden:r02-good"], + "must_miss_names": ["golden:r02-taboo"], + "first_hit_in": "top1" + } +} +``` + +- **种子实体契约**：`name` 必须以 `golden:` 开头（runner 强校验）。执行时 + **跑前先删**自己 seed 的行（上次中断也能幂等重跑）、**跑后必删**（集成库长活， + 留下假实体会破坏其它用例对结果列表的精确断言）。 +- 判定语义： + - `must_hit_names`：这些实体的 chunk 必须出现在结果里； + - `must_miss_names`：**硬过滤红线**——draft / archived / 禁忌体型 / 场景外 / + 类型外实体绝不允许出现在任何位置（零漏是 SQL WHERE 的结构保证，不是排序权重）； + - `first_hit_in`：`must_hit_names` 中最早出现的落位须满足 `top1`（第 1 位）或 `top3`（前 3 位）。 +- **确定性策略**（校准新用例时遵守）： + 1. 嵌入用 `DeterministicEmbedder`（同文本同向量），**query 写成目标 chunk 的原文** + → 余弦距离 0 → 向量路必排第 1，结果与库中其它数据无关； + 2. chunk/query 带各自独立的 marker token（`gq01`、`gq02`…）→ 关键词路只命中本用例的行； + 3. 需要非 top1 的落位时，用"两实体同 chunk 原文"构造并列（引擎按 chunk_id 稳定打破平局）。 +- 检索参数钉死在 runner（维度 1024、top_k 50、默认 RRF 权重），不读 env—— + 同一份用例集在任何机器上口径一致。 + +## 加用例流程（零代码改动） + +1. 在 `cases/*.json` 数组里追加一个用例对象（格式见上）； +2. `make golden` 跑一遍校准 expect（种子用例必须 100% 通过——不过就是 expect 写错 + 或真有退化，二选一，搞清楚再提交）； +3. 提交。CI 防线 5c 自动把它纳入回归口径。 + +注意：runner 对用例/expect 的未知键**一律判 fail**（封闭词汇表）——`must_recomend` +这类 typo 不会被静默忽略，这是防"断言悄悄失效"的刻意设计（与规则 DSL 同一哲学）。 + +## 阈值口径（thresholds.json） + +| 子集 | 指标 | 阈值 | 说明 | +| --- | --- | --- | --- | +| rules | `pass_rate_min` | **1.0** | 硬结论子集，一条不许退（原因见下） | +| retrieval | `pass_rate_min` | 0.8 | 顾问扩充用例后可逐步上调 | +| retrieval | `first_hit_in_top3_min` | 0.8 | 首个 must_hit 落进前 3 的比例 | + +**为什么规则子集阈值必须 1.0**：产品根基本上是"结论由规则引擎产出，LLM 只负责表达" +（防幻觉红线）。规则黄金集里的每一条都是**人工校准过的硬结论**——体型该推荐什么、 +禁忌必须压过什么、合规必须拦什么。这些结论退化了任何一条，就意味着给用户的建议 +直接变味，不存在"可以容忍 20% 出错"的余地；而检索是排序质量问题，允许有起步阈值、 +随用例规模成熟逐步收紧。所以：规则 pass_rate < 1.0 = fail，没有商量。 + +## runner 复用（不依赖 pytest） + +`runner.py` 是普通模块：`run_rules_subset()`（同步）与 `run_retrieval_subset()`（async） +各自返回 `SubsetReport`（含 `pass_rate`、`first_hit_in_top3_rate`、逐用例差异明细 +`summary()`）。将来本地脚本、admin 后台"跑一遍黄金集"直接 import 即可， +`test_golden.py` 只是薄壳。 diff --git a/tests/golden/__init__.py b/tests/golden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/cases/retrieval_cases.json b/tests/golden/cases/retrieval_cases.json new file mode 100644 index 0000000..6eb343d --- /dev/null +++ b/tests/golden/cases/retrieval_cases.json @@ -0,0 +1,348 @@ +[ + { + "name": "exact-text-match-ranks-first", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r01-exact", + "attrs": {}, + "status": "published", + "chunks": ["gq01 crisp white cotton shirt for office wear"] + } + ], + "query": "gq01 crisp white cotton shirt for office wear", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r01-exact"], + "must_miss_names": [], + "first_hit_in": "top1" + } + }, + { + "name": "taboo-body-shape-entity-never-leaks", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r02-taboo", + "attrs": {"occasions": ["wedding"], "avoid_body_shapes": ["apple"]}, + "status": "published", + "chunks": ["gq02 pear body dress styling guide"] + }, + { + "entity_type": "item", + "name": "golden:r02-good", + "attrs": {"occasions": ["wedding"]}, + "status": "published", + "chunks": ["gq02 pear body dress styling guide for apple shapes"] + } + ], + "query": "gq02 pear body dress styling guide for apple shapes", + "user_ctx": {"body_shape": "apple", "occasion": "wedding", "entity_types": ["item"]}, + "expect": { + "must_hit_names": ["golden:r02-good"], + "must_miss_names": ["golden:r02-taboo"], + "first_hit_in": "top1" + } + }, + { + "name": "draft-entity-never-leaks-even-with-near-text", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r03-draft", + "attrs": {}, + "status": "draft", + "chunks": ["gq03 sequin party clutch styling notes"] + }, + { + "entity_type": "item", + "name": "golden:r03-live", + "attrs": {}, + "status": "published", + "chunks": ["gq03 sequin party clutch styling notes published edition"] + } + ], + "query": "gq03 sequin party clutch styling notes published edition", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r03-live"], + "must_miss_names": ["golden:r03-draft"], + "first_hit_in": "top1" + } + }, + { + "name": "archived-entity-never-leaks", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r04-archived", + "attrs": {}, + "status": "archived", + "chunks": ["gq04 vintage brooch care instructions"] + }, + { + "entity_type": "item", + "name": "golden:r04-live", + "attrs": {}, + "status": "published", + "chunks": ["gq04 vintage brooch care instructions revised"] + } + ], + "query": "gq04 vintage brooch care instructions revised", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r04-live"], + "must_miss_names": ["golden:r04-archived"], + "first_hit_in": "top1" + } + }, + { + "name": "undeclared-occasion-entity-excluded-under-filter", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r05-declared", + "attrs": {"occasions": ["interview"]}, + "status": "published", + "chunks": ["gq05 navy blazer pairing guide"] + }, + { + "entity_type": "item", + "name": "golden:r05-undeclared", + "attrs": {}, + "status": "published", + "chunks": ["gq05 navy blazer pairing guide extra tips"] + } + ], + "query": "gq05 navy blazer pairing guide", + "user_ctx": {"occasion": "interview"}, + "expect": { + "must_hit_names": ["golden:r05-declared"], + "must_miss_names": ["golden:r05-undeclared"], + "first_hit_in": "top1" + } + }, + { + "name": "entity-type-filter-restricts-recall", + "seed_entities": [ + { + "entity_type": "style", + "name": "golden:r06-style", + "attrs": {}, + "status": "published", + "chunks": ["gq06 monochrome layering theory deep dive"] + }, + { + "entity_type": "item", + "name": "golden:r06-item", + "attrs": {}, + "status": "published", + "chunks": ["gq06 monochrome layering theory applied outfit"] + } + ], + "query": "gq06 monochrome layering theory applied outfit", + "user_ctx": {"entity_types": ["item"]}, + "expect": { + "must_hit_names": ["golden:r06-item"], + "must_miss_names": ["golden:r06-style"], + "first_hit_in": "top1" + } + }, + { + "name": "multi-chunk-entity-hit-on-second-chunk", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r07-multi", + "attrs": {}, + "status": "published", + "chunks": [ + "gq07 wool coat winter intro", + "gq07 wool coat collar and lapel details", + "gq07 wool coat care and storage" + ] + } + ], + "query": "gq07 wool coat collar and lapel details", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r07-multi"], + "must_miss_names": [], + "first_hit_in": "top1" + } + }, + { + "name": "relevant-chunk-outranks-unrelated-sibling", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r08-relevant", + "attrs": {}, + "status": "published", + "chunks": ["gq08 breathable linen shirt summer comfort"] + }, + { + "entity_type": "item", + "name": "golden:r08-noise", + "attrs": {}, + "status": "published", + "chunks": ["gq08 leather boots winter warm"] + } + ], + "query": "gq08 breathable linen shirt summer comfort", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r08-relevant"], + "must_miss_names": [], + "first_hit_in": "top1" + } + }, + { + "name": "duplicate-content-first-hit-still-top3", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r09-first", + "attrs": {}, + "status": "published", + "chunks": ["gq09 silver pleated midi skirt styling"] + }, + { + "entity_type": "item", + "name": "golden:r09-second", + "attrs": {}, + "status": "published", + "chunks": ["gq09 silver pleated midi skirt styling"] + } + ], + "query": "gq09 silver pleated midi skirt styling", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r09-second"], + "must_miss_names": [], + "first_hit_in": "top3" + } + }, + { + "name": "topic-siblings-all-recalled", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r10-a", + "attrs": {}, + "status": "published", + "chunks": ["gq10 wide brim hat sun protection notes"] + }, + { + "entity_type": "item", + "name": "golden:r10-b", + "attrs": {}, + "status": "published", + "chunks": ["gq10 wide brim hat travel packing tips"] + }, + { + "entity_type": "item", + "name": "golden:r10-c", + "attrs": {}, + "status": "published", + "chunks": ["gq10 wide brim hat material comparison"] + } + ], + "query": "gq10 wide brim hat sun protection notes", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r10-a", "golden:r10-b", "golden:r10-c"], + "must_miss_names": [], + "first_hit_in": "top3" + } + }, + { + "name": "cjk-exact-vector-match", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r11-cjk", + "attrs": {}, + "status": "published", + "chunks": ["gq11 梨形身材 高腰线 连衣裙 穿搭要点"] + } + ], + "query": "gq11 梨形身材 高腰线 连衣裙 穿搭要点", + "user_ctx": {}, + "expect": { + "must_hit_names": ["golden:r11-cjk"], + "must_miss_names": [], + "first_hit_in": "top1" + } + }, + { + "name": "combined-red-line-taboo-draft-archived", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r12-taboo", + "attrs": {"avoid_body_shapes": ["pear"]}, + "status": "published", + "chunks": ["gq12 column line coat structure guide"] + }, + { + "entity_type": "item", + "name": "golden:r12-draft", + "attrs": {}, + "status": "draft", + "chunks": ["gq12 column line coat structure guide draft"] + }, + { + "entity_type": "item", + "name": "golden:r12-archived", + "attrs": {}, + "status": "archived", + "chunks": ["gq12 column line coat structure guide archive"] + }, + { + "entity_type": "item", + "name": "golden:r12-good", + "attrs": {}, + "status": "published", + "chunks": ["gq12 column line coat structure guide final"] + } + ], + "query": "gq12 column line coat structure guide final", + "user_ctx": {"body_shape": "pear"}, + "expect": { + "must_hit_names": ["golden:r12-good"], + "must_miss_names": [ + "golden:r12-taboo", + "golden:r12-draft", + "golden:r12-archived" + ], + "first_hit_in": "top1" + } + }, + { + "name": "occasion-positive-declared-hit", + "seed_entities": [ + { + "entity_type": "item", + "name": "golden:r13-gala", + "attrs": {"occasions": ["gala", "party"]}, + "status": "published", + "chunks": ["gq13 velvet wrap dress evening notes"] + }, + { + "entity_type": "item", + "name": "golden:r13-office", + "attrs": {"occasions": ["office"]}, + "status": "published", + "chunks": ["gq13 velvet wrap dress evening notes office edition"] + } + ], + "query": "gq13 velvet wrap dress evening notes", + "user_ctx": {"occasion": "gala"}, + "expect": { + "must_hit_names": ["golden:r13-gala"], + "must_miss_names": ["golden:r13-office"], + "first_hit_in": "top1" + } + } +] diff --git a/tests/golden/cases/rules_cases.json b/tests/golden/cases/rules_cases.json new file mode 100644 index 0000000..7dc253c --- /dev/null +++ b/tests/golden/cases/rules_cases.json @@ -0,0 +1,514 @@ +[ + { + "name": "unconditional-hard-compliance-blocks-exotic-leather", + "rules": [ + { + "rule_key": "G-RULES-01-HARD-EXOTIC", + "category": "compliance", + "priority": 100, + "confidence": 1.0, + "is_hard": true, + "conditions": {}, + "effects": { + "avoid": [ + {"slot": "accessory", "attrs": {"material": ["真兽皮"]}}, + {"slot": "shoes", "attrs": {"material": ["真皮"]}} + ], + "explanation": "合规红线：任何场景都不推荐真兽皮材质。" + } + } + ], + "context": {}, + "expect": { + "must_avoid": [ + {"slot": "accessory", "attrs_include": {"material": ["真兽皮"]}}, + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}} + ], + "forbidden_recommend": [ + {"slot": "accessory", "attrs_include": {"material": ["真兽皮"]}} + ], + "hard_blocks_nonempty": true + } + }, + { + "name": "hard-compliance-fires-on-formal-occasion", + "rules": [ + { + "rule_key": "G-RULES-02-HARD-FORMAL", + "category": "compliance", + "priority": 100, + "confidence": 1.0, + "is_hard": true, + "conditions": {"occasion": {"include": ["红毯", "晚宴"]}}, + "effects": { + "avoid": [ + {"slot": "shoes", "attrs": {"material": ["真皮"]}}, + {"slot": "accessory", "attrs": {"material": ["真兽皮"]}} + ], + "explanation": "正式场合合规红线：不推荐真皮与真兽皮。" + } + } + ], + "context": {"occasion": "晚宴"}, + "expect": { + "must_avoid": [ + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}} + ], + "forbidden_recommend": [ + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}} + ], + "hard_blocks_nonempty": true + } + }, + { + "name": "hard-compliance-silent-on-daily-occasion", + "rules": [ + { + "rule_key": "G-RULES-03-HARD-FORMAL", + "category": "compliance", + "priority": 100, + "confidence": 1.0, + "is_hard": true, + "conditions": {"occasion": {"include": ["红毯", "晚宴"]}}, + "effects": { + "avoid": [{"slot": "shoes", "attrs": {"material": ["真皮"]}}], + "explanation": "正式场合合规红线：不推荐真皮。" + } + } + ], + "context": {"occasion": "通勤"}, + "expect": { + "forbidden_recommend": [ + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "user-taboo-overrides-trend-recommendation", + "rules": [ + { + "rule_key": "G-RULES-04-TREND-CROP", + "category": "trend", + "priority": 50, + "confidence": 0.7, + "conditions": {"style": "甜酷"}, + "effects": { + "recommend": [{"slot": "top", "attrs": {"length": ["短款"]}}], + "explanation": "短款上衣是当季甜酷风的核心单品。" + } + }, + { + "rule_key": "G-RULES-05-TABOO-MIDRIFF", + "category": "user_taboo", + "priority": 95, + "confidence": 1.0, + "conditions": {"user_taboos": {"include": ["露脐"]}}, + "effects": { + "avoid": [{"slot": "top", "attrs": {"length": ["短款", "露脐"]}}], + "explanation": "用户不接受露脐，已剔除相关单品。" + } + } + ], + "context": {"style": "甜酷", "user_taboos": ["露脐"]}, + "expect": { + "must_avoid": [ + {"slot": "top", "attrs_include": {"length": ["短款", "露脐"]}} + ], + "forbidden_recommend": [ + {"slot": "top", "attrs_include": {"length": ["短款"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "user-taboo-limited-to-conflicting-dimension", + "rules": [ + { + "rule_key": "G-RULES-06-BODY-H-BOTTOM", + "category": "body_shape", + "priority": 90, + "confidence": 0.9, + "conditions": {"body_shape": "H型"}, + "effects": { + "recommend": [{"slot": "bottom", "attrs": {"silhouette": ["直筒", "阔腿"]}}], + "avoid": [{"slot": "bottom", "attrs": {"silhouette": ["紧身"]}}], + "explanation": "H型宜直线轮廓放大腰线比例。" + } + }, + { + "rule_key": "G-RULES-07-TABOO-MIDRIFF", + "category": "user_taboo", + "priority": 95, + "confidence": 1.0, + "conditions": {"user_taboos": {"include": ["露脐"]}}, + "effects": { + "avoid": [{"slot": "top", "attrs": {"length": ["短款", "露脐"]}}], + "explanation": "用户不接受露脐，已剔除相关单品。" + } + } + ], + "context": {"body_shape": "H型", "user_taboos": ["露脐"]}, + "expect": { + "must_recommend": [ + {"slot": "bottom", "attrs_include": {"silhouette": ["阔腿"]}} + ], + "must_avoid": [ + {"slot": "bottom", "attrs_include": {"silhouette": ["紧身"]}} + ], + "forbidden_recommend": [ + {"slot": "top", "attrs_include": {"length": ["短款"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "body-temperature-occasion-stack", + "rules": [ + { + "rule_key": "G-RULES-08-BODY-A-SHOULDER", + "category": "body_shape", + "priority": 90, + "confidence": 0.95, + "conditions": { + "body_shape": "A型", + "features": {"include": ["溜肩"]}, + "goals": {"include": ["修饰溜肩"]} + }, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"shoulder": ["垫肩"], "sleeve": ["泡泡袖"]}} + ], + "avoid": [{"slot": "top", "attrs": {"silhouette": ["修身"]}}], + "explanation": "溜肩需要用垫肩与泡泡袖在肩部建立结构感。" + } + }, + { + "rule_key": "G-RULES-09-TEMP-18-26", + "category": "temperature", + "priority": 75, + "confidence": 0.85, + "conditions": {"temperature": {"min": 18, "max": 26}}, + "effects": { + "recommend": [{"slot": "outer", "attrs": {"thickness": ["薄款"]}}], + "avoid": [{"slot": "outer", "attrs": {"thickness": ["厚款"]}}], + "explanation": "18-26℃ 单层薄外套最舒适。" + } + }, + { + "rule_key": "G-RULES-10-OCC-INTERVIEW", + "category": "occasion", + "priority": 80, + "confidence": 0.9, + "conditions": {"occasion": "面试"}, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"collar": ["衬衫领"]}}, + {"slot": "bottom", "attrs": {"silhouette": ["直筒"]}} + ], + "avoid": [{"slot": "top", "attrs": {"neckline": ["深V"]}}], + "explanation": "面试场合需要干练可信的轮廓。" + } + } + ], + "context": { + "body_shape": "A型", + "features": ["溜肩"], + "goals": ["修饰溜肩"], + "temperature": 22.0, + "occasion": "面试" + }, + "expect": { + "must_recommend": [ + {"slot": "top", "attrs_include": {"shoulder": ["垫肩"]}}, + {"slot": "top", "attrs_include": {"sleeve": ["泡泡袖"]}}, + {"slot": "top", "attrs_include": {"collar": ["衬衫领"]}}, + {"slot": "bottom", "attrs_include": {"silhouette": ["直筒"]}}, + {"slot": "outer", "attrs_include": {"thickness": ["薄款"]}} + ], + "must_avoid": [ + {"slot": "top", "attrs_include": {"silhouette": ["修身"]}}, + {"slot": "top", "attrs_include": {"neckline": ["深V"]}}, + {"slot": "outer", "attrs_include": {"thickness": ["厚款"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "missing-profile-fields-keep-rules-silent", + "rules": [ + { + "rule_key": "G-RULES-11-BODY-A-SHOULDER", + "category": "body_shape", + "priority": 90, + "confidence": 0.95, + "conditions": { + "body_shape": "A型", + "features": {"include": ["溜肩"]}, + "goals": {"include": ["修饰溜肩"]} + }, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"shoulder": ["垫肩"], "sleeve": ["泡泡袖"]}} + ], + "avoid": [{"slot": "top", "attrs": {"silhouette": ["修身"]}}], + "explanation": "溜肩需要用垫肩与泡泡袖在肩部建立结构感。" + } + }, + { + "rule_key": "G-RULES-12-TEMP-18-26", + "category": "temperature", + "priority": 75, + "confidence": 0.85, + "conditions": {"temperature": {"min": 18, "max": 26}}, + "effects": { + "recommend": [{"slot": "outer", "attrs": {"thickness": ["薄款"]}}], + "avoid": [{"slot": "outer", "attrs": {"thickness": ["厚款"]}}], + "explanation": "18-26℃ 单层薄外套最舒适。" + } + } + ], + "context": {"features": ["溜肩"], "goals": ["修饰溜肩"]}, + "expect": { + "forbidden_recommend": [ + {"slot": "top", "attrs_include": {}}, + {"slot": "outer", "attrs_include": {}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "temperature-lower-bound-inclusive", + "rules": [ + { + "rule_key": "G-RULES-13-TEMP-18-26", + "category": "temperature", + "priority": 75, + "confidence": 0.85, + "conditions": {"temperature": {"min": 18, "max": 26}}, + "effects": { + "recommend": [{"slot": "outer", "attrs": {"thickness": ["薄款"]}}], + "avoid": [{"slot": "outer", "attrs": {"thickness": ["厚款"]}}], + "explanation": "18-26℃ 单层薄外套最舒适。" + } + } + ], + "context": {"temperature": 18.0}, + "expect": { + "must_recommend": [ + {"slot": "outer", "attrs_include": {"thickness": ["薄款"]}} + ], + "must_avoid": [ + {"slot": "outer", "attrs_include": {"thickness": ["厚款"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "temperature-below-range-stays-silent", + "rules": [ + { + "rule_key": "G-RULES-14-TEMP-18-26", + "category": "temperature", + "priority": 75, + "confidence": 0.85, + "conditions": {"temperature": {"min": 18, "max": 26}}, + "effects": { + "recommend": [{"slot": "outer", "attrs": {"thickness": ["薄款"]}}], + "avoid": [{"slot": "outer", "attrs": {"thickness": ["厚款"]}}], + "explanation": "18-26℃ 单层薄外套最舒适。" + } + } + ], + "context": {"temperature": 17.5}, + "expect": { + "forbidden_recommend": [ + {"slot": "outer", "attrs_include": {}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "interview-occasion-collar-and-neckline", + "rules": [ + { + "rule_key": "G-RULES-15-OCC-INTERVIEW", + "category": "occasion", + "priority": 80, + "confidence": 0.9, + "conditions": {"occasion": "面试"}, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"collar": ["衬衫领"]}}, + {"slot": "bottom", "attrs": {"silhouette": ["直筒"]}} + ], + "avoid": [{"slot": "top", "attrs": {"neckline": ["深V"]}}], + "explanation": "面试场合需要干练可信的轮廓。" + } + } + ], + "context": {"occasion": "面试"}, + "expect": { + "must_recommend": [ + {"slot": "top", "attrs_include": {"collar": ["衬衫领"]}}, + {"slot": "bottom", "attrs_include": {"silhouette": ["直筒"]}} + ], + "must_avoid": [ + {"slot": "top", "attrs_include": {"neckline": ["深V"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "composite-priority-chain-hard-taboo-body", + "rules": [ + { + "rule_key": "G-RULES-16-HARD-FORMAL", + "category": "compliance", + "priority": 100, + "confidence": 1.0, + "is_hard": true, + "conditions": {"occasion": {"include": ["红毯", "晚宴"]}}, + "effects": { + "avoid": [ + {"slot": "accessory", "attrs": {"material": ["真兽皮"]}}, + {"slot": "shoes", "attrs": {"material": ["真皮"]}} + ], + "explanation": "正式场合合规红线：不推荐真兽皮材质。" + } + }, + { + "rule_key": "G-RULES-17-TABOO-MIDRIFF", + "category": "user_taboo", + "priority": 95, + "confidence": 1.0, + "conditions": {"user_taboos": {"include": ["露脐"]}}, + "effects": { + "avoid": [{"slot": "top", "attrs": {"length": ["短款", "露脐"]}}], + "explanation": "用户不接受露脐，已剔除相关单品。" + } + }, + { + "rule_key": "G-RULES-18-BODY-A-SHOULDER", + "category": "body_shape", + "priority": 90, + "confidence": 0.95, + "conditions": { + "body_shape": "A型", + "features": {"include": ["溜肩"]}, + "goals": {"include": ["修饰溜肩"]} + }, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"shoulder": ["垫肩"], "sleeve": ["泡泡袖"]}} + ], + "avoid": [{"slot": "top", "attrs": {"silhouette": ["修身"]}}], + "explanation": "溜肩需要用垫肩与泡泡袖在肩部建立结构感。" + } + }, + { + "rule_key": "G-RULES-19-TREND-CROP", + "category": "trend", + "priority": 50, + "confidence": 0.7, + "conditions": {"style": "甜酷"}, + "effects": { + "recommend": [{"slot": "top", "attrs": {"length": ["短款"]}}], + "explanation": "短款上衣是当季甜酷风的核心单品。" + } + } + ], + "context": { + "body_shape": "A型", + "features": ["溜肩"], + "goals": ["修饰溜肩"], + "style": "甜酷", + "user_taboos": ["露脐"], + "occasion": "红毯" + }, + "expect": { + "must_recommend": [ + {"slot": "top", "attrs_include": {"shoulder": ["垫肩"]}}, + {"slot": "top", "attrs_include": {"sleeve": ["泡泡袖"]}} + ], + "must_avoid": [ + {"slot": "top", "attrs_include": {"length": ["露脐"]}}, + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}} + ], + "forbidden_recommend": [ + {"slot": "top", "attrs_include": {"length": ["短款"]}}, + {"slot": "shoes", "attrs_include": {"material": ["真皮"]}}, + {"slot": "accessory", "attrs_include": {"material": ["真兽皮"]}} + ], + "hard_blocks_nonempty": true + } + }, + { + "name": "features-superset-still-matches-include", + "rules": [ + { + "rule_key": "G-RULES-20-BODY-A-SHOULDER", + "category": "body_shape", + "priority": 90, + "confidence": 0.95, + "conditions": { + "body_shape": "A型", + "features": {"include": ["溜肩"]}, + "goals": {"include": ["修饰溜肩"]} + }, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"shoulder": ["垫肩"], "sleeve": ["泡泡袖"]}} + ], + "avoid": [{"slot": "top", "attrs": {"silhouette": ["修身"]}}], + "explanation": "溜肩需要用垫肩与泡泡袖在肩部建立结构感。" + } + } + ], + "context": { + "body_shape": "A型", + "features": ["溜肩", "宽胯"], + "goals": ["修饰溜肩"] + }, + "expect": { + "must_recommend": [ + {"slot": "top", "attrs_include": {"shoulder": ["垫肩"]}} + ], + "hard_blocks_nonempty": false + } + }, + { + "name": "goals-partial-miss-blocks-rule", + "rules": [ + { + "rule_key": "G-RULES-21-BODY-A-SHOULDER", + "category": "body_shape", + "priority": 90, + "confidence": 0.95, + "conditions": { + "body_shape": "A型", + "features": {"include": ["溜肩"]}, + "goals": {"include": ["修饰溜肩"]} + }, + "effects": { + "recommend": [ + {"slot": "top", "attrs": {"shoulder": ["垫肩"], "sleeve": ["泡泡袖"]}} + ], + "avoid": [{"slot": "top", "attrs": {"silhouette": ["修身"]}}], + "explanation": "溜肩需要用垫肩与泡泡袖在肩部建立结构感。" + } + } + ], + "context": { + "body_shape": "A型", + "features": ["溜肩"], + "goals": ["显瘦"] + }, + "expect": { + "forbidden_recommend": [ + {"slot": "top", "attrs_include": {}} + ], + "hard_blocks_nonempty": false + } + } +] diff --git a/tests/golden/runner.py b/tests/golden/runner.py new file mode 100644 index 0000000..c97781b --- /dev/null +++ b/tests/golden/runner.py @@ -0,0 +1,533 @@ +"""黄金集回归 runner：JSON 用例加载 → 执行 → 指标计算 → 逐用例明细（T-P2-11）. + +黄金集是"指标退化直接阻断合并"的防线（FR-KB-11 / NFR-MAINT-04）。本模块把 +用例（数据）、阈值（配置）与判定（代码）彻底分开——加用例 = 加 JSON 行， +规模扩充（知识库文档 §7 的四子集 330 例口径，由内容轨 T-P2-16 达成）不需要 +动框架。 + +设计约束： + +1. **不依赖 pytest**：``test_golden.py`` 只是薄壳。将来本地脚本或 admin + 后台跑同一套黄金集，直接 import 本模块即可。 +2. **失败必须可解释**：每条用例的判定携带"期望 vs 实际"的差异摘要， + CI 日志里要能直接看出哪条用例在哪个断言上退化，而不是只有一个通过率。 +3. **种子实体自管理**：检索用例的实体以 ``golden:`` 前缀命名；跑前先删 + （上次运行可能中断在判定之前，保证幂等重跑）、跑后必删（集成库长活， + 留下假实体会破坏其它用例对结果列表的精确断言）。 +4. **参数钉死**：黄金集断言确定性结果，检索的维度/top_k 固定在框架里而 + 不读 env——env 漂移会让同一份用例集在不同机器上口径不一。 +5. **封闭词汇表**：用例/expect 的未知键一律判为用例失败而非静默忽略—— + ``must_recomend`` 这类 typo 若被忽略，意味着断言悄悄失效（与规则 DSL + 同一哲学，FR-KB-02）。 +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession + +from packages.domain.ids import next_id +from packages.retrieval.embedding import DeterministicEmbedder +from packages.retrieval.repo import ChunkInput, hybrid_search, upsert_chunks +from packages.retrieval.rerank import ScoredChunk +from packages.rules import dsl, engine +from packages.shared.db import session_scope +from packages.shared.models import KbChunk, KbEntity + +GOLDEN_DIR = Path(__file__).resolve().parent + +#: 种子实体命名前缀：runner 只清理带此前缀的实体——既不误伤业务数据， +#: 也让"哪些行是黄金集造的"在库内一眼可辨（幂等重跑契约，任务文档 §3.3）。 +SEED_NAME_PREFIX = "golden:" + +#: 检索执行的固定参数（与 Settings 默认值一致但刻意不读 Settings）。 +_VECTOR_DIMENSIONS = 1024 # 与 kb_chunk.embedding 的 Vector(1024) 列宽一致 +_RETRIEVAL_TOP_K = 50 # 与 Settings.retrieval_top_k 默认值一致 + +_RULES_CASE_KEYS = frozenset({"name", "rules", "context", "expect"}) +_RULES_EXPECT_KEYS = frozenset( + {"must_recommend", "must_avoid", "forbidden_recommend", "hard_blocks_nonempty"} +) +_RETRIEVAL_CASE_KEYS = frozenset({"name", "seed_entities", "query", "user_ctx", "expect"}) +_RETRIEVAL_EXPECT_KEYS = frozenset({"must_hit_names", "must_miss_names", "first_hit_in"}) +_USER_CTX_KEYS = frozenset({"body_shape", "occasion", "entity_types"}) +_FIRST_HIT_IN_CHOICES = ("top1", "top3") + + +# ── 结果模型 ──────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True, slots=True) +class CaseResult: + """一条用例的判定结论；problems 是"期望 vs 实际"的差异摘要列表.""" + + name: str + passed: bool + problems: tuple[str, ...] = () + #: 检索子集专用：首个 must_hit 落位是否在 top3（未声明 first_hit_in 为 None）。 + first_hit_in_top3: bool | None = None + + +@dataclass(frozen=True, slots=True) +class SubsetReport: + """一个子集的指标汇总。阈值随报告携带，调用方只需对齐指标与阈值.""" + + subset: str + cases: tuple[CaseResult, ...] + pass_rate_min: float + first_hit_in_top3_min: float | None = None + first_hit_in_top3_total: int = 0 + first_hit_in_top3_passed: int = 0 + + @property + def total(self) -> int: + return len(self.cases) + + @property + def passed(self) -> int: + return sum(1 for case in self.cases if case.passed) + + @property + def pass_rate(self) -> float: + return self.passed / self.total if self.total else 0.0 + + @property + def first_hit_in_top3_rate(self) -> float | None: + """分母 = 声明了 first_hit_in 的用例数（规则子集恒为 None）.""" + if not self.first_hit_in_top3_total: + return None + return self.first_hit_in_top3_passed / self.first_hit_in_top3_total + + def summary(self) -> str: + """CI 日志直读的明细：头部指标行 + 每条失败用例的差异摘要.""" + head = ( + f"[golden:{self.subset}] 通过 {self.passed}/{self.total}，" + f"pass_rate={self.pass_rate:.4f}（阈值 {self.pass_rate_min:.2f}）" + ) + if self.first_hit_in_top3_min is not None: + rate = self.first_hit_in_top3_rate + rate_text = "n/a（无用例声明该指标）" if rate is None else f"{rate:.4f}" + head += ( + f"；first_hit_in_top3={rate_text}" + f"（阈值 {self.first_hit_in_top3_min:.2f}，" + f"{self.first_hit_in_top3_passed}/{self.first_hit_in_top3_total} 例参与）" + ) + lines = [head] + for case in self.cases: + if case.passed: + continue + lines.append(f" [FAIL] {case.name}") + lines.extend(f" - {problem}" for problem in case.problems) + return "\n".join(lines) + + +# ── 配置与用例加载 ────────────────────────────────────────────────────────── + + +@dataclass(frozen=True, slots=True) +class SubsetThresholds: + """thresholds.json 里一个子集的阈值.""" + + pass_rate_min: float + first_hit_in_top3_min: float | None = None + + +def load_thresholds() -> dict[str, SubsetThresholds]: + raw = json.loads((GOLDEN_DIR / "thresholds.json").read_text(encoding="utf-8")) + thresholds: dict[str, SubsetThresholds] = {} + for subset, cfg in raw.items(): + if not isinstance(cfg, dict) or "pass_rate_min" not in cfg: + raise ValueError(f"thresholds.json：子集 {subset} 缺少 pass_rate_min") + thresholds[subset] = SubsetThresholds( + pass_rate_min=float(cfg["pass_rate_min"]), + first_hit_in_top3_min=( + float(cfg["first_hit_in_top3_min"]) if "first_hit_in_top3_min" in cfg else None + ), + ) + return thresholds + + +def _thresholds_for(subset: str) -> SubsetThresholds: + thresholds = load_thresholds() + if subset not in thresholds: + raise ValueError(f"thresholds.json 缺少子集 {subset} 的阈值") + return thresholds[subset] + + +def _load_cases(filename: str) -> list[dict[str, Any]]: + path = GOLDEN_DIR / "cases" / filename + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError(f"{path} 应为 JSON 数组（每个元素一个用例对象）") + if not all(isinstance(case, dict) for case in raw): + raise ValueError(f"{path} 存在非对象的数组元素（用例必须是 JSON 对象）") + return raw + + +def _unknown_keys(actual: Mapping[str, Any], allowed: frozenset[str]) -> list[str]: + return sorted(set(actual) - allowed) + + +def _empty_marker(subset: str) -> tuple[CaseResult, ...]: + """空子集的哨兵失败用例：vacuous green 是回归防线最危险的静默失效.""" + return ( + CaseResult( + f"<{subset}-empty>", + False, + (f"子集 {subset} 没有任何用例：请检查 cases/*.json 是否被清空",), + ), + ) + + +# ── 规则子集（packages.rules DSL + 引擎，纯逻辑无 I/O）───────────────────── + + +def _build_context(raw: Mapping[str, Any]) -> dsl.RuleContext: + """JSON context → RuleContext。集合字段（数组）收成 tuple；未知字段 TypeError.""" + kwargs: dict[str, Any] = {} + for key, value in raw.items(): + if key in ("features", "goals", "user_taboos"): + kwargs[key] = tuple(value) + else: + kwargs[key] = value + return dsl.RuleContext(**kwargs) + + +def _expected_item(item: Any) -> tuple[str, dict[str, tuple[str, ...]]]: + """expect 条目 {slot, attrs_include} → (slot, {维度: 取值元组})；attrs 缺省=整 slot.""" + if not isinstance(item, dict) or not item.get("slot"): + raise ValueError(f"expect 条目缺少 slot：{item!r}") + attrs = item.get("attrs_include") or {} + return str(item["slot"]), {str(dim): tuple(values) for dim, values in attrs.items()} + + +def _covers(actual: dsl.SlotAttrs, slot: str, attrs: dict[str, tuple[str, ...]]) -> bool: + """子集语义（must_*）：同 slot，且期望每个维度的取值都被 actual 包含. + + attrs 为空 = 对整个 slot 的期望（与 DSL"空 attrs 表示对品类推荐/回避"对齐）。 + """ + if actual.slot != slot: + return False + return all( + dim in actual.attrs and set(values) <= set(actual.attrs[dim]) + for dim, values in attrs.items() + ) + + +def _overlaps(actual: dsl.SlotAttrs, slot: str, attrs: dict[str, tuple[str, ...]]) -> bool: + """红线语义（forbidden_*）：同 slot 且任一期望维度与 actual 取值相交即命中.""" + if actual.slot != slot: + return False + return any( + dim in actual.attrs and set(values) & set(actual.attrs[dim]) + for dim, values in attrs.items() + ) + + +def _fmt_slot_attrs(item: dsl.SlotAttrs) -> str: + if not item.attrs: + return item.slot + dims = ", ".join(f"{dim}={list(values)}" for dim, values in item.attrs.items()) + return f"{item.slot}[{dims}]" + + +def _fmt_items(items: Sequence[dsl.SlotAttrs]) -> str: + if not items: + return "（空）" + return "[" + ", ".join(_fmt_slot_attrs(item) for item in items) + "]" + + +def _judge_rules_case(case: Mapping[str, Any]) -> CaseResult: + name = str(case.get("name", "<缺少 name>")) + problems: list[str] = [] + unknown = _unknown_keys(case, _RULES_CASE_KEYS) + if unknown: + problems.append(f"用例含未识别字段 {unknown}（可用：{sorted(_RULES_CASE_KEYS)}）") + if "rules" not in case or "expect" not in case: + problems.append("用例缺少必填字段 rules / expect") + return CaseResult(name, False, tuple(problems)) + expect: Mapping[str, Any] = case["expect"] + if not isinstance(expect, Mapping): + problems.append("expect 应为 JSON 对象") + return CaseResult(name, False, tuple(problems)) + + try: + rules = [dsl.Rule.model_validate(rule) for rule in case["rules"]] + ctx = _build_context(case.get("context") or {}) + must_recommend = [_expected_item(item) for item in expect.get("must_recommend", [])] + must_avoid = [_expected_item(item) for item in expect.get("must_avoid", [])] + forbidden = [_expected_item(item) for item in expect.get("forbidden_recommend", [])] + except Exception as exc: + problems.append(f"用例无法解析：{exc}") + return CaseResult(name, False, tuple(problems)) + + unknown_expect = _unknown_keys(expect, _RULES_EXPECT_KEYS) + if unknown_expect: + problems.append( + f"expect 含未识别字段 {unknown_expect}（可用：{sorted(_RULES_EXPECT_KEYS)}）——" + "静默忽略会让断言悄悄失效" + ) + + verdict = engine.evaluate(rules, ctx) + + for slot, attrs in must_recommend: + if not any(_covers(rec, slot, attrs) for rec in verdict.recommend): + problems.append( + f"must_recommend 未命中：期望 {slot}[{attrs}]，" + f"实际 recommend={_fmt_items(verdict.recommend)}" + ) + for slot, attrs in must_avoid: + if not any(_covers(av, slot, attrs) for av in verdict.avoid): + problems.append( + f"must_avoid 未命中：期望 {slot}[{attrs}]，实际 avoid={_fmt_items(verdict.avoid)}" + ) + for slot, attrs in forbidden: + violation = next((rec for rec in verdict.recommend if _overlaps(rec, slot, attrs)), None) + if violation is not None: + problems.append( + f"forbidden_recommend 违规：{_fmt_slot_attrs(violation)} " + f"与红线 {slot}[{attrs}] 相交" + ) + if "hard_blocks_nonempty" in expect: + want = bool(expect["hard_blocks_nonempty"]) + if want != verdict.blocked: + hit_keys = [block.rule_key for block in verdict.hard_blocks] + problems.append( + f"hard_blocks_nonempty 期望 {want}，实际 {verdict.blocked}" + f"（硬拦截规则：{hit_keys or '无'}）" + ) + return CaseResult(name, not problems, tuple(problems)) + + +def run_rules_subset() -> SubsetReport: + """规则子集：纯逻辑求值，同步执行即可（无 DB / 网络）.""" + cases = _load_cases("rules_cases.json") + results = tuple(_judge_rules_case(case) for case in cases) or _empty_marker("rules") + thresholds = _thresholds_for("rules") + return SubsetReport(subset="rules", cases=results, pass_rate_min=thresholds.pass_rate_min) + + +# ── 检索子集（真实 PG + DeterministicEmbedder + hybrid_search）───────────── + + +@dataclass(frozen=True, slots=True) +class SeedEntity: + """一条检索用例自带的种子实体（跑前清理、跑后删除，见模块 docstring）.""" + + entity_type: str + name: str + attrs: dict[str, Any] + status: str + chunks: tuple[str, ...] + + +def _parse_seeds(raw_seeds: Any) -> list[SeedEntity]: + if not isinstance(raw_seeds, list) or not raw_seeds: + raise ValueError("seed_entities 必须是非空数组") + seeds: list[SeedEntity] = [] + seen: set[str] = set() + for raw in raw_seeds: + if not isinstance(raw, dict): + raise ValueError(f"seed 实体应为 JSON 对象：{raw!r}") + name = str(raw.get("name", "")) + if not name.startswith(SEED_NAME_PREFIX): + raise ValueError(f"种子实体 name 必须以 {SEED_NAME_PREFIX!r} 开头：{name!r}") + if name in seen: + raise ValueError(f"种子实体 name 在用例内重复：{name!r}") + seen.add(name) + chunks = tuple(str(text) for text in raw.get("chunks", [])) + if not chunks or not all(text.strip() for text in chunks): + raise ValueError(f"种子实体 {name!r} 至少要有一个非空 chunk 文本") + seeds.append( + SeedEntity( + entity_type=str(raw.get("entity_type", "item")), + name=name, + attrs=dict(raw.get("attrs") or {}), + status=str(raw.get("status", "published")), + chunks=chunks, + ) + ) + return seeds + + +def _build_user_ctx(raw: Mapping[str, Any]) -> dict[str, Any]: + unknown = _unknown_keys(raw, _USER_CTX_KEYS) + if unknown: + raise ValueError(f"user_ctx 含未识别字段 {unknown}（可用：{sorted(_USER_CTX_KEYS)}）") + ctx: dict[str, Any] = dict(raw) + if ctx.get("entity_types"): + ctx["entity_types"] = tuple(ctx["entity_types"]) + else: + ctx.pop("entity_types", None) # 空数组视为不限类型（与 hybrid_search 语义一致） + return ctx + + +async def _delete_seeded(session: AsyncSession, seeds: Sequence[SeedEntity]) -> int: + """按 golden: 前缀 name 删除本用例 seed 过的实体及其全部向量（幂等）.""" + names = [seed.name for seed in seeds] + entity_ids = list( + (await session.execute(select(KbEntity.entity_id).where(KbEntity.name.in_(names)))) + .scalars() + .all() + ) + if not entity_ids: + return 0 + await session.execute(delete(KbChunk).where(KbChunk.entity_id.in_(entity_ids))) + await session.execute(delete(KbEntity).where(KbEntity.entity_id.in_(entity_ids))) + return len(entity_ids) + + +def _judge_retrieval_expect( + expect: Mapping[str, Any], + results: Sequence[ScoredChunk], + name_by_entity: Mapping[int, str], +) -> tuple[list[str], bool | None]: + """检索用例判定。返回 (差异摘要列表, 首命中是否在 top3 或 None).""" + problems: list[str] = [] + unknown = _unknown_keys(expect, _RETRIEVAL_EXPECT_KEYS) + if unknown: + problems.append(f"expect 含未识别字段 {unknown}（可用：{sorted(_RETRIEVAL_EXPECT_KEYS)}）") + + position_by_name: dict[str, int] = {} + ranked_names: list[str] = [] + for index, chunk in enumerate(results, start=1): + name = name_by_entity.get(chunk.entity_id, f"#实体{chunk.entity_id}") + ranked_names.append(name) + if name not in position_by_name: + position_by_name[name] = index + + must_hit = [str(item) for item in expect.get("must_hit_names", [])] + must_miss = [str(item) for item in expect.get("must_miss_names", [])] + for name in must_hit: + if name not in position_by_name: + problems.append(f"must_hit 未出现：{name!r} 不在结果中（结果实体序列：{ranked_names}）") + for name in must_miss: + position = position_by_name.get(name) + if position is not None: + problems.append( + f"must_miss 红线违规：{name!r} 出现在第 {position} 位——" + "硬过滤零漏（draft/禁忌/场景外实体绝不出现）被破坏" + ) + + first_hit_in_top3: bool | None = None + if "first_hit_in" in expect: + mode = expect["first_hit_in"] + if mode not in _FIRST_HIT_IN_CHOICES: + problems.append(f"first_hit_in 只能是 {_FIRST_HIT_IN_CHOICES}，得到 {mode!r}") + else: + positions = [position_by_name[name] for name in must_hit if name in position_by_name] + if not positions: + problems.append("无法评价 first_hit_in：没有任何 must_hit 实体出现在结果中") + first_hit_in_top3 = False + else: + first = min(positions) + limit = 1 if mode == "top1" else 3 + if first > limit: + problems.append( + f"first_hit_in={mode} 未满足：首个命中在第 {first} 位，" + f"前 {limit} 位序列：{ranked_names[:limit]}" + ) + first_hit_in_top3 = first <= 3 + return problems, first_hit_in_top3 + + +async def run_retrieval_case( + case: Mapping[str, Any], embedder: DeterministicEmbedder +) -> CaseResult: + name = str(case.get("name", "<缺少 name>")) + problems: list[str] = [] + unknown = _unknown_keys(case, _RETRIEVAL_CASE_KEYS) + if unknown: + problems.append(f"用例含未识别字段 {unknown}（可用：{sorted(_RETRIEVAL_CASE_KEYS)}）") + try: + for required in ("seed_entities", "query", "expect"): + if required not in case: + raise ValueError(f"用例缺少必填字段 {required}") + seeds = _parse_seeds(case["seed_entities"]) + query = str(case["query"]) + user_ctx = _build_user_ctx(case.get("user_ctx") or {}) + expect = case["expect"] + if not isinstance(expect, Mapping): + raise ValueError("expect 应为 JSON 对象") + except Exception as exc: + problems.append(f"用例无法解析：{exc}") + return CaseResult(name, False, tuple(problems)) + + first_hit_in_top3: bool | None = None + try: + # 1) 跑前先清自己 seed 的行：上一次运行可能中断在判定之前（幂等重跑）。 + async with session_scope() as session: + await _delete_seeded(session, seeds) + # 2) 重建种子实体并写入向量（DeterministicEmbedder：同文本同向量）。 + entity_ids: dict[str, int] = {} + async with session_scope() as session: + for seed in seeds: + entity_id = next_id() + session.add( + KbEntity( + entity_id=entity_id, + entity_type=seed.entity_type, + name=seed.name, + attrs=seed.attrs, + status=seed.status, + ) + ) + # 先落实体行再写 chunk：kb_chunk 有 entity 外键，而两个模型之间 + # 没有 relationship()，session 又是 autoflush=False——不显式 + # flush 的话 UOW 不保证实体先于 chunk 插入（实测会 FK 违约）。 + await session.flush() + await upsert_chunks( + session, + entity_id, + [ChunkInput(text=text) for text in seed.chunks], + embedder, + ) + entity_ids[seed.name] = entity_id + # 3) 混合检索：硬过滤在 SQL WHERE 里，"零漏"是结构保证而非概率保证。 + query_vec = (await embedder.embed([query]))[0] + async with session_scope() as session: + results = await hybrid_search( + session, + query_vec, + query, + entity_types=user_ctx.get("entity_types"), + occasion=user_ctx.get("occasion"), + body_shape=user_ctx.get("body_shape"), + top_k=_RETRIEVAL_TOP_K, + ) + name_by_entity = {entity_id: seed_name for seed_name, entity_id in entity_ids.items()} + case_problems, first_hit_in_top3 = _judge_retrieval_expect(expect, results, name_by_entity) + problems.extend(case_problems) + finally: + # 4) 跑后必删：集成库长活，留下的假实体会破坏其它用例对结果列表的精确断言。 + try: + async with session_scope() as session: + await _delete_seeded(session, seeds) + except Exception as exc: + problems.append(f"种子清理失败（会污染共享集成库）：{exc}") + return CaseResult(name, not problems, tuple(problems), first_hit_in_top3) + + +async def run_retrieval_subset() -> SubsetReport: + """检索子集：真实 PG + pgvector 上执行（种子自管理，幂等可重跑）.""" + cases = _load_cases("retrieval_cases.json") + embedder = DeterministicEmbedder(dimensions=_VECTOR_DIMENSIONS) + results = tuple([await run_retrieval_case(case, embedder) for case in cases]) + if not results: + results = _empty_marker("retrieval") + thresholds = _thresholds_for("retrieval") + declaring = [case for case in results if case.first_hit_in_top3 is not None] + return SubsetReport( + subset="retrieval", + cases=results, + pass_rate_min=thresholds.pass_rate_min, + first_hit_in_top3_min=thresholds.first_hit_in_top3_min, + first_hit_in_top3_total=len(declaring), + first_hit_in_top3_passed=sum(1 for case in declaring if case.first_hit_in_top3), + ) diff --git a/tests/golden/test_golden.py b/tests/golden/test_golden.py new file mode 100644 index 0000000..27a0979 --- /dev/null +++ b/tests/golden/test_golden.py @@ -0,0 +1,52 @@ +"""黄金集回归 pytest 入口（marker=golden；无 MEIDA_RUN_INTEGRATION 时整体 skip）. + +薄壳（任务文档 §3.4）：加载/执行/指标/明细全部在 ``runner.py``——本文件只把 +``SubsetReport`` 的指标对齐 thresholds，退化时把逐用例明细完整抛给 CI 日志。 + +Run: MEIDA_RUN_INTEGRATION=1 .venv/bin/pytest -m golden --no-cov（即 make golden） +""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator + +import pytest + +from packages.shared.db import dispose_engine +from tests.golden import runner + +pytestmark = [ + pytest.mark.golden, + pytest.mark.skipif( + not os.getenv("MEIDA_RUN_INTEGRATION"), + reason="需要真实 PostgreSQL/pgvector：MEIDA_RUN_INTEGRATION=1（make golden）", + ), +] + + +@pytest.fixture(autouse=True) +async def _hermetic_engine() -> AsyncIterator[None]: + """每个用例前后重建全局 engine：asyncpg 连接绑定创建它的事件循环， + pytest-asyncio 给每个用例一个新 loop，复用池里旧连接会报 + "attached to a different loop"（与 tests/integration 同一处理）。""" + await dispose_engine() + yield + await dispose_engine() + + +def test_rules_subset_meets_thresholds() -> None: + """规则子集阈值 = 1.0：核心结论由规则产出，一条不许退（README 详述原因）.""" + report = runner.run_rules_subset() + assert report.pass_rate >= report.pass_rate_min, f"规则黄金集指标退化：\n{report.summary()}" + + +async def test_retrieval_subset_meets_thresholds() -> None: + """检索子集 0.8 起步：pass_rate 与 first_hit_in_top3 双指标对齐阈值.""" + report = await runner.run_retrieval_subset() + detail = report.summary() + assert report.pass_rate >= report.pass_rate_min, f"检索黄金集指标退化：\n{detail}" + rate = report.first_hit_in_top3_rate + assert rate is not None and rate >= report.first_hit_in_top3_min, ( + f"检索黄金集 first_hit_in_top3 指标退化（或无用例声明该指标）：\n{detail}" + ) diff --git a/tests/golden/thresholds.json b/tests/golden/thresholds.json new file mode 100644 index 0000000..ed3b57b --- /dev/null +++ b/tests/golden/thresholds.json @@ -0,0 +1,9 @@ +{ + "rules": { + "pass_rate_min": 1.0 + }, + "retrieval": { + "pass_rate_min": 0.8, + "first_hit_in_top3_min": 0.8 + } +}