feat(retrieval): hybrid search over pgvector with hard filters and gap tracking
DeterministicEmbedder(可换 BGE-M3 的接口位)+ kb_chunk 双索引写入 + 硬过滤→向量→关键词→RRF 重排管线 + kb_gap 缺口埋点(T-P2-03/04/06,TD-08)。 实现: - embedding.py:EmbeddingProvider Protocol + DeterministicEmbedder (sha256 种子→归一化 1024 维假向量,同文本同向量/异文本近似正交) + build_embedding_provider 工厂(未知模型名抛 CONFIG_ERROR)。 - repo.py:upsert_chunks(整实体先删后插,逐行带 embedding_model/version, 可重跑判据 T-P2-03);hybrid_search(§8.3 CTE:硬过滤 published/ entity_types/occasion/avoid_body_shapes → <=> 余弦 topN → ts_rank topN → RRF);record_gap(咨询锁串行化同 query+filter_ctx 的 hit_count 递增)。 - rerank.py:RRF/权重/新鲜度纯数学(独立于 SQL 以便单测逐项断言)。 - service.py:RetrievalService.search 编排 embed→检索→(空结果)gap 埋点, SearchOutcome.degraded 标记降级。 测试:单测 +30(test_embedding 确定性/维度/归一化/工厂拒绝、test_rerank 公式逐项断言、test_retrieval_service 编排打桩);集成 +5 (test_hybrid_retrieval:禁忌实体同文最近向量仍零出现、draft 零出现、 关键词路径、换模型整组替换、kb_gap hit_count=2、partial 索引 EXPLAIN 验证)。 全绿:ruff / mypy(retrieval+shared+domain) / lint-imports / 268 单测 / 21 集成。 偏差说明: 1) 新增 rerank.py(任务文档只点名 embedding/repo/service,但重排数学必须 不连库可单测,独立模块是最低成本实现,仍属 packages/retrieval/**)。 2) upsert_chunks 的 upsert 粒度是实体(先删后插):kb_chunk 无业务自然键, 行级 ON CONFLICT 不可行;换模型名=新版本整组替换,不留旧版本残向量。 3) 禁忌过滤用 COALESCE(attrs->avoid_body_shapes, '[]')兜底:jsonb NULL 的 ? 运算结果是 NULL,直接 NOT 会把未声明禁忌的好实体一并排除。 4) 关键词路过滤用 tsv @@ tsquery 而非 ts_rank>0:ts_rank 对不匹配文本返回 ~1e-20 正 epsilon,阈值过滤会把无关 chunk 全放进关键词路。 5) RRF 分量按 1/(k+1) 归一到 (0,1],使 0.45/0.2/0.25/0.1 权重量纲可比 (原始倒数 0.45×1/61 与规则项 0.25×1 完全不可比)。
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
"""DeterministicEmbedder 与工厂:纯数学,不连库不联网(T-P2-03 接口位)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.retrieval.embedding import (
|
||||
DETERMINISTIC_MODEL_NAME,
|
||||
DeterministicEmbedder,
|
||||
build_embedding_provider,
|
||||
)
|
||||
from packages.shared.config import get_settings
|
||||
from packages.shared.errors import AppError, ErrorCode
|
||||
|
||||
|
||||
def _dot(a: list[float], b: list[float]) -> float:
|
||||
return sum(x * y for x, y in zip(a, b, strict=True))
|
||||
|
||||
|
||||
async def test_same_text_embeds_to_same_vector() -> None:
|
||||
embedder = DeterministicEmbedder(dimensions=1024)
|
||||
first, second = await embedder.embed(["白衬衫怎么搭配", "白衬衫怎么搭配"])
|
||||
assert first == second
|
||||
|
||||
|
||||
async def test_dimensions_match_declaration() -> None:
|
||||
vec = (await DeterministicEmbedder(dimensions=1024).embed(["hello"]))[0]
|
||||
assert len(vec) == 1024
|
||||
small = (await DeterministicEmbedder(dimensions=8).embed(["hello"]))[0]
|
||||
assert len(small) == 8
|
||||
|
||||
|
||||
async def test_vectors_are_l2_normalized() -> None:
|
||||
for text in ("hello", "中文知识片段", ""):
|
||||
vec = (await DeterministicEmbedder(dimensions=1024).embed([text]))[0]
|
||||
assert math.isclose(_dot(vec, vec), 1.0, rel_tol=1e-6, abs_tol=1e-9)
|
||||
|
||||
|
||||
async def test_different_texts_are_near_orthogonal() -> None:
|
||||
"""不同文本近似正交(|dot| ≪ 1):向量召回管线在无真模型时仍可测."""
|
||||
|
||||
left, right = await DeterministicEmbedder(dimensions=1024).embed(
|
||||
["interview outfit advice", "winter skincare routine"]
|
||||
)
|
||||
assert left != right
|
||||
assert abs(_dot(left, right)) < 0.1
|
||||
|
||||
|
||||
async def test_batch_preserves_order_and_length() -> None:
|
||||
texts = ["alpha", "beta", "gamma"]
|
||||
vectors = await DeterministicEmbedder(dimensions=16).embed(texts)
|
||||
assert len(vectors) == len(texts)
|
||||
singles = [(await DeterministicEmbedder(dimensions=16).embed([t]))[0] for t in texts]
|
||||
assert vectors == singles
|
||||
|
||||
|
||||
def test_metadata_is_stable() -> None:
|
||||
embedder = DeterministicEmbedder(dimensions=8)
|
||||
assert embedder.model_name == DETERMINISTIC_MODEL_NAME
|
||||
assert embedder.version
|
||||
assert embedder.dimensions == 8
|
||||
|
||||
|
||||
def test_invalid_dimensions_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="dimensions"):
|
||||
DeterministicEmbedder(dimensions=0)
|
||||
|
||||
|
||||
def test_factory_builds_deterministic_embedder(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# 带 alias 的 Settings 字段按字段名构造会被静默忽略 —— 统一走
|
||||
# setenv + cache_clear(上批经验,见任务文档 §2)。
|
||||
monkeypatch.setenv("KB_EMBEDDING_MODEL", "deterministic-v1")
|
||||
monkeypatch.setenv("KB_VECTOR_DIMENSIONS", "1024")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
settings = get_settings()
|
||||
provider = build_embedding_provider(settings)
|
||||
assert isinstance(provider, DeterministicEmbedder)
|
||||
assert provider.model_name == "deterministic-v1"
|
||||
assert provider.dimensions == 1024
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_factory_rejects_unknown_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("KB_EMBEDDING_MODEL", "bge-m3-onnx")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
settings = get_settings()
|
||||
assert settings.kb_embedding_model == "bge-m3-onnx"
|
||||
with pytest.raises(AppError) as excinfo:
|
||||
build_embedding_provider(settings)
|
||||
assert excinfo.value.code == ErrorCode.CONFIG_ERROR
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
get_settings.cache_clear()
|
||||
Reference in New Issue
Block a user