xsl
9c1802130d
第三批（T12 后台/T13 黄金集/T14 检索缓存）协调提交： - kb：KbAssetService（登记/审核，tryon_ready 只能随 approved 打开）+ Asset 模型与 repo - retrieval：list_gaps 缺口看板查询 + invalidate_cache() 钩子（T14 背后实现真缓存） - 配置：ADMIN_TOKEN（空=后台 404）+ 检索缓存三参数 - pytest golden marker（默认跳过，CI 强制跑）+ Makefile golden + CI 防线 5c - 测试隔离修复：test_auth_sms 加 autouse 清 sms:* 键夹具（频控计数跨运行 累积会毒化后续运行——随机 IP 空间仅 254 且部分用例固定 device_id）
143 lines
5.4 KiB
Python
143 lines
5.4 KiB
Python
"""检索编排：embed → 混合检索 →（无结果时）缺口埋点（架构 §8.3）.
|
||
|
||
只做编排不做策略：模型、权重、top_k 全部来自 Settings；``rule_boost`` 的
|
||
生产者（规则引擎联动）属后续任务，这里只认接口。检索是内部能力，
|
||
没有 HTTP 路由 —— 被 agent / report 的 service 调用。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Mapping
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from packages.retrieval import repo
|
||
from packages.retrieval.embedding import EmbeddingProvider, build_embedding_provider
|
||
from packages.retrieval.rerank import DEFAULT_RERANK_WEIGHTS, ScoredChunk
|
||
from packages.shared.config import Settings
|
||
from packages.shared.logging import get_logger
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class UserContext:
|
||
"""调用方给的检索上下文（用户画像/场景），驱动硬过滤与规则加权。
|
||
|
||
``body_shape`` 是禁忌过滤的输入：命中 ``avoid_body_shapes`` 的实体
|
||
绝不出现（FR-KB-06），而不是排得靠后。
|
||
"""
|
||
|
||
body_shape: str | None = None
|
||
occasion: str | None = None
|
||
entity_types: tuple[str, ...] = ()
|
||
rule_boost: Mapping[int, float] = field(default_factory=dict)
|
||
|
||
def filter_ctx(self) -> dict[str, Any]:
|
||
"""落进 kb_gap 的过滤条件快照：缺口是在什么条件下出现的。"""
|
||
return {
|
||
"body_shape": self.body_shape,
|
||
"occasion": self.occasion,
|
||
"entity_types": list(self.entity_types),
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class SearchOutcome:
|
||
"""一次检索的结果。``degraded`` 表示无结果并已记入 kb_gap（T-P2-06）。"""
|
||
|
||
results: list[ScoredChunk]
|
||
degraded: bool
|
||
|
||
|
||
class RetrievalService:
|
||
"""混合检索的编排入口（嵌入 + 检索 + 缺口埋点）."""
|
||
|
||
def __init__(
|
||
self,
|
||
session: AsyncSession,
|
||
*,
|
||
embedding: EmbeddingProvider,
|
||
weights: Mapping[str, float] | None = None,
|
||
top_k: int = 50,
|
||
) -> None:
|
||
self._session = session
|
||
self._embedding = embedding
|
||
self._weights: dict[str, float] = (
|
||
dict(weights) if weights is not None else dict(DEFAULT_RERANK_WEIGHTS)
|
||
)
|
||
self._top_k = top_k
|
||
|
||
@classmethod
|
||
def from_settings(cls, session: AsyncSession, settings: Settings) -> RetrievalService:
|
||
"""从 Settings 装配（模型、权重、top_k 一处配置，换模型不发版）."""
|
||
return cls(
|
||
session,
|
||
embedding=build_embedding_provider(settings),
|
||
weights=settings.retrieval_rerank_weights,
|
||
top_k=settings.retrieval_top_k,
|
||
)
|
||
|
||
async def search(
|
||
self, query_text: str, *, user_ctx: UserContext | None = None
|
||
) -> SearchOutcome:
|
||
ctx = user_ctx or UserContext()
|
||
# embed 的输入就是 query 原文（DeterministicEmbedder/未来的 BGE-M3 都
|
||
# 不需要拼提示词），保持「同 query 同向量」，缓存与复跑才有意义。
|
||
query_vec = (await self._embedding.embed([query_text]))[0]
|
||
results = await repo.hybrid_search(
|
||
self._session,
|
||
query_vec,
|
||
query_text,
|
||
entity_types=ctx.entity_types or None,
|
||
occasion=ctx.occasion,
|
||
body_shape=ctx.body_shape,
|
||
rule_boost=ctx.rule_boost,
|
||
weights=self._weights,
|
||
top_k=self._top_k,
|
||
)
|
||
if results:
|
||
return SearchOutcome(results=results, degraded=False)
|
||
|
||
# 无结果也是信号：落 kb_gap 驱动内容生产（T-P2-06）。埋点失败不该
|
||
# 让调用方拿到异常 —— 检索空结果本身是合法结局，这里吞掉埋点异常
|
||
# 只记日志。query 原文进表不进日志（日志里只留长度与过滤条件）。
|
||
try:
|
||
await repo.record_gap(self._session, query_text, ctx.filter_ctx())
|
||
await self._session.commit()
|
||
except Exception:
|
||
await self._session.rollback()
|
||
logger.error("retrieval_gap_record_failed", query_len=len(query_text))
|
||
logger.info("retrieval_gap", query_len=len(query_text), **ctx.filter_ctx())
|
||
return SearchOutcome(results=[], degraded=True)
|
||
|
||
|
||
class GapSummary:
|
||
"""kb_gap 的一行摘要（缺口看板的数据形状）。V1 用普通类：字段还会随
|
||
看板需求演进，先不冻结成 dataclass。"""
|
||
|
||
def __init__(self, row: Any) -> None:
|
||
self.gap_id = row.gap_id
|
||
self.query_text = row.query_text
|
||
self.filter_ctx = row.filter_ctx
|
||
self.hit_count = row.hit_count
|
||
self.first_seen_at = row.first_seen_at
|
||
self.last_seen_at = row.last_seen_at
|
||
|
||
|
||
async def list_gaps(session: AsyncSession, *, limit: int = 50, offset: int = 0) -> list[GapSummary]:
|
||
"""缺口看板：按 hit_count 降序（最常被问到的缺口排最前，T-P2-06）。"""
|
||
rows = await repo.list_gaps(session, limit=limit, offset=offset)
|
||
return [GapSummary(r) for r in rows]
|
||
|
||
|
||
def invalidate_cache() -> None:
|
||
"""发布/归档知识后调用：让检索结果缓存整体失效（T-P2-10）。
|
||
|
||
第三批 T14 会在背后实现真正的进程内缓存；本桩保证调用方（运营后台的
|
||
发布编排）现在就能接上这个钩子，T14 合并后无需再改调用点。
|
||
"""
|
||
repo.INVALIDATION_VERSION += 1
|