feat(gateway): 全局串行化(并发=1) + 记录入参/出参/耗时日志

并发模型从「每worker并发1 + 多worker并行」改为全局串行:同一时间
只处理1个请求,其余排队;多 worker 仅作热备(主 worker 坏了才用备机)。
接口4(face/features) 走豆包、不占 GPU,不纳入串行。

- pool: asyncio.Semaphore(max_global_concurrency=1) + acquire/release_global_slot
  (依赖单进程 uvicorn 部署,已在注释中标注)
- forward: proxy_request 最外层 acquire 全局槽、try/finally 全路径释放;
  入参 multipart 解析挂 request.state;原重试/故障转移逻辑抽到 _dispatch_with_retries
- reqlog(新): 标量入参保留;图片(file/base64)存盘转URL,绝不内嵌base64;
  出参递归摘要截断(landmarks/长串/大数组)
- logging_middleware: RequestLogEntry 加 request_params/response_data,
  jsonl 全量记录;_load_from_logfile 同步映射防重启丢字段;get_stats recent 暴露
- /gateway-health 暴露 global_max/global_busy/global_waiting
- config: dispatch 新增 max_global_concurrency / max_queue_wait_seconds
- tests: test_reqlog + test_gateway_serialization(9 用例)

顺带提交此前未提交的网关统计(daily stats 接口)与耗时看板(api_timing_dashboard.html)。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ubuntu
2026-07-23 22:56:48 +08:00
co-authored by Claude
parent 98b9108837
commit bb9f55e93c
10 changed files with 1037 additions and 7 deletions
+68 -2
View File
@@ -50,6 +50,14 @@ _pool_condition: Optional[asyncio.Condition] = None
_health_task: Optional[asyncio.Task] = None
_shutdown_event: Optional[asyncio.Event] = None
# 全局串行槽:同一时间最多 max_global_concurrency 个请求进入 worker 派发,其余排队。
# ⚠️ 依赖单进程 uvicorn 部署(hair-gateway.service 无 --workers)。
# 若改多 worker / gunicorn,进程内信号量会失效,需换成跨进程锁(文件锁/Redis)。
_global_sem: Optional[asyncio.Semaphore] = None
_global_max: int = 1
_global_busy: int = 0
_global_waiting: int = 0
# ---------------------------------------------------------------------------
# 健康检查后台任务
@@ -162,6 +170,9 @@ async def init_pool(cfg: dict) -> None:
_pool_condition = asyncio.Condition()
_shutdown_event = asyncio.Event()
# 全局串行槽(独立于 worker busy 标志)
init_global_slot(cfg)
# 立即做一轮健康检查以快速上线
hc_cfg = cfg["health_check"]
token = cfg["shared_password"]
@@ -206,6 +217,56 @@ async def shutdown_pool() -> None:
logger.info("Worker 池已关闭")
# ---------------------------------------------------------------------------
# 全局串行槽
# ---------------------------------------------------------------------------
def init_global_slot(cfg: dict) -> None:
"""初始化全局串行信号量。
max_global_concurrency=1 → 同一时间只有一个请求进入 worker 派发,其余在信号量上排队。
依赖单进程 uvicorn 部署;多进程需换跨进程锁。
"""
global _global_sem, _global_max, _global_busy, _global_waiting
n = int(cfg.get("dispatch", {}).get("max_global_concurrency", 1))
if n < 1:
n = 1
_global_sem = asyncio.Semaphore(n)
_global_max = n
_global_busy = 0
_global_waiting = 0
logger.info("全局并发槽初始化: max=%d(单进程生效)", n)
async def acquire_global_slot(cfg: dict) -> bool:
"""获取一个全局处理槽。True=获得;False=排队超时。
超时由 dispatch.max_queue_wait_seconds 控制(需 < nginx proxy_read_timeout 600s)。
"""
global _global_busy, _global_waiting
if _global_sem is None:
return True # 未初始化,不限流
timeout = float(cfg.get("dispatch", {}).get("max_queue_wait_seconds", 590))
_global_waiting += 1
try:
await asyncio.wait_for(_global_sem.acquire(), timeout=timeout)
_global_busy += 1
return True
except asyncio.TimeoutError:
logger.warning("全局排队超时(%.1fs),拒绝请求 | waiting=%d", timeout, _global_waiting)
return False
finally:
_global_waiting -= 1
def release_global_slot() -> None:
"""释放全局处理槽。"""
global _global_busy
if _global_sem is not None:
_global_busy -= 1
_global_sem.release()
# ---------------------------------------------------------------------------
# 派发
# ---------------------------------------------------------------------------
@@ -270,9 +331,14 @@ async def release_worker(w: WorkerState) -> None:
def get_pool_status() -> dict:
"""返回当前池状态(供 /gateway-health 使用)。"""
global_info = {
"global_max": _global_max,
"global_busy": _global_busy,
"global_waiting": _global_waiting,
}
if not _workers:
return {"total": 0, "healthy": 0, "busy": 0}
return {"total": 0, "healthy": 0, "busy": 0, **global_info}
total = len(_workers)
healthy = sum(1 for w in _workers.values() if w.online)
busy = sum(1 for w in _workers.values() if w.busy)
return {"total": total, "healthy": healthy, "busy": busy}
return {"total": total, "healthy": healthy, "busy": busy, **global_info}