feat(gateway): 全局串行化(并发=1) + 记录入参/出参/耗时日志
并发模型从「每worker并发1 + 多worker并行」改为全局串行：同一时间 只处理1个请求，其余排队；多 worker 仅作热备（主 worker 坏了才用备机）。 接口4(face/features) 走豆包、不占 GPU，不纳入串行。 - pool: asyncio.Semaphore(max_global_concurrency=1) + acquire/release_global_slot （依赖单进程 uvicorn 部署，已在注释中标注） - forward: proxy_request 最外层 acquire 全局槽、try/finally 全路径释放； 入参 multipart 解析挂 request.state；原重试/故障转移逻辑抽到 _dispatch_with_retries - reqlog(新): 标量入参保留；图片(file/base64)存盘转URL，绝不内嵌base64； 出参递归摘要截断(landmarks/长串/大数组) - logging_middleware: RequestLogEntry 加 request_params/response_data， jsonl 全量记录；_load_from_logfile 同步映射防重启丢字段；get_stats recent 暴露 - /gateway-health 暴露 global_max/global_busy/global_waiting - config: dispatch 新增 max_global_concurrency / max_queue_wait_seconds - tests: test_reqlog + test_gateway_serialization（9 用例） 顺带提交此前未提交的网关统计(daily stats 接口)与耗时看板(api_timing_dashboard.html)。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ubuntu
@@ -14,6 +14,8 @@ from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from gateway.reqlog import summarize_for_log
|
||||
|
||||
logger = logging.getLogger("gateway.logging_middleware")
|
||||
|
||||
|
||||
@@ -34,6 +36,8 @@ class RequestLogEntry:
|
||||
worker: str = "" # 处理请求的 worker URL（空串表示网关本地处理）
|
||||
response_code: Optional[int] = None # 响应 JSON 中的 code 字段
|
||||
request_id: Optional[str] = None # 响应 JSON 中的 request_id
|
||||
request_params: Optional[dict] = None # 入参（图片用 URL 引用，无 base64）
|
||||
response_data: Optional[Any] = None # 出参摘要（递归截断）
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -195,6 +199,8 @@ def _load_from_logfile(filepath: str, max_entries: int) -> int:
|
||||
worker=data.get("worker", ""),
|
||||
response_code=data.get("response_code"),
|
||||
request_id=data.get("request_id"),
|
||||
request_params=data.get("request_params"),
|
||||
response_data=data.get("response_data"),
|
||||
)
|
||||
_buffer.append(entry)
|
||||
count += 1
|
||||
@@ -248,6 +254,7 @@ async def request_logging_middleware(request, call_next):
|
||||
# 提取响应 body 并解析业务字段（仅 JSON 响应）
|
||||
response_code = None
|
||||
request_id = None
|
||||
response_data = None
|
||||
content_type = response.headers.get("content-type", "")
|
||||
|
||||
if "application/json" in content_type or "application/json" in (response.media_type or ""):
|
||||
@@ -261,6 +268,7 @@ async def request_logging_middleware(request, call_next):
|
||||
data = json.loads(body)
|
||||
response_code = data.get("code")
|
||||
request_id = data.get("request_id")
|
||||
response_data = summarize_for_log(data)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
pass
|
||||
|
||||
@@ -274,6 +282,9 @@ async def request_logging_middleware(request, call_next):
|
||||
media_type=response.media_type,
|
||||
)
|
||||
|
||||
# 入参（由 forward.py / 接口4 handler 挂到 request.state）
|
||||
request_params = getattr(request.state, "log_request_params", None)
|
||||
|
||||
# 记录
|
||||
now = datetime.datetime.utcnow()
|
||||
entry = RequestLogEntry(
|
||||
@@ -287,6 +298,8 @@ async def request_logging_middleware(request, call_next):
|
||||
worker=worker_url,
|
||||
response_code=response_code,
|
||||
request_id=request_id,
|
||||
request_params=request_params,
|
||||
response_data=response_data,
|
||||
)
|
||||
|
||||
_buffer.append(entry)
|
||||
@@ -366,6 +379,8 @@ def get_stats() -> Dict[str, Any]:
|
||||
"worker": e.worker,
|
||||
"response_code": e.response_code,
|
||||
"request_id": e.request_id,
|
||||
"request_params": e.request_params,
|
||||
"response_data": e.response_data,
|
||||
}
|
||||
for e in recent_100
|
||||
]
|
||||
@@ -404,3 +419,250 @@ def get_stats() -> Dict[str, Any]:
|
||||
"recent": recent,
|
||||
"last_updated": datetime.datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 按天统计（读取磁盘日志全量，支持任意历史日期 + 单次换发型耗时拆解）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 接口路径 → 编号/名称（对齐 gateway/app.py 里各路由 docstring 中的「接口N」编号）
|
||||
PATH_INTERFACE_MAP: Dict[str, Dict[str, Any]] = {
|
||||
"/api/v1/face/measure": {"num": 1, "name": "四庭七眼测量标注"},
|
||||
"/api/v1/hair/grow": {"num": 2, "name": "C端生发"},
|
||||
"/api/v1/hair/grow-b": {"num": 3, "name": "B端生发（医生/操作端）"},
|
||||
"/api/v1/face/features": {"num": 4, "name": "用户特征分析"},
|
||||
"/api/v1/hairline/generate": {"num": 5, "name": "发际线PNG生成"},
|
||||
"/api/v1/face/measure-v2": {"num": 6, "name": "四庭七眼测量标注 v2"},
|
||||
"/api/v1/hair/grow-v2": {"num": 7, "name": "C端生发 v2"},
|
||||
"/api/v1/head/mask": {"num": 9, "name": "头发遮罩生成"},
|
||||
"/api/v1/head/band": {"num": 10, "name": "头部外缘膨胀带遮罩"},
|
||||
}
|
||||
|
||||
# 会按「勾选发型数量」串行多次调用 ComfyUI 的接口（耗时 ≈ 固定开销 + 单次换发型耗时 × 发型数）
|
||||
MULTI_STYLE_PATHS = {
|
||||
"/api/v1/hair/grow",
|
||||
"/api/v1/hair/grow-v2",
|
||||
"/api/v1/hairline/generate",
|
||||
}
|
||||
|
||||
|
||||
def _log_file_path() -> Path:
|
||||
"""当前日志文件路径（若尚未 init_logging，用默认路径兜底）。"""
|
||||
if _writer is not None:
|
||||
return _writer.filepath
|
||||
return Path(__file__).resolve().parent.parent / "gateway" / "request_log.jsonl"
|
||||
|
||||
|
||||
def _read_all_log_entries() -> List[dict]:
|
||||
"""读取磁盘上完整的日志历史（含轮转备份 .jsonl.1，按时间顺序：备份在前）。"""
|
||||
log_path = _log_file_path()
|
||||
paths = []
|
||||
backup = log_path.with_suffix(".jsonl.1")
|
||||
if backup.exists():
|
||||
paths.append(backup)
|
||||
if log_path.exists():
|
||||
paths.append(log_path)
|
||||
|
||||
entries: List[dict] = []
|
||||
for p in paths:
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entries.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except Exception:
|
||||
logger.warning("读取日志文件失败: %s", p, exc_info=True)
|
||||
return entries
|
||||
|
||||
|
||||
def _parse_ts(ts: str) -> Optional[datetime.datetime]:
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"):
|
||||
try:
|
||||
return datetime.datetime.strptime(ts, fmt).replace(tzinfo=datetime.timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _percentile(values: List[float], p: float) -> float:
|
||||
"""线性插值百分位数，values 需已排序。"""
|
||||
if not values:
|
||||
return 0.0
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
k = (len(values) - 1) * p
|
||||
f = int(k)
|
||||
c = min(f + 1, len(values) - 1)
|
||||
if f == c:
|
||||
return values[f]
|
||||
return values[f] + (values[c] - values[f]) * (k - f)
|
||||
|
||||
|
||||
def get_available_dates(tz_offset_hours: float = 8) -> List[str]:
|
||||
"""返回日志中出现过的所有本地日期（YYYY-MM-DD），最新在前。"""
|
||||
tz = datetime.timezone(datetime.timedelta(hours=tz_offset_hours))
|
||||
dates = set()
|
||||
for e in _read_all_log_entries():
|
||||
t = _parse_ts(e.get("timestamp", ""))
|
||||
if t is None:
|
||||
continue
|
||||
dates.add(t.astimezone(tz).date().isoformat())
|
||||
return sorted(dates, reverse=True)
|
||||
|
||||
|
||||
def _estimate_per_style(durations_ms: List[float]) -> Optional[Dict[str, Any]]:
|
||||
"""从一组请求耗时里拆解出「单次换发型边际耗时」与「固定开销」。
|
||||
|
||||
原理：这类接口对每个勾选的发型串行跑一次 ComfyUI，总耗时 ≈ 固定开销(人脸检测等)
|
||||
+ 单次换发型耗时 × 发型数量。发型数量未被记录，这里用迭代最小二乘估计"量子"
|
||||
（近似每多选 1 个发型多花多少秒），再据此把样本归类、做线性回归得到最终估计。
|
||||
样本耗时种类过少（不足以分辨固定开销与边际耗时）时返回 None。
|
||||
"""
|
||||
xs = sorted(d / 1000.0 for d in durations_ms if d and d > 0)
|
||||
if len(xs) < 3:
|
||||
return None
|
||||
|
||||
guess = xs[0]
|
||||
if guess <= 0:
|
||||
return None
|
||||
|
||||
def _fit(ns: List[int], xs_: List[float]):
|
||||
n_mean = sum(ns) / len(ns)
|
||||
x_mean = sum(xs_) / len(xs_)
|
||||
num = sum((n - n_mean) * (x - x_mean) for n, x in zip(ns, xs_))
|
||||
den = sum((n - n_mean) ** 2 for n in ns)
|
||||
if den == 0:
|
||||
return None
|
||||
slope = num / den
|
||||
intercept = x_mean - slope * n_mean
|
||||
return slope, intercept
|
||||
|
||||
for _ in range(8):
|
||||
ns = [max(1, round(x / guess)) for x in xs]
|
||||
fit = _fit(ns, xs)
|
||||
if fit is None or fit[0] <= 0:
|
||||
break
|
||||
guess = fit[0]
|
||||
|
||||
ns_final = [max(1, round(x / guess)) for x in xs]
|
||||
if len(set(ns_final)) < 2:
|
||||
return None # 样本都挤在同一档，无法拆分固定开销 / 边际耗时
|
||||
|
||||
fit = _fit(ns_final, xs)
|
||||
if fit is None or fit[0] <= 0:
|
||||
return None
|
||||
slope, intercept = fit
|
||||
|
||||
groups: Dict[int, List[float]] = {}
|
||||
for x, n in zip(xs, ns_final):
|
||||
groups.setdefault(n, []).append(x)
|
||||
clusters = [
|
||||
{"styles": n, "count": len(v), "avg_seconds": round(sum(v) / len(v), 2)}
|
||||
for n, v in sorted(groups.items())
|
||||
]
|
||||
|
||||
return {
|
||||
"per_style_seconds": round(slope, 2),
|
||||
"fixed_overhead_seconds": round(max(intercept, 0.0), 2),
|
||||
"sample_count": len(xs),
|
||||
"clusters": clusters,
|
||||
}
|
||||
|
||||
|
||||
def get_daily_stats(date_str: str, tz_offset_hours: float = 8) -> Dict[str, Any]:
|
||||
"""统计某一天（本地时区，默认 UTC+8）的接口调用情况。
|
||||
|
||||
读取磁盘上的完整日志历史（不受内存环形缓冲区大小限制），返回：
|
||||
- summary：当天总请求数/成功率/总耗时/平均耗时
|
||||
- endpoints：按接口分组的详细统计（含接口编号/名称、耗时分布、
|
||||
对「多发型串行」接口额外给出单次换发型耗时拆解）
|
||||
- hourly：按小时的请求量分布，供画图用
|
||||
"""
|
||||
tz = datetime.timezone(datetime.timedelta(hours=tz_offset_hours))
|
||||
try:
|
||||
target_date = datetime.date.fromisoformat(date_str)
|
||||
except ValueError:
|
||||
return {"error": f"日期格式错误: {date_str!r}，应为 YYYY-MM-DD"}
|
||||
|
||||
day_entries = []
|
||||
for e in _read_all_log_entries():
|
||||
if not str(e.get("path", "")).startswith("/api/"):
|
||||
continue
|
||||
t = _parse_ts(e.get("timestamp", ""))
|
||||
if t is None:
|
||||
continue
|
||||
local = t.astimezone(tz)
|
||||
if local.date() == target_date:
|
||||
e = dict(e)
|
||||
e["_local_hour"] = local.hour
|
||||
day_entries.append(e)
|
||||
|
||||
if not day_entries:
|
||||
return {
|
||||
"date": date_str,
|
||||
"summary": {"total": 0, "success_rate": 0, "avg_duration_ms": 0,
|
||||
"total_duration_seconds": 0, "min_duration_ms": 0, "max_duration_ms": 0},
|
||||
"endpoints": [],
|
||||
"hourly": [],
|
||||
}
|
||||
|
||||
total = len(day_entries)
|
||||
ok_count = sum(1 for e in day_entries
|
||||
if e.get("status_code") == 200 and e.get("response_code") in (0, None))
|
||||
all_durations = sorted(e.get("duration_ms", 0.0) for e in day_entries)
|
||||
|
||||
by_path: Dict[str, List[dict]] = {}
|
||||
for e in day_entries:
|
||||
by_path.setdefault(e["path"], []).append(e)
|
||||
|
||||
endpoints = []
|
||||
for path, items in by_path.items():
|
||||
durs = sorted(it.get("duration_ms", 0.0) for it in items)
|
||||
ok = sum(1 for it in items
|
||||
if it.get("status_code") == 200 and it.get("response_code") in (0, None))
|
||||
info = PATH_INTERFACE_MAP.get(path, {"num": None, "name": path})
|
||||
entry: Dict[str, Any] = {
|
||||
"path": path,
|
||||
"interface_num": info["num"],
|
||||
"interface_name": info["name"],
|
||||
"count": len(items),
|
||||
"ok_count": ok,
|
||||
"fail_count": len(items) - ok,
|
||||
"success_rate": round(ok / len(items) * 100, 1),
|
||||
"avg_duration_ms": round(sum(durs) / len(durs), 1),
|
||||
"median_duration_ms": round(_percentile(durs, 0.5), 1),
|
||||
"p95_duration_ms": round(_percentile(durs, 0.95), 1),
|
||||
"min_duration_ms": round(durs[0], 1),
|
||||
"max_duration_ms": round(durs[-1], 1),
|
||||
"total_duration_seconds": round(sum(durs) / 1000, 1),
|
||||
"per_style": None,
|
||||
}
|
||||
if path in MULTI_STYLE_PATHS:
|
||||
entry["per_style"] = _estimate_per_style(durs)
|
||||
endpoints.append(entry)
|
||||
|
||||
endpoints.sort(key=lambda x: -x["count"])
|
||||
|
||||
hourly_counts: Dict[int, int] = {}
|
||||
for e in day_entries:
|
||||
hourly_counts[e["_local_hour"]] = hourly_counts.get(e["_local_hour"], 0) + 1
|
||||
hourly = [{"hour": h, "count": hourly_counts.get(h, 0)} for h in range(24) if hourly_counts.get(h, 0) > 0]
|
||||
|
||||
return {
|
||||
"date": date_str,
|
||||
"summary": {
|
||||
"total": total,
|
||||
"success_rate": round(ok_count / total * 100, 1),
|
||||
"avg_duration_ms": round(sum(all_durations) / total, 1),
|
||||
"total_duration_seconds": round(sum(all_durations) / 1000, 1),
|
||||
"min_duration_ms": round(all_durations[0], 1),
|
||||
"max_duration_ms": round(all_durations[-1], 1),
|
||||
},
|
||||
"endpoints": endpoints,
|
||||
"hourly": hourly,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user