feat(health): /health 深检——磁盘水位 + 下游依赖 TCP 探测,异常返回 503 degraded

背景: 9·19 磁盘满事故中业务已瘫但 /health 仍 200(只看模型就绪标志),
网关持续把流量打到本机。深检让 worker 在磁盘不足(min_free_gb, 默认5G)或
ComfyUI/change_hair TCP 不通时主动报 503,网关探测两次失败即摘除。

- 配置: worker_config.json "health" 段,env HEALTH_MIN_FREE_GB/
  HEALTH_CHECK_DEPS/HEALTH_DEP_COMFYUI/HEALTH_DEPS_CACHE_SECONDS 可覆盖
- 依赖探测 0.8s 超时并发执行,结果缓存 5s;深检自身异常按就绪返回
- 测试: 深检各分支用例 + 旧用例改为 mock 深检(不依赖本机服务)
This commit is contained in:
xsl
2026-09-20 22:07:48 +08:00
parent 752630e4f5
commit 6b098c0e15
3 changed files with 196 additions and 6 deletions
+65 -1
View File
@@ -36,12 +36,76 @@ def test_auth_missing_token_401(client):
assert r.status_code == 401
def test_health_no_token_200(client):
def _patch_health_ok(monkeypatch):
"""让 /health 深检确定性通过:不依赖本机磁盘水位与 8188/8801 服务存活。"""
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
**app_module._HEALTH_CFG, "min_free_gb": 0.0, "check_deps": True})
async def _deps_ok():
return True, {"comfyui": True, "change_hair": True}
monkeypatch.setattr(app_module, "_check_deps", _deps_ok)
def test_health_no_token_200(client, monkeypatch):
_patch_health_ok(monkeypatch)
r = client.get("/health")
assert r.status_code == 200
assert r.json()["status"] == "ok"
def test_health_disk_low_503(client, monkeypatch):
# 阈值调到不可能的 99999G → 必然触发磁盘水位 503
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
**app_module._HEALTH_CFG, "min_free_gb": 99999.0, "check_deps": False})
r = client.get("/health")
assert r.status_code == 503
body = r.json()
assert body["status"] == "degraded"
assert body["reason"].startswith("disk_free_below")
def test_health_dep_down_503(client, monkeypatch):
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
**app_module._HEALTH_CFG, "min_free_gb": 0.0, "check_deps": True})
async def _deps_down():
return False, {"comfyui": False, "change_hair": True}
monkeypatch.setattr(app_module, "_check_deps", _deps_down)
r = client.get("/health")
assert r.status_code == 503
body = r.json()
assert body["status"] == "degraded"
assert "comfyui" in body["reason"]
def test_health_ok_carries_detail(client, monkeypatch):
_patch_health_ok(monkeypatch)
r = client.get("/health")
assert r.status_code == 200
body = r.json()
assert "disk_free_gb" in body
assert body["deps"] == {"comfyui": True, "change_hair": True}
def test_tcp_probe_real_socket():
# 用进程内真实 socket 验证探测本身(曾因 open_connection 返回元组误判全部不可达)
import asyncio
async def scenario():
server = await asyncio.start_server(lambda r, w: None, host="127.0.0.1", port=0)
port = server.sockets[0].getsockname()[1]
up = await app_module._tcp_probe("127.0.0.1", port)
server.close()
await server.wait_closed()
# 端口 1 无监听,本机 connect 立即被拒
down = await app_module._tcp_probe("127.0.0.1", 1)
return up, down
up, down = asyncio.run(scenario())
assert up is True
assert down is False
def test_param_none_provided_1007(client):
r = client.post(URL, headers=H)
assert r.json()["code"] == 1007