feat(health): /health 深检——磁盘水位 + 下游依赖 TCP 探测,异常返回 503 degraded
背景: 9·19 磁盘满事故中业务已瘫但 /health 仍 200(只看模型就绪标志), 网关持续把流量打到本机。深检让 worker 在磁盘不足(min_free_gb, 默认5G)或 ComfyUI/change_hair TCP 不通时主动报 503,网关探测两次失败即摘除。 - 配置: worker_config.json "health" 段,env HEALTH_MIN_FREE_GB/ HEALTH_CHECK_DEPS/HEALTH_DEP_COMFYUI/HEALTH_DEPS_CACHE_SECONDS 可覆盖 - 依赖探测 0.8s 超时并发执行,结果缓存 5s;深检自身异常按就绪返回 - 测试: 深检各分支用例 + 旧用例改为 mock 深检(不依赖本机服务)
This commit is contained in:
+65
-1
@@ -36,12 +36,76 @@ def test_auth_missing_token_401(client):
|
||||
assert r.status_code == 401
|
||||
|
||||
|
||||
def test_health_no_token_200(client):
|
||||
def _patch_health_ok(monkeypatch):
|
||||
"""让 /health 深检确定性通过:不依赖本机磁盘水位与 8188/8801 服务存活。"""
|
||||
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
|
||||
**app_module._HEALTH_CFG, "min_free_gb": 0.0, "check_deps": True})
|
||||
|
||||
async def _deps_ok():
|
||||
return True, {"comfyui": True, "change_hair": True}
|
||||
monkeypatch.setattr(app_module, "_check_deps", _deps_ok)
|
||||
|
||||
|
||||
def test_health_no_token_200(client, monkeypatch):
|
||||
_patch_health_ok(monkeypatch)
|
||||
r = client.get("/health")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
|
||||
|
||||
def test_health_disk_low_503(client, monkeypatch):
|
||||
# 阈值调到不可能的 99999G → 必然触发磁盘水位 503
|
||||
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
|
||||
**app_module._HEALTH_CFG, "min_free_gb": 99999.0, "check_deps": False})
|
||||
r = client.get("/health")
|
||||
assert r.status_code == 503
|
||||
body = r.json()
|
||||
assert body["status"] == "degraded"
|
||||
assert body["reason"].startswith("disk_free_below")
|
||||
|
||||
|
||||
def test_health_dep_down_503(client, monkeypatch):
|
||||
monkeypatch.setattr(app_module, "_HEALTH_CFG", {
|
||||
**app_module._HEALTH_CFG, "min_free_gb": 0.0, "check_deps": True})
|
||||
|
||||
async def _deps_down():
|
||||
return False, {"comfyui": False, "change_hair": True}
|
||||
monkeypatch.setattr(app_module, "_check_deps", _deps_down)
|
||||
r = client.get("/health")
|
||||
assert r.status_code == 503
|
||||
body = r.json()
|
||||
assert body["status"] == "degraded"
|
||||
assert "comfyui" in body["reason"]
|
||||
|
||||
|
||||
def test_health_ok_carries_detail(client, monkeypatch):
|
||||
_patch_health_ok(monkeypatch)
|
||||
r = client.get("/health")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "disk_free_gb" in body
|
||||
assert body["deps"] == {"comfyui": True, "change_hair": True}
|
||||
|
||||
|
||||
def test_tcp_probe_real_socket():
|
||||
# 用进程内真实 socket 验证探测本身(曾因 open_connection 返回元组误判全部不可达)
|
||||
import asyncio
|
||||
|
||||
async def scenario():
|
||||
server = await asyncio.start_server(lambda r, w: None, host="127.0.0.1", port=0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
up = await app_module._tcp_probe("127.0.0.1", port)
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
# 端口 1 无监听,本机 connect 立即被拒
|
||||
down = await app_module._tcp_probe("127.0.0.1", 1)
|
||||
return up, down
|
||||
|
||||
up, down = asyncio.run(scenario())
|
||||
assert up is True
|
||||
assert down is False
|
||||
|
||||
|
||||
def test_param_none_provided_1007(client):
|
||||
r = client.post(URL, headers=H)
|
||||
assert r.json()["code"] == 1007
|
||||
|
||||
Reference in New Issue
Block a user