feat(gateway): 被动摘除后进入 60s 隔离期,防探测复活横跳

落地 docs/网关摘除隔离期实施方案.md(9·19 worker 磁盘满事故第 3 层防线):
- pool.py: WorkerState.offline_until + _cooldown_seconds();mark_worker_unhealthy
  摘除时设隔离截止时刻,隔离期内探测成功也不复活(_try_revive 守卫)
- pool.py: 删除从未被调用的死代码 _check_worker_health/_mark_failure
- config.py + config.example.json: health_check.cooldown_seconds(默认 60,0=关闭)
- tests/test_gateway_pool.py: 9 用例(摘除设隔离/隔离内不复活/期满复活/整链路等)

网关机已重启验证:启动日志含"隔离期=60s",/gateway-health 正常。
This commit is contained in:
xsl
2026-09-20 23:02:18 +08:00
parent 88fee6474d
commit 6b24bf0b0d
4 changed files with 190 additions and 44 deletions
+33 -43
View File
@@ -18,6 +18,15 @@ import httpx
logger = logging.getLogger("gateway.pool")
def _cooldown_seconds() -> float:
"""被动摘除后的隔离期时长(秒),配置 health_check.cooldown_seconds,默认 60。"""
try:
from gateway.config import get_config
return float(get_config()["health_check"].get("cooldown_seconds", 60))
except Exception: # noqa: BLE001
return 60.0
# ---------------------------------------------------------------------------
# 异常
# ---------------------------------------------------------------------------
@@ -39,6 +48,7 @@ class WorkerState:
busy: bool = False
consecutive_failures: int = 0
consecutive_successes: int = 0
offline_until: float = 0.0 # 被动摘除后的隔离截止时刻(time.monotonic() 基准)
# ---------------------------------------------------------------------------
@@ -63,45 +73,18 @@ _global_waiting: int = 0
# 健康检查后台任务
# ---------------------------------------------------------------------------
async def _check_worker_health(
client: httpx.AsyncClient,
w: WorkerState,
cfg: dict,
) -> None:
"""探测单个 worker 的 /health,更新上下线状态。"""
hc_cfg = cfg["health_check"]
url = f"{w.url}{hc_cfg['path']}"
token = cfg["shared_password"]
def _try_revive(w: WorkerState, healthy_threshold: int) -> bool:
"""探测成功后是否允许将该 worker 复活上线。
try:
resp = await client.get(
url,
headers={"X-Internal-Token": token},
timeout=hc_cfg["timeout_seconds"],
)
if resp.status_code == 200:
w.consecutive_failures = 0
w.consecutive_successes += 1
if w.consecutive_successes >= hc_cfg["healthy_threshold"]:
if not w.online:
w.online = True
logger.info("Worker 上线: %s(连续成功 %d 次)", w.url, w.consecutive_successes)
else:
_mark_failure(w, f"HTTP {resp.status_code}")
except Exception as exc:
_mark_failure(w, str(exc))
def _mark_failure(w: WorkerState, reason: str) -> None:
"""记录一次失败,达到阈值后下线。"""
w.consecutive_successes = 0
w.consecutive_failures += 1
threshold = w.consecutive_failures # used in log
hc_threshold = 2 # default, will be overridden
if w.consecutive_failures >= hc_threshold:
# 实际 threshold 从配置读取,这里先做基本判断
pass
logger.debug("Worker %s 健康检查失败 (%d/%d): %s", w.url, w.consecutive_failures, 99, reason)
条件:未在线 + 连续成功达到阈值 + **不在被动摘除隔离期内**
(offline_until 之前即使 /health 探测成功也压着不上线,防横跳)。
"""
if w.online or w.consecutive_successes < healthy_threshold:
return False
if time.monotonic() < w.offline_until:
logger.debug("Worker %s 探测成功但处于隔离期,暂不复活", w.url)
return False
return True
async def _health_check_loop(cfg: dict) -> None:
@@ -113,8 +96,8 @@ async def _health_check_loop(cfg: dict) -> None:
token = cfg["shared_password"]
logger.info(
"健康检查循环启动 | 间隔=%ds | 下线阈值=%d | 上线阈值=%d | workers=%d",
interval, unhealthy_threshold, healthy_threshold, len(_workers),
"健康检查循环启动 | 间隔=%ds | 下线阈值=%d | 上线阈值=%d | 隔离期=%.0fs | workers=%d",
interval, unhealthy_threshold, healthy_threshold, _cooldown_seconds(), len(_workers),
)
async with httpx.AsyncClient() as client:
@@ -130,7 +113,7 @@ async def _health_check_loop(cfg: dict) -> None:
if resp.status_code == 200:
w.consecutive_failures = 0
w.consecutive_successes += 1
if w.consecutive_successes >= healthy_threshold and not w.online:
if _try_revive(w, healthy_threshold):
w.online = True
logger.info("✅ Worker 上线: %s", w.url)
else:
@@ -272,12 +255,19 @@ def release_global_slot() -> None:
# ---------------------------------------------------------------------------
async def mark_worker_unhealthy(w: WorkerState) -> None:
"""被动标记:转发失败时立即将该 worker 下线。"""
"""被动标记:转发失败时立即将该 worker 下线,并进入隔离期。
隔离期内即使 /health 探测成功也不复活(由 _try_revive 守卫),
避免"摘除→8 秒复活→再失败"的横跳。
"""
async with _pool_condition:
if w.online:
w.online = False
w.consecutive_failures += 1
logger.warning("⚠ Worker 被动下线: %s(转发失败)", w.url)
cooldown = _cooldown_seconds()
w.offline_until = time.monotonic() + cooldown
logger.warning("⚠ Worker 被动下线: %s(转发失败,隔离 %.0fs)",
w.url, cooldown)
async def acquire_worker(cfg: dict) -> WorkerState: