hair/tests/test_gateway_pool.py

"""gateway/pool.py 隔离期逻辑单测（不依赖真实网络）。

覆盖《docs/网关摘除隔离期实施方案.md》§3.3 的三个用例：
- 被动摘除设置隔离期（offline_until > now）
- 隔离期内探测成功不复活
- 隔离期满后探测成功恢复上线
外加：_try_revive 的阈值/在线短路、mark_worker_unhealthy 对已下线 worker 的幂等、
_health_check_loop 整链路验证、_cooldown_seconds 配置读取。

与 test_gateway_serialization.py 同款：asyncio.run 套同步测试，不依赖 pytest-asyncio。
"""

import asyncio
import time
from unittest.mock import patch

from gateway import pool
from gateway.pool import WorkerState

_HC_CFG = {
    "health_check": {
        "path": "/health",
        "interval_seconds": 0.05,  # 测试用：快速轮询
        "timeout_seconds": 3,
        "unhealthy_threshold": 2,
        "healthy_threshold": 1,
    },
    "shared_password": "test-token",
}


# ---------------------------------------------------------------------------
# 被动摘除 → 隔离期
# ---------------------------------------------------------------------------

def test_passive_ejection_sets_cooldown():
    w = WorkerState(url="http://127.0.0.1:9000", online=True)

    async def run():
        pool._pool_condition = asyncio.Condition()
        with patch.object(pool, "_cooldown_seconds", return_value=60.0):
            await pool.mark_worker_unhealthy(w)

    asyncio.run(run())
    assert w.online is False
    assert w.offline_until > time.monotonic(), "被动摘除必须设置未来的隔离截止时刻"
    assert w.consecutive_failures == 1


def test_passive_ejection_idempotent_when_offline():
    """已下线的 worker 再次被标记：不重复计失败、不续期隔离。"""
    w = WorkerState(url="http://127.0.0.1:9000", online=False)
    w.offline_until = time.monotonic() + 30
    before = w.offline_until

    async def run():
        pool._pool_condition = asyncio.Condition()
        with patch.object(pool, "_cooldown_seconds", return_value=60.0):
            await pool.mark_worker_unhealthy(w)

    asyncio.run(run())
    assert w.online is False
    assert w.consecutive_failures == 0
    assert w.offline_until == before


# ---------------------------------------------------------------------------
# _try_revive：探测成功后是否允许复活
# ---------------------------------------------------------------------------

def test_probe_success_blocked_during_cooldown():
    w = WorkerState(url="http://127.0.0.1:9000")  # offline
    w.consecutive_successes = 1
    w.offline_until = time.monotonic() + 60
    assert pool._try_revive(w, healthy_threshold=1) is False
    assert w.online is False


def test_probe_success_revives_after_cooldown():
    w = WorkerState(url="http://127.0.0.1:9000")
    w.consecutive_successes = 1
    w.offline_until = time.monotonic() - 1  # 隔离期已过
    assert pool._try_revive(w, healthy_threshold=1) is True


def test_try_revive_requires_success_threshold():
    w = WorkerState(url="http://127.0.0.1:9000")
    w.consecutive_successes = 1
    w.offline_until = 0.0
    assert pool._try_revive(w, healthy_threshold=2) is False


def test_try_revive_noop_when_already_online():
    w = WorkerState(url="http://127.0.0.1:9000", online=True)
    w.consecutive_successes = 1
    assert pool._try_revive(w, healthy_threshold=1) is False


# ---------------------------------------------------------------------------
# _health_check_loop 整链路：隔离期内探测 200 不上线，期满后上线
# ---------------------------------------------------------------------------

def test_health_check_loop_respects_cooldown(monkeypatch):
    w = WorkerState(url="http://127.0.0.1:9000", online=False)
    w.offline_until = time.monotonic() + 60  # 处于隔离期

    class _FakeResp:
        status_code = 200

    class _FakeClient:
        async def __aenter__(self):
            return self

        async def __aexit__(self, *args):
            return False

        async def get(self, url, headers=None, timeout=None):
            return _FakeResp()

    async def run():
        monkeypatch.setattr(pool, "_workers", {"w1": w})
        monkeypatch.setattr(pool, "_shutdown_event", asyncio.Event())
        monkeypatch.setattr(pool.httpx, "AsyncClient", _FakeClient)
        task = asyncio.create_task(pool._health_check_loop(_HC_CFG))
        try:
            await asyncio.sleep(0.2)  # 跑若干轮探测
            assert w.consecutive_successes >= 1, "探测应已成功过"
            assert w.online is False, "隔离期内即使探测成功也不得复活"

            w.offline_until = time.monotonic() - 1  # 隔离期结束
            await asyncio.sleep(0.2)
            assert w.online is True, "隔离期满后下一次探测成功应上线"
        finally:
            pool._shutdown_event.set()
            await asyncio.wait_for(task, timeout=2)

    asyncio.run(run())


# ---------------------------------------------------------------------------
# _cooldown_seconds 配置读取
# ---------------------------------------------------------------------------

def test_cooldown_seconds_reads_config(monkeypatch):
    from gateway import config as gw_config
    monkeypatch.setattr(gw_config, "_config_cache",
                        {"health_check": {"cooldown_seconds": 120}})
    assert pool._cooldown_seconds() == 120.0


def test_cooldown_seconds_falls_back_to_60(monkeypatch):
    from gateway import config as gw_config
    monkeypatch.setattr(gw_config, "_config_cache", None)  # get_config 会抛 RuntimeError
    assert pool._cooldown_seconds() == 60.0