diff --git a/.run/start-test.log b/.run/start-test.log index 171f66f..b9c6f2b 100644 --- a/.run/start-test.log +++ b/.run/start-test.log @@ -5,42 +5,284 @@ mode: auto bind: 0.0.0.0:8018 listen: https://0.0.0.0:8018 browser test page: https://:8018/ -INFO: Started server process [17367] +INFO: Started server process [20276] INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit) -INFO: 127.0.0.1:38170 - "WebSocket /ws/animation" [accepted] +INFO: 127.0.0.1:35988 - "WebSocket /ws/subtitles" [accepted] INFO: connection open -INFO: 127.0.0.1:38188 - "WebSocket /ws/subtitles" [accepted] -INFO: connection open -INFO: 127.0.0.1:38202 - "GET /health HTTP/1.1" 200 OK -INFO: 127.0.0.1:38220 - "GET / HTTP/1.1" 200 OK -INFO: connection closed -INFO: connection closed -INFO: 127.0.0.1:38220 - "GET /web/app.js?v=20260329k HTTP/1.1" 200 OK -INFO: 127.0.0.1:38220 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK -INFO: 127.0.0.1:38256 - "GET /health HTTP/1.1" 200 OK -INFO: 127.0.0.1:38266 - "GET /events HTTP/1.1" 200 OK -INFO: 127.0.0.1:38286 - "WebSocket /ws/subtitles" [accepted] -INFO: connection open -INFO: 127.0.0.1:38302 - "WebSocket /ws/animation" [accepted] -INFO: connection open -INFO: 127.0.0.1:40412 - "GET / HTTP/1.1" 200 OK -INFO: connection closed -INFO: connection closed -INFO: 127.0.0.1:40412 - "GET /web/app.js?v=20260329k HTTP/1.1" 304 Not Modified -INFO: 127.0.0.1:40412 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK -INFO: 127.0.0.1:40440 - "GET /events HTTP/1.1" 200 OK -INFO: 127.0.0.1:40436 - "GET /health HTTP/1.1" 200 OK -INFO: 127.0.0.1:40462 - "WebSocket /ws/subtitles" [accepted] -INFO: connection open -INFO: 127.0.0.1:40472 - "WebSocket /ws/animation" [accepted] -INFO: connection open -INFO: 127.0.0.1:52378 - "WebSocket /ws/audio" [accepted] +INFO: 127.0.0.1:36002 - "WebSocket /ws/subtitles" [accepted] INFO: connection open INFO: connection closed -INFO: 127.0.0.1:52392 - "WebSocket /ws/audio" [accepted] +INFO: 127.0.0.1:36022 - "WebSocket /ws/animation" [accepted] INFO: connection open +INFO: 127.0.0.1:36036 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36050 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36072 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36078 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:36104 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36118 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36148 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36168 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36194 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36206 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:36224 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54498 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54524 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54542 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54562 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54584 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54596 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54608 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54628 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54642 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54650 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54664 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54684 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54706 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54718 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54748 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54756 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:54788 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59782 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59802 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59812 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59832 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59860 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59874 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59888 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59902 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59906 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59910 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59922 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59928 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59944 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59962 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59976 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:60002 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58794 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58800 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58830 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58854 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58884 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58906 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58930 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58950 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58974 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:58986 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59002 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59026 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59046 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO: 127.0.0.1:59046 - "GET /web/app.js?v=20260329m HTTP/1.1" 200 OK +INFO: 127.0.0.1:59046 - "GET /web/app.js?v=20260329q HTTP/1.1" 200 OK +INFO: 127.0.0.1:59046 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK +INFO: 127.0.0.1:59114 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:59124 - "GET /events HTTP/1.1" 200 OK +INFO: 127.0.0.1:59114 - "GET /events HTTP/1.1" 200 OK +INFO: 127.0.0.1:59150 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:59174 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 127.0.0.1:59184 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 127.0.0.1:59186 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:59190 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:44402 - "WebSocket /ws/audio" [accepted] +INFO: connection open +INFO: connection closed +INFO: 127.0.0.1:44446 - "WebSocket /ws/audio" [accepted] +INFO: connection open +INFO: 127.0.0.1:44450 - "WebSocket /ws/audio" [accepted] +INFO: connection open +INFO: 127.0.0.1:44472 - "WebSocket /ws/audio" [accepted] +INFO: connection closed +INFO: connection closed +INFO: connection open +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 329, in asgi_send + await self.send(data) # type: ignore[arg-type] + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/websockets/legacy/protocol.py", line 618, in send + await self.ensure_open() + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/websockets/legacy/protocol.py", line 916, in ensure_open + raise self.connection_closed_exc() +websockets.exceptions.ConnectionClosedOK: received 1000 (OK) stale-open; then sent 1000 (OK) stale-open + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 86, in send + await self._send(message) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 39, in sender + await send(message) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 342, in asgi_send + raise ClientDisconnected from exc +uvicorn.protocols.utils.ClientDisconnected + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 244, in run_asgi + result = await self.app(self.scope, self.asgi_receive, self.asgi_send) # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 151, in __call__ + await self.app(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/cors.py", line 80, in __call__ + await self.app(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app + raise exc + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 680, in app + await route.handle(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 350, in handle + await self.app(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 160, in app + await wrap_app_handling_exceptions(app, session)(scope, receive, send) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app + raise exc + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 157, in app + await func(session) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 764, in app + await dependant.call(**solved_result.values) + File "/home/xsl/code/product/main.py", line 446, in ws_audio + await websocket.send_text( + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 166, in send_text + await self.send({"type": "websocket.send", "text": data}) + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 89, in send + raise WebSocketDisconnect(code=1006) +starlette.websockets.WebSocketDisconnect INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK" /home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.) return torch._C._cuda_getDeviceCount() > 0 @@ -60,20 +302,19 @@ Building prefix dict from the default dictionary ... DEBUG:jieba:Building prefix dict from the default dictionary ... Loading model from cache /tmp/jieba.cache DEBUG:jieba:Loading model from cache /tmp/jieba.cache -Loading model cost 0.341 seconds. -DEBUG:jieba:Loading model cost 0.341 seconds. +Loading model cost 0.356 seconds. +DEBUG:jieba:Loading model cost 0.356 seconds. Prefix dict has been built successfully. DEBUG:jieba:Prefix dict has been built successfully. WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning. -INFO: 127.0.0.1:42956 - "POST /chat/text HTTP/1.1" 200 OK -INFO: 127.0.0.1:42956 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:42956 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:44470 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 127.0.0.1:44470 - "GET /favicon.ico HTTP/1.1" 404 Not Found INFO:root:download models from model hub: hf INFO:httpx:HTTP Request: GET https://huggingface.co/api/models/FunAudioLLM/SenseVoiceSmall/revision/main "HTTP/1.1 200 OK" funasr version: 1.3.1. Check update of funasr, and it would cost few times. You may disable it by set `disable_update=True` in AutoModel You are using the latest version of funasr-1.3.1 - Fetching 29 files: 0%| | 0/29 [00:00 WARNING:services.asr:ASR running in degraded mode on cpu - 0%| | 0/1 [00:00:8018/` +- `https://:8018/health` + 如果浏览器与服务端跨机器访问，麦克风采集通常需要 HTTPS。可以先生成自签证书： ```bash @@ -133,6 +150,8 @@ bash scripts/gen-self-signed-cert.sh - `WS /ws/animation`: 数字人控制流。 - `GET /avatar/schema`: 当前控制协议 schema。 +说明：三个 WebSocket 通道当前都只保留一个活动客户端。新页面接入后，旧页面会收到 `1012 replaced` 并停止接收数据。 + ## 后续开发建议 优先阅读 [docs/development-guide.md](docs/development-guide.md)。这个文档说明了： diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc index a35129b..c716853 100644 Binary files a/__pycache__/main.cpython-312.pyc and b/__pycache__/main.cpython-312.pyc differ diff --git a/docs/development-guide.md b/docs/development-guide.md index 5504fc1..54be34b 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -49,6 +49,14 @@ 当前实现假设一次只保留一个主音频 WebSocket 连接。新连接到来时会主动关闭旧连接。这是刻意简化，不是 bug。 +现在这个约束已经扩展到三个浏览器通道： + +- `WS /ws/audio` +- `WS /ws/subtitles` +- `WS /ws/animation` + +也就是说，服务端当前只允许一个浏览器页面完整接管数字人会话。新页面连上后，旧页面会被主动断开，关闭码为 `1012 replaced`。 + ### 3.2 会话状态机 [core/state_machine.py](../core/state_machine.py) 的状态很轻： @@ -99,6 +107,7 @@ - 维护聊天消息面板和诊断面板。 - 用 Three.js + VRM 加载和渲染模型。 - 把动画控制帧缓存约 `120ms` 后再播放，尽量和音频保持一致。 +- 处理刷新后的语音恢复：如果浏览器拦截自动恢复，会等待下一次用户手势再恢复麦克风和播放上下文。 前端当前兼容三种形态： @@ -106,6 +115,11 @@ - 通用 GLTF morph target。 - 没有表情 rig 时回退到调试头像。 +当前前端待机动画状态： + +- 已启用随机眨眼。 +- 更激进的身体 idle motion 已回退，因为部分 VRM 模型会出现 T pose 或姿态异常。 + 如果以后想接入真实业务数字人模型，先确认模型是否提供： - 标准 VRM 表情。 @@ -177,10 +191,11 @@ 这些约束需要后续开发明确知晓： - 当前是进程内单例服务，不是多租户、多 session 设计。 -- 浏览器侧只允许一个主音频 WebSocket 会话接管服务端语音输入输出。 +- 浏览器侧只允许一个页面完整接管服务端语音输入输出与字幕/动画订阅。 - 模型推理基本都直接跑在应用事件循环附近，适合研发验证，不适合高并发。 - 当前动画驱动依赖音频能量和谱质心，不具备严格唇音级别精度。 - 语音链路高度依赖 VAD；如果要做电话式长连接交互，建议补更稳的 turn management。 +- 浏览器刷新后不保证零手势恢复语音；这是浏览器媒体策略约束，不是后端故障。 ## 9. 推荐扩展方向 diff --git a/main.py b/main.py index 1b7dceb..99e121b 100644 --- a/main.py +++ b/main.py @@ -103,6 +103,17 @@ async def _broadcast_json(clients: set[WebSocket], payload: dict) -> None: clients.discard(ws) +async def _replace_existing_clients(clients: set[WebSocket]) -> bool: + replaced_previous = len(clients) > 0 + if not replaced_previous: + return False + old_clients = list(clients) + await asyncio.gather(*(client.close(code=1012, reason="replaced") for client in old_clients), return_exceptions=True) + for client in old_clients: + clients.discard(client) + return True + + async def _broadcast_subtitle( role: str, text: str, @@ -392,6 +403,7 @@ async def events(): @app.websocket("/ws/subtitles") async def ws_subtitles(websocket: WebSocket): await websocket.accept() + await _replace_existing_clients(subtitle_clients) subtitle_clients.add(websocket) try: while True: @@ -406,6 +418,7 @@ async def ws_subtitles(websocket: WebSocket): @app.websocket("/ws/animation") async def ws_animation(websocket: WebSocket): await websocket.accept() + await _replace_existing_clients(animation_clients) animation_clients.add(websocket) try: await websocket.send_text(json.dumps({"type": "animation_ready", **avatar_service.schema}, ensure_ascii=False)) @@ -423,12 +436,7 @@ async def ws_animation(websocket: WebSocket): async def ws_audio(websocket: WebSocket): await websocket.accept() - replaced_previous = len(audio_clients) > 0 - if replaced_previous: - old_clients = list(audio_clients) - await asyncio.gather(*(client.close(code=1012, reason="replaced") for client in old_clients), return_exceptions=True) - for client in old_clients: - audio_clients.discard(client) + replaced_previous = await _replace_existing_clients(audio_clients) audio_clients.add(websocket) sample_rate = 48000 diff --git a/scripts/start-public.sh b/scripts/start-public.sh index 97dbe8a..998e331 100755 --- a/scripts/start-public.sh +++ b/scripts/start-public.sh @@ -5,8 +5,6 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python" - if [[ -f ".env" ]]; then set -a # shellcheck disable=SC1091 @@ -14,6 +12,26 @@ if [[ -f ".env" ]]; then set +a fi +pick_python() { + if [[ -n "${VISUAL_CHAT_PYTHON:-}" && -x "${VISUAL_CHAT_PYTHON}" ]]; then + echo "$VISUAL_CHAT_PYTHON" + return 0 + fi + if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then + echo "${VIRTUAL_ENV}/bin/python" + return 0 + fi + if [[ -x "/home/xsl/miniconda3/envs/MuseTalk/bin/python" ]]; then + echo "/home/xsl/miniconda3/envs/MuseTalk/bin/python" + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + command -v python3 + return 0 + fi + command -v python +} + normalize_host() { local host="${1:-0.0.0.0}" case "$host" in @@ -27,10 +45,22 @@ normalize_host() { } HOST="$(normalize_host "${WEBRTC_HOST:-0.0.0.0}")" -ARGS=(main:app --host "$HOST" --port "${WEBRTC_PORT:-8080}") +PORT="${WEBRTC_PORT:-8080}" +PY="$(pick_python)" +SCHEME="http" +ARGS=(main:app --host "$HOST" --port "$PORT") if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then + SCHEME="https" ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE") fi +echo "repo: $ROOT" +echo "python: $PY" +echo "bind: ${HOST}:${PORT}" +echo "listen: ${SCHEME}://${HOST}:${PORT}" +echo "browser test page: ${SCHEME}://:${PORT}/" +echo "note: only one browser page is allowed to own the session; a newer client replaces the older one." +echo "note: after browser refresh, one click may still be required to resume mic/audio due to browser media policy." + exec "$PY" -m uvicorn "${ARGS[@]}" diff --git a/scripts/start-test.sh b/scripts/start-test.sh index e314fb2..7b43953 100644 --- a/scripts/start-test.sh +++ b/scripts/start-test.sh @@ -80,5 +80,7 @@ echo "mode: $MODE" echo "bind: ${HOST}:${PORT}" echo "listen: ${SCHEME}://${HOST}:${PORT}" echo "browser test page: ${SCHEME}://:${PORT}/" +echo "note: only one browser page is allowed to own the session; a newer client replaces the older one." +echo "note: after browser refresh, one click may still be required to resume mic/audio due to browser media policy." exec "$PY" -m uvicorn "${ARGS[@]}" \ No newline at end of file diff --git a/web/app.js b/web/app.js index cc4aa43..4abffea 100644 --- a/web/app.js +++ b/web/app.js @@ -64,6 +64,7 @@ let micProcessorNode = null; let micSinkGain = null; let audioPlaybackCursor = 0; const audioPlaybackSources = new Set(); +const recentSubtitleEvents = new Map(); let reconnectOnGestureArmed = false; let pageGestureBootstrapArmed = false; @@ -127,6 +128,18 @@ const avatarState = { debugRig: null, debugJaw: null, debugMouth: null, + debugLeftEye: null, + debugRightEye: null, + blink: { + value: 0, + phase: "idle", + phaseStartedAtMs: performance.now(), + nextAtMs: performance.now() + 1800, + closeMs: 90, + holdMs: 50, + openMs: 120, + pendingDoubleBlink: false, + }, }; const gltfLoader = new GLTFLoader(); @@ -160,6 +173,27 @@ function addMessage(role, text) { return div; } +function shouldRenderSubtitleEvent(data) { + const key = JSON.stringify({ + role: data?.role || "", + text: data?.text || "", + partial: Boolean(data?.partial), + final: Boolean(data?.final), + source: data?.source || "", + }); + const now = Date.now(); + const lastSeenAt = recentSubtitleEvents.get(key) || 0; + recentSubtitleEvents.set(key, now); + + for (const [entryKey, seenAt] of recentSubtitleEvents.entries()) { + if (now - seenAt > 1500) { + recentSubtitleEvents.delete(entryKey); + } + } + + return now - lastSeenAt > 350; +} + function setConnBadge(label, kind = "neutral") { connBadge.textContent = label; connBadge.className = `badge ${kind}`; @@ -487,6 +521,8 @@ function createDebugRig() { avatarState.debugRig = group; avatarState.debugJaw = jawPivot; avatarState.debugMouth = mouth; + avatarState.debugLeftEye = leftEye; + avatarState.debugRightEye = rightEye; avatarState.headBone = neck; avatarState.neckBone = neck; } @@ -505,6 +541,7 @@ function renderLoop() { const elapsed = CLOCK.getElapsedTime(); controls.update(); flushAnimationQueue(); + updateBlinkState(performance.now()); applyProceduralIdlePose(elapsed); applyAnimationControls(avatarState.activeControls); avatarState.currentVrm?.update?.(delta); @@ -618,6 +655,8 @@ function applyAnimationControls(controls) { setMorphValue("mouthPucker", mouthPucker); } + applyBlinkControls(avatarState.blink?.value || 0); + // For plain head meshes (no VRM expressions and no morph targets), // amplify head motion to keep speaking visually obvious. const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09; @@ -702,6 +741,8 @@ function detachCurrentAvatar() { avatarState.headBone = avatarState.debugRig.children[1]; avatarState.neckBone = avatarState.debugRig.children[1]; avatarState.debugRig.visible = true; + avatarState.debugLeftEye = avatarState.debugLeftEye || avatarState.debugRig.children[1]?.children?.[2] || null; + avatarState.debugRightEye = avatarState.debugRightEye || avatarState.debugRig.children[1]?.children?.[3] || null; modelName.textContent = "debug-avatar"; } @@ -831,6 +872,9 @@ function collectMorphBindings(root) { viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"], viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"], mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"], + blink: ["blink", "eyeblink", "eye_blink", "blinkboth"], + blinkLeft: ["blinkleft", "blink_left", "eyeblinkleft", "eyelid_l", "eyeblink_l"], + blinkRight: ["blinkright", "blink_right", "eyeblinkright", "eyelid_r", "eyeblink_r"], }; root.traverse((node) => { @@ -910,6 +954,82 @@ function applyProceduralIdlePose(time) { applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05); } +function scheduleNextBlink(nowMs, preferSoon = false) { + const quickBlink = Math.random() < 0.18; + const minDelay = preferSoon ? 240 : 1800; + const maxDelay = preferSoon ? 520 : 5200; + avatarState.blink.nextAtMs = nowMs + minDelay + Math.random() * (maxDelay - minDelay); + avatarState.blink.pendingDoubleBlink = !preferSoon && quickBlink; +} + +function updateBlinkState(nowMs) { + const blink = avatarState.blink; + if (!blink) return; + + if (blink.phase === "idle") { + blink.value = 0; + if (nowMs >= blink.nextAtMs) { + blink.phase = "closing"; + blink.phaseStartedAtMs = nowMs; + } + return; + } + + if (blink.phase === "closing") { + const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.closeMs); + blink.value = progress; + if (progress >= 1) { + blink.phase = "hold"; + blink.phaseStartedAtMs = nowMs; + blink.value = 1; + } + return; + } + + if (blink.phase === "hold") { + blink.value = 1; + if (nowMs - blink.phaseStartedAtMs >= blink.holdMs) { + blink.phase = "opening"; + blink.phaseStartedAtMs = nowMs; + } + return; + } + + if (blink.phase === "opening") { + const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.openMs); + blink.value = 1 - progress; + if (progress >= 1) { + blink.phase = "idle"; + blink.phaseStartedAtMs = nowMs; + blink.value = 0; + scheduleNextBlink(nowMs, blink.pendingDoubleBlink); + } + } +} + +function applyBlinkControls(blinkValue) { + const clamped = clampControl(blinkValue); + + if (avatarState.currentVrm?.expressionManager) { + setExpressionValue("blink", clamped); + setExpressionValue("blinkLeft", clamped); + setExpressionValue("blinkRight", clamped); + } + + setMorphValue("blink", clamped); + setMorphValue("blinkLeft", clamped); + setMorphValue("blinkRight", clamped); + + const eyeScaleY = Math.max(0.08, 1 - clamped * 0.92); + const eyeScaleZ = 1 + clamped * 0.35; + if (avatarState.debugLeftEye) { + avatarState.debugLeftEye.scale.set(1, eyeScaleY, eyeScaleZ); + } + if (avatarState.debugRightEye) { + avatarState.debugRightEye.scale.set(1, eyeScaleY, eyeScaleZ); + } +} + function applyBoneRotation(bone, x = 0, y = 0, z = 0) { if (!bone) return; const baseQuaternion = avatarState.idleRig?.base?.get(bone); @@ -956,6 +1076,9 @@ function subscribeSubtitles() { subtitleWs.onmessage = (event) => { try { const data = JSON.parse(event.data); + if (!shouldRenderSubtitleEvent(data)) { + return; + } if (data.role === "user") { aiStreamingEl = null; addMessage(data.role, data.text || ""); @@ -973,7 +1096,14 @@ function subscribeSubtitles() { // no-op } }; - subtitleWs.onclose = () => { + subtitleWs.onclose = (event) => { + if (subtitleWs && subtitleWs.readyState === WebSocket.CLOSED) { + subtitleWs = null; + } + if (event.code === 1012) { + addMessage("system", "字幕通道已被新客户端接管。当前页面停止接收字幕。"); + return; + } setTimeout(subscribeSubtitles, 1000); }; } @@ -1001,8 +1131,15 @@ function subscribeAnimation() { // no-op } }; - animationWs.onclose = () => { + animationWs.onclose = (event) => { + if (animationWs && animationWs.readyState === WebSocket.CLOSED) { + animationWs = null; + } animationConn.textContent = "已断开"; + if (event.code === 1012) { + animationConn.textContent = "已被新客户端接管"; + return; + } setTimeout(subscribeAnimation, 1000); }; } diff --git a/web/index.html b/web/index.html index f887116..6e9f87a 100644 --- a/web/index.html +++ b/web/index.html @@ -96,6 +96,7 @@ } } - + +