diff --git a/.env b/.env new file mode 100644 index 0000000..d16657c --- /dev/null +++ b/.env @@ -0,0 +1,22 @@ +# LLM (DeepSeek, OpenAI-compatible) +# Fill your real key below. +LLM_API_KEY=sk-aac48115300845be97a032444fc11560 +LLM_BASE_URL=https://api.deepseek.com/v1 +LLM_MODEL=deepseek-chat +LLM_TIMEOUT=20 +LLM_MAX_TOKENS=300 + +# Web service +WEBRTC_HOST=0.0.0.0 +WEBRTC_PORT=8018 +STUN_URL=stun:stun.l.google.com:19302 +CORS_ORIGINS=* + +# HTTPS (for cross-device mic/WebRTC) +SSL_CERTFILE=/home/xsl/code/product/certs/dev-cert.pem +SSL_KEYFILE=/home/xsl/code/product/certs/dev-key.pem + +# 3D avatar driver mode +AVATAR_DRIVER_MODE=blendshape_stream +AVATAR_CONTROL_PROTOCOL=ws +AVATAR_BLENDSHAPE_SCHEMA=arkit diff --git a/.run-8018.log b/.run-8018.log new file mode 100644 index 0000000..29ad387 --- /dev/null +++ b/.run-8018.log @@ -0,0 +1,386 @@ +INFO: Started server process [401194] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit) +INFO: 127.0.0.1:43090 - "GET /health HTTP/1.1" 200 OK +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.) + return torch._C._cuda_getDeviceCount() > 0 +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect" +INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1 + super().__init__("LSTM", *args, **kwargs) +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + WeightNorm.apply(module, name, dim) +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found" +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + import pkg_resources +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found" +Building prefix dict from the default dictionary ... +DEBUG:jieba:Building prefix dict from the default dictionary ... +Loading model from cache /tmp/jieba.cache +DEBUG:jieba:Loading model from cache /tmp/jieba.cache +Loading model cost 0.358 seconds. +DEBUG:jieba:Loading model cost 0.358 seconds. +Prefix dict has been built successfully. +DEBUG:jieba:Prefix dict has been built successfully. +WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning. +INFO: 127.0.0.1:60258 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 127.0.0.1:40524 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:12419 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:9602 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:6375 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:6375 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:6375 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(5xwO9ewTsb 1 udp 1853759231 172.28.96.1 60702 typ prflx) +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(0) ICE completed +INFO: 172.28.96.1:9899 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(0) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:1631 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:6354 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:11967 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:2728 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO: 172.28.96.1:2728 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13395 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:4941 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:4941 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5793 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:3627 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(2) Discovered peer reflexive candidate Candidate(Ly5ELFkmha 1 udp 1853759231 172.28.96.1 50062 typ prflx) +INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(2) ICE completed +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(2) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:1723 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:14573 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:7534 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(3) Discovered peer reflexive candidate Candidate(sCNsxRJ8b8 1 udp 1853759231 172.28.96.1 64421 typ prflx) +INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(3) ICE completed +INFO: 172.28.96.1:7236 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 172.28.96.1:10460 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(3) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:14796 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:2032 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:14707 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:13486 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(4) Discovered peer reflexive candidate Candidate(Qady0BDpmC 1 udp 1853759231 172.28.96.1 53980 typ prflx) +INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(4) ICE completed +INFO: 172.28.96.1:13486 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:13486 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10285 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5486 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:13787 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:10285 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:10285 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(5) Discovered peer reflexive candidate Candidate(BLyMgNoiBf 1 udp 1853759231 172.28.96.1 55730 typ prflx) +INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(5) ICE completed +INFO: 172.28.96.1:8337 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(5) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:7750 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:1059 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:14463 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:14634 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:14634 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(6) Discovered peer reflexive candidate Candidate(dbehuF1olz 1 udp 1853759231 172.28.96.1 56094 typ prflx) +INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(6) ICE completed +INFO: connection closed +INFO: connection closed +INFO: 172.28.96.1:14493 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10905 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:10117 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:6762 - "GET /meta HTTP/1.1" 200 OK +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:6762 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(7) Discovered peer reflexive candidate Candidate(8K1dQt9yU0 1 udp 1853759231 172.28.96.1 49725 typ prflx) +INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(7) ICE completed +INFO: 172.28.96.1:14132 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:14132 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:2807 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:6331 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:5113 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:2807 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:2807 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(8) Discovered peer reflexive candidate Candidate(ROwaVxsM9g 1 udp 1853759231 172.28.96.1 52410 typ prflx) +INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(8) ICE completed +INFO: 172.28.96.1:10113 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:10113 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10113 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5903 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:3565 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:10113 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:10113 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(9) Discovered peer reflexive candidate Candidate(E8HKWSMFTW 1 udp 1853759231 172.28.96.1 50992 typ prflx) +INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(9) ICE completed +INFO: 172.28.96.1:13074 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: connection closed +INFO: 172.28.96.1:13074 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13074 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:7285 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:7285 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:13646 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:9569 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:2629 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:13646 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:13646 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(10) Discovered peer reflexive candidate Candidate(AbT1vCac8Y 1 udp 1853759231 172.28.96.1 62912 typ prflx) +INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(10) ICE completed +INFO: 172.28.96.1:6176 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(10) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 127.0.0.1:51958 - "GET /health HTTP/1.1" 200 OK +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [401194] +INFO: Started server process [404442] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit) +INFO: 127.0.0.1:34264 - "GET /health HTTP/1.1" 200 OK +INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.) + return torch._C._cuda_getDeviceCount() > 0 +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect" +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1 + super().__init__("LSTM", *args, **kwargs) +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + WeightNorm.apply(module, name, dim) +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + import pkg_resources +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found" +Building prefix dict from the default dictionary ... +DEBUG:jieba:Building prefix dict from the default dictionary ... +Loading model from cache /tmp/jieba.cache +DEBUG:jieba:Loading model from cache /tmp/jieba.cache +Loading model cost 0.384 seconds. +DEBUG:jieba:Loading model cost 0.384 seconds. +Prefix dict has been built successfully. +DEBUG:jieba:Prefix dict has been built successfully. +WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning. +INFO: 127.0.0.1:35522 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:14179 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:12729 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:3368 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:3368 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(jQ7HV4xKqV 1 udp 1853759231 172.28.96.1 52186 typ prflx) +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(0) ICE completed +INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK" +INFO: 172.28.96.1:8927 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(0) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [404442] diff --git a/AVATAR_3D_REFACTOR.md b/AVATAR_3D_REFACTOR.md new file mode 100644 index 0000000..675f261 --- /dev/null +++ b/AVATAR_3D_REFACTOR.md @@ -0,0 +1,189 @@ +# 3D 数字人驱动重构方案 + +## 1. 目标变更 + +原链路： + +ASR -> LLM -> TTS -> MuseTalk/图片或视频驱动 -> WebRTC 视频轨 -> 前端播放 + +新链路： + +ASR -> LLM -> TTS -> 音频特征/viseme/blendshape 控制数据 -> 前端 3D 渲染引擎 -> 浏览器展示 + +核心变化： +- 后端不再负责生成数字人视频帧 +- 后端保留实时语音、时序仲裁、字幕、音频下行 +- 后端新增 3D 控制流输出能力 +- 3D 模型渲染放到前端或独立渲染端 + +## 2. 为什么要这样改 + +MuseTalk 这类方案适合“2D 说话视频合成”，但不适合 3D 模型系统： +- 输出形态是视频帧，不是骨骼或 blendshape 控制数据 +- 一旦换 3D 模型，原本的视频生成结果无法直接复用 +- 每轮都做视频推理，延迟、GPU 占用、链路复杂度都偏高 +- 3D 系统通常需要的是时间序列控制参数，而不是渲染后的像素 + +因此，3D 化的正确边界是： +- 后端负责“说什么、怎么说、何时说” +- 渲染端负责“模型如何动、如何显示” + +## 3. 本次代码重构内容 + +当前代码已调整为： +- WebRTC 下行仅保留音频轨 +- 新增 `/ws/animation` WebSocket，下发 3D 动画控制数据 +- `services/avatar.py` 改为音频到控制参数的驱动服务 +- `/avatar/schema` 暴露控制协议说明 +- 前端采用 Three.js + GLTFLoader + VRM loader，直接在浏览器渲染 3D 模型 +- 支持加载本地 VRM/GLB 文件，也支持加载 URL 形式的 VRM/GLB/GLTF +- 若未加载模型，页面会回退到一个调试用的 3D 头像，便于联调口型和头部姿态 + +当前控制协议字段包括： +- `jawOpen` +- `mouthClose` +- `mouthFunnel` +- `mouthPucker` +- `viseme_aa` +- `viseme_ee` +- `viseme_oh` +- `headYaw` +- `headPitch` +- `headRoll` + +输出包结构示例： + +```json +{ + "type": "animation_chunk", + "driver": "arkit-blendshape-v1", + "mode": "blendshape_stream", + "schema": "arkit", + "chunk_index": 0, + "total_chunks": 3, + "sample_rate": 24000, + "fps": 25, + "text": "你好，欢迎使用系统。", + "frame_count": 18, + "duration_ms": 720, + "frames": [ + { + "seq": 102, + "time_ms": 0, + "speaking": true, + "controls": { + "jawOpen": 0.41, + "mouthClose": 0.66, + "mouthFunnel": 0.27, + "mouthPucker": 0.19, + "viseme_aa": 0.41, + "viseme_ee": 0.24, + "viseme_oh": 0.31, + "headYaw": 0.004, + "headPitch": 0.009, + "headRoll": 0.001 + } + } + ] +} +``` + +## 4. 当前实现的定位 + +当前实现是“联调级骨架”，作用是： +- 先把系统边界改对 +- 让前后端可以并行开发 +- 让 3D 引擎接入方尽快有稳定协议可以消费 + +它不是最终口型算法，原因很明确： +- 目前控制值来自音频能量和频谱特征 +- 没有做 phoneme 强制对齐 +- 没有做语言相关 viseme 映射 +- 没有结合 3D 模型自身 rig 约束做校准 + +所以这版适合： +- 打通链路 +- 调接口 +- 验证时序 +- 校验驱动字段 + +不适合直接作为最终高保真口型方案。 + +## 5. 推荐的生产化方案 + +### 方案 A：前端 Three.js / React Three Fiber + +适合： +- 浏览器直接渲染 WebGL 角色 +- glTF/VRM 模型 +- ARKit blendshape 或自定义 morph target + +建议做法： +- 前端加载 3D 模型 +- 建立 `controlsKey -> morphTargetInfluence` 映射表 +- `/ws/animation` 按时间戳缓冲一小段再播放 +- 音频继续使用 WebRTC 远端音频轨 +- 前端用远端音频播放时间作为动画时钟基准 + +当前仓库已经按这个方向落地了第一版： +- 静态页面通过 import map 从 CDN 引入 `three`、`GLTFLoader`、`@pixiv/three-vrm` +- VRM 优先使用 `expressionManager` 写入 `aa/ee/oh/ou` +- 普通 glTF 则尝试把控制字段映射到 morph target +- 头部姿态优先驱动 `head/neck` 骨骼，找不到时回退调试头像 + +### 方案 B：Unity/Unreal 渲染端 + +适合： +- 更复杂的材质、灯光、骨骼、表情系统 +- 要求更强的 3D 表现力 + +建议做法： +- 浏览器只做 UI 和音频交互 +- Unity/Unreal 作为单独渲染客户端消费 `/ws/animation` +- 浏览器与渲染端通过流媒体或共享画面集成 + +### 方案 C：专用音频驱动服务 + +适合： +- 追求更准的嘴型 +- 后端可以接受增加一个模型服务 + +可以考虑接入： +- phoneme 对齐模型 +- viseme 预测模型 +- NVIDIA Audio2Face 类服务 +- 自研 `audio -> blendshape` 网络 + +推荐协议保持不变： +- 外部模型服务可以替换 `services/avatar.py` +- 对外仍输出统一 `frames[].controls` + +## 6. 推荐的下一步实施顺序 + +1. 先确定 3D 模型和渲染端技术栈 +2. 固定一份 blendshape 映射表 +3. 把当前前端调试面板替换成真实 3D 场景 +4. 增加时间同步与缓冲策略 +5. 再替换掉启发式控制算法，接入更强的 viseme 模型 + +## 7. 接口约束建议 + +为了后续可替换，建议保持这些约束不变： +- 下行动画通道继续使用 `/ws/animation` +- 每条消息包含 `frame_count`、`fps`、`frames` +- 每帧保留 `seq` 和 `time_ms` +- `controls` 使用稳定字段名，不轻易改动 +- 新增字段时只追加，不破坏已有字段 + +## 8. 风险与注意事项 + +- 真实 3D 模型的 blendshape 命名不一定和当前字段一致，需要做一次映射层 +- 如果模型是骨骼嘴型而不是 morph target，需要把 `controls` 转成骨骼驱动参数 +- 浏览器渲染时要注意音频时钟和动画时钟一致，否则会出现“听起来对，但嘴型慢半拍” +- 如果后续改成 phoneme/viseme 对齐，建议在包里增加 phoneme 片段和置信度字段 + +## 9. 结论 + +这次重构的重点不是“把 2D 数字人继续硬改下去”，而是把系统边界调整到适合 3D 模型的形态。 + +现在的系统已经从“视频合成型数字人”切到了“控制流型数字人”，后续无论接 Three.js、Unity、Unreal，还是专门的 Audio2Face/viseme 服务，都可以在这个边界上继续演进。 diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md index 486221e..05f72dd 100644 --- a/DEPLOYMENT_GUIDE.md +++ b/DEPLOYMENT_GUIDE.md @@ -15,7 +15,7 @@ - 用户语音上行（VAD + ASR） - 文本调用在线 LLM - 本地 TTS 合成语音 -- 数字人视频下行（MuseTalk 优先，失败回退写实底片） +- WebRTC 音频下行 + 3D 数字人控制流下行（WebSocket） - 前端字幕与指标面板（SSE + WebSocket） - 单工仲裁（支持 barge-in 打断） @@ -36,10 +36,10 @@ - `llm.py`：在线模型 + fallback - `tts.py`：Kokoro - `vad.py`：Silero VAD - - `avatar.py`：MuseTalk 推理调度 + 写实回退帧 + - `avatar.py`：音频转 3D 控制数据（blendshape / 头部姿态） - `webrtc/` - - `tracks.py`：实际音视频轨道实现 - - `gateway.py`、`video_track.py`、`audio_track.py`：结构化封装 + - `tracks.py`：实际音频轨道实现 + - `gateway.py`、`audio_track.py`：结构化封装 - `web/` - `index.html`、`app.js`、`style.css` - `scripts/` @@ -192,11 +192,18 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c ## 7. 前端功能说明 网页包含： -- 视频窗口（数字人） +- Three.js 3D 数字人舞台 - 文本输入框 - 字幕消息区（WebSocket） - 指标面板（SSE） +当前前端能力： +- 浏览器通过 import map 直接加载 Three.js、GLTFLoader、three-vrm +- 可加载本地 `VRM`/`GLB` 文件 +- 可加载 URL 形式的 `VRM`/`GLB`/`GLTF` 模型 +- 页面默认会自动尝试加载官方 `three-vrm` 示例模型：`VRM1_Constraint_Twist_Sample.vrm` +- 未加载真实模型时会显示调试头像，仍可联调动画控制流 + 当前已支持： - 页面刷新自动重连 - 单连接策略（新连接踢掉旧连接） @@ -207,9 +214,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c ## 8. 当前已知边界 -- MuseTalk 结果生成可能有一定延迟，期间使用写实底片回退。 -- 部分“主观体验类”验收项（如口型同步主观评分）需要人工实测判定。 -- Qwen3-TTS 对比方案尚未并入主链路（可后续做 A/B 开关）。 +- 当前后端输出的是可直接对接 3D 引擎的控制数据，不负责最终 3D 渲染。 +- 当前控制数据基于音频能量和频谱特征做启发式推断，适合作为联调骨架，不等同于生产级口型模型。 +- 若要达到更高口型精度，建议后续接入 phoneme/viseme 对齐模型或 Audio2Face 类推理服务。 --- @@ -224,8 +231,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c - 系统是单连接模式，新连接会断开旧连接（预期） 3) 口型不动 -- 查看 `/health` 中 `avatar.queued_frames` 和 `avatar.last_error` -- 若 MuseTalk 未产帧，会回退写实底片 + 口型叠加 +- 查看 `/health` 中 `avatar.last_frame_count` 和 `avatar.last_error` +- 查看前端 `/ws/animation` 是否已连接，控制流预览是否持续刷新 +- 若模型已加载但不动，先检查该模型是否带有 VRM expression 或 morph target 4) 模型异常 - 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD diff --git a/TECHNICAL_DOCUMENTATION.md b/TECHNICAL_DOCUMENTATION.md new file mode 100644 index 0000000..89e15a6 --- /dev/null +++ b/TECHNICAL_DOCUMENTATION.md @@ -0,0 +1,278 @@ +# 3D Digital Human System Technical Documentation + +## 1. Project Summary + +This project implements a real-time conversational digital human system with these core capabilities: + +- Text and voice interaction +- LLM response generation (DeepSeek online, with fallback) +- TTS synthesis for reply audio +- Audio-driven 3D animation control stream +- Browser-side 3D rendering (Three.js + VRM/GLTF) +- WebRTC audio transport and WebSocket animation transport + +Current workspace root: + +- `/home/xsl/code/product` + +## 2. Runtime Architecture + +### 2.1 High-Level Pipeline + +1. User input enters through `/chat/text` or voice pipeline. +2. LLM generates reply text. +3. TTS synthesizes reply waveform. +4. Avatar service converts audio chunks to animation control frames. +5. Backend pushes: + - reply audio to WebRTC audio track + - animation frames to `/ws/animation` +6. Frontend renders 3D model and applies mouth/head/body motions. + +### 2.2 Transport Split + +- WebRTC: audio only +- WebSocket (`/ws/animation`): animation control data +- SSE (`/events`): system metrics and runtime status + +This split removes server-side video generation and makes rendering native in the browser. + +## 3. Core Modules + +### 3.1 Backend Entry + +- `main.py` + +Responsibilities: + +- FastAPI app and route registration +- WebRTC offer/peer management +- Animation WebSocket broadcasting +- Runtime metrics and `/health` +- Session reset and chat endpoints + +### 3.2 Pipeline Orchestration + +- `core/pipeline.py` +- `core/state_machine.py` + +Responsibilities: + +- Turn-based orchestration (input -> LLM -> TTS -> output) +- State transitions (idle, thinking, speaking) +- Latency accounting and result metadata + +### 3.3 Services + +- `services/llm.py` +- `services/tts.py` +- `services/asr.py` +- `services/vad.py` +- `services/avatar.py` + +Responsibilities: + +- `LLMService`: online model calls via OpenAI-compatible client +- `TTSService`: speech synthesis +- `AvatarService`: audio-to-controls mapping for mouth/viseme/head motion + +### 3.4 Frontend + +- `web/index.html` +- `web/app.js` +- `web/style.css` + +Responsibilities: + +- Three.js scene setup and camera controls +- VRM/GLTF model loading (URL or local file) +- Animation buffering and frame application +- Idle body pose + talking expressions +- WebRTC audio playback + +## 4. Animation Data Contract + +Backend pushes animation chunks shaped like: + +- `type`: `animation_chunk` / `animation_reset` / `animation_state` +- `fps`, `duration_ms`, `frame_count` +- `frames[]` with: + - `seq` + - `time_ms` + - `controls` + +Typical controls include: + +- `jawOpen` +- `viseme_aa`, `viseme_ee`, `viseme_oh` +- `mouthPucker` +- `headYaw`, `headPitch`, `headRoll` + +Frontend behavior: + +- buffers frames for stable playback +- applies VRM expressions when available +- falls back to morph targets or bone motion + +## 5. Configuration + +Configuration file: + +- `.env` (root) + +### 5.1 LLM Settings + +Supported API key aliases: + +- `LLM_API_KEY` +- `DEEPSEEK_API_KEY` +- `OPENAI_API_KEY` + +Recommended DeepSeek settings: + +- `LLM_BASE_URL=https://api.deepseek.com/v1` +- `LLM_MODEL=deepseek-chat` + +### 5.2 Server Settings + +- `WEBRTC_HOST` (default `0.0.0.0`) +- `WEBRTC_PORT` (default `8018`) +- `SSL_CERTFILE` +- `SSL_KEYFILE` + +### 5.3 Avatar Settings + +- `AVATAR_DRIVER_MODE=blendshape_stream` +- `AVATAR_CONTROL_PROTOCOL=ws` +- `AVATAR_BLENDSHAPE_SCHEMA=arkit` + +## 6. Startup and Process Control + +New process script: + +- `scripts/service-8018.sh` + +Usage: + +```bash +bash scripts/service-8018.sh start +bash scripts/service-8018.sh status +bash scripts/service-8018.sh logs +bash scripts/service-8018.sh restart +bash scripts/service-8018.sh stop +``` + +Behavior: + +- Loads environment from `.env` +- Starts uvicorn from project venv if available +- Supports HTTPS cert/key from `.env` +- Writes pid and log to `.run/` +- Cleans stale python listeners on target port + +Runtime files: + +- `.run/visual-chat-8018.pid` +- `.run/visual-chat-8018.log` + +## 7. API and Web Endpoints + +### 7.1 Health and Metadata + +- `GET /health` +- `GET /meta` + +### 7.2 Chat and Session + +- `POST /chat/text` +- `POST /chat/reset` + +### 7.3 Streaming Channels + +- `GET /events` (SSE) +- `GET /ws/subtitles` (WebSocket) +- `GET /ws/animation` (WebSocket) + +### 7.4 WebRTC + +- `POST /webrtc/offer` + +## 8. Validation Checklist + +### 8.1 Service Up + +- `bash scripts/service-8018.sh status` should show running +- `GET /health` should return valid JSON + +### 8.2 LLM Online + +Health fields should indicate: + +- `llm.ready = true` +- `llm.error = null` +- `last_llm_source = online` after at least one chat turn + +### 8.3 Frontend Motion and Audio + +- Model loads successfully +- Mouth motion follows replies +- Body remains in non-T-pose idle stance +- Remote audio is audible via WebRTC + +## 9. Troubleshooting Guide + +### 9.1 LLM stays fallback + +Symptoms: + +- `last_llm_source = fallback` +- `llm.ready = false` + +Checks: + +1. Confirm `.env` exists in workspace root. +2. Confirm key value is set in `LLM_API_KEY` (or alias). +3. Restart service after any `.env` update. +4. Verify `llm.base_url` and `llm.model` in `/health`. + +### 9.2 Model load errors + +Symptoms: + +- Web UI shows model fetch failures + +Checks: + +1. Use reachable URL or local `.vrm` file. +2. Confirm browser can access external CDN. +3. Prefer VRM full-body models for body-pose control. + +### 9.3 Mouth not moving + +Checks: + +1. Confirm animation websocket connected. +2. Confirm animation frame count increases. +3. Confirm TTS is ready and not in silence fallback. + +### 9.4 HTTPS/microphone issues on LAN + +Checks: + +1. Use valid cert paths in `.env`. +2. Trust self-signed cert on browser/device. +3. Ensure firewall allows target port. + +## 10. Security and Operations Notes + +- Keep `.env` out of source control. +- Rotate API keys regularly. +- Restrict exposed port by network policy if internet-facing. +- Consider reverse proxy and managed TLS for production. + +## 11. Suggested Next Production Steps + +1. Add structured request tracing (request id, turn id). +2. Add persistent metrics export (Prometheus/OpenTelemetry). +3. Add automatic reconnect and health watchdog. +4. Add integration tests for LLM online/fallback branches. +5. Add model asset mirror to avoid public CDN instability. diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..b7995bc Binary files /dev/null and b/__pycache__/config.cpython-312.pyc differ diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..8af30ff Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/config.py b/config.py index 4592c08..07d6aaf 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,4 @@ +from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -5,20 +6,33 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") hf_token: str = "" - llm_api_key: str = "" - llm_base_url: str = "https://api.openai.com/v1" - llm_model: str = "gpt-4o-mini" + llm_api_key: str = Field( + default="", + validation_alias=AliasChoices("LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY"), + ) + llm_base_url: str = Field( + default="https://api.deepseek.com/v1", + validation_alias=AliasChoices("LLM_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"), + ) + llm_model: str = Field( + default="deepseek-chat", + validation_alias=AliasChoices("LLM_MODEL", "DEEPSEEK_MODEL", "OPENAI_MODEL"), + ) llm_timeout: int = 8 llm_max_tokens: int = 150 webrtc_host: str = "0.0.0.0" webrtc_port: int = 8080 avatar_fps: int = 25 + avatar_driver_mode: str = "blendshape_stream" + avatar_control_protocol: str = "ws" + avatar_blendshape_schema: str = "arkit" stun_url: str = "stun:stun.l.google.com:19302" cors_origins: str = "*" ssl_certfile: str = "" ssl_keyfile: str = "" + # Legacy MuseTalk settings retained for migration reference only. musetalk_enabled: bool = True musetalk_repo_dir: str = "/home/xsl/work/MuseTalk" musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py" diff --git a/core/__pycache__/__init__.cpython-312.pyc b/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..9f4d1f8 Binary files /dev/null and b/core/__pycache__/__init__.cpython-312.pyc differ diff --git a/core/__pycache__/pipeline.cpython-312.pyc b/core/__pycache__/pipeline.cpython-312.pyc new file mode 100644 index 0000000..b932f40 Binary files /dev/null and b/core/__pycache__/pipeline.cpython-312.pyc differ diff --git a/core/__pycache__/state_machine.cpython-312.pyc b/core/__pycache__/state_machine.cpython-312.pyc new file mode 100644 index 0000000..47f367c Binary files /dev/null and b/core/__pycache__/state_machine.cpython-312.pyc differ diff --git a/core/pipeline.py b/core/pipeline.py index 299792f..bf699cf 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -11,6 +11,9 @@ from services.llm import LLMService from services.tts import TTSService +AudioChunkCallback = Callable[[np.ndarray, int, str, int, int], Awaitable[None]] + + class ChatPipeline: def __init__( self, @@ -75,7 +78,7 @@ class ChatPipeline: async def _synthesize_sentence_first( self, reply_text: str, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, should_interrupt: Optional[Callable[[], bool]] = None, ) -> tuple[np.ndarray, int, int, int]: @@ -94,7 +97,7 @@ class ChatPipeline: if i == 0: first_chunk_ms = int((time.perf_counter() - t0) * 1000) if on_audio_chunk is not None: - await on_audio_chunk(seg_audio, seg_sr) + await on_audio_chunk(seg_audio, seg_sr, seg, i, len(segments)) audio_chunks.append(seg_audio) tts_total_ms = int((time.perf_counter() - t0) * 1000) audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32) @@ -103,7 +106,7 @@ class ChatPipeline: async def process_turn( self, audio_16k: np.ndarray, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_user_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, @@ -145,7 +148,7 @@ class ChatPipeline: async def process_text_turn( self, user_text: str, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, ) -> tuple[dict, np.ndarray, int]: diff --git a/main.py b/main.py index 611c7c6..90420b1 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,9 @@ from __future__ import annotations -import os -import logging import asyncio import json +import logging +import os import time from dataclasses import dataclass, field @@ -11,8 +11,8 @@ import numpy as np from aiortc import RTCPeerConnection, RTCSessionDescription from dotenv import load_dotenv from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect -from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel @@ -24,7 +24,7 @@ from services.avatar import AvatarService from services.llm import LLMService from services.tts import TTSService from services.vad import VADService -from webrtc.tracks import AudioBus, AvatarAudioTrack, AvatarVideoTrack +from webrtc.tracks import AudioBus, AvatarAudioTrack load_dotenv() if settings.hf_token: @@ -51,6 +51,7 @@ pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service) audio_bus = AudioBus(sample_rate=48000) pcs: set[RTCPeerConnection] = set() subtitle_clients: set[WebSocket] = set() +animation_clients: set[WebSocket] = set() @dataclass @@ -69,6 +70,8 @@ class RuntimeState: last_input_mode: str = "none" vad_start_count: int = 0 vad_end_count: int = 0 + last_animation_mode: str = settings.avatar_driver_mode + last_animation_frame_count: int = 0 busy: bool = False buffer_16k: list[np.ndarray] = field(default_factory=list) @@ -80,6 +83,18 @@ class TextChatRequest(BaseModel): text: str +async def _broadcast_json(clients: set[WebSocket], payload: dict) -> None: + stale: list[WebSocket] = [] + message = json.dumps(payload, ensure_ascii=False) + for ws in list(clients): + try: + await ws.send_text(message) + except Exception: + stale.append(ws) + for ws in stale: + clients.discard(ws) + + async def _broadcast_subtitle( role: str, text: str, @@ -90,22 +105,54 @@ async def _broadcast_subtitle( ) -> None: if not text: return - payload = { - "role": role, - "text": text, - "source": source, - "partial": partial, - "final": final, - "ts_ms": int(time.time() * 1000), + await _broadcast_json( + subtitle_clients, + { + "role": role, + "text": text, + "source": source, + "partial": partial, + "final": final, + "ts_ms": int(time.time() * 1000), + }, + ) + + +async def _broadcast_animation(payload: dict) -> None: + runtime.last_animation_mode = str(payload.get("driver", settings.avatar_driver_mode)) + runtime.last_animation_frame_count = int(payload.get("frame_count", 0)) + await _broadcast_json(animation_clients, payload) + + +def _runtime_payload() -> dict: + return { + "state": str(arbitrator.state), + "peers": len(pcs), + "subtitle_clients": len(subtitle_clients), + "animation_clients": len(animation_clients), + "pipeline_runs": runtime.pipeline_runs, + "last_latency_ms": runtime.last_latency_ms, + "last_asr_latency_ms": runtime.last_asr_latency_ms, + "last_llm_latency_ms": runtime.last_llm_latency_ms, + "last_llm_source": runtime.last_llm_source, + "last_tts_latency_ms": runtime.last_tts_latency_ms, + "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, + "last_barge_in_ms": runtime.last_barge_in_ms, + "barge_in_count": runtime.barge_in_count, + "last_input_mode": runtime.last_input_mode, + "vad_start_count": runtime.vad_start_count, + "vad_end_count": runtime.vad_end_count, + "last_user_text": runtime.last_user_text, + "last_reply_text": runtime.last_reply_text, + "last_animation_mode": runtime.last_animation_mode, + "last_animation_frame_count": runtime.last_animation_frame_count, + "pipeline_busy": runtime.busy, + "llm": llm_service.health, + "asr": asr_service.health, + "tts": tts_service.health, + "vad": vad_service.health, + "avatar": avatar_service.health, } - stale: list[WebSocket] = [] - for ws in list(subtitle_clients): - try: - await ws.send_text(json.dumps(payload, ensure_ascii=False)) - except Exception: - stale.append(ws) - for ws in stale: - subtitle_clients.discard(ws) def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: @@ -119,7 +166,6 @@ def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray: - # aiortc audio ndarray is typically (channels, samples) if pcm.ndim == 2: mono = pcm.mean(axis=0) else: @@ -135,10 +181,28 @@ def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> n return np.interp(x_new, x_old, mono).astype(np.float32) +async def _enqueue_audio_and_animation( + seg_audio: np.ndarray, + seg_sr: int, + seg_text: str, + idx: int, + total: int, +) -> None: + audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) + pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_seg) + animation_payload = await avatar_service.build_controls_from_audio( + seg_audio, + seg_sr, + text=seg_text, + chunk_index=idx, + total_chunks=total, + ) + await _broadcast_animation(animation_payload) + + async def _run_pipeline_from_buffer() -> None: - if runtime.busy: - return - if not runtime.buffer_16k: + if runtime.busy or not runtime.buffer_16k: return runtime.busy = True @@ -147,10 +211,6 @@ async def _run_pipeline_from_buffer() -> None: runtime.last_input_mode = "voice" audio_16k = np.concatenate(runtime.buffer_16k, axis=0) runtime.buffer_16k.clear() - async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: - audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) - pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) - await audio_bus.enqueue(pcm_seg) async def on_user_text_now(user_text: str) -> None: runtime.last_user_text = user_text @@ -164,12 +224,11 @@ async def _run_pipeline_from_buffer() -> None: result, audio, sr = await pipeline.process_turn( audio_16k, - on_audio_chunk=on_audio_chunk, + on_audio_chunk=_enqueue_audio_and_animation, on_user_text=on_user_text_now, on_reply_text=on_reply_text_now, on_reply_segment=on_reply_segment_now, ) - asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) runtime.last_user_text = result["user_text"] runtime.last_reply_text = result["reply_text"] runtime.pipeline_runs += 1 @@ -179,11 +238,11 @@ async def _run_pipeline_from_buffer() -> None: runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) - # Fallback: if chunk callback path produced nothing, enqueue full audio. if result.get("audio_samples", 0) <= 1: audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await audio_bus.enqueue(pcm_int16) + await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"])) finally: runtime.busy = False @@ -200,7 +259,6 @@ async def _consume_user_audio(track) -> None: runtime.buffer_16k.append(mono_16k) vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0) - # Silero VAD requires exact 512 samples at 16k. while vad_buffer.shape[0] >= 512: chunk = vad_buffer[:512] vad_buffer = vad_buffer[512:] @@ -214,6 +272,7 @@ async def _consume_user_audio(track) -> None: was_avatar_speaking = str(arbitrator.state) == "avatar_speaking" await arbitrator.on_speech_start() await audio_bus.clear() + await _broadcast_animation(await avatar_service.build_reset_payload(reason="speech-start")) if was_avatar_speaking: runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000) runtime.barge_in_count += 1 @@ -226,30 +285,7 @@ async def _consume_user_audio(track) -> None: @app.get("/health") async def health(): - return { - "state": arbitrator.state, - "peers": len(pcs), - "pipeline_runs": runtime.pipeline_runs, - "last_latency_ms": runtime.last_latency_ms, - "last_asr_latency_ms": runtime.last_asr_latency_ms, - "last_llm_latency_ms": runtime.last_llm_latency_ms, - "last_llm_source": runtime.last_llm_source, - "last_tts_latency_ms": runtime.last_tts_latency_ms, - "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, - "last_barge_in_ms": runtime.last_barge_in_ms, - "barge_in_count": runtime.barge_in_count, - "last_input_mode": runtime.last_input_mode, - "vad_start_count": runtime.vad_start_count, - "vad_end_count": runtime.vad_end_count, - "last_user_text": runtime.last_user_text, - "last_reply_text": runtime.last_reply_text, - "pipeline_busy": runtime.busy, - "llm": llm_service.health, - "asr": asr_service.health, - "tts": tts_service.health, - "vad": vad_service.health, - "avatar": avatar_service.health, - } + return _runtime_payload() @app.get("/meta") @@ -259,34 +295,20 @@ async def meta(): "https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile), "host": settings.webrtc_host, "port": settings.webrtc_port, + "avatar_protocol": settings.avatar_control_protocol, } +@app.get("/avatar/schema") +async def avatar_schema(): + return avatar_service.schema + + @app.get("/events") async def events(): async def gen(): while True: - payload = { - "state": str(arbitrator.state), - "peers": len(pcs), - "pipeline_runs": runtime.pipeline_runs, - "last_latency_ms": runtime.last_latency_ms, - "last_asr_latency_ms": runtime.last_asr_latency_ms, - "last_llm_latency_ms": runtime.last_llm_latency_ms, - "last_llm_source": runtime.last_llm_source, - "last_tts_latency_ms": runtime.last_tts_latency_ms, - "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, - "last_barge_in_ms": runtime.last_barge_in_ms, - "barge_in_count": runtime.barge_in_count, - "last_input_mode": runtime.last_input_mode, - "vad_start_count": runtime.vad_start_count, - "vad_end_count": runtime.vad_end_count, - "last_user_text": runtime.last_user_text, - "last_reply_text": runtime.last_reply_text, - "pipeline_busy": runtime.busy, - "avatar": avatar_service.health, - } - yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + yield f"data: {json.dumps(_runtime_payload(), ensure_ascii=False)}\n\n" await asyncio.sleep(1.0) return StreamingResponse(gen(), media_type="text/event-stream") @@ -306,9 +328,24 @@ async def ws_subtitles(websocket: WebSocket): subtitle_clients.discard(websocket) +@app.websocket("/ws/animation") +async def ws_animation(websocket: WebSocket): + await websocket.accept() + animation_clients.add(websocket) + try: + await websocket.send_text(json.dumps({"type": "animation_ready", **avatar_service.schema}, ensure_ascii=False)) + await websocket.send_text(json.dumps(await avatar_service.build_idle_payload(reason="client-connected"), ensure_ascii=False)) + while True: + await websocket.send_text(json.dumps({"type": "ping"}, ensure_ascii=False)) + await asyncio.sleep(15) + except WebSocketDisconnect: + animation_clients.discard(websocket) + except Exception: + animation_clients.discard(websocket) + + @app.post("/demo/run-once") async def run_once(): - # 3 seconds of silence placeholder; replace with real microphone stream path. fake_audio = np.zeros(16000 * 3, dtype=np.float32) await arbitrator.on_speech_start() result = await pipeline.run_once(fake_audio) @@ -326,11 +363,6 @@ async def chat_text(req: TextChatRequest): runtime.busy = True start_t = time.perf_counter() try: - async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: - audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) - pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) - await audio_bus.enqueue(pcm_seg) - async def on_reply_text_now(reply_text: str) -> None: runtime.last_reply_text = reply_text @@ -338,13 +370,13 @@ async def chat_text(req: TextChatRequest): await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1)) await _broadcast_subtitle("user", text, "text") + await _broadcast_animation(await avatar_service.build_reset_payload(reason="text-turn-start")) result, audio, sr = await pipeline.process_text_turn( text, - on_audio_chunk=on_audio_chunk, + on_audio_chunk=_enqueue_audio_and_animation, on_reply_text=on_reply_text_now, on_reply_segment=on_reply_segment_now, ) - asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) runtime.last_input_mode = "text" runtime.last_user_text = result["user_text"] runtime.last_reply_text = result["reply_text"] @@ -355,11 +387,11 @@ async def chat_text(req: TextChatRequest): runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) - # Fallback: if no chunk was enqueued for some reason, enqueue full audio. if result.get("audio_samples", 0) <= 1: audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await audio_bus.enqueue(pcm_int16) + await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"])) return {"ok": True, **result} finally: runtime.busy = False @@ -382,13 +414,14 @@ async def chat_reset(): runtime.last_input_mode = "none" runtime.vad_start_count = 0 runtime.vad_end_count = 0 + runtime.last_animation_frame_count = 0 + await _broadcast_animation(await avatar_service.build_reset_payload(reason="chat-reset")) return {"ok": True} @app.get("/demo/frame-idle-shape") async def frame_idle_shape(): - frame = await avatar_service.idle_frame() - return {"shape": list(frame.shape)} + return await avatar_service.build_idle_payload(reason="demo") @app.get("/") @@ -401,7 +434,6 @@ async def webrtc_offer(request: Request): params = await request.json() offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) - # Single-connection mode: kick existing peers before accepting new one. replaced_previous = len(pcs) > 0 if replaced_previous: old_peers = list(pcs) @@ -423,7 +455,6 @@ async def webrtc_offer(request: Request): if track.kind == "audio": asyncio.create_task(_consume_user_audio(track)) - pc.addTrack(AvatarVideoTrack(avatar_service, arbitrator, audio_bus, settings.avatar_fps)) pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus)) await pc.setRemoteDescription(offer) diff --git a/scripts/service-8018.sh b/scripts/service-8018.sh new file mode 100644 index 0000000..c73d4de --- /dev/null +++ b/scripts/service-8018.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUN_DIR="$ROOT_DIR/.run" +PID_FILE="$RUN_DIR/visual-chat-8018.pid" +LOG_FILE="$RUN_DIR/visual-chat-8018.log" +ENV_FILE="$ROOT_DIR/.env" + +mkdir -p "$RUN_DIR" + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +HOST="${WEBRTC_HOST:-0.0.0.0}" +PORT="${WEBRTC_PORT:-8018}" +SSL_CERTFILE="${SSL_CERTFILE:-$ROOT_DIR/certs/dev-cert.pem}" +SSL_KEYFILE="${SSL_KEYFILE:-$ROOT_DIR/certs/dev-key.pem}" + +if [[ -x "$ROOT_DIR/.venv/bin/python" ]]; then + PYTHON_BIN="$ROOT_DIR/.venv/bin/python" +else + PYTHON_BIN="$(command -v python3 || true)" +fi + +if [[ -z "$PYTHON_BIN" ]]; then + echo "python3 not found" + exit 1 +fi + +port_pids() { + ss -ltnp 2>/dev/null \ + | sed -n "s/.*:${PORT} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" \ + | tr '\n' ' ' +} + +is_running() { + if [[ ! -f "$PID_FILE" ]]; then + return 1 + fi + local pid + pid="$(cat "$PID_FILE")" + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null +} + +start() { + if is_running; then + echo "already running: pid=$(cat "$PID_FILE")" + return 0 + fi + + local stale + stale="$(port_pids)" + if [[ -n "${stale// /}" ]]; then + echo "killing stale python on port ${PORT}: $stale" + # shellcheck disable=SC2086 + kill $stale || true + sleep 1 + fi + + local args + args=(main:app --host "$HOST" --port "$PORT") + if [[ -n "$SSL_CERTFILE" && -n "$SSL_KEYFILE" ]]; then + args+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE") + fi + + nohup "$PYTHON_BIN" -m uvicorn "${args[@]}" >>"$LOG_FILE" 2>&1 & + local pid=$! + echo "$pid" >"$PID_FILE" + sleep 1 + + if kill -0 "$pid" 2>/dev/null; then + echo "started: pid=$pid" + echo "url: https://$HOST:$PORT" + echo "log: $LOG_FILE" + else + echo "failed to start, see log: $LOG_FILE" + rm -f "$PID_FILE" + return 1 + fi +} + +stop() { + if ! is_running; then + echo "not running" + rm -f "$PID_FILE" + return 0 + fi + + local pid + pid="$(cat "$PID_FILE")" + kill "$pid" || true + + for _ in {1..20}; do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.2 + done + + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" || true + fi + + rm -f "$PID_FILE" + echo "stopped" +} + +status() { + if is_running; then + echo "running: pid=$(cat "$PID_FILE")" + return 0 + fi + echo "not running" + return 1 +} + +logs() { + if [[ -f "$LOG_FILE" ]]; then + tail -n 120 "$LOG_FILE" + else + echo "no log file yet" + fi +} + +restart() { + stop || true + start +} + +case "${1:-}" in + start) start ;; + stop) stop ;; + status) status ;; + restart) restart ;; + logs) logs ;; + *) + echo "usage: $0 {start|stop|status|restart|logs}" + exit 2 + ;; +esac diff --git a/services/__pycache__/__init__.cpython-312.pyc b/services/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..02f4042 Binary files /dev/null and b/services/__pycache__/__init__.cpython-312.pyc differ diff --git a/services/__pycache__/asr.cpython-312.pyc b/services/__pycache__/asr.cpython-312.pyc new file mode 100644 index 0000000..78649ba Binary files /dev/null and b/services/__pycache__/asr.cpython-312.pyc differ diff --git a/services/__pycache__/avatar.cpython-312.pyc b/services/__pycache__/avatar.cpython-312.pyc new file mode 100644 index 0000000..1fad129 Binary files /dev/null and b/services/__pycache__/avatar.cpython-312.pyc differ diff --git a/services/__pycache__/llm.cpython-312.pyc b/services/__pycache__/llm.cpython-312.pyc new file mode 100644 index 0000000..8ab10f5 Binary files /dev/null and b/services/__pycache__/llm.cpython-312.pyc differ diff --git a/services/__pycache__/tts.cpython-312.pyc b/services/__pycache__/tts.cpython-312.pyc new file mode 100644 index 0000000..2b7e5ea Binary files /dev/null and b/services/__pycache__/tts.cpython-312.pyc differ diff --git a/services/__pycache__/vad.cpython-312.pyc b/services/__pycache__/vad.cpython-312.pyc new file mode 100644 index 0000000..b83adb8 Binary files /dev/null and b/services/__pycache__/vad.cpython-312.pyc differ diff --git a/services/avatar.py b/services/avatar.py index b19a35d..0c1251e 100644 --- a/services/avatar.py +++ b/services/avatar.py @@ -1,193 +1,213 @@ from __future__ import annotations import asyncio -import os -import sys -import uuid -import wave -from collections import deque -from pathlib import Path +import math +import time +from typing import Any -import cv2 import numpy as np from config import settings class AvatarService: - def __init__(self, width: int = 640, height: int = 360) -> None: - self.width = width - self.height = height - self._frames: deque[np.ndarray] = deque() + def __init__(self) -> None: + self.driver_name = "arkit-blendshape-v1" self._lock = asyncio.Lock() - self._job_lock = asyncio.Lock() + self._sequence = 0 self._last_error: str | None = None - self._base_frame = self._load_base_frame() + self._last_frame_count = 0 + self._last_chunk_duration_ms = 0 - async def render_frame(self, mouth_open: float = 0.0, speaking: bool = False) -> np.ndarray: - frame = self._base_frame.copy() - if speaking: - alpha = float(np.clip(mouth_open, 0.0, 1.0)) - # Slightly boost contrast/brightness while speaking to keep a "live" feeling. - frame = cv2.convertScaleAbs(frame, alpha=1.0 + 0.10 * alpha, beta=2 + int(8 * alpha)) - # Fallback mouth animation on realistic base frame. - h, w = frame.shape[:2] - cx = w // 2 - cy = int(h * 0.70) - half_w = max(28, int(w * 0.055)) - half_h = max(4, int(4 + alpha * h * 0.035)) + async def build_idle_payload(self, reason: str = "idle") -> dict[str, Any]: + frame = self._build_frame(seq=await self._next_sequence(), time_ms=0, controls=self._neutral_controls()) + self._last_frame_count = 1 + self._last_chunk_duration_ms = 0 + self._last_error = None + return { + "type": "animation_state", + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "schema": settings.avatar_blendshape_schema, + "fps": settings.avatar_fps, + "reason": reason, + "frame_count": 1, + "frames": [frame], + "ts_ms": int(time.time() * 1000), + } - lip_color = (26, 26, 92) - lip_border = (70, 70, 160) - cv2.ellipse(frame, (cx, cy), (half_w + 2, half_h + 2), 0, 0, 360, lip_border, -1) - cv2.ellipse(frame, (cx, cy), (half_w, half_h), 0, 0, 360, lip_color, -1) - return frame + async def build_reset_payload(self, reason: str = "reset") -> dict[str, Any]: + payload = await self.build_idle_payload(reason=reason) + payload["type"] = "animation_reset" + return payload - async def idle_frame(self) -> np.ndarray: - return await self.render_frame(mouth_open=0.0, speaking=False) + async def build_controls_from_audio( + self, + audio: np.ndarray, + sr: int, + *, + text: str = "", + chunk_index: int = 0, + total_chunks: int = 1, + ) -> dict[str, Any]: + if sr <= 0: + self._last_error = f"invalid sample rate: {sr}" + return await self.build_idle_payload(reason="invalid-sample-rate") - async def speaking_frame(self, mouth_open: float = 0.5) -> np.ndarray: - return await self.render_frame(mouth_open=mouth_open, speaking=True) + mono = np.asarray(audio, dtype=np.float32).reshape(-1) + if mono.size <= 1: + return await self.build_idle_payload(reason="empty-audio") - async def pop_generated_frame(self) -> np.ndarray | None: - async with self._lock: - if self._frames: - return self._frames.popleft() - return None + mono = np.clip(mono, -1.0, 1.0) + frame_count = max(1, int(math.ceil((mono.shape[0] / float(sr)) * settings.avatar_fps))) + seqs = await self._reserve_sequences(frame_count) + frames = self._audio_to_frames(mono, sr, seqs) - async def enqueue_musetalk_from_audio(self, audio: np.ndarray, sr: int) -> None: - if not settings.musetalk_enabled: - return - if audio.size <= 1 or sr <= 0: - return - - async with self._job_lock: - try: - job_id = f"job_{uuid.uuid4().hex[:10]}" - repo_dir = Path(settings.musetalk_repo_dir) - work_root = repo_dir / "results" / "visual-chat-jobs" / job_id - work_root.mkdir(parents=True, exist_ok=True) - wav_path = work_root / "input.wav" - cfg_path = work_root / "inference.yaml" - output_name = f"{job_id}.mp4" - output_path = Path(settings.musetalk_result_dir) / "v15" / output_name - - self._write_wav(wav_path, audio.astype(np.float32, copy=False), sr) - cfg_path.write_text( - ( - "task_0:\n" - f" video_path: \"{settings.musetalk_source_video}\"\n" - f" audio_path: \"{wav_path}\"\n" - f" result_name: \"{output_name}\"\n" - ), - encoding="utf-8", - ) - - env = os.environ.copy() - env.setdefault("HF_HOME", str(repo_dir / "models")) - env["PYTHONPATH"] = ( - f"{repo_dir}:{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(repo_dir) - ) - proc = await asyncio.create_subprocess_exec( - sys.executable, - str(settings.musetalk_infer_script), - "--version", - "v15", - "--inference_config", - str(cfg_path), - "--result_dir", - str(settings.musetalk_result_dir), - "--unet_model_path", - str(settings.musetalk_unet_model), - "--unet_config", - str(settings.musetalk_unet_config), - "--whisper_dir", - str(settings.musetalk_whisper_dir), - "--batch_size", - "8", - "--use_float16", - cwd=str(repo_dir), - env=env, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - self._last_error = stderr.decode("utf-8", errors="ignore")[-500:] - return - frames: list[np.ndarray] = [] - if output_path.exists(): - frames = self._read_video_frames(str(output_path)) - if not frames: - # Some MuseTalk runs leave image frames instead of final mp4. - fallback_dir = Path(settings.musetalk_result_dir) / "v15" / "yongen" - if fallback_dir.exists(): - frames = self._read_image_frames(str(fallback_dir)) - if not frames: - self._last_error = f"MuseTalk produced no readable frames: {output_path}" - return - - async with self._lock: - self._frames.clear() - self._frames.extend(frames) - self._last_error = None - except Exception as exc: # pragma: no cover - self._last_error = str(exc) - - def _read_video_frames(self, path: str) -> list[np.ndarray]: - cap = cv2.VideoCapture(path) - frames: list[np.ndarray] = [] - while True: - ok, frame = cap.read() - if not ok or frame is None: - break - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - frames.append(frame) - cap.release() - return frames - - def _read_image_frames(self, dir_path: str) -> list[np.ndarray]: - frames: list[np.ndarray] = [] - for name in sorted(os.listdir(dir_path)): - if not name.lower().endswith(".png"): - continue - frame = cv2.imread(os.path.join(dir_path, name)) - if frame is None: - continue - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - frames.append(frame) - return frames - - def _load_base_frame(self) -> np.ndarray: - src = settings.musetalk_source_video - if src and os.path.exists(src): - cap = cv2.VideoCapture(src) - ok, frame = cap.read() - cap.release() - if ok and frame is not None: - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - return frame - # Last-resort dark background (should rarely happen). - fallback = np.zeros((self.height, self.width, 3), dtype=np.uint8) - fallback[:, :, :] = (20, 20, 20) - return fallback - - @staticmethod - def _write_wav(path: Path, audio: np.ndarray, sr: int) -> None: - pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16) - with wave.open(str(path), "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sr) - wf.writeframes(pcm.tobytes()) + self._last_frame_count = len(frames) + self._last_chunk_duration_ms = int(round(1000.0 * mono.shape[0] / float(sr))) + self._last_error = None + return { + "type": "animation_chunk", + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "schema": settings.avatar_blendshape_schema, + "chunk_index": chunk_index, + "total_chunks": total_chunks, + "sample_rate": sr, + "fps": settings.avatar_fps, + "text": text, + "frame_count": len(frames), + "duration_ms": self._last_chunk_duration_ms, + "frames": frames, + "ts_ms": int(time.time() * 1000), + } @property - def health(self) -> dict: + def schema(self) -> dict[str, Any]: return { - "enabled": settings.musetalk_enabled, - "queued_frames": len(self._frames), + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "protocol": settings.avatar_control_protocol, + "schema": settings.avatar_blendshape_schema, + "controls": [ + "jawOpen", + "mouthClose", + "mouthFunnel", + "mouthPucker", + "viseme_aa", + "viseme_ee", + "viseme_oh", + "headYaw", + "headPitch", + "headRoll", + ], + } + + def _audio_to_frames(self, audio: np.ndarray, sr: int, sequences: range) -> list[dict[str, Any]]: + samples_per_frame = max(1, int(round(sr / max(1, settings.avatar_fps)))) + frames: list[dict[str, Any]] = [] + jaw_prev = 0.0 + for index, seq in enumerate(sequences): + start = index * samples_per_frame + end = min(audio.shape[0], start + samples_per_frame) + window = audio[start:end] + if window.size == 0: + window = np.zeros(1, dtype=np.float32) + energy, brightness = self._extract_features(window, sr) + jaw_open = 0.70 * jaw_prev + 0.30 * energy + jaw_prev = jaw_open + phase = index / max(1, len(sequences) - 1) + frames.append( + self._build_frame( + seq=seq, + time_ms=int(round(index * 1000.0 / settings.avatar_fps)), + controls=self._controls_from_features(jaw_open, brightness, phase), + ) + ) + return frames + + def _extract_features(self, window: np.ndarray, sr: int) -> tuple[float, float]: + rms = float(np.sqrt(np.mean(window**2))) if window.size else 0.0 + energy = float(np.clip((rms - 0.01) * 10.0, 0.0, 1.0)) + if window.size < 16: + return energy, 0.5 + spectrum = np.abs(np.fft.rfft(window * np.hanning(window.size))) + if spectrum.size <= 1: + return energy, 0.5 + freqs = np.fft.rfftfreq(window.size, d=1.0 / float(sr)) + weights = spectrum[1:] + freq_values = freqs[1:] + denom = float(np.sum(weights)) + 1e-6 + centroid = float(np.sum(freq_values * weights) / denom) + brightness = float(np.clip((centroid - 500.0) / 2500.0, 0.0, 1.0)) + return energy, brightness + + def _controls_from_features(self, energy: float, brightness: float, phase: float) -> dict[str, float]: + jaw_open = float(np.clip(energy, 0.0, 1.0)) + mouth_close = float(np.clip(1.0 - jaw_open * 0.82, 0.0, 1.0)) + mouth_funnel = float(np.clip(0.16 + jaw_open * (0.40 - brightness * 0.12), 0.0, 1.0)) + mouth_pucker = float(np.clip(0.06 + jaw_open * (0.20 + (1.0 - brightness) * 0.30), 0.0, 1.0)) + viseme_aa = jaw_open + viseme_ee = float(np.clip(jaw_open * (0.25 + brightness * 0.95), 0.0, 1.0)) + viseme_oh = float(np.clip(jaw_open * (0.30 + (1.0 - brightness) * 0.90), 0.0, 1.0)) + head_yaw = float(np.clip(math.sin(phase * math.pi * 1.7) * 0.02 * max(jaw_open, 0.18), -0.25, 0.25)) + head_pitch = float(np.clip(math.cos(phase * math.pi * 2.0) * 0.03 * max(jaw_open, 0.15), -0.25, 0.25)) + head_roll = float(np.clip(math.sin(phase * math.pi) * 0.015, -0.15, 0.15)) + return { + "jawOpen": round(jaw_open, 4), + "mouthClose": round(mouth_close, 4), + "mouthFunnel": round(mouth_funnel, 4), + "mouthPucker": round(mouth_pucker, 4), + "viseme_aa": round(viseme_aa, 4), + "viseme_ee": round(viseme_ee, 4), + "viseme_oh": round(viseme_oh, 4), + "headYaw": round(head_yaw, 4), + "headPitch": round(head_pitch, 4), + "headRoll": round(head_roll, 4), + } + + def _build_frame(self, seq: int, time_ms: int, controls: dict[str, float]) -> dict[str, Any]: + return { + "seq": seq, + "time_ms": time_ms, + "speaking": controls["jawOpen"] > 0.04, + "controls": controls, + } + + def _neutral_controls(self) -> dict[str, float]: + return { + "jawOpen": 0.0, + "mouthClose": 1.0, + "mouthFunnel": 0.0, + "mouthPucker": 0.0, + "viseme_aa": 0.0, + "viseme_ee": 0.0, + "viseme_oh": 0.0, + "headYaw": 0.0, + "headPitch": 0.0, + "headRoll": 0.0, + } + + async def _next_sequence(self) -> int: + async with self._lock: + seq = self._sequence + self._sequence += 1 + return seq + + async def _reserve_sequences(self, count: int) -> range: + async with self._lock: + start = self._sequence + self._sequence += count + return range(start, start + count) + + @property + def health(self) -> dict[str, Any]: + return { + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "protocol": settings.avatar_control_protocol, + "schema": settings.avatar_blendshape_schema, + "last_frame_count": self._last_frame_count, + "last_chunk_duration_ms": self._last_chunk_duration_ms, "last_error": self._last_error, } diff --git a/services/llm.py b/services/llm.py index cc1f9b3..2105fe1 100644 --- a/services/llm.py +++ b/services/llm.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import os from typing import Optional from collections import deque @@ -15,23 +16,37 @@ class LLMService: self._error: Optional[str] = None self._client = None self._model = (settings.llm_model or "").strip() + self._base_url = (settings.llm_base_url or "").strip() self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant) self._system_prompt = "请用中文口语化简短回复，1-2句。" if not settings.llm_api_key: + self._error = "missing LLM api key: set LLM_API_KEY or DEEPSEEK_API_KEY" return + + # If DeepSeek key is provided but no explicit base URL override, + # route to DeepSeek-compatible OpenAI endpoint by default. + if ( + os.getenv("DEEPSEEK_API_KEY") + and not os.getenv("LLM_BASE_URL") + and not os.getenv("DEEPSEEK_BASE_URL") + and (self._base_url.lower() == "https://api.openai.com/v1" or not self._base_url) + ): + self._base_url = "https://api.deepseek.com/v1" + if not self._model: - base = (settings.llm_base_url or "").lower() - self._model = "deepseek-chat" if "deepseek.com" in base else "gpt-4o-mini" + base = self._base_url.lower() + self._model = "deepseek-chat" if "deepseek" in base else "gpt-4o-mini" try: from openai import AsyncOpenAI self._client = AsyncOpenAI( api_key=settings.llm_api_key, - base_url=settings.llm_base_url, + base_url=self._base_url, timeout=settings.llm_timeout, ) self._ready = True + self._error = None except Exception as exc: # pragma: no cover self._error = str(exc) logger.warning("LLM online mode unavailable: %s", exc) @@ -67,6 +82,7 @@ class LLMService: return { "ready": self._ready, "model": self._model, + "base_url": self._base_url, "history_items": len(self._history), "error": self._error, } diff --git a/web/app.js b/web/app.js index 45c72b1..43a2a09 100644 --- a/web/app.js +++ b/web/app.js @@ -1,3 +1,8 @@ +import * as THREE from "three"; +import { OrbitControls } from "three/addons/controls/OrbitControls.js"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"; + const statusEl = document.getElementById("status"); const outputEl = document.getElementById("output"); const healthBtn = document.getElementById("healthBtn"); @@ -5,11 +10,25 @@ const offerBtn = document.getElementById("offerBtn"); const resetBtn = document.getElementById("resetBtn"); const sendBtn = document.getElementById("sendBtn"); const chatInput = document.getElementById("chatInput"); -const remoteVideo = document.getElementById("remoteVideo"); -remoteVideo.muted = false; -remoteVideo.volume = 1.0; +const remoteAudio = document.getElementById("remoteAudio"); +const modelFile = document.getElementById("modelFile"); +const modelUrl = document.getElementById("modelUrl"); +const loadModelBtn = document.getElementById("loadModelBtn"); +const resetViewBtn = document.getElementById("resetViewBtn"); +const avatarCanvas = document.getElementById("avatarCanvas"); +const modelStatus = document.getElementById("modelStatus"); +const modelName = document.getElementById("modelName"); +const animationBuffer = document.getElementById("animationBuffer"); + +remoteAudio.muted = false; +remoteAudio.volume = 1.0; + const messagesEl = document.getElementById("messages"); const connBadge = document.getElementById("connBadge"); +const animationConn = document.getElementById("animationConn"); +const animationDriver = document.getElementById("animationDriver"); +const animationFrames = document.getElementById("animationFrames"); +const animationPreview = document.getElementById("animationPreview"); const mPeers = document.getElementById("mPeers"); const mBusy = document.getElementById("mBusy"); const mLatency = document.getElementById("mLatency"); @@ -23,6 +42,9 @@ const mTts = document.getElementById("mTts"); const mBargeIn = document.getElementById("mBargeIn"); const mInputMode = document.getElementById("mInputMode"); const mVadCount = document.getElementById("mVadCount"); +const mAnimClients = document.getElementById("mAnimClients"); +const mAnimDriver = document.getElementById("mAnimDriver"); +const mAnimFrames = document.getElementById("mAnimFrames"); const dConn = document.getElementById("dConn"); const dVad = document.getElementById("dVad"); const dAsr = document.getElementById("dAsr"); @@ -32,11 +54,15 @@ const dTts = document.getElementById("dTts"); let pc = null; let evt = null; let subtitleWs = null; +let animationWs = null; let localMicStream = null; -let lastSeenRun = 0; -let lastBusy = false; let aiStreamingEl = null; + const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect"; +const ANIMATION_BUFFER_MS = 120; +const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; +const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; +const CLOCK = new THREE.Clock(); const stateMap = { "SessionState.IDLE": "空闲", @@ -49,6 +75,56 @@ const stateMap = { avatar_speaking: "数字人说话", }; +const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true }); +renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); +renderer.outputColorSpace = THREE.SRGBColorSpace; + +const scene = new THREE.Scene(); +scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5); + +const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100); +camera.position.set(0, 1.45, 2.65); + +const controls = new OrbitControls(camera, avatarCanvas); +controls.target.set(0, 1.15, 0); +controls.enableDamping = true; +controls.minDistance = 1.3; +controls.maxDistance = 4.5; +controls.maxPolarAngle = Math.PI * 0.58; +controls.minPolarAngle = Math.PI * 0.22; + +const avatarState = { + currentRoot: null, + currentVrm: null, + morphBindings: new Map(), + pendingFrames: [], + scheduleCursorMs: performance.now(), + lastAppliedSeq: -1, + activeControls: { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }, + headBone: null, + neckBone: null, + idleRig: null, + debugRig: null, + debugJaw: null, + debugMouth: null, +}; + +const gltfLoader = new GLTFLoader(); +gltfLoader.register((parser) => new VRMLoaderPlugin(parser)); +gltfLoader.crossOrigin = "anonymous"; + +const TMP_EULER = new THREE.Euler(); +const TMP_QUATERNION = new THREE.Quaternion(); + function fmtTime() { return new Date().toLocaleTimeString("zh-CN", { hour12: false }); } @@ -82,6 +158,132 @@ function setDot(el, kind) { el.className = `dot ${kind}`; } +function initStage() { + scene.background = new THREE.Color(0x0b1118); + + const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65); + hemiLight.position.set(0, 2.6, 0); + scene.add(hemiLight); + + const keyLight = new THREE.DirectionalLight(0xffffff, 1.7); + keyLight.position.set(1.4, 2.2, 2.0); + scene.add(keyLight); + + const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8); + rimLight.position.set(-2.0, 1.4, -1.5); + scene.add(rimLight); + + const floor = new THREE.Mesh( + new THREE.CircleGeometry(2.8, 48), + new THREE.MeshStandardMaterial({ + color: 0x101b28, + roughness: 0.94, + metalness: 0.08, + transparent: true, + opacity: 0.9, + }), + ); + floor.rotation.x = -Math.PI / 2; + floor.position.y = -0.02; + scene.add(floor); + + const halo = new THREE.Mesh( + new THREE.RingGeometry(1.2, 1.95, 64), + new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }), + ); + halo.rotation.x = -Math.PI / 2; + halo.position.y = 0.01; + scene.add(halo); + + createDebugRig(); + resizeStage(); + renderLoop(); +} + +function createDebugRig() { + const group = new THREE.Group(); + + const bustMaterial = new THREE.MeshStandardMaterial({ + color: 0xc7d2e0, + roughness: 0.72, + metalness: 0.05, + }); + const accentMaterial = new THREE.MeshStandardMaterial({ + color: 0x0f172a, + roughness: 0.35, + metalness: 0.18, + }); + const lipMaterial = new THREE.MeshStandardMaterial({ + color: 0x8b425f, + roughness: 0.56, + metalness: 0.02, + }); + + const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial); + torso.position.set(0, 0.42, 0); + group.add(torso); + + const neck = new THREE.Group(); + neck.position.set(0, 0.95, 0); + group.add(neck); + + const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial); + head.scale.set(0.92, 1.03, 0.92); + neck.add(head); + + const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial); + hair.position.set(0, 0.06, 0); + neck.add(hair); + + const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); + leftEye.position.set(-0.1, 0.03, 0.28); + neck.add(leftEye); + + const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); + rightEye.position.set(0.1, 0.03, 0.28); + neck.add(rightEye); + + const jawPivot = new THREE.Group(); + jawPivot.position.set(0, -0.06, 0.15); + neck.add(jawPivot); + + const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial); + jaw.position.set(0, -0.08, 0.02); + jawPivot.add(jaw); + + const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial); + mouth.position.set(0, -0.02, 0.31); + neck.add(mouth); + + scene.add(group); + avatarState.currentRoot = group; + avatarState.debugRig = group; + avatarState.debugJaw = jawPivot; + avatarState.debugMouth = mouth; + avatarState.headBone = neck; + avatarState.neckBone = neck; +} + +function resizeStage() { + const rect = avatarCanvas.getBoundingClientRect(); + if (!rect.width || !rect.height) return; + renderer.setSize(rect.width, rect.height, false); + camera.aspect = rect.width / rect.height; + camera.updateProjectionMatrix(); +} + +function renderLoop() { + requestAnimationFrame(renderLoop); + const delta = CLOCK.getDelta(); + const elapsed = CLOCK.getElapsedTime(); + controls.update(); + flushAnimationQueue(); + applyProceduralIdlePose(elapsed); + applyAnimationControls(avatarState.activeControls); + avatarState.currentVrm?.update?.(delta); + renderer.render(scene, camera); +} + function updateMetrics(data) { mPeers.textContent = String(data.peers ?? 0); mBusy.textContent = data.pipeline_busy ? "是" : "否"; @@ -96,6 +298,9 @@ function updateMetrics(data) { mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`; mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none"; mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`; + mAnimClients.textContent = String(data.animation_clients ?? 0); + mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--"; + mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0); setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn"); setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn"); @@ -104,6 +309,394 @@ function updateMetrics(data) { setDot(dTts, data?.tts?.ready ? "ok" : "warn"); } +function queueAnimationFrames(payload) { + const frames = Array.isArray(payload.frames) ? payload.frames : []; + const now = performance.now(); + const frameWindowMs = (1000 / Math.max(1, payload.fps || 25)); + + if (payload.type === "animation_reset" || payload.type === "animation_state") { + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS; + avatarState.activeControls = { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }; + } + + const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs); + frames.forEach((frame, index) => { + const fallbackTime = index * frameWindowMs; + avatarState.pendingFrames.push({ + dueAt: baseTime + (frame.time_ms ?? fallbackTime), + seq: frame.seq ?? index, + controls: frame.controls || {}, + }); + }); + + avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq); + avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs); + animationBuffer.textContent = String(avatarState.pendingFrames.length); +} + +function flushAnimationQueue() { + const now = performance.now(); + let frameToApply = null; + while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) { + frameToApply = avatarState.pendingFrames.shift(); + } + + if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) { + avatarState.lastAppliedSeq = frameToApply.seq; + avatarState.activeControls = { + ...avatarState.activeControls, + ...frameToApply.controls, + }; + } + + animationBuffer.textContent = String(avatarState.pendingFrames.length); +} + +function applyAnimationControls(controls) { + const safeControls = controls || {}; + const jawOpen = clampControl(safeControls.jawOpen); + const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen); + const visemeEe = clampControl(safeControls.viseme_ee); + const visemeOh = clampControl(safeControls.viseme_oh); + const mouthPucker = clampControl(safeControls.mouthPucker); + const headYaw = clampSigned(safeControls.headYaw, 0.35); + const headPitch = clampSigned(safeControls.headPitch, 0.3); + const headRoll = clampSigned(safeControls.headRoll, 0.24); + const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0; + const t = performance.now() * 0.001; + + if (avatarState.currentVrm?.expressionManager) { + setExpressionValue("aa", visemeAa); + setExpressionValue("ee", visemeEe); + setExpressionValue("oh", visemeOh); + setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker)); + } + + if (avatarState.morphBindings.size > 0) { + setMorphValue("jawOpen", jawOpen); + setMorphValue("viseme_aa", visemeAa); + setMorphValue("viseme_ee", visemeEe); + setMorphValue("viseme_oh", visemeOh); + setMorphValue("mouthPucker", mouthPucker); + } + + // For plain head meshes (no VRM expressions and no morph targets), + // amplify head motion to keep speaking visually obvious. + const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09; + const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12; + const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04; + + applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied); + if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) { + applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4); + } + + if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { + const baseScale = 1.0 + jawOpen * 0.03; + avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale); + } + + if (avatarState.debugJaw) { + avatarState.debugJaw.rotation.x = jawOpen * 0.5; + avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015; + } + if (avatarState.debugMouth) { + avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45; + avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8; + avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01; + } +} + +function setExpressionValue(name, value) { + try { + avatarState.currentVrm.expressionManager.setValue(name, value); + } catch (_) { + // no-op: expression name may not exist on the current model + } +} + +function setMorphValue(controlName, value) { + const bindings = avatarState.morphBindings.get(controlName) || []; + bindings.forEach((binding) => { + binding.mesh.morphTargetInfluences[binding.index] = value; + }); +} + +function clampControl(value) { + return Math.min(1, Math.max(0, Number(value) || 0)); +} + +function clampSigned(value, limit) { + return Math.min(limit, Math.max(-limit, Number(value) || 0)); +} + +function resetView() { + camera.position.set(0, 1.45, 2.65); + controls.target.set(0, 1.15, 0); + controls.update(); +} + +function resetAvatarPose() { + avatarState.activeControls = { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }; + applyAnimationControls(avatarState.activeControls); +} + +function detachCurrentAvatar() { + if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { + scene.remove(avatarState.currentRoot); + } + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + avatarState.lastAppliedSeq = -1; + avatarState.idleRig = null; + avatarState.currentRoot = avatarState.debugRig; + avatarState.currentVrm = null; + avatarState.morphBindings.clear(); + avatarState.headBone = avatarState.debugRig.children[1]; + avatarState.neckBone = avatarState.debugRig.children[1]; + avatarState.debugRig.visible = true; + modelName.textContent = "debug-avatar"; +} + +async function loadModelFromUrl(url) { + if (!url) return false; + modelStatus.textContent = "模型加载中..."; + try { + const gltf = await gltfLoader.loadAsync(url); + attachLoadedAsset(gltf, url.split("/").pop() || url); + modelStatus.textContent = "模型已加载"; + return true; + } catch (error) { + console.error(error); + modelStatus.textContent = "模型加载失败，已回退调试头像"; + addMessage("system", `模型加载失败：${error?.message || error}`); + detachCurrentAvatar(); + return false; + } +} + +async function loadDefaultAvatarModel() { + modelUrl.value = DEFAULT_BODY_MODEL_URL; + const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL); + if (!ok) { + modelUrl.value = DEFAULT_VRM_FALLBACK_URL; + await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL); + } +} + +async function loadModelFromFile(file) { + if (!file) return; + const url = URL.createObjectURL(file); + modelStatus.textContent = "本地模型加载中..."; + try { + const gltf = await gltfLoader.loadAsync(url); + attachLoadedAsset(gltf, file.name); + modelStatus.textContent = "本地模型已加载"; + } catch (error) { + console.error(error); + modelStatus.textContent = "本地模型加载失败，已回退调试头像"; + addMessage("system", `本地模型加载失败：${error?.message || error}`); + detachCurrentAvatar(); + } finally { + URL.revokeObjectURL(url); + } +} + +function attachLoadedAsset(gltf, label) { + detachCurrentAvatar(); + + const vrm = gltf.userData?.vrm || null; + const root = vrm?.scene || gltf.scene; + if (!root) { + throw new Error("模型中未找到可渲染的 scene"); + } + + if (vrm) { + VRMUtils.removeUnnecessaryVertices(gltf.scene); + VRMUtils.combineSkeletons(gltf.scene); + VRMUtils.combineMorphs(vrm); + VRMUtils.rotateVRM0(vrm); + } + + root.traverse((obj) => { + obj.frustumCulled = false; + }); + + root.position.set(0, 0, 0); + root.rotation.set(0, 0, 0); + root.scale.setScalar(1); + scene.add(root); + + avatarState.currentRoot = root; + avatarState.currentVrm = vrm; + avatarState.debugRig.visible = false; + avatarState.morphBindings = collectMorphBindings(root); + avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1]; + avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone; + avatarState.idleRig = buildIdleRig(vrm, root); + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + avatarState.lastAppliedSeq = -1; + + fitCameraToObject(root); + resetAvatarPose(); + modelName.textContent = label; +} + +function fitCameraToObject(root) { + const box = new THREE.Box3().setFromObject(root); + const center = new THREE.Vector3(); + const size = new THREE.Vector3(); + box.getCenter(center); + box.getSize(size); + + const radius = Math.max(size.x, size.y, size.z, 1.2); + controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0)); + camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9); + controls.update(); +} + +function resolveHeadBone(vrm, root) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.("head") || + vrm?.humanoid?.getRawBoneNode?.("head") || + root.getObjectByName("Head") || + root.getObjectByName("head") || + root + ); +} + +function resolveNeckBone(vrm, root) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.("neck") || + vrm?.humanoid?.getRawBoneNode?.("neck") || + root.getObjectByName("Neck") || + root.getObjectByName("neck") || + resolveHeadBone(vrm, root) + ); +} + +function collectMorphBindings(root) { + const bindings = new Map(); + const aliases = { + jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"], + viseme_aa: ["viseme_aa", "aa", "a", "moutha"], + viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"], + viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"], + mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"], + }; + + root.traverse((node) => { + if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return; + const entries = Object.entries(node.morphTargetDictionary); + Object.entries(aliases).forEach(([controlName, names]) => { + entries.forEach(([targetName, index]) => { + const normalized = targetName.toLowerCase(); + if (!names.includes(normalized)) return; + if (!bindings.has(controlName)) bindings.set(controlName, []); + bindings.get(controlName).push({ mesh: node, index }); + }); + }); + }); + return bindings; +} + +function buildIdleRig(vrm, root) { + const lookup = { + hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]), + spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]), + chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]), + upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]), + neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]), + head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]), + leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]), + rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]), + leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]), + rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]), + leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]), + rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]), + leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]), + rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]), + }; + + const base = new Map(); + Object.values(lookup).forEach((bone) => { + if (bone && !base.has(bone)) { + base.set(bone, bone.quaternion.clone()); + } + }); + + return { bones: lookup, base }; +} + +function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) || + vrm?.humanoid?.getRawBoneNode?.(humanoidName) || + fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) || + null + ); +} + +function applyProceduralIdlePose(time) { + const rig = avatarState.idleRig; + if (!rig) return; + + const breathe = Math.sin(time * 1.9) * 0.018; + const sway = Math.sin(time * 1.15) * 0.05; + const armDrift = Math.sin(time * 1.55) * 0.07; + const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06; + const handDrift = Math.sin(time * 2.3) * 0.04; + + applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04); + applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06); + applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08); + applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1); + + applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18); + applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18); + applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0); + applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0); + applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12); + applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12); + applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05); + applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05); +} + +function applyBoneRotation(bone, x = 0, y = 0, z = 0) { + if (!bone) return; + const baseQuaternion = avatarState.idleRig?.base?.get(bone); + if (baseQuaternion) { + bone.quaternion.copy(baseQuaternion); + TMP_EULER.set(x, y, z, "XYZ"); + TMP_QUATERNION.setFromEuler(TMP_EULER); + bone.quaternion.multiply(TMP_QUATERNION); + return; + } + bone.rotation.x = x; + bone.rotation.y = y; + bone.rotation.z = z; +} + async function checkHealth() { const res = await fetch("/health"); const data = await res.json(); @@ -123,8 +716,6 @@ function subscribeEvents() { outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); - lastBusy = !!data.pipeline_busy; - if (typeof data.pipeline_runs === "number" && data.pipeline_runs > lastSeenRun) lastSeenRun = data.pipeline_runs; } catch (_) { // no-op } @@ -160,6 +751,36 @@ function subscribeSubtitles() { }; } +function subscribeAnimation() { + if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return; + const scheme = location.protocol === "https:" ? "wss" : "ws"; + animationConn.textContent = "连接中"; + animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`); + animationWs.onopen = () => { + animationConn.textContent = "已连接"; + }; + animationWs.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + if (data.type === "ping") return; + if (data.driver) { + animationDriver.textContent = data.driver; + } + const frameCount = data.frame_count ?? data.frames?.length ?? 0; + animationFrames.textContent = String(frameCount); + const preview = data.frames?.[0]?.controls || null; + animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2); + queueAnimationFrames(data); + } catch (_) { + // no-op + } + }; + animationWs.onclose = () => { + animationConn.textContent = "已断开"; + setTimeout(subscribeAnimation, 1000); + }; +} + async function connectWebRTC() { if (pc) { outputEl.textContent = "WebRTC 已连接或正在连接"; @@ -175,21 +796,22 @@ async function connectWebRTC() { }); const currentPc = pc; let seenConnected = false; + currentPc.onconnectionstatechange = () => { const st = currentPc.connectionState; if (st === "connected") { seenConnected = true; setConnBadge("已连接", "ok"); localStorage.setItem(AUTO_RECONNECT_KEY, "1"); - } - else if (st === "connecting") setConnBadge("连接中...", "warn"); - else if (st === "failed" || st === "disconnected" || st === "closed") { + } else if (st === "connecting") { + setConnBadge("连接中...", "warn"); + } else if (st === "failed" || st === "disconnected" || st === "closed") { setConnBadge(`连接${st}`, "warn"); if (seenConnected && (st === "disconnected" || st === "failed")) { addMessage("system", "连接已断开，可能被新的客户端连接接管。"); } if (localMicStream) { - localMicStream.getTracks().forEach((t) => t.stop()); + localMicStream.getTracks().forEach((track) => track.stop()); localMicStream = null; } if (pc === currentPc) { @@ -198,13 +820,12 @@ async function connectWebRTC() { offerBtn.disabled = false; } }; - // Explicitly request remote media m-lines so server can attach tracks. - pc.addTransceiver("video", { direction: "recvonly" }); + pc.ontrack = (event) => { if (event.streams && event.streams[0]) { - remoteVideo.srcObject = event.streams[0]; - remoteVideo.play().catch(() => { - addMessage("system", "远端音视频已到达，如未自动播放请点击视频区域后重试。"); + remoteAudio.srcObject = event.streams[0]; + remoteAudio.play().catch(() => { + addMessage("system", "远端语音已到达，如未自动播放请先与页面交互后重试。"); }); } }; @@ -220,12 +841,8 @@ async function connectWebRTC() { offerBtn.disabled = false; throw err; } - stream.getTracks().forEach((track) => pc.addTrack(track, stream)); - const sendAudioTrack = stream.getAudioTracks()[0]; - if (sendAudioTrack) { - sendAudioTrack.enabled = true; - } + stream.getTracks().forEach((track) => pc.addTrack(track, stream)); const offer = await pc.createOffer(); await pc.setLocalDescription(offer); @@ -240,11 +857,7 @@ async function connectWebRTC() { const answer = await res.json(); await pc.setRemoteDescription(answer); statusEl.textContent = "状态：WebRTC 已连接"; - outputEl.textContent = JSON.stringify( - { connected: true, iceConnectionState: pc.iceConnectionState }, - null, - 2 - ); + outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2); setConnBadge("已连接", "ok"); addMessage("system", "WebRTC 已连接，可开始对话。"); if (answer.replaced_previous) { @@ -256,8 +869,7 @@ async function connectWebRTC() { async function sendText() { const text = (chatInput.value || "").trim(); - if (!text) return; - if (sendBtn.disabled) return; + if (!text || sendBtn.disabled) return; sendBtn.disabled = true; const res = await fetch("/chat/text", { method: "POST", @@ -277,6 +889,9 @@ async function sendText() { async function resetChat() { await fetch("/chat/reset", { method: "POST" }); messagesEl.innerHTML = ""; + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + resetAvatarPose(); addMessage("system", "会话已重置。"); await checkHealth(); } @@ -284,24 +899,56 @@ async function resetChat() { function tryAutoReconnect() { if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return; setTimeout(() => { - if (!pc) connectWebRTC().catch(() => { - // no-op: user can reconnect manually if permission/policy blocks auto flow - }); + if (!pc) { + connectWebRTC().catch(() => { + // no-op + }); + } }, 300); } +function handleModelFileChange(event) { + const [file] = event.target.files || []; + if (!file) return; + loadModelFromFile(file); +} + +function handleModelUrlLoad() { + const url = modelUrl.value.trim(); + if (!url) return; + loadModelFromUrl(url); +} + +new ResizeObserver(() => resizeStage()).observe(avatarCanvas); +window.addEventListener("resize", resizeStage); + healthBtn.addEventListener("click", checkHealth); offerBtn.addEventListener("click", connectWebRTC); resetBtn.addEventListener("click", resetChat); sendBtn.addEventListener("click", sendText); -chatInput.addEventListener("keydown", (e) => { - if (e.key === "Enter") { - e.preventDefault(); +loadModelBtn.addEventListener("click", handleModelUrlLoad); +resetViewBtn.addEventListener("click", resetView); +modelFile.addEventListener("change", handleModelFileChange); +chatInput.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault(); sendText(); } }); +modelUrl.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault(); + handleModelUrlLoad(); + } +}); + +initStage(); +resetView(); +resetAvatarPose(); +loadDefaultAvatarModel(); checkHealth(); subscribeEvents(); subscribeSubtitles(); +subscribeAnimation(); setConnBadge("未连接", "neutral"); tryAutoReconnect(); diff --git a/web/index.html b/web/index.html index 455fc10..945662e 100644 --- a/web/index.html +++ b/web/index.html @@ -8,11 +8,44 @@
-

可视化语音聊天系统

+

3D 数字人语音聊天系统

状态：启动中

-
- +
+
+ Three.js + VRM 驱动舞台 + 未连接 +
+
+ + + + + +
+
+ +
+
+ 模型状态 + 使用调试头像 +
+
+ 驱动器 + -- +
+
+ 最近帧数 + 0 +
+
+
+
+
当前模型debug-avatar
+
动画缓冲0
+
+ 
等待动画控制数据...
+
@@ -45,12 +78,24 @@
打断耗时0 ms
输入模式none
VAD触发0/0
+
动画客户端0
+
动画驱动--
+
最近动画帧0

运行状态


     

   

-  
+  
+  
 
 
diff --git a/web/style.css b/web/style.css
index 561c1fe..9734d8a 100644
--- a/web/style.css
+++ b/web/style.css
@@ -1,32 +1,167 @@
 body {
   margin: 0;
-  font-family: sans-serif;
-  background: #0f1115;
+  font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
+  background:
+    radial-gradient(circle at top, rgba(42, 93, 255, 0.16), transparent 36%),
+    radial-gradient(circle at right, rgba(15, 173, 129, 0.12), transparent 28%),
+    linear-gradient(180deg, #091018 0%, #111827 46%, #0b1018 100%);
   color: #f2f2f2;
 }
 
 .wrap {
-  max-width: 900px;
+  max-width: 1120px;
   margin: 24px auto;
   padding: 0 16px;
 }
 
-.video-box {
+.stage-box {
   width: 100%;
-  aspect-ratio: 16 / 9;
-  border: 1px solid #333;
-  border-radius: 10px;
-  background: #1a1f27;
+  border: 1px solid rgba(116, 139, 166, 0.22);
+  border-radius: 24px;
+  background: linear-gradient(180deg, rgba(18, 25, 36, 0.94), rgba(12, 18, 28, 0.96));
   overflow: hidden;
+  padding: 18px;
+  box-sizing: border-box;
+  display: flex;
+  flex-direction: column;
+  gap: 14px;
+  box-shadow: 0 28px 70px rgba(0, 0, 0, 0.32);
 }
 
-video {
+.stage-copy {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  gap: 12px;
+}
+
+.stage-copy strong {
+  font-size: 20px;
+  letter-spacing: 0.01em;
+}
+
+.stage-copy span {
+  font-size: 13px;
+  color: #7dd3fc;
+}
+
+.stage-toolbar {
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr) auto auto;
+  gap: 12px;
+  align-items: center;
+}
+
+.stage-toolbar input[type="text"] {
+  min-width: 0;
+  background: rgba(10, 15, 24, 0.78);
+  color: #e5eef8;
+  border: 1px solid rgba(112, 146, 190, 0.3);
+  border-radius: 12px;
+  padding: 10px 12px;
+}
+
+.stage-toolbar input[type="file"] {
+  position: absolute;
+  width: 1px;
+  height: 1px;
+  opacity: 0;
+  pointer-events: none;
+}
+
+.file-pill {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  min-height: 40px;
+  border-radius: 999px;
+  border: 1px solid rgba(102, 199, 255, 0.28);
+  background: rgba(9, 60, 86, 0.32);
+  color: #d7f2ff;
+  padding: 0 16px;
+  cursor: pointer;
+  white-space: nowrap;
+}
+
+.canvas-shell {
+  position: relative;
+  aspect-ratio: 16 / 9;
+  border-radius: 18px;
+  overflow: hidden;
+  background:
+    radial-gradient(circle at 50% 28%, rgba(148, 196, 255, 0.18), transparent 24%),
+    linear-gradient(180deg, #152235 0%, #0c131d 52%, #0a1017 100%);
+  border: 1px solid rgba(95, 128, 160, 0.18);
+}
+
+#avatarCanvas {
   width: 100%;
   height: 100%;
-  object-fit: cover;
   display: block;
 }
 
+.canvas-overlay {
+  position: absolute;
+  left: 14px;
+  right: 14px;
+  bottom: 14px;
+  display: grid;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 10px;
+  pointer-events: none;
+}
+
+.overlay-card {
+  border: 1px solid rgba(144, 170, 197, 0.18);
+  background: rgba(7, 11, 18, 0.68);
+  backdrop-filter: blur(10px);
+  border-radius: 14px;
+  padding: 10px 12px;
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+}
+
+.overlay-card span {
+  color: #8ea6c3;
+  font-size: 12px;
+}
+
+.overlay-card strong {
+  font-size: 14px;
+}
+
+.stage-metrics {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 12px;
+}
+
+.stage-metrics div {
+  border: 1px solid rgba(52, 72, 96, 0.7);
+  border-radius: 14px;
+  background: rgba(14, 22, 33, 0.7);
+  padding: 12px;
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+}
+
+.stage-metrics span {
+  color: #94a3b8;
+  font-size: 12px;
+}
+
+.stage-metrics strong {
+  font-size: 16px;
+}
+
+#animationPreview {
+  flex: 1;
+  min-height: 180px;
+  max-height: 260px;
+}
+
 .actions {
   margin-top: 16px;
   display: flex;
@@ -42,10 +177,10 @@ video {
 
 .chat-box input {
   flex: 1;
-  background: #111827;
+  background: rgba(13, 19, 29, 0.9);
   color: #f3f4f6;
   border: 1px solid #374151;
-  border-radius: 8px;
+  border-radius: 12px;
   padding: 10px;
 }
 
@@ -55,9 +190,9 @@ video {
   max-height: 360px;
   overflow: auto;
   border: 1px solid #1f2937;
-  border-radius: 10px;
+  border-radius: 16px;
   padding: 10px;
-  background: #0b1220;
+  background: rgba(10, 18, 32, 0.82);
   display: flex;
   flex-direction: column;
   gap: 8px;
@@ -66,8 +201,8 @@ video {
 .diag {
   margin-top: 12px;
   border: 1px solid #1f2937;
-  border-radius: 10px;
-  background: #0b0d12;
+  border-radius: 16px;
+  background: rgba(8, 13, 18, 0.78);
   padding: 10px;
   display: flex;
   gap: 16px;
@@ -122,12 +257,12 @@ video {
 
 .msg.user {
   align-self: flex-end;
-  background: #1d4ed8;
+  background: linear-gradient(135deg, #2563eb, #0ea5e9);
 }
 
 .msg.ai {
   align-self: flex-start;
-  background: #374151;
+  background: linear-gradient(135deg, #334155, #475569);
 }
 
 .panel {
@@ -143,8 +278,8 @@ video {
 
 .metric {
   border: 1px solid #1f2937;
-  background: #0b0d12;
-  border-radius: 8px;
+  background: rgba(8, 13, 18, 0.78);
+  border-radius: 14px;
   padding: 8px 10px;
   display: flex;
   flex-direction: column;
@@ -167,10 +302,10 @@ video {
 }
 
 button {
-  background: #2563eb;
+  background: linear-gradient(135deg, #2563eb, #0891b2);
   color: white;
   border: 0;
-  border-radius: 8px;
+  border-radius: 12px;
   padding: 8px 12px;
   cursor: pointer;
 }
@@ -231,9 +366,31 @@ button:disabled {
 
 pre {
   margin-top: 0;
-  background: #0b0d12;
+  background: rgba(8, 13, 18, 0.84);
   border: 1px solid #222;
-  border-radius: 8px;
+  border-radius: 14px;
   padding: 12px;
   overflow: auto;
 }
+
+audio {
+  display: none;
+}
+
+@media (max-width: 720px) {
+  .metrics {
+    grid-template-columns: repeat(2, minmax(120px, 1fr));
+  }
+
+  .stage-toolbar {
+    grid-template-columns: 1fr;
+  }
+
+  .stage-metrics {
+    grid-template-columns: 1fr;
+  }
+
+  .canvas-overlay {
+    grid-template-columns: 1fr;
+  }
+}
diff --git a/webrtc/__pycache__/__init__.cpython-312.pyc b/webrtc/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000..4deb15a
Binary files /dev/null and b/webrtc/__pycache__/__init__.cpython-312.pyc differ
diff --git a/webrtc/__pycache__/tracks.cpython-312.pyc b/webrtc/__pycache__/tracks.cpython-312.pyc
new file mode 100644
index 0000000..f748168
Binary files /dev/null and b/webrtc/__pycache__/tracks.cpython-312.pyc differ
diff --git a/webrtc/gateway.py b/webrtc/gateway.py
index 20d8e6c..00fd135 100644
--- a/webrtc/gateway.py
+++ b/webrtc/gateway.py
@@ -3,21 +3,18 @@ from __future__ import annotations
 from aiortc import RTCPeerConnection
 from fastapi import FastAPI
 
-from config import settings
-from core.state_machine import Arbitrator
-from services.avatar import AvatarService
 from webrtc.audio_track import build_audio_track
 from webrtc.tracks import AudioBus
-from webrtc.video_track import build_video_track
 
 
 def attach_webrtc_tracks(
     pc: RTCPeerConnection,
-    avatar_service: AvatarService,
-    arbitrator: Arbitrator,
+    avatar_service,
+    arbitrator,
     audio_bus: AudioBus,
 ) -> None:
-    pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
+    _ = avatar_service
+    _ = arbitrator
     pc.addTrack(build_audio_track(audio_bus))
 
 
diff --git a/webrtc/tracks.py b/webrtc/tracks.py
index 10246c1..1af68d9 100644
--- a/webrtc/tracks.py
+++ b/webrtc/tracks.py
@@ -6,10 +6,7 @@ from collections import deque
 
 import numpy as np
 from aiortc import MediaStreamTrack
-from av import AudioFrame, VideoFrame
-
-from core.state_machine import Arbitrator, SessionState
-from services.avatar import AvatarService
+from av import AudioFrame
 
 
 class AudioBus:
@@ -55,41 +52,6 @@ class AudioBus:
             self._level = 0.0
 
 
-class AvatarVideoTrack(MediaStreamTrack):
-    kind = "video"
-
-    def __init__(
-        self,
-        avatar_service: AvatarService,
-        arbitrator: Arbitrator,
-        audio_bus: AudioBus,
-        fps: int = 25,
-    ) -> None:
-        super().__init__()
-        self.avatar_service = avatar_service
-        self.arbitrator = arbitrator
-        self.audio_bus = audio_bus
-        self.fps = fps
-        self._pts = 0
-        self._time_base = Fraction(1, 90000)
-
-    async def recv(self) -> VideoFrame:
-        await asyncio.sleep(1 / self.fps)
-        generated = await self.avatar_service.pop_generated_frame()
-        if generated is not None:
-            arr = generated
-        else:
-            level = await self.audio_bus.level()
-            speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
-            arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
-
-        frame = VideoFrame.from_ndarray(arr, format="bgr24")
-        frame.pts = self._pts
-        frame.time_base = self._time_base
-        self._pts += int(90000 / self.fps)
-        return frame
-
-
 class AvatarAudioTrack(MediaStreamTrack):
     kind = "audio"
 
diff --git a/webrtc/video_track.py b/webrtc/video_track.py
index f5ac09b..54fc003 100644
--- a/webrtc/video_track.py
+++ b/webrtc/video_track.py
@@ -1,21 +1,7 @@
 from __future__ import annotations
 
-from core.state_machine import Arbitrator
-from services.avatar import AvatarService
-from webrtc.tracks import AudioBus, AvatarVideoTrack
-
-__all__ = ["AvatarVideoTrack", "build_video_track"]
+__all__: list[str] = []
 
 
-def build_video_track(
-    avatar_service: AvatarService,
-    arbitrator: Arbitrator,
-    audio_bus: AudioBus,
-    fps: int = 25,
-) -> AvatarVideoTrack:
-    return AvatarVideoTrack(
-        avatar_service=avatar_service,
-        arbitrator=arbitrator,
-        audio_bus=audio_bus,
-        fps=fps,
-    )
+def build_video_track(*_args, **_kwargs):
+    raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")