From f24de38e94abd3fbfc3ee66f7dc0fe3f10b710be Mon Sep 17 00:00:00 2001 From: xsl Date: Fri, 27 Mar 2026 17:10:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=A7=E5=8A=9F=E5=91=8A=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 22 + .run-8018.log | 386 ++++++++++ AVATAR_3D_REFACTOR.md | 189 +++++ DEPLOYMENT_GUIDE.md | 28 +- TECHNICAL_DOCUMENTATION.md | 278 +++++++ __pycache__/config.cpython-312.pyc | Bin 0 -> 2535 bytes __pycache__/main.cpython-312.pyc | Bin 0 -> 30272 bytes config.py | 20 +- core/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 136 bytes core/__pycache__/pipeline.cpython-312.pyc | Bin 0 -> 9934 bytes .../__pycache__/state_machine.cpython-312.pyc | Bin 0 -> 2964 bytes core/pipeline.py | 11 +- main.py | 209 ++--- scripts/service-8018.sh | 145 ++++ services/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 140 bytes services/__pycache__/asr.cpython-312.pyc | Bin 0 -> 2530 bytes services/__pycache__/avatar.cpython-312.pyc | Bin 0 -> 12389 bytes services/__pycache__/llm.cpython-312.pyc | Bin 0 -> 5117 bytes services/__pycache__/tts.cpython-312.pyc | Bin 0 -> 2455 bytes services/__pycache__/vad.cpython-312.pyc | Bin 0 -> 2018 bytes services/avatar.py | 360 ++++----- services/llm.py | 22 +- web/app.js | 715 +++++++++++++++++- web/index.html | 53 +- web/style.css | 205 ++++- webrtc/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 138 bytes webrtc/__pycache__/tracks.cpython-312.pyc | Bin 0 -> 5956 bytes webrtc/gateway.py | 11 +- webrtc/tracks.py | 40 +- webrtc/video_track.py | 20 +- 30 files changed, 2310 insertions(+), 404 deletions(-) create mode 100644 .env create mode 100644 .run-8018.log create mode 100644 AVATAR_3D_REFACTOR.md create mode 100644 TECHNICAL_DOCUMENTATION.md create mode 100644 __pycache__/config.cpython-312.pyc create mode 100644 __pycache__/main.cpython-312.pyc create mode 100644 core/__pycache__/__init__.cpython-312.pyc create mode 100644 core/__pycache__/pipeline.cpython-312.pyc create mode 100644 core/__pycache__/state_machine.cpython-312.pyc create mode 100644 scripts/service-8018.sh create mode 100644 services/__pycache__/__init__.cpython-312.pyc create mode 100644 services/__pycache__/asr.cpython-312.pyc create mode 100644 services/__pycache__/avatar.cpython-312.pyc create mode 100644 services/__pycache__/llm.cpython-312.pyc create mode 100644 services/__pycache__/tts.cpython-312.pyc create mode 100644 services/__pycache__/vad.cpython-312.pyc create mode 100644 webrtc/__pycache__/__init__.cpython-312.pyc create mode 100644 webrtc/__pycache__/tracks.cpython-312.pyc diff --git a/.env b/.env new file mode 100644 index 0000000..d16657c --- /dev/null +++ b/.env @@ -0,0 +1,22 @@ +# LLM (DeepSeek, OpenAI-compatible) +# Fill your real key below. +LLM_API_KEY=sk-aac48115300845be97a032444fc11560 +LLM_BASE_URL=https://api.deepseek.com/v1 +LLM_MODEL=deepseek-chat +LLM_TIMEOUT=20 +LLM_MAX_TOKENS=300 + +# Web service +WEBRTC_HOST=0.0.0.0 +WEBRTC_PORT=8018 +STUN_URL=stun:stun.l.google.com:19302 +CORS_ORIGINS=* + +# HTTPS (for cross-device mic/WebRTC) +SSL_CERTFILE=/home/xsl/code/product/certs/dev-cert.pem +SSL_KEYFILE=/home/xsl/code/product/certs/dev-key.pem + +# 3D avatar driver mode +AVATAR_DRIVER_MODE=blendshape_stream +AVATAR_CONTROL_PROTOCOL=ws +AVATAR_BLENDSHAPE_SCHEMA=arkit diff --git a/.run-8018.log b/.run-8018.log new file mode 100644 index 0000000..29ad387 --- /dev/null +++ b/.run-8018.log @@ -0,0 +1,386 @@ +INFO: Started server process [401194] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit) +INFO: 127.0.0.1:43090 - "GET /health HTTP/1.1" 200 OK +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.) + return torch._C._cuda_getDeviceCount() > 0 +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect" +INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1 + super().__init__("LSTM", *args, **kwargs) +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + WeightNorm.apply(module, name, dim) +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found" +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + import pkg_resources +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found" +Building prefix dict from the default dictionary ... +DEBUG:jieba:Building prefix dict from the default dictionary ... +Loading model from cache /tmp/jieba.cache +DEBUG:jieba:Loading model from cache /tmp/jieba.cache +Loading model cost 0.358 seconds. +DEBUG:jieba:Loading model cost 0.358 seconds. +Prefix dict has been built successfully. +DEBUG:jieba:Prefix dict has been built successfully. +WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning. +INFO: 127.0.0.1:60258 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 127.0.0.1:40524 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:2190 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:12419 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:9602 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:6375 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:6375 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:6375 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(5xwO9ewTsb 1 udp 1853759231 172.28.96.1 60702 typ prflx) +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(0) ICE completed +INFO: 172.28.96.1:9899 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(0) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:1631 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:1631 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:6354 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:11967 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:2728 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO: 172.28.96.1:2728 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13395 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:4941 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:4941 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5793 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:3627 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(2) Discovered peer reflexive candidate Candidate(Ly5ELFkmha 1 udp 1853759231 172.28.96.1 50062 typ prflx) +INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(2) ICE completed +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(2) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:1723 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /web/app.js HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:1723 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:14573 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:7534 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(3) Discovered peer reflexive candidate Candidate(sCNsxRJ8b8 1 udp 1853759231 172.28.96.1 64421 typ prflx) +INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(3) ICE completed +INFO: 172.28.96.1:7236 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 172.28.96.1:10460 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(3) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:14796 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14796 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:2032 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:14707 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:13486 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(4) Discovered peer reflexive candidate Candidate(Qady0BDpmC 1 udp 1853759231 172.28.96.1 53980 typ prflx) +INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(4) ICE completed +INFO: 172.28.96.1:13486 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:13486 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:13486 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10285 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5486 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:13787 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:10285 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:10285 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(5) Discovered peer reflexive candidate Candidate(BLyMgNoiBf 1 udp 1853759231 172.28.96.1 55730 typ prflx) +INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(5) ICE completed +INFO: 172.28.96.1:8337 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(5) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:7750 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:7750 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:1059 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:14463 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:14634 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:14634 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(6) Discovered peer reflexive candidate Candidate(dbehuF1olz 1 udp 1853759231 172.28.96.1 56094 typ prflx) +INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(6) ICE completed +INFO: connection closed +INFO: connection closed +INFO: 172.28.96.1:14493 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14493 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10905 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:10117 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:6762 - "GET /meta HTTP/1.1" 200 OK +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:6762 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(7) Discovered peer reflexive candidate Candidate(8K1dQt9yU0 1 udp 1853759231 172.28.96.1 49725 typ prflx) +INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(7) ICE completed +INFO: 172.28.96.1:14132 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:14132 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:14132 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:2807 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:6331 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:5113 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:2807 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:2807 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(8) Discovered peer reflexive candidate Candidate(ROwaVxsM9g 1 udp 1853759231 172.28.96.1 52410 typ prflx) +INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(8) ICE completed +INFO: 172.28.96.1:10113 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 172.28.96.1:10113 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:4914 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:10113 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:5903 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:3565 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:10113 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:10113 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(9) Discovered peer reflexive candidate Candidate(E8HKWSMFTW 1 udp 1853759231 172.28.96.1 50992 typ prflx) +INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(9) ICE completed +INFO: 172.28.96.1:13074 - "GET / HTTP/1.1" 200 OK +INFO: connection closed +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: connection closed +INFO: 172.28.96.1:13074 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:13074 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:7285 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:7285 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:13646 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 172.28.96.1:9569 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:2629 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:13646 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:13646 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(10) Discovered peer reflexive candidate Candidate(AbT1vCac8Y 1 udp 1853759231 172.28.96.1 62912 typ prflx) +INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(10) ICE completed +INFO: 172.28.96.1:6176 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(10) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: 127.0.0.1:51958 - "GET /health HTTP/1.1" 200 OK +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [401194] +INFO: Started server process [404442] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit) +INFO: 127.0.0.1:34264 - "GET /health HTTP/1.1" 200 OK +INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.) + return torch._C._cuda_getDeviceCount() > 0 +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect" +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1 + super().__init__("LSTM", *args, **kwargs) +/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + WeightNorm.apply(module, name, dim) +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found" +/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + import pkg_resources +INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found" +Building prefix dict from the default dictionary ... +DEBUG:jieba:Building prefix dict from the default dictionary ... +Loading model from cache /tmp/jieba.cache +DEBUG:jieba:Loading model from cache /tmp/jieba.cache +Loading model cost 0.384 seconds. +DEBUG:jieba:Loading model cost 0.384 seconds. +Prefix dict has been built successfully. +DEBUG:jieba:Prefix dict has been built successfully. +WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning. +INFO: 127.0.0.1:35522 - "POST /chat/text HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET / HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /web/style.css HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /health HTTP/1.1" 200 OK +INFO: 172.28.96.1:6932 - "GET /events HTTP/1.1" 200 OK +INFO: 172.28.96.1:14179 - "WebSocket /ws/subtitles" [accepted] +INFO: connection open +INFO: 172.28.96.1:12729 - "WebSocket /ws/animation" [accepted] +INFO: connection open +INFO: 172.28.96.1:3368 - "GET /meta HTTP/1.1" 200 OK +INFO: 172.28.96.1:3368 - "POST /webrtc/offer HTTP/1.1" 200 OK +INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(jQ7HV4xKqV 1 udp 1853759231 172.28.96.1 52186 typ prflx) +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.WAITING -> State.IN_PROGRESS +INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.IN_PROGRESS -> State.SUCCEEDED +INFO:aioice.ice:Connection(0) ICE completed +INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK" +INFO: 172.28.96.1:8927 - "POST /chat/text HTTP/1.1" 200 OK +INFO: connection closed +INFO: connection closed +INFO:aioice.ice:Connection(0) Consent to send expired +ERROR:asyncio:Task exception was never retrieved +future: exception=MediaStreamError()> +Traceback (most recent call last): + File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio + frame = await track.recv() + ^^^^^^^^^^^^^^^^^^ + File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv + raise MediaStreamError +aiortc.mediastreams.MediaStreamError +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [404442] diff --git a/AVATAR_3D_REFACTOR.md b/AVATAR_3D_REFACTOR.md new file mode 100644 index 0000000..675f261 --- /dev/null +++ b/AVATAR_3D_REFACTOR.md @@ -0,0 +1,189 @@ +# 3D 数字人驱动重构方案 + +## 1. 目标变更 + +原链路： + +ASR -> LLM -> TTS -> MuseTalk/图片或视频驱动 -> WebRTC 视频轨 -> 前端播放 + +新链路： + +ASR -> LLM -> TTS -> 音频特征/viseme/blendshape 控制数据 -> 前端 3D 渲染引擎 -> 浏览器展示 + +核心变化： +- 后端不再负责生成数字人视频帧 +- 后端保留实时语音、时序仲裁、字幕、音频下行 +- 后端新增 3D 控制流输出能力 +- 3D 模型渲染放到前端或独立渲染端 + +## 2. 为什么要这样改 + +MuseTalk 这类方案适合“2D 说话视频合成”，但不适合 3D 模型系统： +- 输出形态是视频帧，不是骨骼或 blendshape 控制数据 +- 一旦换 3D 模型，原本的视频生成结果无法直接复用 +- 每轮都做视频推理，延迟、GPU 占用、链路复杂度都偏高 +- 3D 系统通常需要的是时间序列控制参数，而不是渲染后的像素 + +因此，3D 化的正确边界是： +- 后端负责“说什么、怎么说、何时说” +- 渲染端负责“模型如何动、如何显示” + +## 3. 本次代码重构内容 + +当前代码已调整为： +- WebRTC 下行仅保留音频轨 +- 新增 `/ws/animation` WebSocket，下发 3D 动画控制数据 +- `services/avatar.py` 改为音频到控制参数的驱动服务 +- `/avatar/schema` 暴露控制协议说明 +- 前端采用 Three.js + GLTFLoader + VRM loader，直接在浏览器渲染 3D 模型 +- 支持加载本地 VRM/GLB 文件，也支持加载 URL 形式的 VRM/GLB/GLTF +- 若未加载模型，页面会回退到一个调试用的 3D 头像，便于联调口型和头部姿态 + +当前控制协议字段包括： +- `jawOpen` +- `mouthClose` +- `mouthFunnel` +- `mouthPucker` +- `viseme_aa` +- `viseme_ee` +- `viseme_oh` +- `headYaw` +- `headPitch` +- `headRoll` + +输出包结构示例： + +```json +{ + "type": "animation_chunk", + "driver": "arkit-blendshape-v1", + "mode": "blendshape_stream", + "schema": "arkit", + "chunk_index": 0, + "total_chunks": 3, + "sample_rate": 24000, + "fps": 25, + "text": "你好，欢迎使用系统。", + "frame_count": 18, + "duration_ms": 720, + "frames": [ + { + "seq": 102, + "time_ms": 0, + "speaking": true, + "controls": { + "jawOpen": 0.41, + "mouthClose": 0.66, + "mouthFunnel": 0.27, + "mouthPucker": 0.19, + "viseme_aa": 0.41, + "viseme_ee": 0.24, + "viseme_oh": 0.31, + "headYaw": 0.004, + "headPitch": 0.009, + "headRoll": 0.001 + } + } + ] +} +``` + +## 4. 当前实现的定位 + +当前实现是“联调级骨架”，作用是： +- 先把系统边界改对 +- 让前后端可以并行开发 +- 让 3D 引擎接入方尽快有稳定协议可以消费 + +它不是最终口型算法，原因很明确： +- 目前控制值来自音频能量和频谱特征 +- 没有做 phoneme 强制对齐 +- 没有做语言相关 viseme 映射 +- 没有结合 3D 模型自身 rig 约束做校准 + +所以这版适合： +- 打通链路 +- 调接口 +- 验证时序 +- 校验驱动字段 + +不适合直接作为最终高保真口型方案。 + +## 5. 推荐的生产化方案 + +### 方案 A：前端 Three.js / React Three Fiber + +适合： +- 浏览器直接渲染 WebGL 角色 +- glTF/VRM 模型 +- ARKit blendshape 或自定义 morph target + +建议做法： +- 前端加载 3D 模型 +- 建立 `controlsKey -> morphTargetInfluence` 映射表 +- `/ws/animation` 按时间戳缓冲一小段再播放 +- 音频继续使用 WebRTC 远端音频轨 +- 前端用远端音频播放时间作为动画时钟基准 + +当前仓库已经按这个方向落地了第一版： +- 静态页面通过 import map 从 CDN 引入 `three`、`GLTFLoader`、`@pixiv/three-vrm` +- VRM 优先使用 `expressionManager` 写入 `aa/ee/oh/ou` +- 普通 glTF 则尝试把控制字段映射到 morph target +- 头部姿态优先驱动 `head/neck` 骨骼，找不到时回退调试头像 + +### 方案 B：Unity/Unreal 渲染端 + +适合： +- 更复杂的材质、灯光、骨骼、表情系统 +- 要求更强的 3D 表现力 + +建议做法： +- 浏览器只做 UI 和音频交互 +- Unity/Unreal 作为单独渲染客户端消费 `/ws/animation` +- 浏览器与渲染端通过流媒体或共享画面集成 + +### 方案 C：专用音频驱动服务 + +适合： +- 追求更准的嘴型 +- 后端可以接受增加一个模型服务 + +可以考虑接入： +- phoneme 对齐模型 +- viseme 预测模型 +- NVIDIA Audio2Face 类服务 +- 自研 `audio -> blendshape` 网络 + +推荐协议保持不变： +- 外部模型服务可以替换 `services/avatar.py` +- 对外仍输出统一 `frames[].controls` + +## 6. 推荐的下一步实施顺序 + +1. 先确定 3D 模型和渲染端技术栈 +2. 固定一份 blendshape 映射表 +3. 把当前前端调试面板替换成真实 3D 场景 +4. 增加时间同步与缓冲策略 +5. 再替换掉启发式控制算法，接入更强的 viseme 模型 + +## 7. 接口约束建议 + +为了后续可替换，建议保持这些约束不变： +- 下行动画通道继续使用 `/ws/animation` +- 每条消息包含 `frame_count`、`fps`、`frames` +- 每帧保留 `seq` 和 `time_ms` +- `controls` 使用稳定字段名，不轻易改动 +- 新增字段时只追加，不破坏已有字段 + +## 8. 风险与注意事项 + +- 真实 3D 模型的 blendshape 命名不一定和当前字段一致，需要做一次映射层 +- 如果模型是骨骼嘴型而不是 morph target，需要把 `controls` 转成骨骼驱动参数 +- 浏览器渲染时要注意音频时钟和动画时钟一致，否则会出现“听起来对，但嘴型慢半拍” +- 如果后续改成 phoneme/viseme 对齐，建议在包里增加 phoneme 片段和置信度字段 + +## 9. 结论 + +这次重构的重点不是“把 2D 数字人继续硬改下去”，而是把系统边界调整到适合 3D 模型的形态。 + +现在的系统已经从“视频合成型数字人”切到了“控制流型数字人”，后续无论接 Three.js、Unity、Unreal，还是专门的 Audio2Face/viseme 服务，都可以在这个边界上继续演进。 diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md index 486221e..05f72dd 100644 --- a/DEPLOYMENT_GUIDE.md +++ b/DEPLOYMENT_GUIDE.md @@ -15,7 +15,7 @@ - 用户语音上行（VAD + ASR） - 文本调用在线 LLM - 本地 TTS 合成语音 -- 数字人视频下行（MuseTalk 优先，失败回退写实底片） +- WebRTC 音频下行 + 3D 数字人控制流下行（WebSocket） - 前端字幕与指标面板（SSE + WebSocket） - 单工仲裁（支持 barge-in 打断） @@ -36,10 +36,10 @@ - `llm.py`：在线模型 + fallback - `tts.py`：Kokoro - `vad.py`：Silero VAD - - `avatar.py`：MuseTalk 推理调度 + 写实回退帧 + - `avatar.py`：音频转 3D 控制数据（blendshape / 头部姿态） - `webrtc/` - - `tracks.py`：实际音视频轨道实现 - - `gateway.py`、`video_track.py`、`audio_track.py`：结构化封装 + - `tracks.py`：实际音频轨道实现 + - `gateway.py`、`audio_track.py`：结构化封装 - `web/` - `index.html`、`app.js`、`style.css` - `scripts/` @@ -192,11 +192,18 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c ## 7. 前端功能说明 网页包含： -- 视频窗口（数字人） +- Three.js 3D 数字人舞台 - 文本输入框 - 字幕消息区（WebSocket） - 指标面板（SSE） +当前前端能力： +- 浏览器通过 import map 直接加载 Three.js、GLTFLoader、three-vrm +- 可加载本地 `VRM`/`GLB` 文件 +- 可加载 URL 形式的 `VRM`/`GLB`/`GLTF` 模型 +- 页面默认会自动尝试加载官方 `three-vrm` 示例模型：`VRM1_Constraint_Twist_Sample.vrm` +- 未加载真实模型时会显示调试头像，仍可联调动画控制流 + 当前已支持： - 页面刷新自动重连 - 单连接策略（新连接踢掉旧连接） @@ -207,9 +214,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c ## 8. 当前已知边界 -- MuseTalk 结果生成可能有一定延迟，期间使用写实底片回退。 -- 部分“主观体验类”验收项（如口型同步主观评分）需要人工实测判定。 -- Qwen3-TTS 对比方案尚未并入主链路（可后续做 A/B 开关）。 +- 当前后端输出的是可直接对接 3D 引擎的控制数据，不负责最终 3D 渲染。 +- 当前控制数据基于音频能量和频谱特征做启发式推断，适合作为联调骨架，不等同于生产级口型模型。 +- 若要达到更高口型精度，建议后续接入 phoneme/viseme 对齐模型或 Audio2Face 类推理服务。 --- @@ -224,8 +231,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c - 系统是单连接模式，新连接会断开旧连接（预期） 3) 口型不动 -- 查看 `/health` 中 `avatar.queued_frames` 和 `avatar.last_error` -- 若 MuseTalk 未产帧，会回退写实底片 + 口型叠加 +- 查看 `/health` 中 `avatar.last_frame_count` 和 `avatar.last_error` +- 查看前端 `/ws/animation` 是否已连接，控制流预览是否持续刷新 +- 若模型已加载但不动，先检查该模型是否带有 VRM expression 或 morph target 4) 模型异常 - 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD diff --git a/TECHNICAL_DOCUMENTATION.md b/TECHNICAL_DOCUMENTATION.md new file mode 100644 index 0000000..89e15a6 --- /dev/null +++ b/TECHNICAL_DOCUMENTATION.md @@ -0,0 +1,278 @@ +# 3D Digital Human System Technical Documentation + +## 1. Project Summary + +This project implements a real-time conversational digital human system with these core capabilities: + +- Text and voice interaction +- LLM response generation (DeepSeek online, with fallback) +- TTS synthesis for reply audio +- Audio-driven 3D animation control stream +- Browser-side 3D rendering (Three.js + VRM/GLTF) +- WebRTC audio transport and WebSocket animation transport + +Current workspace root: + +- `/home/xsl/code/product` + +## 2. Runtime Architecture + +### 2.1 High-Level Pipeline + +1. User input enters through `/chat/text` or voice pipeline. +2. LLM generates reply text. +3. TTS synthesizes reply waveform. +4. Avatar service converts audio chunks to animation control frames. +5. Backend pushes: + - reply audio to WebRTC audio track + - animation frames to `/ws/animation` +6. Frontend renders 3D model and applies mouth/head/body motions. + +### 2.2 Transport Split + +- WebRTC: audio only +- WebSocket (`/ws/animation`): animation control data +- SSE (`/events`): system metrics and runtime status + +This split removes server-side video generation and makes rendering native in the browser. + +## 3. Core Modules + +### 3.1 Backend Entry + +- `main.py` + +Responsibilities: + +- FastAPI app and route registration +- WebRTC offer/peer management +- Animation WebSocket broadcasting +- Runtime metrics and `/health` +- Session reset and chat endpoints + +### 3.2 Pipeline Orchestration + +- `core/pipeline.py` +- `core/state_machine.py` + +Responsibilities: + +- Turn-based orchestration (input -> LLM -> TTS -> output) +- State transitions (idle, thinking, speaking) +- Latency accounting and result metadata + +### 3.3 Services + +- `services/llm.py` +- `services/tts.py` +- `services/asr.py` +- `services/vad.py` +- `services/avatar.py` + +Responsibilities: + +- `LLMService`: online model calls via OpenAI-compatible client +- `TTSService`: speech synthesis +- `AvatarService`: audio-to-controls mapping for mouth/viseme/head motion + +### 3.4 Frontend + +- `web/index.html` +- `web/app.js` +- `web/style.css` + +Responsibilities: + +- Three.js scene setup and camera controls +- VRM/GLTF model loading (URL or local file) +- Animation buffering and frame application +- Idle body pose + talking expressions +- WebRTC audio playback + +## 4. Animation Data Contract + +Backend pushes animation chunks shaped like: + +- `type`: `animation_chunk` / `animation_reset` / `animation_state` +- `fps`, `duration_ms`, `frame_count` +- `frames[]` with: + - `seq` + - `time_ms` + - `controls` + +Typical controls include: + +- `jawOpen` +- `viseme_aa`, `viseme_ee`, `viseme_oh` +- `mouthPucker` +- `headYaw`, `headPitch`, `headRoll` + +Frontend behavior: + +- buffers frames for stable playback +- applies VRM expressions when available +- falls back to morph targets or bone motion + +## 5. Configuration + +Configuration file: + +- `.env` (root) + +### 5.1 LLM Settings + +Supported API key aliases: + +- `LLM_API_KEY` +- `DEEPSEEK_API_KEY` +- `OPENAI_API_KEY` + +Recommended DeepSeek settings: + +- `LLM_BASE_URL=https://api.deepseek.com/v1` +- `LLM_MODEL=deepseek-chat` + +### 5.2 Server Settings + +- `WEBRTC_HOST` (default `0.0.0.0`) +- `WEBRTC_PORT` (default `8018`) +- `SSL_CERTFILE` +- `SSL_KEYFILE` + +### 5.3 Avatar Settings + +- `AVATAR_DRIVER_MODE=blendshape_stream` +- `AVATAR_CONTROL_PROTOCOL=ws` +- `AVATAR_BLENDSHAPE_SCHEMA=arkit` + +## 6. Startup and Process Control + +New process script: + +- `scripts/service-8018.sh` + +Usage: + +```bash +bash scripts/service-8018.sh start +bash scripts/service-8018.sh status +bash scripts/service-8018.sh logs +bash scripts/service-8018.sh restart +bash scripts/service-8018.sh stop +``` + +Behavior: + +- Loads environment from `.env` +- Starts uvicorn from project venv if available +- Supports HTTPS cert/key from `.env` +- Writes pid and log to `.run/` +- Cleans stale python listeners on target port + +Runtime files: + +- `.run/visual-chat-8018.pid` +- `.run/visual-chat-8018.log` + +## 7. API and Web Endpoints + +### 7.1 Health and Metadata + +- `GET /health` +- `GET /meta` + +### 7.2 Chat and Session + +- `POST /chat/text` +- `POST /chat/reset` + +### 7.3 Streaming Channels + +- `GET /events` (SSE) +- `GET /ws/subtitles` (WebSocket) +- `GET /ws/animation` (WebSocket) + +### 7.4 WebRTC + +- `POST /webrtc/offer` + +## 8. Validation Checklist + +### 8.1 Service Up + +- `bash scripts/service-8018.sh status` should show running +- `GET /health` should return valid JSON + +### 8.2 LLM Online + +Health fields should indicate: + +- `llm.ready = true` +- `llm.error = null` +- `last_llm_source = online` after at least one chat turn + +### 8.3 Frontend Motion and Audio + +- Model loads successfully +- Mouth motion follows replies +- Body remains in non-T-pose idle stance +- Remote audio is audible via WebRTC + +## 9. Troubleshooting Guide + +### 9.1 LLM stays fallback + +Symptoms: + +- `last_llm_source = fallback` +- `llm.ready = false` + +Checks: + +1. Confirm `.env` exists in workspace root. +2. Confirm key value is set in `LLM_API_KEY` (or alias). +3. Restart service after any `.env` update. +4. Verify `llm.base_url` and `llm.model` in `/health`. + +### 9.2 Model load errors + +Symptoms: + +- Web UI shows model fetch failures + +Checks: + +1. Use reachable URL or local `.vrm` file. +2. Confirm browser can access external CDN. +3. Prefer VRM full-body models for body-pose control. + +### 9.3 Mouth not moving + +Checks: + +1. Confirm animation websocket connected. +2. Confirm animation frame count increases. +3. Confirm TTS is ready and not in silence fallback. + +### 9.4 HTTPS/microphone issues on LAN + +Checks: + +1. Use valid cert paths in `.env`. +2. Trust self-signed cert on browser/device. +3. Ensure firewall allows target port. + +## 10. Security and Operations Notes + +- Keep `.env` out of source control. +- Rotate API keys regularly. +- Restrict exposed port by network policy if internet-facing. +- Consider reverse proxy and managed TLS for production. + +## 11. Suggested Next Production Steps + +1. Add structured request tracing (request id, turn id). +2. Add persistent metrics export (Prometheus/OpenTelemetry). +3. Add automatic reconnect and health watchdog. +4. Add integration tests for LLM online/fallback branches. +5. Add model asset mirror to avoid public CDN instability. diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7995bca6e8545040dec4980c20bf801e5cff61b GIT binary patch literal 2535 zcmb7`%TF6e9LLx8+88ikUVdQSBqW63cy&a*vuu7dg7LdoO_p7lAr{Arj+xkzMNn6`#Af}Ku%WWTu=@AB)@b| zf`RWOC?okvz#FTE{1OU1sU%YC4ud11Q%)m(8ksWEW{jPC^M~xAy@~))ipn@uZ?IB?uMGXQMeF|S*TQpp)fTEm6)_yTqaCXLSutEtl0y~C@ zXlu0vkfqy@6%LhXIJ*NJdkBeOQQRArXA z4x$~cU^+CqubBqa9K*H%YiWcwnr04Yc?0aB!(uAyIC(OwC@heM2;~VvduiRyDf<&N z!e_lRx0wMC?yu0++O)M$OJ&|#Ai{HRv+4&0Y!;iU3CHxT=FobcZgqVjvqGC|xjc?k z5_we5v*c`u%v1hwcq?{{9I^|}>5n|pEF9V+Z+l3wZagi%wDACKV@C&Bn>faCA0J$a z8;-ZLDUsPw;^QO}~BR_rED!J6fgHq*Pdy=x-A#j_JPr^Y8~n0N5~qP{CM zqT84N8ymZZMQDT&6X*y#qDC4yv75$Wi$-$=f*j4s1}8+nH)hP8t*t3{N~AJWcR0uctDsW^6C5>{`TvTs0S zD~GnVi>!1mf9;(t#4cbR0pTc4otH&a08y%Qe>yR(6fESV^G=qwSEGQ3ykaTd_#c_7 zl_ z3~M7pGz@^H%k#j)Df{;z*{+v@KAsiG;IMuvn<;YCjbEe z%d89YcN=cw664avrJD&27_7K^IkIO*!u?zZxm@5f%;hqd5iX-#u5uaUl4e5XZQC}n z!g-v_1eZxJQ(UfbnP&2%W%*ki$CHcMd zy#3-Y`+vlrwSRK7^0Fg#wD?#r%dzM2Za3QYJbCfwTxt4VY3X4(slEvY5}~8ztR#X=bUk{CPR~1@k0BM=QreRZlGCM*FyDuQWCP+n43!7PG`cM@x^1 zu)GROi6l>2mJ@DQ&(YfPd|B>!8SnRE{cb$vwNh>@arE(VfcttUN^pV zvRRfdxfe!Da`y>a{ixe9>0QU9n@I81l;xD0=<-@!ZoD#K@>NLc9W2Qm$FM99xIHr^ zx$~HmwUICl=MSM~Ifm|43$Gj6T8#mA_K3eXnBNsqg5n;r)2!NYn|FweAMv5#TVDCz w#tRJDU6jZWlUD(s&-X&Q{6f0@CN%6@^*#RPjl_Rn-}gz8jvr>f4T~iI0xr|}qyPW_ literal 0 HcmV?d00001 diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8af30ff70f3fb896abcd7e273523032baf416ab6 GIT binary patch literal 30272 zcmd^odw5jWb?-SdXWpaHXlC?&zcoS+#M^+e1wt6fmTd$k7RKX2X9N)8Ai^GCrU%x+_uiw?|!oJ_r=hBH}|*p znRDi77_gi6-hb`^t+V#tYd_9@uC?~sdw!IYW99I46#j7dmzz26pU9y+W*OnH9XD~@ zS#B36av~q(M)?7rCtMfQ4d~cgKcHuCVL)JS!+?RkjRQu!^+D69dB8ks8L*_{SqE(J z7lQWDoPnHC$AE+R8-lr`&H*RGjlsN8*MMu(J>VYo40uNK2lAP}DOfOCI8exNbFgT% zc%XQ+WT0fUbfA>^S%PJwX9L(O4DztIJR0*LY00i^?OEl(j$QOC^x1P%lGqQ1%tpY zR=#AA>Dl+!D=`M{#5Xpou1#<)Q(bq#)vCJghO1q5b;9LUU7O*m4cr;n6u2vJ_e+AD z)4&$^c8FCMtCg=A=z`ByHCDG+6WAu!rdl)Ye~(xP|N5K!w=@3+@(-AG+%`^ZJkE(t zFX`m`)O350OONX6Wt3h}HU}z|)*HB2+$k~!?&Q`s8Zjlp&kJB_q8RKadDGM{fKzi zaX~HlBe$TxTSMQeq2CMozs!>UW|e-6rZm1=P(l9g}2#WB_+p%-Y1A&0FWqfQbFi51ye0cQ-LLuN?fzY5dJVD-+UU1wm`o!^Y zVC)I6kTiAsL*dQ`wkJ(H0*_AyLgA!kcVJ)t_~5}nIO$f=uHn$2oH&wjY`K3&|9!)v z7z`ZpO98JwY408m26hBO6DVXLY47R3zfX1L^@pW^e{^_kzv4xu=%-H_q-Y^HEt~zJ zzGtiszjtdN zvM>Y#PXvNfc@GVTCjG&xhsKd}6%~=f!o%TUU`l8|6xf&41HjKB4ohf|ap?#g!kB+F zkmOsEeDhA!e2zaD96#h6mxlKbkA;$XvU5-hh=H;2us;}5C`JR}1LLCVJP`1U0V(vs z%wNd;Bjg{Q2nKwplfYB~s&3CsondcMKQS1(;h;dZMfWD1p~-!$%)Y_kF!H1_j17;n zuBLbmYCEmM8(eDvw{eyWa09%^4d}ov^56<}0YTJLZ->jkTmo|$z!@45W=gqC%wel%tac?DI?e1HR!g@^Pq+!Et;oI{f%pAeGJV*u-SmH;N&i%zXkA zPzWOxi7~ny@(7HHDQD_a)UL=PmB`{7Srriq&$(kll@=|nlf&z@p0#xmq4XS5s!NoWM+DF5K}xiwRL-b0QCclW zs@5c*&a$#GUb#M6xjvQM`b0xxM5vh4$Ard2UcpS)Y+Xz!c)zykRnL6wrkS3Ivua-0 zBt58nTlPt+2*FDysQ@M^HX!wwesgzX#L?89TKA+Cpic@k4*R@<)I*8)61bPZeFPpL zz;fL|*iHqID+IwAb(^r}9N#X}Q1D=ic?ZgI)PzrQ9DmJ->9d#PQM&!yQC&DCx^diy zmYPN3xM^PGFX|K_;HW;F5_Ay4DeO@fx3>)9lVIVdg%K)8#xt!~D4&8b|9MQ~U(=t} zf7xI_8BSIR+_VlUpd%g?Ab{m$M22%(P~w9=U*L{#Yq^t^wOlx*HsH8APV(?bXT)~< z*bj3jA3DSx)<4W0;=iLm#C!F9-7jL54~&H-rGU>L8XO+>nv%ju2rP_2oE)79N%tek zq$Nc21Dhl*TMrKg7&ntN`X?sfpEQYJPyCYTH6%^ST4tK?A0bvJX~0wy3?xmX;D-GB z14-SXP>3qAiXMsD4#Rq%1kyp8jVWtMgzrQ63z7KxuiPgyT%x>kX6u5(b!z0q$hpD` zlk<+%Guz-kz4j}kv${o{-cp`$dCs<+X*qW&=4y=F8>9BdMFW>xcFoG=dS-PAbMC3! z6S?Pf@0m+K)Wd7h#N{+yU&t-`h~p92o_Ffr6Zf8T{ls3jsE5a|Lo{TMHErhkmpq-- zy1(W-D~*>q0qA9+6uy@$dBUqpIyV_EZ_ts?ojl>2^uW7c)Fn{`5^)m5D31AqgJ^j6 zP&c#Vk9Ee=K;+Ii&4>6ST)I1k)$TZrZaA%*)=vxR?iW@&+K7>*x&$aPlE_b?u9fa1 z>dqULuB>)NozfLqw=$%<#Rvb2(t3?aK^hMRq+LinX_S{KiBv4oe)2J3RT>Rpv{HLX ztprw+zd$Rk*C3Gsg~gQ)5~fDXY=5_={S~H28bg$Q_zTfg!1}<4PR{O3m>mn&;stZj zRZmd@{|m|zg%yeXl7;-D^PcC6pDX@)>0CHkyeyXA@?nn2V!dXt+Kj*AtOnyp1a5?K zk=(J3oksqz_)bCiYa_5hbqYWj)h271e*+@7Ou9`lK$TWRsPc$>IHi@qB$BN)QZ1>p zLwex))afLw4*Xl>p%e8=+hs{d?G52PrBq>;f?=vkmEg9mF7zdh@)DS|Y1Vhf1$nLV z1g+m62(v{~>O<;D6SyI2YFC^mBJt(i138x8yJ(_Ss*rE82 zI2A1YMh->|h1&c}C`=j(PdY@wkcL@9QK0yz-=MH)C}^3_ctBy*P?Tf@!wCwTh61Z+ z<^u}5hN2=foTZVYp{UAA;m}ajWTnW}P}F6maB3(TvQp$}D4-b0e4gPo;mw($9H$9w zxos#7sEkJwmw3&LC%i)O7+IxY>35hIXZBLtsnlAzUT(kb%I@ zFgd_mDbICQzpCtzo}eW3!N8cbj8bH49V=*voK%s&tpq})%_u7qIiM*b(NI>eB#lrz z1j7fQj)I7(c-lxTR2@vLOOZQL5=|;FgLO_iNs6U;+n3_fq%WfsU&LSN8O+o(+@h}3 zZUoEptozx4cz%5}zdn}Nu&9UURp#0MOwpnNu0ivlO3ijF&8r zmMmW^C7&{F%H`x!q4lXGpDOKFRFhAQHl14dd|bzs)yiMBP}v}V%tAwJyy2c`!#xYd zRnM0_SGJH_63?xU=2qVhaaF}#jZs(QN2@6xuGC;;GL6bXqMX7t#A{CTQ}9z3Erstm zDuhv1`594{HpK@i8ARi>9;GlT`#VPspCN{@B!*EFLtQA-c}yFp4d8@Q<@k@+k@8KL zxXes!5(dX1;uB}}0af_FC|RYD#k(L{nIRudj6;LuHDh`Sg^x@?0vel0nuZ`8g;%b? z1bDy?KmDlxu*^K0p=u3H_@NInB3eM2fIw#OlYSJDq;X6M5*Y7sdV#=)DJf|hhi-=u z!(rce5F*!M-&o)f6N#CWrN~(hpA<-`D@Vu2#-(qd#D9pt(C-64#xglh*IzLeCUQNe zrcX?t-~4>fb3GS!%}&R1S4PY$6FII^drs^*mv`dPneGKg-s$RR`_C23)jwBpp)p$A z5zAi@bF73=W_O(GJ<)qkf3AP7;nM1P`yCM6G=7bjw$9tv!_Ss;>W&k4oZfYA>)h&? z%^SBZkJ^^M;`)Bc%Ox?}#)z=-HhumFa-Zi&!pM9?o{a^y&!ut?LXU$kz7(4IDOl zFiZsh;o;DJxgM_GJv;+_o%A~t{<{QD0(48i2k5mzt0@kTCWX;}e@r?>p(H^|X9!R? zmYyNNB#~zc`zip&9yIWy67gP1^WXvK5ChQf3v3WPM^t}6;Ohj=1B9r(QzIuOwB)CU z4#pnzme3Mn1brKX7bphpTha)!K7IJf!>2or9ZlrAP8~jR_}qiDhhw>wCxolHu2~^r z%RRN>#D-_3vr}iLVz!#O-BDZ1Ojp9>Ivu!TDoy0NXKhz=@=gt$7&ynD*qb@bmPLzK z$MV<29BY5OVYVt}Yl_?2qqg=}blnAAM1&3UFw-`s2_N$mNkhjl%s!h4e0u>{ zyS!kas9gw?yqa&eK{jo`@B#%b4{Enl7ZQ{*^=NX>GVGM0dej(Jm!1)I*ePjX=`t#m zH;c((Ro)tLr(-BsT0w&aEcHd1X&g0$^HqA&t#e{(nDx-`mEh09)G=6Yq*=t3yZ~%< zdF?kpzxxAj<~P6j;5Xpfyn0zfW0N-rhR4Lf;iNr`ZOfp%Ul8)z*kqqEV3Go~brVSg z)|rkqNef$KeBh3grob3BBPRpWmyltyc;Do3kah>g!q5g`BZ5{TM$Dvkj4r4L_sc%e z5&^~9gk8k_vcgb5EFPv63#Gx98ZBhLRqGCdZ`}_K*&wmF`?Wh?DULMD7%amk1uK@E z6}AFI{H1oxBy7iVui`KCJpiUa_}Iba*UsG$uU#3fUHQu9@Atgi^V+UR?aEkf@7oVW zb`8WlduHxia5zuxKe7Mx=-mEjLFPC^kKAF-EnJa z)LMFOTg+M)7ocvhTc~J>S9C`!x)bI+Xr&T4`3X-UG*OE-u5ufHjWbq4?_?}CMudEn z#8w%pTpqJ_#D$Ki(6N}u8R|b-)FJp2@VH$o&s9P8Y;`7ZwQCVulB!sWZFfxlkG zZ{8rhUhN`$CBOMj;q_HU!Z+{)?<6>gnCe4Q7aO~OfPR=7yJ}!I4R^>%su%gmrr#i? zB(xz43FlLpOU+_Ixn?{srQAo!P(`PSpBf)vNmZ5CNEUW29V1(*MEylUWiBISoR7wYvvQp7_TsW!|%S1EYda+!z;4P%pXCvykO~;?+!v%^r>5h&Z90XY@-^;sY|(ZQnl9IMPjiM=TmZ5^-1ZZlomN^38(c*v|*d^ zOj{I6byZb&(89E>%bxJvDTEu@qG0JaSXUZdHB0ArG#%=B)ZkLb~i2Xzjl)%rAl z@*39aZTPFkUtuk3H5=`cowtg`4lguSB+SU?oC$vF&!Y1p@{CU%^AGz;}+1rHv+huw|H9 zN%|z8Y9hfH1bK_UZDVkJ5XLl{+A<@>Db*?fd4I3I?c}z3d(DEYes1W(@T-L}S7+Sb zdBxs&BScf=v9bpK5BWObd1C{tkkTd^OX`XGF{`Hg3O*k?|rg zy^fdmOY>0ZbYARJSADimT^j4BNcS4e&GNT4EI+yJJ$vo-YXZEE<>c~96g#9|VPA1C zkWB7dZdVcF=ClsHG3M-pk8s>Y!%b^Xww5;bC3XH`*3LU+@hX5U@ZDqd(o27em-J@@ zNSq1L5P=jV@6meHssO(>3Ueh^8*Bd~wV{dl_mna}vKq{<=Jz%UAtbYvLrI5KQ@tV|WltaK!rsFIEhD;-xB1DzDFD=S<=8V+815lKk@ zjR3JBsQ+DfVH=eOnyh*@N`dgCG?o69H*RHsQhG?<9W`rIR)=`h#TIM3z#9!C=4-ZeAJNh$W2= zVk_TANlJ(%M7;DKX5uj;bw#i{7-9XNb1ejqCsi0)3gh}v!LeB3JxJ2M( z0yLE}K}7lVw*X1hUkD~h($~nLgE*2Rh2W92kkG+I zo)8I(ikz=>^x|7~Y3Y|B|1thTUd%Ld&ow!oo<25By6n~~rL7PHuDT14Z##Z3OcUac z%BZ6<=BS?8wy3K&lqYJuCr$I_$^}os*}gM0@Het_SC|#Q<_9jY}CCWP>a@4Ma z#8GR7#8GaInu}+Rrwykih!!!OY8VvXxx-5qW0dRvG#+#9R8Pma*wz0mO8<`taPaLFFoqyK1AZ+PUp9S6kfP7O}T2xEd}# z8gJbdZQT`lXisFXFESL2j2?*&O+{LF#acffZ~T1BH66E4N9@yA3(HQLe$T$xifX^U z*udqqf0AhY{3i&xo(lSP=m5IRu};3%#$OgXH}q`aUcZa)Ss}c^b+^LtPAlKDT6m|e z5%|0Le9v0*y9GAFtNETggm-HM!W-;}{q72$@YM{iW$+FLH{|r1^*=J}L9=8%B);J( z;v2|(EpsKwCJSoiylj$`#uAE$wG)@k+DVbDRL&~35*^jSKFNX-WjqQ!HWqA35Y{%m zC@9>^VLk-IY>{L#Ed>rfDHKNy(*~&;5&$g{8IP2&QL$#FP0a?+_YcEwlj!}!E=m0S74-p(#<4Xe`1XREtu8BRvY1euwW0l&iZ zw$46ff5RZs&L7)=U`pJiG3Y;nT|1cvokIzRVZ+W%RpMAg6=PA zUsaDYCye(=<|_O969NB0kWAS_u%H+`(8_|bp#lcTPnML!NII#= z@$L2BTtA=JHq(=^IuZq?XGhMAJQIvNYi11zhdb`5h&n1_j;dMxLP7btz}((TTVn<5 zuM})~H85*i$SXRRKWDyB7t3p(HNL;BRg$GZ<)1Tbri|DpL;y& zsGjSOIhqoligV#PeZ*5YyERc72YZ-B^=CUi}2Q(8sI-H;(OZ7KPFBKjq(UAddCz#!fpN@2u7nTyN=V(7&rM>Z#VhTWW;QyVVBrX%Gl+p=j^6 zGYBRYIPzJ`Pnp_fv!+3mQGL+iWv;aXNG8{+tP1^1gCo5O5@@K%dMU%kNqUm}2vlGw zF{@uTjDBG&joENSqWpSDK;SW34osxmamLHPq!6_C{}i##3qVqZ046`#Sm(6rjV7%W zooYd#_;B1|Uc$M14UIV+reMmFr=cQAHRI8QW?_3Zp^l7*aEknSgItAPT_4Q$&7m+9 zc#{%#=njO#6Cv!h`S)SXNeTzXaVAli7?*H7Dm9^DWeSfEjt9MF*%H`}3YKW?PufEv z>^KIb@DLWPq>Wq$14m>BPUK19K_5%cmU6IU3scFa_7{Ch@)-?ETmCq-RDjHCsYUNc zhO9*)7U!48J@rvf{X%j1`2+EamC=foS1RuPfjPeJ-srk}W5xF^IE&)Wx~Q`*jg-Pt zYeTej!-qB_*;pA3#_J(!FR~PF;inw!aii z^CNSVMT6xk72`|W=xd41O7-93xUDK`!`Yy!R2R;3(@0|LP-safrVh?J@hY{PB7YnQ z{nNJ1Iy%RKP@IIhf#43i2UK$+|!s!-+p9d%joM6?7pG@ zj_@`9ERS(vqRi>ZW)2`4l#!9iq~l;r7uYWO^n$T(%1Ebl*H?8*yOGqC&T3^gBsh}r zFx=7rfiDyI-vHpLf`Pz9W=p`*eQe4t*J@ge4bRoo9-9%7wMW9?N_a}nuX=v{bL;0D z*2Egty*4>tdS9fZ?`<*S>W|s`7hJ_>ThFxOIF+j|Zm)~j>*!pH`>MT8F-_+>t^8#j z-)R;u>y5yt@ay+%Gt}G>;L+$Nc-k_+m?v1CDn%tiukFV_Cuj|<@`lxi6(&t2J1;*Lz;5K zXE!rcn)v1h(ik98H_CX@r>S^$RYG>%LfZ5&t9cbH@P}btlHM7hSjquN_!BX84WZLq%t(FL~DukC;?*SAxc;< zS%=txbnVuN{SZu#QwNwsQWIj{N!{%t;;?aXl`JMoqSlg_wJa`_MTD{i!4nsXqC(NR z;(4KVQD-u=(BL5jTz%A4|ANRgg$=JZL^?Vn5A2$6cqr!D9k=g}*mo;KNEdU}#_hEc zdo3G6IKpKp{{$AtIiKS{Nnk@|!B&COT!t1L50cfWbHzUauWIvT@x>`GL(8v2`%vIx zo-G#sYy4(TI9m+t<(ADx?s7T5StnesC?kA5zgZA2Z!iFVo#R3Bx{fDYV9;2x#i)P1 zPDlO?<-p(26aI#QC)}t9E{`qMSY^rBqA8D!tv^H0`M*E5e*2N7*-6gAQhe((j=bL{ ze@)J+b`G_cdPKv;n=Q9ee(HQ<`polIf$lqJ%4=xXkJ4OB8^~6=$_&MZhpFzfko6x0<}9 z2E@pmwA2zxbd5qUQ4oi9%SkIgE)++F;&a}4p>9!UF|;Neg>gqU1l$+uzTN!I<_klS zHQOSs{vU6O_dXWweGF)1VsgIbiJ0S1+m0n_X5+o4{V)&{e?Ugk6^%C+yY&f4z?H za@b$5&m(*d-{lluU%MRm8;yLIOL(KH4EUP@-{leBG#CiC@<`@QJ5RWSK_`PQ20aWG zRBSEKzqv|Blxxa?zvU+Ut$d#F0zGi>?qXncyd5?Pt=P&8EMZ|)p7;vBAjP|*`Q;}Y z`wI3}3Fba!PlVH$jNfLbLmlJl9(`C9@3Nl%RxPqJW6~Z$Kb;1z#h;mR0s20sV3ht1 zh4z|c<7jlNl!f#c2>W*e{|JyYli3DU5iYT@4*3t_2sqhiO8<<&5b-f79hF2Qgn>OY zUPwnP3nM9=l=tF_@2-ClI;k}?JquP(+*%j4*2Sz1aiJlCBbn~fxVtXuuA3W(xtGUt zmd|u!KX2wftl@^nPZBxHZ-i=);;}nB8+jbFt`#nGWk4^N@tyU;<#H3@^*rE|wH?OW z;dbJVmn@+qL$f9H3Rp{Z3593olkN1b!m?6j!WdFzPDQYjqW(P0EMQ(im4T;U+r(KU zcWG&SBa3g+#>eqWVy4;Sqt3{l^u|`UEiAQ!xJOK^5v|Nju~xJ(Gey;qS+rvVC`Z|@ zQoD+}kfb-Z(pv@DOdCbV#oSvl&bRUh+p8``BiWW7KCCV_w5@j0shm>CEN9wSQq0lR zl*(|xge#9S(1fWqfG|r&n97`|>9J8k-7Vu0U8xNfmB9$tDyZcTn_d5UgZ z9<>jAYC6TYO()%Jve!yUI=1La8APg^e$&R0JCxc;zcr?Y8`HiDhAr4qrG8)#S*DC9 z7(}WYha;O5>hv21k>$)FvO-kv0sNHQ?@q^7Fr{T-5V=|L8`-L0>Gy37BGWyXv_|Y1 z*`{FWcgi4=jhKs7%6DdtN2NrW{@5O?7R|%_v;mAiujz^EoJI>1HsLlgSixXE+aM!m zb{GEI@mF6<`lIxCP-8D)FYEfcr*df4uNn?j1x6>rN3dY&#t%yS@HxY$u_i%3V#D57 zcrt|D6frQR-q%z`r+tUN1CpK0ifYNaL9_ZZq~)avL0LD_%YE9^T{akG>SpOJ&`Li9 z&~TQ_^k}M%Rws7oA+0b{t(y&NNq`iN|k_G`la z5g?hD+OcIFKw}=)NfavytRk?60Nc5wc}g~qlL(`!O5Pi#i6*O!+-V9pL*P#ce3!r_ z0xuKz9stba^itq)_9@aY$oFpu{4IfhCh#=^|3ctj348#M)eO#}w!X9u^+dxaB%;b? zImzZYweG~aQ=3j~irGqMy56_tF65TZt-j=omTg6|&*$!rn0GJ8#&KCfZeb)>WhlR< zOPRY3k-1xL&0NEUx=Ti!c#E6YM$Bs$95oj!;@;kQ!+M!dZ-Zy;V9h&hJh=Anpr zXrZP#Ub8V;voT?Jl7XCT9cSTc@4Utt$U4rTwT`oMwX#WGqPp?F%1RD*XZ?R~6qj&% zPK%M;`g217iiO-t&a&=$qBd|H&2)WHhla?om&08GMdf5n_v_F%&?}F1w)BGWZ!hXC zM`&LwKXCyAqCfHt@Z9!aExqfxlbJ z_qxsR)=>oPLH6bg?=H6xeuo_y{K&uq|B;zN8-or8^B8n9n9pEgPH&z5M|FC{nzG4~ z2(w36B43jDz(kQ>V#K86OL)o8?Zgi(sG4hpe0n9$5?^(B#EB{uPhTRm#8+KZm-v#l zkv&U%RZ#|a)TGn3WTjPQ)g>v|veJ?=nLSHrRoy!-5J=~yviM7AX&$~g1(mH|LYroM z(yV)f5`To&fs7}s<}f9d-^BQ+(mEZr9+u^94_Fv0le?h->p<%As>If!yWyO&(lpx2 zx{0jwqD5z?H(#1TXflUcSNjg&T0z`VDlaz~D@A&dVCzH1(m;%8)_Vu}rgv;0HZl#h$MHRhEamOINixU;h zzcf)(OYTbbUC}7hDgF)2U5gtF$-gX7UO{i9S0~dc=_u)Xo5>${IaR#yXuN(~w0_&$ z(~U$8{uHnZ5cHf}sDbs{B7LA!fh7%6Ik33o*Brgk3f4z@eOtOMG|=~~bw zEIQ3%tOY927EfCgCOzBr9g3PHv)+&L>5D<2B4PHkc(cnG(mUR`36kb+_M`<8r2<6Z zU>CdQK`-5(-W9l+PHUr_O_d`Wj^jSZuJoR}QN$j?SNO~_!`cy4dkg90XgZgyr`8Qh z8LXg@pSnbs*3H`gNatIQyWQz?BW{ZTXMt8b=H z+e>H0fa0rGp+{Vrd@`PF*Bak~vMD1foVXMx=e&Ah2@|RLlOs%wc4O_qrBf##!QWOk zt2p{_udz76wqm?GX$yuP!`Y&=WK>}+aad^hYoq~D#?)R<%IX~PG9RVJWHXJE1#}B3 z<;g^1Y!c&~(0GW{$hx-WFc+fRyhuJ%Y#c2cVK@;uJRxnGs!ZAbu%VuolU0IOX~LjB z!sfO}#pXY^ToTTOpPzbe>iOwEx1K>4f!M`1?I$Ujw$lBpO=p^(X^z{=BKEQcdoH^N zNcVg9L(y;V<`ecZ`S#7HYC84&xA;z8_~SBQ5{G1$^iG%(*s2!n?zp`&YOkEfxumVJs`c;LH-Lk&S2N}e8Inj3 z^@lossi#i_aV3-pR!{d`7aF7pE5s96b?X+13xSyQ=8z5=i?++iqxxPoqhZa$` zuL`BvSkespkDX-nER~Tz<(Bk-3)}&QBplHETtU(<+e>$jg$@Ozq#HE%(rI1|3Dc9{ zF7ixowd{0W4+;tP(*{Odi0j8i?LXz{LZj=tLPTEU;U&D*!n zZ2d(MZm;{gd9LF*JFe+?-&C~VDLU_dW;@Q~Ir2}9ofwPK{1dC#9C7R+qhQCLxOq>+ zyyvH$^4Tt>hYRo7%h7)M6(1WpYiZ0ona$|c@_zSSqozJKG{3(th#u|ymR%uu;!}O z9=BFUt<`h&F>6y?Xo?6;33vY4yUyGdbJxXl>aep$Bvnys)m(u}@(WwxRd;^e-4Jy* z#NEwNcXPs15cf1jJ&kcsOVrbns9y1*Nx)^fdc_uU@lMWBLng$QNBHZDO`O4tjk>x| z5<=6B5Out#Jl#34*z4wnbDhONFL&_WFp*iYxdx6mJ$$!Gcr(9{@D{$?BD~pJ1pF-n z-)$4#GHwL^LxCq~V$i~%jUY@>4@`zd=t&2Wjryo;;2*#%lTXdQZF`)%9Yd#K!LZAh zodwgdO{yrYmPlnHXd1yJUd$$O7)+x3sSi!)|8?uEv=wP%Vc&+AJS;S8th61zQx%w- zlnTtUxc(G0n#Hwb{mYgMHRo15zwWtp&jY_M^-$8i06SqUNfU|mIr3ri3Y%g6f&4Ng zNt-59=>X{8#~-`UMwU;kd8gV=v_(tSzaqTS|B5A6vOZ$%e_M=N`{P1?MChl{vi-^J zU+TpuGVJ72cQm(l+W7DCofhFz2{8JR0J>G*=fm%v=yCD5723L+rgv#fAEsA5BebraaHVm~Ep0RTs|A0zxA<(Kx=$vFAK zB$^bgH>~MjJCM#&6sGVcqY2H~?4TUs1@a|aK5N1#zmuH*i9-R7%PtW+|I=D-^{3 z3=fh?dPvHnxaNr?q92!`4oWU^8o?K!~Dj}~NdBJOA z%tc5lg&$kOljZ|MzVP@#x(X6MD>N)&k+<#hhlU5Yu#ZU!+xxojmq=z#8b;Y~emKAb z_|kQWl^&I;{uquSG!ewl{E(%yd{FBcWi>A+!pxIv#_Dcf9e91QmJV{Qn`)B6x%(PkVz+`pGthnfa z0=8|Kl?v@vFnt!&lrS9x;|&PGj6nH10eJy-Kjoa3jQ?WY7S zKz`0O{S)WPV-BxtW(!}2D}3`y60U*|b3D8e zza6ru$NMVY2E378j&spOn3Z$57Hx#t@e>4#M2)KzjMi(pz^=9F`8C(_&cw8 za`=slReAi1L~ik-9`CF6+(iT4NHEv6Xd=waArlK>Fv!ncv=L^f1ap90Yg)#aUweeF zhKRHXrdVfqy70-u+2R=Q{!qvB74&7`5rH9{RA6C@&-+kkLKzU~f^juU=t%`eEQne{ zQL8Dml%iIrqsr+T`7S<{FrkYsid_C=`Ro^CeC0GG93BiU8puWbhahN7k;Gk;!6=&v`E+@c#TwOYm6`Y5RF$v zp~cjUMT^Z8h?8StZKgm61vblJDAa5_s^TMEynwH0#Hb$E^!hJMqu1^TLGuw#E hj|$c3(n9rOAt#hv{|JXg-S_b9K+>D*3j6M~{vVcm=%fGu literal 0 HcmV?d00001 diff --git a/config.py b/config.py index 4592c08..07d6aaf 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,4 @@ +from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -5,20 +6,33 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") hf_token: str = "" - llm_api_key: str = "" - llm_base_url: str = "https://api.openai.com/v1" - llm_model: str = "gpt-4o-mini" + llm_api_key: str = Field( + default="", + validation_alias=AliasChoices("LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY"), + ) + llm_base_url: str = Field( + default="https://api.deepseek.com/v1", + validation_alias=AliasChoices("LLM_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"), + ) + llm_model: str = Field( + default="deepseek-chat", + validation_alias=AliasChoices("LLM_MODEL", "DEEPSEEK_MODEL", "OPENAI_MODEL"), + ) llm_timeout: int = 8 llm_max_tokens: int = 150 webrtc_host: str = "0.0.0.0" webrtc_port: int = 8080 avatar_fps: int = 25 + avatar_driver_mode: str = "blendshape_stream" + avatar_control_protocol: str = "ws" + avatar_blendshape_schema: str = "arkit" stun_url: str = "stun:stun.l.google.com:19302" cors_origins: str = "*" ssl_certfile: str = "" ssl_keyfile: str = "" + # Legacy MuseTalk settings retained for migration reference only. musetalk_enabled: bool = True musetalk_repo_dir: str = "/home/xsl/work/MuseTalk" musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py" diff --git a/core/__pycache__/__init__.cpython-312.pyc b/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f4d1f8798be2899532b39ff7dfc2665e2157765 GIT binary patch literal 136 zcmX@j%ge<81Y1OoWikTk#~=H}F+3}Sp_W@Kb6 HVg|ARbg&*@ literal 0 HcmV?d00001 diff --git a/core/__pycache__/pipeline.cpython-312.pyc b/core/__pycache__/pipeline.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b932f4046b5c29bb0913275bf92991d1259a68c6 GIT binary patch literal 9934 zcmdT~eQX;?cHbp;$>sN_sBh}Sk}TTRhhxW%bBc75?a1dOa&mUkiyVic$(2NzAG+O@ zZK=zPodiAlsyoYVTFFg-$U%VEbq;47(AGfR!XI%jz5XK}eI>oop$5|8F6bZOodTR= zpg`Z-<&qRlyS}!Eq9g5_H?y;E-ptOt-Y35_jGtq{M@ptZEEX36LO34dM|@hzFnBBw76OMOtiA&i zLpo^3D76(OIYN49VF)>4E~!!D4CQl4=A9D(;rZ|s8wtnQAe~nTGn{EHL6EC`z$_3V zDH49Hmy@W05?Lr|p_G3!1R+vTf+v+ULtG$7e3WDi6sBC#26#@=Mj}y3D+s)gmMET$ zjBz%2g0mxWAaMfWQKM!|_ty7J#G`D_aX!)$jEC5sDJ~wG4hm446&V<8-Ppy zL+vrrrx_Xli?f(64EPv{62ehdGEcGGm_HbwjtML$uN=u048($L#2+{s5CWW^J<7%e zNf+k*JS#~006!B8hT{^=M_6`Bvc_X-!%#fN`m~%2)RT-;k-*I1K=6p4muSJy#zH*4 zb69_z8}B#z$6#&o6Y8^pG3ef}J~X>w?Sgw)TQ7*?PXYNAarXqVVs~fkEh&3T+P-dS zGG*^g(koVH#@dv!Hl?jCOS^w&^<{0Y)32X;J!5+;WqT}R+mW*ExMllV)?T&ZtX=3j z)3rpUo$VQO`?9(HOFdz8l{}rc`f?0ZosOT1XRMo2)=g<^cZTUsG2J;WVd=P=WxDV2 z8$g>k1~&~oMqcYChq|e2J@vp}H#H*dB!_yb>pmUwy(G{z8$%C|!bFj)Ias&=C1gSq z(j>IwM1q2g7KvTNp=V(O=tO$5uwmetd=ho)0wf7hCu)uoJULez(oW*!mnD%_+fJ66 zt9&bUQ6==Eo^yy)#oqL)1d>4Q=0+%hxBo$3jL1B&FQ(X+Nbz7O$4Plk3?c<{@(Ij| z9&|8@dMN!so*`pJol9m&z}c3QXjG*qOfYZSqXf52G=-=muYmd3EE+0Fdca5vk{&jz zYH#wq8RuRWO(KbX!Q1lx-g}#;2+;^Vz4QC*Y3`@ci(waW+8or3gYiEEbDc1YX6RQZ zn$KwF3>4HR%=>|oGIxyd(Ie9cTc4eHDsYa75C^rv*tHoaQ7w5LpPx^A_kCY*;ZV*39nb zry_uqJXlH=Y$re4xYmG`-R=M}{I_8L0&*IOs?MY#>-HuMcO0&yE^D_vSh~e(I^6*JSOk1>+gx;$v@HSLzxUgYR_C>$9$^#o99iOUEv? zyf0?m)eC#i>|NyEJ9hTi690~vY1*7>+MI6Ma?8CnTi0-AXSS+gNuRD-|6jMvIf&}I z-#7`otI+Lx+s|%a+VW0croJmx-<7WK`MJF}S4+6-e%(McG^K5AXSd7;-f#I6a&b$_ z*0#*Fef2Y28;{_8vrWHy3w3i71$1PMONOren`mV}0xPSyAK)TXvZn<=PtagpbYPG5 zLcx|+aw-aTlL5PJD6udFYYX=2hXnBo^qeq)wV^nJ23$S$ISzJ1>^)I?vL0%+U>A%+ z!6hkL?2CCF?uUFBhkQ_h5tgJ$T)1UPG^tjoT#BrlIbjhkVEw__KGF$RaGg+aJsxca zL}nH=32A2mUDTcu22pde5k{sF&7$Rjqs!w_xn78AljzcxCDE+PCyBWqQN;g>rdCnM zRSD?K$Y~PYmavb90nvK_;A7m%ZF!KNgIv{Tst<^#HJ)uFdhp+;}~2LJuRIcX&M!Y9>5zryW;pfu1vs%rupuqquy{BD?Z+1&&S>#oVTI#aIB^L1(0)+Cdy zu1%V=7H3kkVsT_F-jv0gwlpU-+1k3Kb;Z-P#Qd|zck1cnV3NOWYf80kz16nuw!JEA z^Df&q-1f9D1yt?S0vf4IfUNo5*zRNOkPEye{3*pK0z-nH!Ve zm_M3zR0B}h?DK=)JG#=+_CYAqu|3tXJq-+vdwqf)3)^T zCEaE1?6zek*v8UV{S^Bo0beeH*d&Q6V)5OtRZUZ-`$mI=t?zo zr5k!KwWb=jUZzs@ed+3_lIGjJeQ#?PtY@r?;w}58k6VAzd9Cx4&9@G`lsWKf>cFeF zhWtr7Yje*ZUhF=4{qJE@d^c~@PZ5(KzdTiP zP*wALzsV5_&T*ylqWWAy2YsPmQkEu7Wff2q3!qZo8-Po}P(NC~qE0rlJyfPHDOEGAFzh1ic(chT?^&XV6LmxHUI=@mgN+p1&Aan zNwiEhsC+RG>9Da1ZY8a)YTYcXFBLk}JmCdQ_&nWNHU+`wzITz_lQQ95ce9 zT?neGVUBlKhwS^ z)xIawzAx3j?^gQ@9J&k8|4zubT}XBV`K1!5`Nx4@LIINd>oVl~43c4*he!!xEH?x< zB#Q!~d>}d%VR^}*@$I6KFtN8En;a&TPK#+Lb1=O z1c)pE^86Ib1}8AEQ*<`un4@HbcyNpla^b@)_Y`O-2O%Y!;xrr!3lsh*3vs3AQF(M1 z@?j=8i*az`f)g7MJkQBq2OR4r0uZ9Y%>m8{$HHLzLV!QQwLp!}rFbL;WezrPJ0ee` zWCpQ?X_8Rg5|_nZD5^e+4yQ@z_f5#NdU=fNN}r3 z`l-MSI7uNMjm~T3-iI3WGM6ru+1dxLA_Oh|?_lZORh$faA>y<#W!;#zc4e5ZWu|Mz zQI&DDrW~zlM;n|Q>%8xEp6$%kb*1XM&WF==Ph{LrBn>OI4ey3B-ky}VC+*#Q>FJdB z$;+Ws<50SGXVRKARWF;?-FA8xXVOkz#_U@*`^u!Q-}vG2bH_952U6<8S}2UOej{DAH);G=#+G5~Q%wCLxH0{yhW^X5>4v?x zm}l)nL^b!|vnxBl8?bQzQSoQJSiFVc(L zvaMyAX~_+egrWPZJMJgG%IOHphOh3l?EUJFdE?z21tYi%Y8!g*@d{YU)Papau#bH;}`%)W?k- zz~9u8!}ZinYKVctPpl+ThfLivt(GY~4fXX%E1!9P047p(kRB~4ouI+k=AjbCR7jJB z_*tRqA5v;<*>0Jlk8@F5{uLGQ@v?qFzFRZ;$%}B;P{vG7@x1e2juEGMpU5T8Q=LF-1m~ z*RY6~;aWgtufIMZ{Fx6c)KLB;!aXkc5#SOmYYx3Ow3mcvSk9{fj8xiXoXo z@`&)peH$9{xY)~}O>vAI1z1z(Xx97iIeZ)CeZ2s1Us?&LrxeQme}m4-e+zU9h~qHG zh~r+1I6Xw`GvxgkGYm8N=9b+w`PbyGKI(7Wz+E$uy9TIh=0Ob-NuQR1d% zh_lJ$(5B|Xk2emYx`z%Ol1~=1Md`&yq4nW-JOWK*4V#5&I7}RjO?AgY0gemI6y++y zeH17@;xlvj>h2kf4V7aIdOlnPZ*8coXGQH~<8%Tal`YhFkVAFuH7M$Y!a*2Yv9G); zM#a7rrjL_*e~>WlBuIo;C$G8J)h;9oh5JnSn}=8+%KH71(eIDK4GoAEa5?012T07%Q6rR~%6R%^ZNt!yORZ7|tyHVUKEm17mvQ?tCv;;&7Qmu;B;$5g+XI*AD zZ5?_DZG?(ciI$?80~f>rRaGyD103B07fwb+ww7{2LU2R4QY+L0@2z)jCm5+zoEXdR zy?Jlu&CL7Wo7wLoksbo=tATHGk9b0U#!k>AUuevNP$mX3Xn|yqa|OD{WjIPWa)=oG zDlr6;H@RCvb4Et+^djh`Tb!>GOEe+>hMoBYb%i-=X))V0H7WHwjS&#a#3UJNkPK(g z3~z85!QeBZA!MYIn2=m$!L;pM(OPhH$8?39Q83-el5H~0K4a>qbJp8#$T^j>JSnQ5 z)g8TUnor1XNYgBR(bP1zN7ELI#!>-gMbq9{(hI&tI5K-|&W*gbF!!pq@XDNe^vL`n zH+1~)k$F#ws;{fZ)wby-Y2XR|oJgH2E}E(Hb|IB58fNMY1NT`6I?PPjxM6Kk&%)-+ z$ulJ;!W567U~dGnO6uXhmD&2h*vd=wXnf^xedDw5z05WQ1J?x24=!o!0dq&tWP@83 zAaF8(BS7$ELjW!U7lBK_CEzkZ5^{S~_GZpux>IEH*#HG_2jCI{IYAm90Vxv)XtU*m zWdjB93q)&a9XT-MLxI%hw|o(1;GwQ>e)XnsnVsJsc;Xq$z%Y&`oY>qlRZA2}=MXxegr zJ35nr5|^`+Kvu}T!L7By9o4}dmxU`m_XbDGC$F8n{)X2XOMbEMvwfc*xGek-8ujp7 z6J0#XCHOZ+VcnyILWD1nJSf(;J>(Kp!n=UP4X(_;FN!c0Mz*5e9`E?|M@T-X-L{dO z9*0yLhM8S3$%i`z`eG1uN;>>@K?iVbWewa7@C$1G61RTeB{EH}=s zJl8Z4iA~SJt#SfRNJ)vD=TDGx^frHvvLV=1LS|0`W1En8g%d+*GZ2@zO^5YjcMFhC zD5FISZv``ZN`nMp&hl^vqU&r}WcC7(U&+H2Qtuxsr_1T<>3X#P!{rZ_KRsF7JX75~ z^L=!t9vi5|#;dXMT5O^ko46aBycgeIi%(VKQhP@-#&{BOMIOl>_u3r`#bj$aiEJ z^xf0w7jXaIq*10JoLqE!PHqZC!(<9W{RxO zgh(50(=wJfb(G1FOM7QXrT$;$A> zV)z`fQ6ytXb|8UgjenpKI4~L(ezADEBkWMtmD>+}-ayrSwPA`x6eMH~eFDOR-XXg44+6wn^e>S0P8|RM literal 0 HcmV?d00001 diff --git a/core/pipeline.py b/core/pipeline.py index 299792f..bf699cf 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -11,6 +11,9 @@ from services.llm import LLMService from services.tts import TTSService +AudioChunkCallback = Callable[[np.ndarray, int, str, int, int], Awaitable[None]] + + class ChatPipeline: def __init__( self, @@ -75,7 +78,7 @@ class ChatPipeline: async def _synthesize_sentence_first( self, reply_text: str, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, should_interrupt: Optional[Callable[[], bool]] = None, ) -> tuple[np.ndarray, int, int, int]: @@ -94,7 +97,7 @@ class ChatPipeline: if i == 0: first_chunk_ms = int((time.perf_counter() - t0) * 1000) if on_audio_chunk is not None: - await on_audio_chunk(seg_audio, seg_sr) + await on_audio_chunk(seg_audio, seg_sr, seg, i, len(segments)) audio_chunks.append(seg_audio) tts_total_ms = int((time.perf_counter() - t0) * 1000) audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32) @@ -103,7 +106,7 @@ class ChatPipeline: async def process_turn( self, audio_16k: np.ndarray, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_user_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, @@ -145,7 +148,7 @@ class ChatPipeline: async def process_text_turn( self, user_text: str, - on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_audio_chunk: Optional[AudioChunkCallback] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, ) -> tuple[dict, np.ndarray, int]: diff --git a/main.py b/main.py index 611c7c6..90420b1 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,9 @@ from __future__ import annotations -import os -import logging import asyncio import json +import logging +import os import time from dataclasses import dataclass, field @@ -11,8 +11,8 @@ import numpy as np from aiortc import RTCPeerConnection, RTCSessionDescription from dotenv import load_dotenv from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect -from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel @@ -24,7 +24,7 @@ from services.avatar import AvatarService from services.llm import LLMService from services.tts import TTSService from services.vad import VADService -from webrtc.tracks import AudioBus, AvatarAudioTrack, AvatarVideoTrack +from webrtc.tracks import AudioBus, AvatarAudioTrack load_dotenv() if settings.hf_token: @@ -51,6 +51,7 @@ pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service) audio_bus = AudioBus(sample_rate=48000) pcs: set[RTCPeerConnection] = set() subtitle_clients: set[WebSocket] = set() +animation_clients: set[WebSocket] = set() @dataclass @@ -69,6 +70,8 @@ class RuntimeState: last_input_mode: str = "none" vad_start_count: int = 0 vad_end_count: int = 0 + last_animation_mode: str = settings.avatar_driver_mode + last_animation_frame_count: int = 0 busy: bool = False buffer_16k: list[np.ndarray] = field(default_factory=list) @@ -80,6 +83,18 @@ class TextChatRequest(BaseModel): text: str +async def _broadcast_json(clients: set[WebSocket], payload: dict) -> None: + stale: list[WebSocket] = [] + message = json.dumps(payload, ensure_ascii=False) + for ws in list(clients): + try: + await ws.send_text(message) + except Exception: + stale.append(ws) + for ws in stale: + clients.discard(ws) + + async def _broadcast_subtitle( role: str, text: str, @@ -90,22 +105,54 @@ async def _broadcast_subtitle( ) -> None: if not text: return - payload = { - "role": role, - "text": text, - "source": source, - "partial": partial, - "final": final, - "ts_ms": int(time.time() * 1000), + await _broadcast_json( + subtitle_clients, + { + "role": role, + "text": text, + "source": source, + "partial": partial, + "final": final, + "ts_ms": int(time.time() * 1000), + }, + ) + + +async def _broadcast_animation(payload: dict) -> None: + runtime.last_animation_mode = str(payload.get("driver", settings.avatar_driver_mode)) + runtime.last_animation_frame_count = int(payload.get("frame_count", 0)) + await _broadcast_json(animation_clients, payload) + + +def _runtime_payload() -> dict: + return { + "state": str(arbitrator.state), + "peers": len(pcs), + "subtitle_clients": len(subtitle_clients), + "animation_clients": len(animation_clients), + "pipeline_runs": runtime.pipeline_runs, + "last_latency_ms": runtime.last_latency_ms, + "last_asr_latency_ms": runtime.last_asr_latency_ms, + "last_llm_latency_ms": runtime.last_llm_latency_ms, + "last_llm_source": runtime.last_llm_source, + "last_tts_latency_ms": runtime.last_tts_latency_ms, + "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, + "last_barge_in_ms": runtime.last_barge_in_ms, + "barge_in_count": runtime.barge_in_count, + "last_input_mode": runtime.last_input_mode, + "vad_start_count": runtime.vad_start_count, + "vad_end_count": runtime.vad_end_count, + "last_user_text": runtime.last_user_text, + "last_reply_text": runtime.last_reply_text, + "last_animation_mode": runtime.last_animation_mode, + "last_animation_frame_count": runtime.last_animation_frame_count, + "pipeline_busy": runtime.busy, + "llm": llm_service.health, + "asr": asr_service.health, + "tts": tts_service.health, + "vad": vad_service.health, + "avatar": avatar_service.health, } - stale: list[WebSocket] = [] - for ws in list(subtitle_clients): - try: - await ws.send_text(json.dumps(payload, ensure_ascii=False)) - except Exception: - stale.append(ws) - for ws in stale: - subtitle_clients.discard(ws) def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: @@ -119,7 +166,6 @@ def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray: - # aiortc audio ndarray is typically (channels, samples) if pcm.ndim == 2: mono = pcm.mean(axis=0) else: @@ -135,10 +181,28 @@ def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> n return np.interp(x_new, x_old, mono).astype(np.float32) +async def _enqueue_audio_and_animation( + seg_audio: np.ndarray, + seg_sr: int, + seg_text: str, + idx: int, + total: int, +) -> None: + audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) + pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_seg) + animation_payload = await avatar_service.build_controls_from_audio( + seg_audio, + seg_sr, + text=seg_text, + chunk_index=idx, + total_chunks=total, + ) + await _broadcast_animation(animation_payload) + + async def _run_pipeline_from_buffer() -> None: - if runtime.busy: - return - if not runtime.buffer_16k: + if runtime.busy or not runtime.buffer_16k: return runtime.busy = True @@ -147,10 +211,6 @@ async def _run_pipeline_from_buffer() -> None: runtime.last_input_mode = "voice" audio_16k = np.concatenate(runtime.buffer_16k, axis=0) runtime.buffer_16k.clear() - async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: - audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) - pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) - await audio_bus.enqueue(pcm_seg) async def on_user_text_now(user_text: str) -> None: runtime.last_user_text = user_text @@ -164,12 +224,11 @@ async def _run_pipeline_from_buffer() -> None: result, audio, sr = await pipeline.process_turn( audio_16k, - on_audio_chunk=on_audio_chunk, + on_audio_chunk=_enqueue_audio_and_animation, on_user_text=on_user_text_now, on_reply_text=on_reply_text_now, on_reply_segment=on_reply_segment_now, ) - asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) runtime.last_user_text = result["user_text"] runtime.last_reply_text = result["reply_text"] runtime.pipeline_runs += 1 @@ -179,11 +238,11 @@ async def _run_pipeline_from_buffer() -> None: runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) - # Fallback: if chunk callback path produced nothing, enqueue full audio. if result.get("audio_samples", 0) <= 1: audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await audio_bus.enqueue(pcm_int16) + await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"])) finally: runtime.busy = False @@ -200,7 +259,6 @@ async def _consume_user_audio(track) -> None: runtime.buffer_16k.append(mono_16k) vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0) - # Silero VAD requires exact 512 samples at 16k. while vad_buffer.shape[0] >= 512: chunk = vad_buffer[:512] vad_buffer = vad_buffer[512:] @@ -214,6 +272,7 @@ async def _consume_user_audio(track) -> None: was_avatar_speaking = str(arbitrator.state) == "avatar_speaking" await arbitrator.on_speech_start() await audio_bus.clear() + await _broadcast_animation(await avatar_service.build_reset_payload(reason="speech-start")) if was_avatar_speaking: runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000) runtime.barge_in_count += 1 @@ -226,30 +285,7 @@ async def _consume_user_audio(track) -> None: @app.get("/health") async def health(): - return { - "state": arbitrator.state, - "peers": len(pcs), - "pipeline_runs": runtime.pipeline_runs, - "last_latency_ms": runtime.last_latency_ms, - "last_asr_latency_ms": runtime.last_asr_latency_ms, - "last_llm_latency_ms": runtime.last_llm_latency_ms, - "last_llm_source": runtime.last_llm_source, - "last_tts_latency_ms": runtime.last_tts_latency_ms, - "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, - "last_barge_in_ms": runtime.last_barge_in_ms, - "barge_in_count": runtime.barge_in_count, - "last_input_mode": runtime.last_input_mode, - "vad_start_count": runtime.vad_start_count, - "vad_end_count": runtime.vad_end_count, - "last_user_text": runtime.last_user_text, - "last_reply_text": runtime.last_reply_text, - "pipeline_busy": runtime.busy, - "llm": llm_service.health, - "asr": asr_service.health, - "tts": tts_service.health, - "vad": vad_service.health, - "avatar": avatar_service.health, - } + return _runtime_payload() @app.get("/meta") @@ -259,34 +295,20 @@ async def meta(): "https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile), "host": settings.webrtc_host, "port": settings.webrtc_port, + "avatar_protocol": settings.avatar_control_protocol, } +@app.get("/avatar/schema") +async def avatar_schema(): + return avatar_service.schema + + @app.get("/events") async def events(): async def gen(): while True: - payload = { - "state": str(arbitrator.state), - "peers": len(pcs), - "pipeline_runs": runtime.pipeline_runs, - "last_latency_ms": runtime.last_latency_ms, - "last_asr_latency_ms": runtime.last_asr_latency_ms, - "last_llm_latency_ms": runtime.last_llm_latency_ms, - "last_llm_source": runtime.last_llm_source, - "last_tts_latency_ms": runtime.last_tts_latency_ms, - "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, - "last_barge_in_ms": runtime.last_barge_in_ms, - "barge_in_count": runtime.barge_in_count, - "last_input_mode": runtime.last_input_mode, - "vad_start_count": runtime.vad_start_count, - "vad_end_count": runtime.vad_end_count, - "last_user_text": runtime.last_user_text, - "last_reply_text": runtime.last_reply_text, - "pipeline_busy": runtime.busy, - "avatar": avatar_service.health, - } - yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + yield f"data: {json.dumps(_runtime_payload(), ensure_ascii=False)}\n\n" await asyncio.sleep(1.0) return StreamingResponse(gen(), media_type="text/event-stream") @@ -306,9 +328,24 @@ async def ws_subtitles(websocket: WebSocket): subtitle_clients.discard(websocket) +@app.websocket("/ws/animation") +async def ws_animation(websocket: WebSocket): + await websocket.accept() + animation_clients.add(websocket) + try: + await websocket.send_text(json.dumps({"type": "animation_ready", **avatar_service.schema}, ensure_ascii=False)) + await websocket.send_text(json.dumps(await avatar_service.build_idle_payload(reason="client-connected"), ensure_ascii=False)) + while True: + await websocket.send_text(json.dumps({"type": "ping"}, ensure_ascii=False)) + await asyncio.sleep(15) + except WebSocketDisconnect: + animation_clients.discard(websocket) + except Exception: + animation_clients.discard(websocket) + + @app.post("/demo/run-once") async def run_once(): - # 3 seconds of silence placeholder; replace with real microphone stream path. fake_audio = np.zeros(16000 * 3, dtype=np.float32) await arbitrator.on_speech_start() result = await pipeline.run_once(fake_audio) @@ -326,11 +363,6 @@ async def chat_text(req: TextChatRequest): runtime.busy = True start_t = time.perf_counter() try: - async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: - audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) - pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) - await audio_bus.enqueue(pcm_seg) - async def on_reply_text_now(reply_text: str) -> None: runtime.last_reply_text = reply_text @@ -338,13 +370,13 @@ async def chat_text(req: TextChatRequest): await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1)) await _broadcast_subtitle("user", text, "text") + await _broadcast_animation(await avatar_service.build_reset_payload(reason="text-turn-start")) result, audio, sr = await pipeline.process_text_turn( text, - on_audio_chunk=on_audio_chunk, + on_audio_chunk=_enqueue_audio_and_animation, on_reply_text=on_reply_text_now, on_reply_segment=on_reply_segment_now, ) - asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) runtime.last_input_mode = "text" runtime.last_user_text = result["user_text"] runtime.last_reply_text = result["reply_text"] @@ -355,11 +387,11 @@ async def chat_text(req: TextChatRequest): runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) - # Fallback: if no chunk was enqueued for some reason, enqueue full audio. if result.get("audio_samples", 0) <= 1: audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await audio_bus.enqueue(pcm_int16) + await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"])) return {"ok": True, **result} finally: runtime.busy = False @@ -382,13 +414,14 @@ async def chat_reset(): runtime.last_input_mode = "none" runtime.vad_start_count = 0 runtime.vad_end_count = 0 + runtime.last_animation_frame_count = 0 + await _broadcast_animation(await avatar_service.build_reset_payload(reason="chat-reset")) return {"ok": True} @app.get("/demo/frame-idle-shape") async def frame_idle_shape(): - frame = await avatar_service.idle_frame() - return {"shape": list(frame.shape)} + return await avatar_service.build_idle_payload(reason="demo") @app.get("/") @@ -401,7 +434,6 @@ async def webrtc_offer(request: Request): params = await request.json() offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) - # Single-connection mode: kick existing peers before accepting new one. replaced_previous = len(pcs) > 0 if replaced_previous: old_peers = list(pcs) @@ -423,7 +455,6 @@ async def webrtc_offer(request: Request): if track.kind == "audio": asyncio.create_task(_consume_user_audio(track)) - pc.addTrack(AvatarVideoTrack(avatar_service, arbitrator, audio_bus, settings.avatar_fps)) pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus)) await pc.setRemoteDescription(offer) diff --git a/scripts/service-8018.sh b/scripts/service-8018.sh new file mode 100644 index 0000000..c73d4de --- /dev/null +++ b/scripts/service-8018.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUN_DIR="$ROOT_DIR/.run" +PID_FILE="$RUN_DIR/visual-chat-8018.pid" +LOG_FILE="$RUN_DIR/visual-chat-8018.log" +ENV_FILE="$ROOT_DIR/.env" + +mkdir -p "$RUN_DIR" + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +HOST="${WEBRTC_HOST:-0.0.0.0}" +PORT="${WEBRTC_PORT:-8018}" +SSL_CERTFILE="${SSL_CERTFILE:-$ROOT_DIR/certs/dev-cert.pem}" +SSL_KEYFILE="${SSL_KEYFILE:-$ROOT_DIR/certs/dev-key.pem}" + +if [[ -x "$ROOT_DIR/.venv/bin/python" ]]; then + PYTHON_BIN="$ROOT_DIR/.venv/bin/python" +else + PYTHON_BIN="$(command -v python3 || true)" +fi + +if [[ -z "$PYTHON_BIN" ]]; then + echo "python3 not found" + exit 1 +fi + +port_pids() { + ss -ltnp 2>/dev/null \ + | sed -n "s/.*:${PORT} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" \ + | tr '\n' ' ' +} + +is_running() { + if [[ ! -f "$PID_FILE" ]]; then + return 1 + fi + local pid + pid="$(cat "$PID_FILE")" + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null +} + +start() { + if is_running; then + echo "already running: pid=$(cat "$PID_FILE")" + return 0 + fi + + local stale + stale="$(port_pids)" + if [[ -n "${stale// /}" ]]; then + echo "killing stale python on port ${PORT}: $stale" + # shellcheck disable=SC2086 + kill $stale || true + sleep 1 + fi + + local args + args=(main:app --host "$HOST" --port "$PORT") + if [[ -n "$SSL_CERTFILE" && -n "$SSL_KEYFILE" ]]; then + args+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE") + fi + + nohup "$PYTHON_BIN" -m uvicorn "${args[@]}" >>"$LOG_FILE" 2>&1 & + local pid=$! + echo "$pid" >"$PID_FILE" + sleep 1 + + if kill -0 "$pid" 2>/dev/null; then + echo "started: pid=$pid" + echo "url: https://$HOST:$PORT" + echo "log: $LOG_FILE" + else + echo "failed to start, see log: $LOG_FILE" + rm -f "$PID_FILE" + return 1 + fi +} + +stop() { + if ! is_running; then + echo "not running" + rm -f "$PID_FILE" + return 0 + fi + + local pid + pid="$(cat "$PID_FILE")" + kill "$pid" || true + + for _ in {1..20}; do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.2 + done + + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" || true + fi + + rm -f "$PID_FILE" + echo "stopped" +} + +status() { + if is_running; then + echo "running: pid=$(cat "$PID_FILE")" + return 0 + fi + echo "not running" + return 1 +} + +logs() { + if [[ -f "$LOG_FILE" ]]; then + tail -n 120 "$LOG_FILE" + else + echo "no log file yet" + fi +} + +restart() { + stop || true + start +} + +case "${1:-}" in + start) start ;; + stop) stop ;; + status) status ;; + restart) restart ;; + logs) logs ;; + *) + echo "usage: $0 {start|stop|status|restart|logs}" + exit 2 + ;; +esac diff --git a/services/__pycache__/__init__.cpython-312.pyc b/services/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02f4042273f83a53e9c087ea00f5a930b335aa75 GIT binary patch literal 140 zcmX@j%ge<81Y1OoWikTk#~=JN6bn? zQbi{ZDFPzGSVPsx@=Vlu97#i(fSyIRbOPD(yin1_3xclP;)0G@o$0zc-}Ey%*Gq$} zK9}db=~$6mz@gM_}Cv4~YUU}(S(20P&LfzZ2+CXrG36F~q#s#)6@q{>BLjZ2lYRV*u^M5ust zA-!LhnPOyfHgTA2U~$9reUi=l#AbCy#?AN!p)^NNVNrubWKaYsI6sw`@2FUgCY!36C7&>LNdlp?ipSJ^tIqDFH&5H zLRAZ+2B-o&30ZN9V@$9mN6N}m0!9Lw#I_6xjh#{FE1`uT17f2wDS#BKN?`)30)%`s z@>f;GEr@nciGFico)oL71P@^?@D;zMx<1xc<(6k#gTE>_!ssOc!<6`%2)7C6aXgMD z#Fx-G*5m2Jl-mJsd4tLOX;3q|qXtS4{2?-2TAML zc*xW1DZd}B1HzQyzy*v@9tXA|61oCE<9J;hkClTMp@y#Ezp0#P++2HR!Xn|JndteH zau_ki$&HK<%3|Xtbu;dWE>YeC!VN+POe7PQ#|xnY25sa&H5hQWfD*&WnKrSDZ7V@~ z*AhRcHx2}#TG#k~-`jmBx6GwJN|h2#rS=URrAtD+w(Dx*;j4)ker?}eYI&lxc4KKx z>t8V`*)YBDor6np)ZPQk0dBwV-Gl#JX>7fX1Z~%qjhjwA@$tm;zMm88t|Xe?PrsGE zoP7M;-o-@ELZautB^hk)cxz$rPao;8N0;igJ&Jg#rE?D!zr!-{)=F@5NWRIt#x-CL zA{>=P4dS4zjA|Fcylhr_sI1d8=xcqW`WHg4?*V-c_$%}~?{7cGb9T>|#2F|(TR0=m zS9X|!0vf9o(6ad?6d|twTjsP{Dubw?oD~7!u{0Bkqla~g$r(2v_)NA;Ym_k6G2M~C z93dr1WcmNeA7wT$HEhUvXa3ACnlruT#1lh&yRXNOO{IP3n@y!~=w`;vlIb^q(} z^u5vlXnz;JtDW*(gzYqXaUdh0xOaf|K4`gr?rr>uILKcd-YB96Mq7Sgwv67Hs7jSlVmkh4arKJoj8r?g<$55;(Y3bqbIIA{xg8gFpWpq%;l+WM7vO6= za#=k>SA#`b7nnkcA5fP`>6}ZTLFk|_#@&3EYnzms1^O5$=sF;}5b{j2GnTKb{2g=6 zEHMmtAo;@?IGj%y#w&s8L@SiH_y{%IfMY83usorDfp&v<8il6_FGP9xL&963w-v|< zR2F18Rc=@$mYFhvnppNj7IG~Eq4-aNb%kR=gg sd=+iFjvB6_3WCyQsAMEVFTYroQ4kn2McFM@Bjb+ literal 0 HcmV?d00001 diff --git a/services/__pycache__/avatar.cpython-312.pyc b/services/__pycache__/avatar.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fad1291132e146c5b4d9a185317f11ea1e7a95a GIT binary patch literal 12389 zcmcgS32+PSeDlxE-w5EzvF0aVKdK_Zm7%8`tf$eeW-J z0YYm>$#nV&{{Q~D@BjDzyZ--vS(%N3u)g}c(f?jSQ9s0t5)3jx-#rb$1jSGc9ixWm zlQfO6E~Y!FBdPwRo}`A821xZWcs!ksL zq|_tMI%*=Rs|cRvhMTY zWSBd~a_6HFHbNK83l~ODQS{w3nHEE*sgPQogc>;v=BKaZF;&oo9%?&A2RXGZLqX1{ z^dWVoWjV8w8>NE!-jplMosA~fJ{@D@3_lPaX4jry&mo3Ei)3fG=y{e4#lu6aWDfJA z@klfw86HVQ&Pv8m3=?aJXU~nW@dzu~L$NTQ46z)S;G{}9)5k&AP$V%DPfFEtQDk5w zem2C6aAXsNhWI&JGVpAykH6f0$7+$pH{56o2 z=i1hgx?E!uruDhDRT;xySytYKo>U~ij}fLE@c#mmt`CrXn4)1G_qM;Kdo@4Wn2ymeBR+1PPb0noMn#NELK2sS zgux00Dx^iVRfSRoDynrWWrZ|N19rN8I$5ohCH)FC=u@CVs;mcNOu_75&95V4nzkOJ zI;j`G`1|?`bkN*O_T!vh(!*Yp%*iNhsUcpnL=y2Nmx%EnVn4|ljz@>&J;eiqVL$vS zELqSh8Iq&JtYnh6vSb*7Jtdj=$N)POmh^qYykz3oFrSD^_TtTIA{kyXCVBW)uuO7< zo*s$Dm=GzK%0qGXVzRLJDoQWD6OiI?eK}li&WaI|SgMjYa43Jc2tvMm zS>YIl@+jK!HHfJ`3dhKR^AGAdBo5b%0~%#qafdWCG;l8R5gFSk6JtZe;n7$k%%mEY zACh}pc%dgh1JO^ZkKx2rxECrKg^I@6y$kJKLVK6k-o4P;{m%OAlcyG*91@-!5}!%l9g)PO@G4Ux<0>R z_j>v|y}QHmwcX_aE<32W7%$*KJq*BS9#r}kt!%u~V+n^%mxcrPIT$0YBU{o=wxpqV zj+WQHXxDKp2r;f6+6HOv0ZfXCWX@B8Uq&XEOa#(ZrBkC8ot8CegVFg8h-5R3cQAw364ys~I84rj;Csp1zXepE{&@H2)g$&x2vO)?vN}l74 znbIaLPRW?bY-&CQbIW~VuJYT$x6>xZ%Ggw{Zc0`vb&NTQCzyn!M}Y?MY?cJTS%arC zG_u?zQj~%gEeXY3Sm~v#;FuX!p|PX`WYcD) zv_+k}HQA!HNn4e%VBZv0p#TMymW=(gW{>muv=vakprr66b(M+oqAscWH894twp?#)GgyV2^&?Z<$xGD$c z3&k~HgxiqYiqSTRB)g14h@jAf#hnT+tb~F%ouWZ=ap@jw@uPt*r|>%Uq%Mlzu?-Rb?nCh zE?mAao}7GaW~1nA%NQ2D{>jGaV>jG4{n!1od%tmT?%?fnbBDgU?@rU791z;~z3W>z za9V)hn$u$A8L{pu(HqLxa~}VKr(N*0&z=%J+ZLSL;Nz8^h00c;vUMgod-S$jtlW|@ z{it!}%oDE!h<&iww))oQ+ur$=Z>@fF^_?Sct`*kzyxWpJeDwQ4Vf`_2&0}KQg@#v1Fm# zJtV2v{s}RCo^R=Cqi@nZD-5&UwE%wGK=(8mzHQu1L*_dsx~IkP9rGSPWWLvgMc-?| zc$vY9k{4t=efKv3AQlg#r6H->Nm_<$QLTm?FRIm$;{9MBeT`;x)5c?Hvd-Bg^I-VG zqr+@mvJE9hk^_5U2_EcE0`DJ*$Jtmub9f|jmgOYt`6$nV(GU)+Aj_&?Vn8wvuwmvz z_=03b;BYh<8IUXp9!yo2L21>b?A39n2e;ivPD|?PwGEvomQZ?Gwu;AM&__nJ0^v`s5fB8rDO578b zRvU+cEuv~jkU{;AP%;D5nxK+-{fB}TE;t?vIu}T);ub39v~(>+Pe3h0k*#W)-y36w zern9f=*CP*XeEcVUTL+LI`i)obsE}^na3IA&M{WT7}F-49b-y6Fh$JA%G2ebiJF(Kc|ng;*NLH1z|xfwalOZ!Y2zS@%8~&1 zpwKJ{X^SSU&5Ri_Nq#1kwWteUL4jNn(&g#0v{TXYZmIg7LT#FQ@nZv(#xrg0C0asW z9{`fzVsSel>hB}r$6Y_!NP^hKbzvR)7lM`C0fbQ;b4M@&>?v9DTG{x>5R3W%i0o*b zdjxBGFxmr=WK6MKg6H-jTp0qJhYLrNp*}X8907^V9maA~co;Yg!?j_iTD9wCNvSjq zR>*evQE0>+!9>TiEu9uYGGY9^V(k{ey(P0R=cu}J=<*@eqmlCkV6r!iGwf{N_4w--7dj6L@M}FNHaslW zK9Z|zT&UY7)NPwzb?0FA@Z)0L6H8{Qvgs~msI2-oy{W8x$wLJ?mYS#vZ>CJGYznNTV@BYbg#lEEu@I}jFcVdRYDu3#)p}G1_!L|iV}T0Gt@2w| zg_{HaH!^Ue0#eHiJ^~ri-Qr zePmykGvnonsmO zVx{+GZsNk#3p4r|=DKZ`5ms!ty=UGfRzE0K?gWl*ue`G3@{Y->_w0e3%Rjksd`H$% zzv%KzY`MB+y6IZ$RO_`hQ)_1WZgJm8&86mB-&*tL8nI))(7Io&=@DHIWh^&k8feMdDS^%x@o^|pXFbFe15|po|t=f+$mOcf%~|sW^(=HhVgdr zTY3YN-IKe=qZ#{Rh3Dm`C;G1TO+NWjLab<=bqf`1GsZ=iZ@TK5cgj0spY0bKHjDl( z^B%#!Q*?EKMDo;3Z@AVu)j9Lf?1->()4WM&-g!rV=a?8cAbJjFY#?*SQ(0$Y*3x*d zevFzX;oRSXGheJ97xulDb%Q*ZQSWG!vo~faXmX$3AH1ao;Jd)1$eGa^%Xr^oA>RTY z6GQY%9FL3(6Vt@ebl>BGJ`Vf{{JI23t1_yRYK{(l#w_>y1sLB3c-xf`G1VN^`YViR zRWVle>`xuncE95$&9Iy#Xi8*#kq)iF8_D-eyl_}nWak(dry>zZ>xzP}SlwOrUjthW zSqt9Ag2w-8*j6GI%9oj{d$4u=QfwW+2DYe2m4y4jR>jjz*3QGY?$Ke&3jTK3bzTMg zqEe$~sY$cctXXQ&EVXKu+B8e;nxziS(y~G+Q$1}1sp{+HP@od=+5ezz;g#VB|FJ7C zUcb}T|GQHd{-&#czW%rV>B#)I6yWPh)t`59C`)B|N@ct9w%EUR_5a1Y82-7dUpC7A zQyKN;u72_v+xHadoqa3co!XWE&R3OQf2w@&chr)%K15B6I}TCM%Bgaj$ZGB&l!7#l z#wFcwlvsX}J_1G@T7JQDSwN#iCX$uAh|mS>BD;k`5#l3q5Fb&fPj(XJbHl_-#GSz| zPho^tR_hX0`P4gmg|5TG*26;k5uxQs_Sh5I6*Wv7A z$Azxr!q($L`w5}tME2BE+24qWb&OceW*j+pZN|C?hE>zGwyCz6RkO_PlXq4MoA(Rr z_6tot@3`KHh&4w=_p#Dek@ZytTgT zzS)-7SH98qTH9R5{2pO-m)N-L&L*L;=iTOa2C~ObiwDmLjc2kUCfheC`p=54SdpB+ zvFYa4>sx2{y}tL2p4WQj9-WT}?cHMY?mLeO%?IAylRbPqd-5sqk&w_F%7**012Hi$ zBzoc*n>wo-D{rp4zG`;!>zm%_e64e?OKjRP&k9X@?mYg^Ua{tg=sudU{;0Zk^7za- zu{yX|vjX&vBVtX*VxVEBcJ`bYSYHJB>nBHN8%2Nnq83yb1!_~X>qY|QW`z;K&3z_B;(N@D}LBr+Hb2$gC<%5GB2-1R-^LPw#Ys>m*8{|?oB^z44KOkO>IjHBGAbO7a zz~#+V*XMi;pI^#9$v>GBnxwG$}^i4E<<1E(e(m6 z)hqb|OG9D~CuM|XyP%_~6BG@9&qC{q`V-Uznq2rh$K%^S#y;ap>3{KiV7TWiyIbg& z=x)2|Wj#O>P2EoYn|6TSHqhN=hPRDogv)4*ou=+4^V=S}+vj-OYeu-v0_DqIji?m; zF%15HL{WGGp-s{BuT0T`>HaLJ;m{wjROnA=y%%Wh`Q@~Jv89O8bT7P9QHcz9kl}d0 z^1etilBatd2OnDYaE0rIlF|o7_uh8?8TLajWC`8fzKPzez37*d>COs{W|i(g2fBO4 zAD`Sevv2B<=v)jPPnRlrOvGLT!_9qCW=U|NoW?7=niO4W%01Gbq*2AuqD^JdZBf<(7HQUQ@>ENTBuq5k<)_m#gdmVy+Iv5>H}YX zTr%_~;w(HcHsDJ*kao@RMbILX9CrhtPs*8SBzX$5?G5k{`3y8Pl808{@i~h1w)pV6 zI740r5XX;v*+5Q4wqZmd9GIJh2pDDowvYk2-@{6DA<1py2z>=3Y?*S$ zqI~kya-7ZtO|s*uq$}y*N{+yjj8o*{L3;y9&H%-T9*m1CE5igL8Zm*e=O-0~uc+OK z7C`2J%7Tm`>IvMd7!~G{pBI)YbHrY-ykEtJWCYS0XBMo)ce?n5I24j>p%A=x9DzrP zA$U<4IyVxI$u%6_ypdOP+-8KHfQVeS!re)7lnFaW(nIbRZuqH-AaUx(v+$EXSyn*?XDi6B?7JT z7`=Dbl^dJxHh7`p0TVL sf^UUrNl(-C52&UeQssY3+2QvC%KJmA>K~}UFH8^VY1_{zj0uka3yx4<82|tP literal 0 HcmV?d00001 diff --git a/services/__pycache__/llm.cpython-312.pyc b/services/__pycache__/llm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ab10f5d208afb6d2e95a6ad7bc5802e859d1d24 GIT binary patch literal 5117 zcma)Adu&tJ89(>s_bVi^V>=H+NWhffHiCc{O&3T)v!n?sR9nNUE8}~U*tvf6-fM`{ zq(d$1!bHU)bxl~==(bL41+4tFp{d$XMbWfPnoTmxuUSB&< zx1AO4`Ofz`=X~co-|rm$Vz*lm{C4->h&I|0dJlK>AFUC@+6)j2h(jD9pg1u~5SS(f za+FkCYLtSO66iQH%BV70FvN|cMwMm+Q`|giCJ>2uGFGQMue7}NBBAw0Z6rE~IKwH# z884D0o;^bbO&{V;kWj4QL?S7N1zDH>K_;z(#|&dgEJGQ5GaaP%qqV#&v-5i}W)m@(Hs6nr+t@cW*L`XL1YB7^eRoW>+A|aY& z%v#~;ls-f7UG}lgn?u*DwEw5K@;u~P-Ik*+(fV$|cxvuo8OK1cJhyeDH|Um?Rx)vh z^>N4`*{@4ujXG6nAs0$96Gy*B)1+%keP5$#E^9tGgJQ1AXgU>Z(Wy$SN1C%tZbeA4 zRkxIkoE6qrjvi>)Dn4{d+EU&6tHzxrINJvydqy znjyb>7pjlG&StYd(@J}(=3?m-!N}#z;2_jfG+^?Vyapa*S7~
TY51MNtr_(V9mBa)1Vjz2W*UoYZ#J|*$| zWJ#!)bjGw@r#d1NVR_m*o{~FuCOhKML{!94gIHL5q)Z|*0^WNDf|O!}u~=A?VmcO< zcs4BxicyZn`D9u?GTnk#olFSP1RsbeIX;k1gpY@#Lim`#?+vs`L62f!MLx`Bpyfp| zDJn)bB1HLwEaKEu%z_YSv8A9ek}O73pr>_hSW+I+TqZ?aLKPD`5tZbmm{A<8l#ygU z&ZfjL_SHx=Pkyih2Zi((Lx zl(vMI;P_0r-919-2tT`OuUkgd(t}5KM_nN8}J_ zVjuCw;+Ge3N$ zQ0rOfp6|}rZpqbdS*i`r^%gpIU)`DOc;sB@+EaMgB55-pAa2>V{mb9J(h$fuY|k}p zFL;|SG@olO_wO3%Mh9pQ6)ni$qR!UWJlFf;;Abn&hEEU)9#(u!i;-8hV%dsq!`b1} z!&(2ntC75If6li5KSc)Xd@gw*yw`X4So%qHgEaSlgSugD@9&~+bTL3j%4G|*jPUPy z4Ay@D?jRxnu%Rph^ZuWROZ|GwlEO(706g&u0P{~Njv6BmBQy3;MKe~qOs)4ED;Mty zH)gKTIt3ofK&$fra%tdK0TzxqPmU9^Q`ZEoD)6Qk-d_*^#ss#RC*%#fO|@$pXS`&( zSX#yNP;Q?AD0rD#pk81YG)vBqF}H4AX*o0Aj98URK+IRs(y7w5#1d=LIh9t2-dSpf z62Dn}P1T@NU!A^Q0`E_icvQ8u&QLfq>uf2`rtR|m`@S2UIR#J`&}Xbd9WA%&{h9UG zrgEGs&=@O&qHX~@YzM1SE|s8;ewIGd4Uh+I1?nzS0C$ezG)|q{Fa36P?#f4R{UqHC zJpiyfe>``4@y*-kFRh-xvikk=tH1xT267-$Ok-g|I2MjfDh92_21$h$lLD_8Bgq65 zRav3aP}|1G)Zgt#ph7ZQ^)|(MNeYkil41d9W#!}~pO6$Q)aev2hUK)#t8gRY#8u2; zfKW*egFk}?Cev*gSVVkbm*l`0APUcIhgO?T00!k$c&Qib!GY+(UTBPHiWTrMCGhGC zKruuBdt_e3$t2YEqk#j)(>{n^?B*G99oBU#%>k#?8| z2wX!O&o$<~+j8D*d2d(F+m-h|obx_haQhY}=O+uT4_u}5dj@mx=Nl>_s~{Q&+#c@>Yz{; zZK$DfVP<~j)t>q9H%4&KY2yf&U|8(wye6g!3y+aAr?9#!FCfcUSpAA-=cBG~YwxS4-f=O&f2W=S&O1H|=zDMkUy4;6 z`4$jm94TLDfm!;^5PB^sS5Q)`px=daK1)J%0}O#$V#a6s4FaCHVlUVRX*KgiJa;wa z6oM_^O0;}q)SQ}j-JfMcU?SlZ)^f|=lJ|#l{?HYA$-gUKzw5MR&iJF6g3~?cC{oDX z^|^#0<+=J+Vu9F5FIrlOd!Dx#v0$~t+8hvHhL%uaOH~V$%0J*%!9W>fu=)de`N^d7 z5YpoX(rOq;oM#TBAUPal6bJxZL&cq7E8_R6VlUkw*(g*cNnzCMY0#im6t!aD$EHHT zyFe=#__V)4sF9ZN=9NyeVS*0}^2Bt_y3lH@L!cz#!+R1H4G50h(fOl=dav3vG+uC= zbF4H4Uak9e)6bihnnESX$5w;OQbY!`WeqIRJ>kV+Q^UIlh%XCo45!E$tr3D7M3J1!c)(J8M5}&zt-?lmG9gCK0cG z7!boC@=)&+3O$@m@Q|r$lB(tN2xv?@I51fRi09hh09BOM!P3@0#VJZEqT4k&N%V?shG z$;5=jA;OK+mYu9o61a-}RYK;_t4J57kuF~4O0w!4r$~RWg@WCnW}23xIVsDudtn`T zGtGF-Ncz_P%=-ivUjSwn5tP6>f}kWFPYVeiCmSRIFhZL^&h$6VM znogyOkur%Zsg!7X&aGB8$01{Bhv;rlO_?c2C6rq93U&pX7=t!@L+s(M_^>rb;#u2> zCoP@C)6~*4NhfamQQL9Hu{+W^H=z1H)clsUJ$00YA@F6lz;qfFnxZqp+1kSP2#{c* zxdlkL@N~zFa3|1MWSe{>695c96CA`d3%eU(CL4ww0(m6|Yf6f98?q$>cywfNlXwD; zRIOQ-Ii0^Il%WEdEWxmp^Rf}-bq8`h@G?;wqPr(Kr>3l);MRGlyu+z0>qoYfxizYb z@Kk9B7@dG~oaClBh~A^)cpPQ<6KEVOQg1(Hb}r5XHf2LIhg5bj`E8K)*q~+@?`g@= z*cgajjP10QYAS;@Wx=}As5NR)%axD4RYC&|FuD9nHc32?yS(jC8uW|}YiNj2w`yFY zX388=c*=H2w=yM9vPql4EEbDVR&Q>tN=!RLiE3DyPV{`!dK5gOst0p>6ehqzFnn(R z$NT5n=Hs8n3!(Z#^AijWt6Y`b{cEUcCAwoN+P)ZVzrim=+iyh=%=Db?Wi~yZ9DA@F zZoH2;xqG>#_0rzYvNJumLys?q>d*C_>0OBIx&HD}sB1CQ_3x?(HV^Fvc;A<;heUK! zln))`Z`STQ)Xm@Q7J*N$2M#;%k67V#0o;xu>_e_nY@jWTav@^|$@LyedfE^23Vg~> zA@sq~AvjXdF<@^P6IjRBDlq6l0}%DQkmX4pKEedI8DJ+WaK-3slYk~T+TgGw_r^#C zq-6;Li!wZQevs=!>#SLsZWy}YFpPL(v(9?0(nWo5-|!wt-C%xD&89Sq{Q#)qDGPdB z>W2Y@QY#%^pA~f{mnKSpM!>m`QV)%imQE8>cR4fd29uVV)Er{M&lU58TDDs?2oQ7j z?omV!kgnj6tm8__;fy(IyI6HaEu*I_n#jS`XPf9Kmr$E2j>6!;pULs9FRKx zobO}zgu=U`e>KQ!H-qTi2~hi$F5-0u>NRxlhVaW@8YY_I4CfzPSN6W>8Hm%4!gLy~ zG)3nl7bDj@zkli5mwxJBIzF%n-|)$Wz)8xwW3R%w5+zQCny%1mnM4ujPLNYpy)+I} ze!G)N8{{1`-C^n)rCRRUSjuz;6wXt-f}Tn`N`T!)rZz@Y6|OlIdSS+3d`MN_&uE6P zp<(9K!=3`18-SZFO{kNj08exUc@cS=qwFZWi8V(1DVU~FkrTvtF%%TLi_O(y*J>k@ zYVUJcRB@39_D_%XeXz%sAd6%$1ABu=C~KW=73;9jSzQSV0Vri>>xyP(ES;lIGF~1c z&Kq7Erh8yb8`8%`**4 aa`U3xJlAzgZaXdgEwy45-$!ir9R3GIR2cRE literal 0 HcmV?d00001 diff --git a/services/__pycache__/vad.cpython-312.pyc b/services/__pycache__/vad.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b83adb87d7f27497edc628e5b206c6a505caf9b5 GIT binary patch literal 2018 zcmZuyO>7%Q6rR~1$A7UC5*lL5X~1pB(kKQgh)M-Vkpc)JnL{e1xQCUsJFU0u+Gcj0 z#8HBRltVzRs-DsUQV_Q)96%s(?gfE3kg7;5t0IaJhu%Vr1S%Ka+x6N>)j7<(_wBrS z^WOXB?GOF^aRTG(;ctzvBtrf`B^;pRpzY^j-XS{CsYxtarW9qt6fChUQX-Joi7svt zUD^|T15%b+vJ!cKibB0arBIFF+f_-q7eHx}s;-1KCOIH%^XbzT?dm`&=nUk=j-P6jpbT!e&xQ`_W z?fnE1xYZ7Xc^S7nGVCPa-PH6xa@qNt8otz#+w# zeF}zHuA4aw$0L{;aWCB*d5k zFAv7hPV!<-l{33qH+8Syu2>Dzs9jQl8_XNAjGBK&O=GIw1PtDSYT2-_!I)N68{osT zl{m+w@B~a=yyue-Av%mx6eoKMag^vnu@(DLHhU7XB`eXUulC!^O zl)ITT+w)fze#uR`1H(Is>j^iLbI0=T)^#2nx$&hX6+pt{e~5YH=1@ez}KEUlKme1H0-vq0Ym1a$)BUyJJIY<(Og)Cncd>e z;(qo;cW7j%c)j?ka(8HIKRM;5hd91~W6vA|^HUV`K?n(eN5|t6^Z2yr^b>Ir`=BCv za?tW(y^bgQ4G%KXWwOsq-VZK=_7|`)Qe_p>sjdwki9djz#hqE0Hp%_b+-~+}_KTTs zX1|>M?);AzzQ1rMy|f=)^202_Ta9q$Gn>4xUOuL1gJtzt1%g`SN4 zx@n@ERMmHz6*Kt42f!ex9_HMOLN9JG?zH#>sJDpk$iFeiKIBg**yw1Bv;|pyuALf` z^M@0XJbaKOk<_6;lt%d?^Y literal 0 HcmV?d00001 diff --git a/services/avatar.py b/services/avatar.py index b19a35d..0c1251e 100644 --- a/services/avatar.py +++ b/services/avatar.py @@ -1,193 +1,213 @@ from __future__ import annotations import asyncio -import os -import sys -import uuid -import wave -from collections import deque -from pathlib import Path +import math +import time +from typing import Any -import cv2 import numpy as np from config import settings class AvatarService: - def __init__(self, width: int = 640, height: int = 360) -> None: - self.width = width - self.height = height - self._frames: deque[np.ndarray] = deque() + def __init__(self) -> None: + self.driver_name = "arkit-blendshape-v1" self._lock = asyncio.Lock() - self._job_lock = asyncio.Lock() + self._sequence = 0 self._last_error: str | None = None - self._base_frame = self._load_base_frame() + self._last_frame_count = 0 + self._last_chunk_duration_ms = 0 - async def render_frame(self, mouth_open: float = 0.0, speaking: bool = False) -> np.ndarray: - frame = self._base_frame.copy() - if speaking: - alpha = float(np.clip(mouth_open, 0.0, 1.0)) - # Slightly boost contrast/brightness while speaking to keep a "live" feeling. - frame = cv2.convertScaleAbs(frame, alpha=1.0 + 0.10 * alpha, beta=2 + int(8 * alpha)) - # Fallback mouth animation on realistic base frame. - h, w = frame.shape[:2] - cx = w // 2 - cy = int(h * 0.70) - half_w = max(28, int(w * 0.055)) - half_h = max(4, int(4 + alpha * h * 0.035)) + async def build_idle_payload(self, reason: str = "idle") -> dict[str, Any]: + frame = self._build_frame(seq=await self._next_sequence(), time_ms=0, controls=self._neutral_controls()) + self._last_frame_count = 1 + self._last_chunk_duration_ms = 0 + self._last_error = None + return { + "type": "animation_state", + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "schema": settings.avatar_blendshape_schema, + "fps": settings.avatar_fps, + "reason": reason, + "frame_count": 1, + "frames": [frame], + "ts_ms": int(time.time() * 1000), + } - lip_color = (26, 26, 92) - lip_border = (70, 70, 160) - cv2.ellipse(frame, (cx, cy), (half_w + 2, half_h + 2), 0, 0, 360, lip_border, -1) - cv2.ellipse(frame, (cx, cy), (half_w, half_h), 0, 0, 360, lip_color, -1) - return frame + async def build_reset_payload(self, reason: str = "reset") -> dict[str, Any]: + payload = await self.build_idle_payload(reason=reason) + payload["type"] = "animation_reset" + return payload - async def idle_frame(self) -> np.ndarray: - return await self.render_frame(mouth_open=0.0, speaking=False) + async def build_controls_from_audio( + self, + audio: np.ndarray, + sr: int, + *, + text: str = "", + chunk_index: int = 0, + total_chunks: int = 1, + ) -> dict[str, Any]: + if sr <= 0: + self._last_error = f"invalid sample rate: {sr}" + return await self.build_idle_payload(reason="invalid-sample-rate") - async def speaking_frame(self, mouth_open: float = 0.5) -> np.ndarray: - return await self.render_frame(mouth_open=mouth_open, speaking=True) + mono = np.asarray(audio, dtype=np.float32).reshape(-1) + if mono.size <= 1: + return await self.build_idle_payload(reason="empty-audio") - async def pop_generated_frame(self) -> np.ndarray | None: - async with self._lock: - if self._frames: - return self._frames.popleft() - return None + mono = np.clip(mono, -1.0, 1.0) + frame_count = max(1, int(math.ceil((mono.shape[0] / float(sr)) * settings.avatar_fps))) + seqs = await self._reserve_sequences(frame_count) + frames = self._audio_to_frames(mono, sr, seqs) - async def enqueue_musetalk_from_audio(self, audio: np.ndarray, sr: int) -> None: - if not settings.musetalk_enabled: - return - if audio.size <= 1 or sr <= 0: - return - - async with self._job_lock: - try: - job_id = f"job_{uuid.uuid4().hex[:10]}" - repo_dir = Path(settings.musetalk_repo_dir) - work_root = repo_dir / "results" / "visual-chat-jobs" / job_id - work_root.mkdir(parents=True, exist_ok=True) - wav_path = work_root / "input.wav" - cfg_path = work_root / "inference.yaml" - output_name = f"{job_id}.mp4" - output_path = Path(settings.musetalk_result_dir) / "v15" / output_name - - self._write_wav(wav_path, audio.astype(np.float32, copy=False), sr) - cfg_path.write_text( - ( - "task_0:\n" - f" video_path: \"{settings.musetalk_source_video}\"\n" - f" audio_path: \"{wav_path}\"\n" - f" result_name: \"{output_name}\"\n" - ), - encoding="utf-8", - ) - - env = os.environ.copy() - env.setdefault("HF_HOME", str(repo_dir / "models")) - env["PYTHONPATH"] = ( - f"{repo_dir}:{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(repo_dir) - ) - proc = await asyncio.create_subprocess_exec( - sys.executable, - str(settings.musetalk_infer_script), - "--version", - "v15", - "--inference_config", - str(cfg_path), - "--result_dir", - str(settings.musetalk_result_dir), - "--unet_model_path", - str(settings.musetalk_unet_model), - "--unet_config", - str(settings.musetalk_unet_config), - "--whisper_dir", - str(settings.musetalk_whisper_dir), - "--batch_size", - "8", - "--use_float16", - cwd=str(repo_dir), - env=env, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - self._last_error = stderr.decode("utf-8", errors="ignore")[-500:] - return - frames: list[np.ndarray] = [] - if output_path.exists(): - frames = self._read_video_frames(str(output_path)) - if not frames: - # Some MuseTalk runs leave image frames instead of final mp4. - fallback_dir = Path(settings.musetalk_result_dir) / "v15" / "yongen" - if fallback_dir.exists(): - frames = self._read_image_frames(str(fallback_dir)) - if not frames: - self._last_error = f"MuseTalk produced no readable frames: {output_path}" - return - - async with self._lock: - self._frames.clear() - self._frames.extend(frames) - self._last_error = None - except Exception as exc: # pragma: no cover - self._last_error = str(exc) - - def _read_video_frames(self, path: str) -> list[np.ndarray]: - cap = cv2.VideoCapture(path) - frames: list[np.ndarray] = [] - while True: - ok, frame = cap.read() - if not ok or frame is None: - break - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - frames.append(frame) - cap.release() - return frames - - def _read_image_frames(self, dir_path: str) -> list[np.ndarray]: - frames: list[np.ndarray] = [] - for name in sorted(os.listdir(dir_path)): - if not name.lower().endswith(".png"): - continue - frame = cv2.imread(os.path.join(dir_path, name)) - if frame is None: - continue - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - frames.append(frame) - return frames - - def _load_base_frame(self) -> np.ndarray: - src = settings.musetalk_source_video - if src and os.path.exists(src): - cap = cv2.VideoCapture(src) - ok, frame = cap.read() - cap.release() - if ok and frame is not None: - if frame.shape[1] != self.width or frame.shape[0] != self.height: - frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) - return frame - # Last-resort dark background (should rarely happen). - fallback = np.zeros((self.height, self.width, 3), dtype=np.uint8) - fallback[:, :, :] = (20, 20, 20) - return fallback - - @staticmethod - def _write_wav(path: Path, audio: np.ndarray, sr: int) -> None: - pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16) - with wave.open(str(path), "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sr) - wf.writeframes(pcm.tobytes()) + self._last_frame_count = len(frames) + self._last_chunk_duration_ms = int(round(1000.0 * mono.shape[0] / float(sr))) + self._last_error = None + return { + "type": "animation_chunk", + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "schema": settings.avatar_blendshape_schema, + "chunk_index": chunk_index, + "total_chunks": total_chunks, + "sample_rate": sr, + "fps": settings.avatar_fps, + "text": text, + "frame_count": len(frames), + "duration_ms": self._last_chunk_duration_ms, + "frames": frames, + "ts_ms": int(time.time() * 1000), + } @property - def health(self) -> dict: + def schema(self) -> dict[str, Any]: return { - "enabled": settings.musetalk_enabled, - "queued_frames": len(self._frames), + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "protocol": settings.avatar_control_protocol, + "schema": settings.avatar_blendshape_schema, + "controls": [ + "jawOpen", + "mouthClose", + "mouthFunnel", + "mouthPucker", + "viseme_aa", + "viseme_ee", + "viseme_oh", + "headYaw", + "headPitch", + "headRoll", + ], + } + + def _audio_to_frames(self, audio: np.ndarray, sr: int, sequences: range) -> list[dict[str, Any]]: + samples_per_frame = max(1, int(round(sr / max(1, settings.avatar_fps)))) + frames: list[dict[str, Any]] = [] + jaw_prev = 0.0 + for index, seq in enumerate(sequences): + start = index * samples_per_frame + end = min(audio.shape[0], start + samples_per_frame) + window = audio[start:end] + if window.size == 0: + window = np.zeros(1, dtype=np.float32) + energy, brightness = self._extract_features(window, sr) + jaw_open = 0.70 * jaw_prev + 0.30 * energy + jaw_prev = jaw_open + phase = index / max(1, len(sequences) - 1) + frames.append( + self._build_frame( + seq=seq, + time_ms=int(round(index * 1000.0 / settings.avatar_fps)), + controls=self._controls_from_features(jaw_open, brightness, phase), + ) + ) + return frames + + def _extract_features(self, window: np.ndarray, sr: int) -> tuple[float, float]: + rms = float(np.sqrt(np.mean(window**2))) if window.size else 0.0 + energy = float(np.clip((rms - 0.01) * 10.0, 0.0, 1.0)) + if window.size < 16: + return energy, 0.5 + spectrum = np.abs(np.fft.rfft(window * np.hanning(window.size))) + if spectrum.size <= 1: + return energy, 0.5 + freqs = np.fft.rfftfreq(window.size, d=1.0 / float(sr)) + weights = spectrum[1:] + freq_values = freqs[1:] + denom = float(np.sum(weights)) + 1e-6 + centroid = float(np.sum(freq_values * weights) / denom) + brightness = float(np.clip((centroid - 500.0) / 2500.0, 0.0, 1.0)) + return energy, brightness + + def _controls_from_features(self, energy: float, brightness: float, phase: float) -> dict[str, float]: + jaw_open = float(np.clip(energy, 0.0, 1.0)) + mouth_close = float(np.clip(1.0 - jaw_open * 0.82, 0.0, 1.0)) + mouth_funnel = float(np.clip(0.16 + jaw_open * (0.40 - brightness * 0.12), 0.0, 1.0)) + mouth_pucker = float(np.clip(0.06 + jaw_open * (0.20 + (1.0 - brightness) * 0.30), 0.0, 1.0)) + viseme_aa = jaw_open + viseme_ee = float(np.clip(jaw_open * (0.25 + brightness * 0.95), 0.0, 1.0)) + viseme_oh = float(np.clip(jaw_open * (0.30 + (1.0 - brightness) * 0.90), 0.0, 1.0)) + head_yaw = float(np.clip(math.sin(phase * math.pi * 1.7) * 0.02 * max(jaw_open, 0.18), -0.25, 0.25)) + head_pitch = float(np.clip(math.cos(phase * math.pi * 2.0) * 0.03 * max(jaw_open, 0.15), -0.25, 0.25)) + head_roll = float(np.clip(math.sin(phase * math.pi) * 0.015, -0.15, 0.15)) + return { + "jawOpen": round(jaw_open, 4), + "mouthClose": round(mouth_close, 4), + "mouthFunnel": round(mouth_funnel, 4), + "mouthPucker": round(mouth_pucker, 4), + "viseme_aa": round(viseme_aa, 4), + "viseme_ee": round(viseme_ee, 4), + "viseme_oh": round(viseme_oh, 4), + "headYaw": round(head_yaw, 4), + "headPitch": round(head_pitch, 4), + "headRoll": round(head_roll, 4), + } + + def _build_frame(self, seq: int, time_ms: int, controls: dict[str, float]) -> dict[str, Any]: + return { + "seq": seq, + "time_ms": time_ms, + "speaking": controls["jawOpen"] > 0.04, + "controls": controls, + } + + def _neutral_controls(self) -> dict[str, float]: + return { + "jawOpen": 0.0, + "mouthClose": 1.0, + "mouthFunnel": 0.0, + "mouthPucker": 0.0, + "viseme_aa": 0.0, + "viseme_ee": 0.0, + "viseme_oh": 0.0, + "headYaw": 0.0, + "headPitch": 0.0, + "headRoll": 0.0, + } + + async def _next_sequence(self) -> int: + async with self._lock: + seq = self._sequence + self._sequence += 1 + return seq + + async def _reserve_sequences(self, count: int) -> range: + async with self._lock: + start = self._sequence + self._sequence += count + return range(start, start + count) + + @property + def health(self) -> dict[str, Any]: + return { + "driver": self.driver_name, + "mode": settings.avatar_driver_mode, + "protocol": settings.avatar_control_protocol, + "schema": settings.avatar_blendshape_schema, + "last_frame_count": self._last_frame_count, + "last_chunk_duration_ms": self._last_chunk_duration_ms, "last_error": self._last_error, } diff --git a/services/llm.py b/services/llm.py index cc1f9b3..2105fe1 100644 --- a/services/llm.py +++ b/services/llm.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import os from typing import Optional from collections import deque @@ -15,23 +16,37 @@ class LLMService: self._error: Optional[str] = None self._client = None self._model = (settings.llm_model or "").strip() + self._base_url = (settings.llm_base_url or "").strip() self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant) self._system_prompt = "请用中文口语化简短回复，1-2句。" if not settings.llm_api_key: + self._error = "missing LLM api key: set LLM_API_KEY or DEEPSEEK_API_KEY" return + + # If DeepSeek key is provided but no explicit base URL override, + # route to DeepSeek-compatible OpenAI endpoint by default. + if ( + os.getenv("DEEPSEEK_API_KEY") + and not os.getenv("LLM_BASE_URL") + and not os.getenv("DEEPSEEK_BASE_URL") + and (self._base_url.lower() == "https://api.openai.com/v1" or not self._base_url) + ): + self._base_url = "https://api.deepseek.com/v1" + if not self._model: - base = (settings.llm_base_url or "").lower() - self._model = "deepseek-chat" if "deepseek.com" in base else "gpt-4o-mini" + base = self._base_url.lower() + self._model = "deepseek-chat" if "deepseek" in base else "gpt-4o-mini" try: from openai import AsyncOpenAI self._client = AsyncOpenAI( api_key=settings.llm_api_key, - base_url=settings.llm_base_url, + base_url=self._base_url, timeout=settings.llm_timeout, ) self._ready = True + self._error = None except Exception as exc: # pragma: no cover self._error = str(exc) logger.warning("LLM online mode unavailable: %s", exc) @@ -67,6 +82,7 @@ class LLMService: return { "ready": self._ready, "model": self._model, + "base_url": self._base_url, "history_items": len(self._history), "error": self._error, } diff --git a/web/app.js b/web/app.js index 45c72b1..43a2a09 100644 --- a/web/app.js +++ b/web/app.js @@ -1,3 +1,8 @@ +import * as THREE from "three"; +import { OrbitControls } from "three/addons/controls/OrbitControls.js"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"; + const statusEl = document.getElementById("status"); const outputEl = document.getElementById("output"); const healthBtn = document.getElementById("healthBtn"); @@ -5,11 +10,25 @@ const offerBtn = document.getElementById("offerBtn"); const resetBtn = document.getElementById("resetBtn"); const sendBtn = document.getElementById("sendBtn"); const chatInput = document.getElementById("chatInput"); -const remoteVideo = document.getElementById("remoteVideo"); -remoteVideo.muted = false; -remoteVideo.volume = 1.0; +const remoteAudio = document.getElementById("remoteAudio"); +const modelFile = document.getElementById("modelFile"); +const modelUrl = document.getElementById("modelUrl"); +const loadModelBtn = document.getElementById("loadModelBtn"); +const resetViewBtn = document.getElementById("resetViewBtn"); +const avatarCanvas = document.getElementById("avatarCanvas"); +const modelStatus = document.getElementById("modelStatus"); +const modelName = document.getElementById("modelName"); +const animationBuffer = document.getElementById("animationBuffer"); + +remoteAudio.muted = false; +remoteAudio.volume = 1.0; + const messagesEl = document.getElementById("messages"); const connBadge = document.getElementById("connBadge"); +const animationConn = document.getElementById("animationConn"); +const animationDriver = document.getElementById("animationDriver"); +const animationFrames = document.getElementById("animationFrames"); +const animationPreview = document.getElementById("animationPreview"); const mPeers = document.getElementById("mPeers"); const mBusy = document.getElementById("mBusy"); const mLatency = document.getElementById("mLatency"); @@ -23,6 +42,9 @@ const mTts = document.getElementById("mTts"); const mBargeIn = document.getElementById("mBargeIn"); const mInputMode = document.getElementById("mInputMode"); const mVadCount = document.getElementById("mVadCount"); +const mAnimClients = document.getElementById("mAnimClients"); +const mAnimDriver = document.getElementById("mAnimDriver"); +const mAnimFrames = document.getElementById("mAnimFrames"); const dConn = document.getElementById("dConn"); const dVad = document.getElementById("dVad"); const dAsr = document.getElementById("dAsr"); @@ -32,11 +54,15 @@ const dTts = document.getElementById("dTts"); let pc = null; let evt = null; let subtitleWs = null; +let animationWs = null; let localMicStream = null; -let lastSeenRun = 0; -let lastBusy = false; let aiStreamingEl = null; + const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect"; +const ANIMATION_BUFFER_MS = 120; +const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; +const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; +const CLOCK = new THREE.Clock(); const stateMap = { "SessionState.IDLE": "空闲", @@ -49,6 +75,56 @@ const stateMap = { avatar_speaking: "数字人说话", }; +const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true }); +renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); +renderer.outputColorSpace = THREE.SRGBColorSpace; + +const scene = new THREE.Scene(); +scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5); + +const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100); +camera.position.set(0, 1.45, 2.65); + +const controls = new OrbitControls(camera, avatarCanvas); +controls.target.set(0, 1.15, 0); +controls.enableDamping = true; +controls.minDistance = 1.3; +controls.maxDistance = 4.5; +controls.maxPolarAngle = Math.PI * 0.58; +controls.minPolarAngle = Math.PI * 0.22; + +const avatarState = { + currentRoot: null, + currentVrm: null, + morphBindings: new Map(), + pendingFrames: [], + scheduleCursorMs: performance.now(), + lastAppliedSeq: -1, + activeControls: { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }, + headBone: null, + neckBone: null, + idleRig: null, + debugRig: null, + debugJaw: null, + debugMouth: null, +}; + +const gltfLoader = new GLTFLoader(); +gltfLoader.register((parser) => new VRMLoaderPlugin(parser)); +gltfLoader.crossOrigin = "anonymous"; + +const TMP_EULER = new THREE.Euler(); +const TMP_QUATERNION = new THREE.Quaternion(); + function fmtTime() { return new Date().toLocaleTimeString("zh-CN", { hour12: false }); } @@ -82,6 +158,132 @@ function setDot(el, kind) { el.className = `dot ${kind}`; } +function initStage() { + scene.background = new THREE.Color(0x0b1118); + + const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65); + hemiLight.position.set(0, 2.6, 0); + scene.add(hemiLight); + + const keyLight = new THREE.DirectionalLight(0xffffff, 1.7); + keyLight.position.set(1.4, 2.2, 2.0); + scene.add(keyLight); + + const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8); + rimLight.position.set(-2.0, 1.4, -1.5); + scene.add(rimLight); + + const floor = new THREE.Mesh( + new THREE.CircleGeometry(2.8, 48), + new THREE.MeshStandardMaterial({ + color: 0x101b28, + roughness: 0.94, + metalness: 0.08, + transparent: true, + opacity: 0.9, + }), + ); + floor.rotation.x = -Math.PI / 2; + floor.position.y = -0.02; + scene.add(floor); + + const halo = new THREE.Mesh( + new THREE.RingGeometry(1.2, 1.95, 64), + new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }), + ); + halo.rotation.x = -Math.PI / 2; + halo.position.y = 0.01; + scene.add(halo); + + createDebugRig(); + resizeStage(); + renderLoop(); +} + +function createDebugRig() { + const group = new THREE.Group(); + + const bustMaterial = new THREE.MeshStandardMaterial({ + color: 0xc7d2e0, + roughness: 0.72, + metalness: 0.05, + }); + const accentMaterial = new THREE.MeshStandardMaterial({ + color: 0x0f172a, + roughness: 0.35, + metalness: 0.18, + }); + const lipMaterial = new THREE.MeshStandardMaterial({ + color: 0x8b425f, + roughness: 0.56, + metalness: 0.02, + }); + + const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial); + torso.position.set(0, 0.42, 0); + group.add(torso); + + const neck = new THREE.Group(); + neck.position.set(0, 0.95, 0); + group.add(neck); + + const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial); + head.scale.set(0.92, 1.03, 0.92); + neck.add(head); + + const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial); + hair.position.set(0, 0.06, 0); + neck.add(hair); + + const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); + leftEye.position.set(-0.1, 0.03, 0.28); + neck.add(leftEye); + + const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); + rightEye.position.set(0.1, 0.03, 0.28); + neck.add(rightEye); + + const jawPivot = new THREE.Group(); + jawPivot.position.set(0, -0.06, 0.15); + neck.add(jawPivot); + + const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial); + jaw.position.set(0, -0.08, 0.02); + jawPivot.add(jaw); + + const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial); + mouth.position.set(0, -0.02, 0.31); + neck.add(mouth); + + scene.add(group); + avatarState.currentRoot = group; + avatarState.debugRig = group; + avatarState.debugJaw = jawPivot; + avatarState.debugMouth = mouth; + avatarState.headBone = neck; + avatarState.neckBone = neck; +} + +function resizeStage() { + const rect = avatarCanvas.getBoundingClientRect(); + if (!rect.width || !rect.height) return; + renderer.setSize(rect.width, rect.height, false); + camera.aspect = rect.width / rect.height; + camera.updateProjectionMatrix(); +} + +function renderLoop() { + requestAnimationFrame(renderLoop); + const delta = CLOCK.getDelta(); + const elapsed = CLOCK.getElapsedTime(); + controls.update(); + flushAnimationQueue(); + applyProceduralIdlePose(elapsed); + applyAnimationControls(avatarState.activeControls); + avatarState.currentVrm?.update?.(delta); + renderer.render(scene, camera); +} + function updateMetrics(data) { mPeers.textContent = String(data.peers ?? 0); mBusy.textContent = data.pipeline_busy ? "是" : "否"; @@ -96,6 +298,9 @@ function updateMetrics(data) { mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`; mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none"; mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`; + mAnimClients.textContent = String(data.animation_clients ?? 0); + mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--"; + mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0); setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn"); setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn"); @@ -104,6 +309,394 @@ function updateMetrics(data) { setDot(dTts, data?.tts?.ready ? "ok" : "warn"); } +function queueAnimationFrames(payload) { + const frames = Array.isArray(payload.frames) ? payload.frames : []; + const now = performance.now(); + const frameWindowMs = (1000 / Math.max(1, payload.fps || 25)); + + if (payload.type === "animation_reset" || payload.type === "animation_state") { + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS; + avatarState.activeControls = { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }; + } + + const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs); + frames.forEach((frame, index) => { + const fallbackTime = index * frameWindowMs; + avatarState.pendingFrames.push({ + dueAt: baseTime + (frame.time_ms ?? fallbackTime), + seq: frame.seq ?? index, + controls: frame.controls || {}, + }); + }); + + avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq); + avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs); + animationBuffer.textContent = String(avatarState.pendingFrames.length); +} + +function flushAnimationQueue() { + const now = performance.now(); + let frameToApply = null; + while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) { + frameToApply = avatarState.pendingFrames.shift(); + } + + if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) { + avatarState.lastAppliedSeq = frameToApply.seq; + avatarState.activeControls = { + ...avatarState.activeControls, + ...frameToApply.controls, + }; + } + + animationBuffer.textContent = String(avatarState.pendingFrames.length); +} + +function applyAnimationControls(controls) { + const safeControls = controls || {}; + const jawOpen = clampControl(safeControls.jawOpen); + const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen); + const visemeEe = clampControl(safeControls.viseme_ee); + const visemeOh = clampControl(safeControls.viseme_oh); + const mouthPucker = clampControl(safeControls.mouthPucker); + const headYaw = clampSigned(safeControls.headYaw, 0.35); + const headPitch = clampSigned(safeControls.headPitch, 0.3); + const headRoll = clampSigned(safeControls.headRoll, 0.24); + const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0; + const t = performance.now() * 0.001; + + if (avatarState.currentVrm?.expressionManager) { + setExpressionValue("aa", visemeAa); + setExpressionValue("ee", visemeEe); + setExpressionValue("oh", visemeOh); + setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker)); + } + + if (avatarState.morphBindings.size > 0) { + setMorphValue("jawOpen", jawOpen); + setMorphValue("viseme_aa", visemeAa); + setMorphValue("viseme_ee", visemeEe); + setMorphValue("viseme_oh", visemeOh); + setMorphValue("mouthPucker", mouthPucker); + } + + // For plain head meshes (no VRM expressions and no morph targets), + // amplify head motion to keep speaking visually obvious. + const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09; + const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12; + const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04; + + applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied); + if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) { + applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4); + } + + if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { + const baseScale = 1.0 + jawOpen * 0.03; + avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale); + } + + if (avatarState.debugJaw) { + avatarState.debugJaw.rotation.x = jawOpen * 0.5; + avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015; + } + if (avatarState.debugMouth) { + avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45; + avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8; + avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01; + } +} + +function setExpressionValue(name, value) { + try { + avatarState.currentVrm.expressionManager.setValue(name, value); + } catch (_) { + // no-op: expression name may not exist on the current model + } +} + +function setMorphValue(controlName, value) { + const bindings = avatarState.morphBindings.get(controlName) || []; + bindings.forEach((binding) => { + binding.mesh.morphTargetInfluences[binding.index] = value; + }); +} + +function clampControl(value) { + return Math.min(1, Math.max(0, Number(value) || 0)); +} + +function clampSigned(value, limit) { + return Math.min(limit, Math.max(-limit, Number(value) || 0)); +} + +function resetView() { + camera.position.set(0, 1.45, 2.65); + controls.target.set(0, 1.15, 0); + controls.update(); +} + +function resetAvatarPose() { + avatarState.activeControls = { + jawOpen: 0, + viseme_aa: 0, + viseme_ee: 0, + viseme_oh: 0, + mouthPucker: 0, + headYaw: 0, + headPitch: 0, + headRoll: 0, + }; + applyAnimationControls(avatarState.activeControls); +} + +function detachCurrentAvatar() { + if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { + scene.remove(avatarState.currentRoot); + } + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + avatarState.lastAppliedSeq = -1; + avatarState.idleRig = null; + avatarState.currentRoot = avatarState.debugRig; + avatarState.currentVrm = null; + avatarState.morphBindings.clear(); + avatarState.headBone = avatarState.debugRig.children[1]; + avatarState.neckBone = avatarState.debugRig.children[1]; + avatarState.debugRig.visible = true; + modelName.textContent = "debug-avatar"; +} + +async function loadModelFromUrl(url) { + if (!url) return false; + modelStatus.textContent = "模型加载中..."; + try { + const gltf = await gltfLoader.loadAsync(url); + attachLoadedAsset(gltf, url.split("/").pop() || url); + modelStatus.textContent = "模型已加载"; + return true; + } catch (error) { + console.error(error); + modelStatus.textContent = "模型加载失败，已回退调试头像"; + addMessage("system", `模型加载失败：${error?.message || error}`); + detachCurrentAvatar(); + return false; + } +} + +async function loadDefaultAvatarModel() { + modelUrl.value = DEFAULT_BODY_MODEL_URL; + const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL); + if (!ok) { + modelUrl.value = DEFAULT_VRM_FALLBACK_URL; + await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL); + } +} + +async function loadModelFromFile(file) { + if (!file) return; + const url = URL.createObjectURL(file); + modelStatus.textContent = "本地模型加载中..."; + try { + const gltf = await gltfLoader.loadAsync(url); + attachLoadedAsset(gltf, file.name); + modelStatus.textContent = "本地模型已加载"; + } catch (error) { + console.error(error); + modelStatus.textContent = "本地模型加载失败，已回退调试头像"; + addMessage("system", `本地模型加载失败：${error?.message || error}`); + detachCurrentAvatar(); + } finally { + URL.revokeObjectURL(url); + } +} + +function attachLoadedAsset(gltf, label) { + detachCurrentAvatar(); + + const vrm = gltf.userData?.vrm || null; + const root = vrm?.scene || gltf.scene; + if (!root) { + throw new Error("模型中未找到可渲染的 scene"); + } + + if (vrm) { + VRMUtils.removeUnnecessaryVertices(gltf.scene); + VRMUtils.combineSkeletons(gltf.scene); + VRMUtils.combineMorphs(vrm); + VRMUtils.rotateVRM0(vrm); + } + + root.traverse((obj) => { + obj.frustumCulled = false; + }); + + root.position.set(0, 0, 0); + root.rotation.set(0, 0, 0); + root.scale.setScalar(1); + scene.add(root); + + avatarState.currentRoot = root; + avatarState.currentVrm = vrm; + avatarState.debugRig.visible = false; + avatarState.morphBindings = collectMorphBindings(root); + avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1]; + avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone; + avatarState.idleRig = buildIdleRig(vrm, root); + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + avatarState.lastAppliedSeq = -1; + + fitCameraToObject(root); + resetAvatarPose(); + modelName.textContent = label; +} + +function fitCameraToObject(root) { + const box = new THREE.Box3().setFromObject(root); + const center = new THREE.Vector3(); + const size = new THREE.Vector3(); + box.getCenter(center); + box.getSize(size); + + const radius = Math.max(size.x, size.y, size.z, 1.2); + controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0)); + camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9); + controls.update(); +} + +function resolveHeadBone(vrm, root) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.("head") || + vrm?.humanoid?.getRawBoneNode?.("head") || + root.getObjectByName("Head") || + root.getObjectByName("head") || + root + ); +} + +function resolveNeckBone(vrm, root) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.("neck") || + vrm?.humanoid?.getRawBoneNode?.("neck") || + root.getObjectByName("Neck") || + root.getObjectByName("neck") || + resolveHeadBone(vrm, root) + ); +} + +function collectMorphBindings(root) { + const bindings = new Map(); + const aliases = { + jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"], + viseme_aa: ["viseme_aa", "aa", "a", "moutha"], + viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"], + viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"], + mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"], + }; + + root.traverse((node) => { + if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return; + const entries = Object.entries(node.morphTargetDictionary); + Object.entries(aliases).forEach(([controlName, names]) => { + entries.forEach(([targetName, index]) => { + const normalized = targetName.toLowerCase(); + if (!names.includes(normalized)) return; + if (!bindings.has(controlName)) bindings.set(controlName, []); + bindings.get(controlName).push({ mesh: node, index }); + }); + }); + }); + return bindings; +} + +function buildIdleRig(vrm, root) { + const lookup = { + hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]), + spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]), + chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]), + upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]), + neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]), + head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]), + leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]), + rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]), + leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]), + rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]), + leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]), + rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]), + leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]), + rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]), + }; + + const base = new Map(); + Object.values(lookup).forEach((bone) => { + if (bone && !base.has(bone)) { + base.set(bone, bone.quaternion.clone()); + } + }); + + return { bones: lookup, base }; +} + +function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) { + return ( + vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) || + vrm?.humanoid?.getRawBoneNode?.(humanoidName) || + fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) || + null + ); +} + +function applyProceduralIdlePose(time) { + const rig = avatarState.idleRig; + if (!rig) return; + + const breathe = Math.sin(time * 1.9) * 0.018; + const sway = Math.sin(time * 1.15) * 0.05; + const armDrift = Math.sin(time * 1.55) * 0.07; + const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06; + const handDrift = Math.sin(time * 2.3) * 0.04; + + applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04); + applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06); + applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08); + applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1); + + applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18); + applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18); + applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0); + applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0); + applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12); + applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12); + applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05); + applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05); +} + +function applyBoneRotation(bone, x = 0, y = 0, z = 0) { + if (!bone) return; + const baseQuaternion = avatarState.idleRig?.base?.get(bone); + if (baseQuaternion) { + bone.quaternion.copy(baseQuaternion); + TMP_EULER.set(x, y, z, "XYZ"); + TMP_QUATERNION.setFromEuler(TMP_EULER); + bone.quaternion.multiply(TMP_QUATERNION); + return; + } + bone.rotation.x = x; + bone.rotation.y = y; + bone.rotation.z = z; +} + async function checkHealth() { const res = await fetch("/health"); const data = await res.json(); @@ -123,8 +716,6 @@ function subscribeEvents() { outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); - lastBusy = !!data.pipeline_busy; - if (typeof data.pipeline_runs === "number" && data.pipeline_runs > lastSeenRun) lastSeenRun = data.pipeline_runs; } catch (_) { // no-op } @@ -160,6 +751,36 @@ function subscribeSubtitles() { }; } +function subscribeAnimation() { + if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return; + const scheme = location.protocol === "https:" ? "wss" : "ws"; + animationConn.textContent = "连接中"; + animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`); + animationWs.onopen = () => { + animationConn.textContent = "已连接"; + }; + animationWs.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + if (data.type === "ping") return; + if (data.driver) { + animationDriver.textContent = data.driver; + } + const frameCount = data.frame_count ?? data.frames?.length ?? 0; + animationFrames.textContent = String(frameCount); + const preview = data.frames?.[0]?.controls || null; + animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2); + queueAnimationFrames(data); + } catch (_) { + // no-op + } + }; + animationWs.onclose = () => { + animationConn.textContent = "已断开"; + setTimeout(subscribeAnimation, 1000); + }; +} + async function connectWebRTC() { if (pc) { outputEl.textContent = "WebRTC 已连接或正在连接"; @@ -175,21 +796,22 @@ async function connectWebRTC() { }); const currentPc = pc; let seenConnected = false; + currentPc.onconnectionstatechange = () => { const st = currentPc.connectionState; if (st === "connected") { seenConnected = true; setConnBadge("已连接", "ok"); localStorage.setItem(AUTO_RECONNECT_KEY, "1"); - } - else if (st === "connecting") setConnBadge("连接中...", "warn"); - else if (st === "failed" || st === "disconnected" || st === "closed") { + } else if (st === "connecting") { + setConnBadge("连接中...", "warn"); + } else if (st === "failed" || st === "disconnected" || st === "closed") { setConnBadge(`连接${st}`, "warn"); if (seenConnected && (st === "disconnected" || st === "failed")) { addMessage("system", "连接已断开，可能被新的客户端连接接管。"); } if (localMicStream) { - localMicStream.getTracks().forEach((t) => t.stop()); + localMicStream.getTracks().forEach((track) => track.stop()); localMicStream = null; } if (pc === currentPc) { @@ -198,13 +820,12 @@ async function connectWebRTC() { offerBtn.disabled = false; } }; - // Explicitly request remote media m-lines so server can attach tracks. - pc.addTransceiver("video", { direction: "recvonly" }); + pc.ontrack = (event) => { if (event.streams && event.streams[0]) { - remoteVideo.srcObject = event.streams[0]; - remoteVideo.play().catch(() => { - addMessage("system", "远端音视频已到达，如未自动播放请点击视频区域后重试。"); + remoteAudio.srcObject = event.streams[0]; + remoteAudio.play().catch(() => { + addMessage("system", "远端语音已到达，如未自动播放请先与页面交互后重试。"); }); } }; @@ -220,12 +841,8 @@ async function connectWebRTC() { offerBtn.disabled = false; throw err; } - stream.getTracks().forEach((track) => pc.addTrack(track, stream)); - const sendAudioTrack = stream.getAudioTracks()[0]; - if (sendAudioTrack) { - sendAudioTrack.enabled = true; - } + stream.getTracks().forEach((track) => pc.addTrack(track, stream)); const offer = await pc.createOffer(); await pc.setLocalDescription(offer); @@ -240,11 +857,7 @@ async function connectWebRTC() { const answer = await res.json(); await pc.setRemoteDescription(answer); statusEl.textContent = "状态：WebRTC 已连接"; - outputEl.textContent = JSON.stringify( - { connected: true, iceConnectionState: pc.iceConnectionState }, - null, - 2 - ); + outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2); setConnBadge("已连接", "ok"); addMessage("system", "WebRTC 已连接，可开始对话。"); if (answer.replaced_previous) { @@ -256,8 +869,7 @@ async function connectWebRTC() { async function sendText() { const text = (chatInput.value || "").trim(); - if (!text) return; - if (sendBtn.disabled) return; + if (!text || sendBtn.disabled) return; sendBtn.disabled = true; const res = await fetch("/chat/text", { method: "POST", @@ -277,6 +889,9 @@ async function sendText() { async function resetChat() { await fetch("/chat/reset", { method: "POST" }); messagesEl.innerHTML = ""; + avatarState.pendingFrames = []; + avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; + resetAvatarPose(); addMessage("system", "会话已重置。"); await checkHealth(); } @@ -284,24 +899,56 @@ async function resetChat() { function tryAutoReconnect() { if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return; setTimeout(() => { - if (!pc) connectWebRTC().catch(() => { - // no-op: user can reconnect manually if permission/policy blocks auto flow - }); + if (!pc) { + connectWebRTC().catch(() => { + // no-op + }); + } }, 300); } +function handleModelFileChange(event) { + const [file] = event.target.files || []; + if (!file) return; + loadModelFromFile(file); +} + +function handleModelUrlLoad() { + const url = modelUrl.value.trim(); + if (!url) return; + loadModelFromUrl(url); +} + +new ResizeObserver(() => resizeStage()).observe(avatarCanvas); +window.addEventListener("resize", resizeStage); + healthBtn.addEventListener("click", checkHealth); offerBtn.addEventListener("click", connectWebRTC); resetBtn.addEventListener("click", resetChat); sendBtn.addEventListener("click", sendText); -chatInput.addEventListener("keydown", (e) => { - if (e.key === "Enter") { - e.preventDefault(); +loadModelBtn.addEventListener("click", handleModelUrlLoad); +resetViewBtn.addEventListener("click", resetView); +modelFile.addEventListener("change", handleModelFileChange); +chatInput.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault(); sendText(); } }); +modelUrl.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault(); + handleModelUrlLoad(); + } +}); + +initStage(); +resetView(); +resetAvatarPose(); +loadDefaultAvatarModel(); checkHealth(); subscribeEvents(); subscribeSubtitles(); +subscribeAnimation(); setConnBadge("未连接", "neutral"); tryAutoReconnect(); diff --git a/web/index.html b/web/index.html index 455fc10..945662e 100644 --- a/web/index.html +++ b/web/index.html @@ -8,11 +8,44 @@
-

可视化语音聊天系统

+

3D 数字人语音聊天系统

状态：启动中

-
- +
+
+ Three.js + VRM 驱动舞台 + 未连接 +
+
+ + + + + +
+
+ +
+
+ 模型状态 + 使用调试头像 +
+
+ 驱动器 + -- +
+
+ 最近帧数 + 0 +
+
+
+
+
当前模型debug-avatar
+
动画缓冲0
+
+ 
等待动画控制数据...
+
@@ -45,12 +78,24 @@
打断耗时0 ms
输入模式none
VAD触发0/0
+
动画客户端0
+
动画驱动--
+
最近动画帧0

运行状态


     

   

-  
+  
+  
 
 
diff --git a/web/style.css b/web/style.css
index 561c1fe..9734d8a 100644
--- a/web/style.css
+++ b/web/style.css
@@ -1,32 +1,167 @@
 body {
   margin: 0;
-  font-family: sans-serif;
-  background: #0f1115;
+  font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
+  background:
+    radial-gradient(circle at top, rgba(42, 93, 255, 0.16), transparent 36%),
+    radial-gradient(circle at right, rgba(15, 173, 129, 0.12), transparent 28%),
+    linear-gradient(180deg, #091018 0%, #111827 46%, #0b1018 100%);
   color: #f2f2f2;
 }
 
 .wrap {
-  max-width: 900px;
+  max-width: 1120px;
   margin: 24px auto;
   padding: 0 16px;
 }
 
-.video-box {
+.stage-box {
   width: 100%;
-  aspect-ratio: 16 / 9;
-  border: 1px solid #333;
-  border-radius: 10px;
-  background: #1a1f27;
+  border: 1px solid rgba(116, 139, 166, 0.22);
+  border-radius: 24px;
+  background: linear-gradient(180deg, rgba(18, 25, 36, 0.94), rgba(12, 18, 28, 0.96));
   overflow: hidden;
+  padding: 18px;
+  box-sizing: border-box;
+  display: flex;
+  flex-direction: column;
+  gap: 14px;
+  box-shadow: 0 28px 70px rgba(0, 0, 0, 0.32);
 }
 
-video {
+.stage-copy {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  gap: 12px;
+}
+
+.stage-copy strong {
+  font-size: 20px;
+  letter-spacing: 0.01em;
+}
+
+.stage-copy span {
+  font-size: 13px;
+  color: #7dd3fc;
+}
+
+.stage-toolbar {
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr) auto auto;
+  gap: 12px;
+  align-items: center;
+}
+
+.stage-toolbar input[type="text"] {
+  min-width: 0;
+  background: rgba(10, 15, 24, 0.78);
+  color: #e5eef8;
+  border: 1px solid rgba(112, 146, 190, 0.3);
+  border-radius: 12px;
+  padding: 10px 12px;
+}
+
+.stage-toolbar input[type="file"] {
+  position: absolute;
+  width: 1px;
+  height: 1px;
+  opacity: 0;
+  pointer-events: none;
+}
+
+.file-pill {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  min-height: 40px;
+  border-radius: 999px;
+  border: 1px solid rgba(102, 199, 255, 0.28);
+  background: rgba(9, 60, 86, 0.32);
+  color: #d7f2ff;
+  padding: 0 16px;
+  cursor: pointer;
+  white-space: nowrap;
+}
+
+.canvas-shell {
+  position: relative;
+  aspect-ratio: 16 / 9;
+  border-radius: 18px;
+  overflow: hidden;
+  background:
+    radial-gradient(circle at 50% 28%, rgba(148, 196, 255, 0.18), transparent 24%),
+    linear-gradient(180deg, #152235 0%, #0c131d 52%, #0a1017 100%);
+  border: 1px solid rgba(95, 128, 160, 0.18);
+}
+
+#avatarCanvas {
   width: 100%;
   height: 100%;
-  object-fit: cover;
   display: block;
 }
 
+.canvas-overlay {
+  position: absolute;
+  left: 14px;
+  right: 14px;
+  bottom: 14px;
+  display: grid;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 10px;
+  pointer-events: none;
+}
+
+.overlay-card {
+  border: 1px solid rgba(144, 170, 197, 0.18);
+  background: rgba(7, 11, 18, 0.68);
+  backdrop-filter: blur(10px);
+  border-radius: 14px;
+  padding: 10px 12px;
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+}
+
+.overlay-card span {
+  color: #8ea6c3;
+  font-size: 12px;
+}
+
+.overlay-card strong {
+  font-size: 14px;
+}
+
+.stage-metrics {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 12px;
+}
+
+.stage-metrics div {
+  border: 1px solid rgba(52, 72, 96, 0.7);
+  border-radius: 14px;
+  background: rgba(14, 22, 33, 0.7);
+  padding: 12px;
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+}
+
+.stage-metrics span {
+  color: #94a3b8;
+  font-size: 12px;
+}
+
+.stage-metrics strong {
+  font-size: 16px;
+}
+
+#animationPreview {
+  flex: 1;
+  min-height: 180px;
+  max-height: 260px;
+}
+
 .actions {
   margin-top: 16px;
   display: flex;
@@ -42,10 +177,10 @@ video {
 
 .chat-box input {
   flex: 1;
-  background: #111827;
+  background: rgba(13, 19, 29, 0.9);
   color: #f3f4f6;
   border: 1px solid #374151;
-  border-radius: 8px;
+  border-radius: 12px;
   padding: 10px;
 }
 
@@ -55,9 +190,9 @@ video {
   max-height: 360px;
   overflow: auto;
   border: 1px solid #1f2937;
-  border-radius: 10px;
+  border-radius: 16px;
   padding: 10px;
-  background: #0b1220;
+  background: rgba(10, 18, 32, 0.82);
   display: flex;
   flex-direction: column;
   gap: 8px;
@@ -66,8 +201,8 @@ video {
 .diag {
   margin-top: 12px;
   border: 1px solid #1f2937;
-  border-radius: 10px;
-  background: #0b0d12;
+  border-radius: 16px;
+  background: rgba(8, 13, 18, 0.78);
   padding: 10px;
   display: flex;
   gap: 16px;
@@ -122,12 +257,12 @@ video {
 
 .msg.user {
   align-self: flex-end;
-  background: #1d4ed8;
+  background: linear-gradient(135deg, #2563eb, #0ea5e9);
 }
 
 .msg.ai {
   align-self: flex-start;
-  background: #374151;
+  background: linear-gradient(135deg, #334155, #475569);
 }
 
 .panel {
@@ -143,8 +278,8 @@ video {
 
 .metric {
   border: 1px solid #1f2937;
-  background: #0b0d12;
-  border-radius: 8px;
+  background: rgba(8, 13, 18, 0.78);
+  border-radius: 14px;
   padding: 8px 10px;
   display: flex;
   flex-direction: column;
@@ -167,10 +302,10 @@ video {
 }
 
 button {
-  background: #2563eb;
+  background: linear-gradient(135deg, #2563eb, #0891b2);
   color: white;
   border: 0;
-  border-radius: 8px;
+  border-radius: 12px;
   padding: 8px 12px;
   cursor: pointer;
 }
@@ -231,9 +366,31 @@ button:disabled {
 
 pre {
   margin-top: 0;
-  background: #0b0d12;
+  background: rgba(8, 13, 18, 0.84);
   border: 1px solid #222;
-  border-radius: 8px;
+  border-radius: 14px;
   padding: 12px;
   overflow: auto;
 }
+
+audio {
+  display: none;
+}
+
+@media (max-width: 720px) {
+  .metrics {
+    grid-template-columns: repeat(2, minmax(120px, 1fr));
+  }
+
+  .stage-toolbar {
+    grid-template-columns: 1fr;
+  }
+
+  .stage-metrics {
+    grid-template-columns: 1fr;
+  }
+
+  .canvas-overlay {
+    grid-template-columns: 1fr;
+  }
+}
diff --git a/webrtc/__pycache__/__init__.cpython-312.pyc b/webrtc/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4deb15a3e4bf8c038065573cd3cd569ce2ecc92a
GIT binary patch
literal 138
zcmX@j%ge<81Y1OoWikTk#~=~B5jM<
zx@_>|B3zX3;u(*R2vM<1%=JrMcIX!(j;OQC$#`PK6?J#HDY23LM3&AI*?!$-yrrv%
z@f^T&D$eU%j$1rn^SSP0!$);jC>Dz+L&=_aOzVWUeV-Z%V+&|OR!;OQK$JeK$UUJK
zlByDl9)kWp;5q&Ma!(wjqe|GA*r;=Fp$l7~Jw_DLMP&lJl4QF~&kJ3g?2v8eVRy1q
z=HMyFE}4g?D7$3=p3;EmE7I+3_TBwjIJXxzInppcT}ZIGQ#3__tOKiL?#+Z;&+(ww
z8l99(!Ct#b<)4d2Wd_JKTj#50%O~kBEfh^el%N_)Dk=`rrC_+bKh~$|
zQb-$!g?r*Ue=r{I(}iFJA7U_~oKzw{o6c)W>Oc;?#1#DH!O276*X$zX7>%siouCaVZ1To2V5lC4=c#Lw7Q
z&TVMDzJ7?m;GXl=jvXC6`pzrRaAsFj4e@We7C@9NIbXEfb9laoU|k0Y1^uN+FN!TI
zPmpt<)z{&~e`Xuwek=$eo1(qeY3Idd+P94mwG4W#lU>wnyH1VPX$lgSj`9*oOIy9{
zR^@w;lDQkaY3-+NuvR?OWu>=d83URr>%FjUTW{$%C1jy50PB7|Sz-1i9VXT5F{%7>
zQSXX8*QDSb-@dn%reywUo{TIGx?rc$Ae@BBTmC%F$#&R%b$+x-4Z^9*j^rw{Ejv@3
z>_V=2Zb%~u$af2PDaz@(+R9w6Jug@40OT0`0h>*v81IJAXrL?9YHWuxR}
zunI~B$ZOooyjwc3Gw42=B`mEzVYi(iWZ@XSKh&#J-
z{qrWVw!>GVqT+NuCig^jNmaD&P(sngkd_>Pr!AJyg`QZlZL5m=0)8T)#ALXAJ%fra
z9FN39NobrkuFYzqj2wFs$eu8i|Ycr=sKrHH4XL+
zH@LoDMMKdgB?ge$uMC!2*E7e#Ztc%deL)_=1+AzYYaeTWr#)M;YPO^?Q_`3YeOj_U
z+t51Our1TD?Urr2Vf#pN*3&ra@nt-|8;u_{Uu(YQz3cH!dv*=&&ALl39vnV6>t3C4
zuTBdy?seG`&&9#v!EyhaKbl)n^NZb=_D}48fA^LBm-kJA8mg)(*6+FKidAawiotvwk*(0-=TpL@m&t6u|4Yp72KVj
zE}*9zG*HG*IYp$)DApBXph=plqJdiXRJDY3g9Q8~P=wcoLD3Eiijp~~Afyn`q-YX`
z7^N4LvaH;ora3d9uz)=;RtVH&+i5PvrDz|lLLG-IX2DT@oO+=!U$#|&&}lvi*VicE
zu(&~hu@qkjs0?{)l|qV_NlK^&fPxipDFH8O4Pg`*G^`6G{`Wk6&9LH{!YeBTiKYvr
zBvzi{Q-Vz3%?0->+jhW8tWzl}ETz&57i13l3VvXwPssnJy0Kl;W9Kj*0&
zm(mRrF1Wv4m%DD#%Rij*ul|*v!3RRr};0Q4WF&saJB8~=5+T+0Qu>%(dXvNNQK8-#Ct9C
z<)mhH+Hi(axL<1rVAE%}25RXo8nB7C
zIiNn;y*eOrcWOOYZ*l?W&UPB$`8zu#pr?313OXf4q^x|5o?NtaZcx;T~zHk27qr5jMcg?HXm~rr|boPDWsX
z5z~vm4IJZ34ef@zpE19=+XZ#D}`dmJYca%
z-HQGnr5qN!Z2j)aQ4z)=B0xMdA_^fKQ9>%yk_tgCL_JU$I#Sz#T%;i;*@iOTgKDXU
zEMH!iao5eb>$BCX*liLDINLoi8kDsLl;wA#w${+GMt?DP+Xd7lPyKFwQV@}LQ>=?c
zf4wy60o}PLy%N&367T^M@1OY%+YUy|ItRgxE~-khUybQ}XFR6BM%P2nSI$^sV@!r!
zh6dE9fuZh%3OY9-jefx3?n183pwVO~p@C>3LHRG4o&+sx&gT^v
z2C!(JVm@Gk5y-+^Mv-x(71$)Lya(K{%?v9RyWIg$WM|J8@Gj;U)k>Jg=gSG~I@d*sO)MtRt#_M(_
z9*k($osZSaFd#0wP`npJnCSz0~rVS%PgPc=@Sh
zuyoTnPTLk6jSPlb$PeBS%uh)e8yAZ2WDz6ZUMqFZzYP*$yv)mrc@<2fhNQgU$)&jb
z#Q=#yi8dFsJ^_G4^|BX16Xqy3xhc5S1!pO^??E`duq|k5BE>O3|5tD&H(7Ekb+Rj@
zVluCDTH97gEFoR=ak_Xsu0|o<6eFPlh!zd>#2XJeEUpIbTtrb4>i2;*E*iTs3$0pB
z0@U!yLGL%N9I|)ADD6*B9vhFAWJD4
z@s-vHTROWbP)?GV~{TG*q9(
zX5JGrkC*d=s5PIYcA$hGD?Ch{@9T-lhPyk0?V=nKhLuPp81zxK6Sypv=r90rFAl|u
z1sV87;^BFcb(Nn9{EHWbEe}eag4b|$DC8?aCwsgfd^tE-6+MYzV-8R~`?Q+_YF9WO
zi6}V^BAzb9`lIkUhGj)!s3#8J9#sS|-4;5@OwjPZY(0B$Udu7-_8o>>->E9lfq{^=
z8LD}XQhJXxeol(-k-B?i)#s%A9$9mbR6P)zY27~vR9_yZUh2JDx&EI7o@~zl0%-cC
AZvX%Q

literal 0
HcmV?d00001

diff --git a/webrtc/gateway.py b/webrtc/gateway.py
index 20d8e6c..00fd135 100644
--- a/webrtc/gateway.py
+++ b/webrtc/gateway.py
@@ -3,21 +3,18 @@ from __future__ import annotations
 from aiortc import RTCPeerConnection
 from fastapi import FastAPI
 
-from config import settings
-from core.state_machine import Arbitrator
-from services.avatar import AvatarService
 from webrtc.audio_track import build_audio_track
 from webrtc.tracks import AudioBus
-from webrtc.video_track import build_video_track
 
 
 def attach_webrtc_tracks(
     pc: RTCPeerConnection,
-    avatar_service: AvatarService,
-    arbitrator: Arbitrator,
+    avatar_service,
+    arbitrator,
     audio_bus: AudioBus,
 ) -> None:
-    pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
+    _ = avatar_service
+    _ = arbitrator
     pc.addTrack(build_audio_track(audio_bus))
 
 
diff --git a/webrtc/tracks.py b/webrtc/tracks.py
index 10246c1..1af68d9 100644
--- a/webrtc/tracks.py
+++ b/webrtc/tracks.py
@@ -6,10 +6,7 @@ from collections import deque
 
 import numpy as np
 from aiortc import MediaStreamTrack
-from av import AudioFrame, VideoFrame
-
-from core.state_machine import Arbitrator, SessionState
-from services.avatar import AvatarService
+from av import AudioFrame
 
 
 class AudioBus:
@@ -55,41 +52,6 @@ class AudioBus:
             self._level = 0.0
 
 
-class AvatarVideoTrack(MediaStreamTrack):
-    kind = "video"
-
-    def __init__(
-        self,
-        avatar_service: AvatarService,
-        arbitrator: Arbitrator,
-        audio_bus: AudioBus,
-        fps: int = 25,
-    ) -> None:
-        super().__init__()
-        self.avatar_service = avatar_service
-        self.arbitrator = arbitrator
-        self.audio_bus = audio_bus
-        self.fps = fps
-        self._pts = 0
-        self._time_base = Fraction(1, 90000)
-
-    async def recv(self) -> VideoFrame:
-        await asyncio.sleep(1 / self.fps)
-        generated = await self.avatar_service.pop_generated_frame()
-        if generated is not None:
-            arr = generated
-        else:
-            level = await self.audio_bus.level()
-            speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
-            arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
-
-        frame = VideoFrame.from_ndarray(arr, format="bgr24")
-        frame.pts = self._pts
-        frame.time_base = self._time_base
-        self._pts += int(90000 / self.fps)
-        return frame
-
-
 class AvatarAudioTrack(MediaStreamTrack):
     kind = "audio"
 
diff --git a/webrtc/video_track.py b/webrtc/video_track.py
index f5ac09b..54fc003 100644
--- a/webrtc/video_track.py
+++ b/webrtc/video_track.py
@@ -1,21 +1,7 @@
 from __future__ import annotations
 
-from core.state_machine import Arbitrator
-from services.avatar import AvatarService
-from webrtc.tracks import AudioBus, AvatarVideoTrack
-
-__all__ = ["AvatarVideoTrack", "build_video_track"]
+__all__: list[str] = []
 
 
-def build_video_track(
-    avatar_service: AvatarService,
-    arbitrator: Arbitrator,
-    audio_bus: AudioBus,
-    fps: int = 25,
-) -> AvatarVideoTrack:
-    return AvatarVideoTrack(
-        avatar_service=avatar_service,
-        arbitrator=arbitrator,
-        audio_bus=audio_bus,
-        fps=fps,
-    )
+def build_video_track(*_args, **_kwargs):
+    raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")