xsl/product
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: xsl/product:main
Branches Tags
xsl/product:main xsl/product:js3d
...
pull from: xsl/product:js3d
Branches Tags
xsl/product:js3d xsl/product:main
6 Commits
main ... js3d
Author SHA1 Message Date
xsl 64c894b673 独立依赖 2026-04-08 14:38:04 +08:00
xsl 760cf864eb save code 2026-03-30 21:37:46 +08:00
xsl 66c742864c 测试通过 2026-03-29 23:33:22 +08:00
xsl 4dcae11c1b save code 2026-03-29 23:01:39 +08:00
xsl 36e838caec save code 2026-03-29 20:50:53 +08:00
xsl f24de38e94 大功告成 2026-03-27 17:10:41 +08:00
62 changed files with 90303 additions and 2351 deletions
Expand all files Collapse all files
.env
+30
View File
@@ -0,0 +1,30 @@
# LLM (DeepSeek, OpenAI-compatible)
# Fill your real key below.
LLM_API_KEY=sk-aac48115300845be97a032444fc11560
LLM_BASE_URL=https://api.deepseek.com/v1
LLM_MODEL=deepseek-chat
LLM_TIMEOUT=20
LLM_MAX_TOKENS=300
LLM_TEMPERATURE=0.55
LLM_SYSTEM_PROMPT=你是一个自然、友好的中文语音助手。请像真人当面聊天一样回答，口语化、简洁、自然，通常1到2句。不要使用书面公告腔，不要分点，不要长段解释，不要夸张语气词。不要虚构身份关系，不要自称家人、恋人或其他私人身份。避免客服套话，比如“随时为你服务”、“很高兴为你服务”、“请问有什么可以帮您”。优先使用短句和自然停顿，适合直接语音播报。
TTS_VOICE=zf_xiaoxiao
TTS_SPEED=0.90
TTS_SENTENCE_MAX_LEN=64
TTS_SEGMENT_PAUSE_MS=90
TTS_FADE_MS=12
# Web service（与当前代码一致，仅 HTTP_HOST / HTTP_PORT）
HTTP_HOST=0.0.0.0
HTTP_PORT=8018
CORS_ORIGINS=*
# HTTPS（跨设备访问麦克风/页面时常需 TLS；路径相对项目根目录）
SSL_CERTFILE=certs/dev-cert.pem
SSL_KEYFILE=certs/dev-key.pem
# 3D avatar driver mode
AVATAR_DRIVER_MODE=blendshape_stream
AVATAR_CONTROL_PROTOCOL=ws
AVATAR_BLENDSHAPE_SCHEMA=arkit
SUDO_PASSWORD=asdf
.run-8018.log
+386
View File
@@ -0,0 +1,386 @@
INFO: Started server process [401194]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
INFO: 127.0.0.1:43090 - "GET /health HTTP/1.1" 200 OK
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
return torch._C._cuda_getDeviceCount() > 0
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
super().__init__("LSTM", *args, **kwargs)
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
WeightNorm.apply(module, name, dim)
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found"
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
import pkg_resources
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found"
Building prefix dict from the default dictionary ...
DEBUG:jieba:Building prefix dict from the default dictionary ...
Loading model from cache /tmp/jieba.cache
DEBUG:jieba:Loading model from cache /tmp/jieba.cache
Loading model cost 0.358 seconds.
DEBUG:jieba:Loading model cost 0.358 seconds.
Prefix dict has been built successfully.
DEBUG:jieba:Prefix dict has been built successfully.
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
INFO: 127.0.0.1:60258 - "POST /chat/text HTTP/1.1" 200 OK
INFO: 127.0.0.1:40524 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:2190 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:2190 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:2190 - "GET /web/app.js HTTP/1.1" 200 OK
INFO: 172.28.96.1:2190 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:2190 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:12419 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:9602 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:6375 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:6375 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:6375 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(5xwO9ewTsb 1 udp 1853759231 172.28.96.1 60702 typ prflx)
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(0) ICE completed
INFO: 172.28.96.1:9899 - "POST /chat/text HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(0) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-27' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:1631 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:1631 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:1631 - "GET /web/app.js HTTP/1.1" 200 OK
INFO: 172.28.96.1:1631 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:1631 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:6354 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:11967 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:2728 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO: 172.28.96.1:2728 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:13395 - "GET /web/app.js HTTP/1.1" 200 OK
INFO: 172.28.96.1:4941 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:4941 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:5793 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:3627 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(2) Discovered peer reflexive candidate Candidate(Ly5ELFkmha 1 udp 1853759231 172.28.96.1 50062 typ prflx)
INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(2) ICE completed
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(2) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-941' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:1723 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:1723 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:1723 - "GET /web/app.js HTTP/1.1" 200 OK
INFO: 172.28.96.1:1723 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:1723 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:14573 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:7534 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(3) Discovered peer reflexive candidate Candidate(sCNsxRJ8b8 1 udp 1853759231 172.28.96.1 64421 typ prflx)
INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(3) ICE completed
INFO: 172.28.96.1:7236 - "POST /chat/text HTTP/1.1" 200 OK
INFO: 172.28.96.1:10460 - "POST /chat/text HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(3) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-1283' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:14796 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:14796 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:14796 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK
INFO: 172.28.96.1:14796 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:14796 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:2032 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:14707 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:13486 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:13486 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(4) Discovered peer reflexive candidate Candidate(Qady0BDpmC 1 udp 1853759231 172.28.96.1 53980 typ prflx)
INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(4) ICE completed
INFO: 172.28.96.1:13486 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-2812' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:13486 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:13486 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK
INFO: 172.28.96.1:13486 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:13486 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:10285 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:5486 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:13787 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:10285 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:10285 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(5) Discovered peer reflexive candidate Candidate(BLyMgNoiBf 1 udp 1853759231 172.28.96.1 55730 typ prflx)
INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(5) ICE completed
INFO: 172.28.96.1:8337 - "POST /chat/text HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(5) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-2887' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:7750 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:7750 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:7750 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
INFO: 172.28.96.1:7750 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:7750 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:1059 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:14463 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:14634 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:14634 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(6) Discovered peer reflexive candidate Candidate(dbehuF1olz 1 udp 1853759231 172.28.96.1 56094 typ prflx)
INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(6) ICE completed
INFO: connection closed
INFO: connection closed
INFO: 172.28.96.1:14493 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:14493 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:14493 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
INFO: 172.28.96.1:14493 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:14493 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:10905 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:10117 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:6762 - "GET /meta HTTP/1.1" 200 OK
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-4222' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:6762 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(7) Discovered peer reflexive candidate Candidate(8K1dQt9yU0 1 udp 1853759231 172.28.96.1 49725 typ prflx)
INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(7) ICE completed
INFO: 172.28.96.1:14132 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-4834' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:14132 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:14132 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
INFO: 172.28.96.1:14132 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:14132 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:2807 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:6331 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:5113 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:2807 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:2807 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(8) Discovered peer reflexive candidate Candidate(ROwaVxsM9g 1 udp 1853759231 172.28.96.1 52410 typ prflx)
INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(8) ICE completed
INFO: 172.28.96.1:10113 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-6923' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 172.28.96.1:10113 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:4914 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:10113 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:5903 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:3565 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:10113 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:10113 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(9) Discovered peer reflexive candidate Candidate(E8HKWSMFTW 1 udp 1853759231 172.28.96.1 50992 typ prflx)
INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(9) ICE completed
INFO: 172.28.96.1:13074 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-15396' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: connection closed
INFO: 172.28.96.1:13074 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:13074 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
INFO: 172.28.96.1:7285 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:7285 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:13646 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 172.28.96.1:9569 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:2629 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:13646 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:13646 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(10) Discovered peer reflexive candidate Candidate(AbT1vCac8Y 1 udp 1853759231 172.28.96.1 62912 typ prflx)
INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(10) ICE completed
INFO: 172.28.96.1:6176 - "POST /chat/text HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(10) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-16263' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: 127.0.0.1:51958 - "GET /health HTTP/1.1" 200 OK
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [401194]
INFO: Started server process [404442]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
INFO: 127.0.0.1:34264 - "GET /health HTTP/1.1" 200 OK
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
return torch._C._cuda_getDeviceCount() > 0
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
super().__init__("LSTM", *args, **kwargs)
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
WeightNorm.apply(module, name, dim)
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
import pkg_resources
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found"
Building prefix dict from the default dictionary ...
DEBUG:jieba:Building prefix dict from the default dictionary ...
Loading model from cache /tmp/jieba.cache
DEBUG:jieba:Loading model from cache /tmp/jieba.cache
Loading model cost 0.384 seconds.
DEBUG:jieba:Loading model cost 0.384 seconds.
Prefix dict has been built successfully.
DEBUG:jieba:Prefix dict has been built successfully.
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
INFO: 127.0.0.1:35522 - "POST /chat/text HTTP/1.1" 200 OK
INFO: 172.28.96.1:6932 - "GET / HTTP/1.1" 200 OK
INFO: 172.28.96.1:6932 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 172.28.96.1:6932 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
INFO: 172.28.96.1:6932 - "GET /health HTTP/1.1" 200 OK
INFO: 172.28.96.1:6932 - "GET /events HTTP/1.1" 200 OK
INFO: 172.28.96.1:14179 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 172.28.96.1:12729 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 172.28.96.1:3368 - "GET /meta HTTP/1.1" 200 OK
INFO: 172.28.96.1:3368 - "POST /webrtc/offer HTTP/1.1" 200 OK
INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(jQ7HV4xKqV 1 udp 1853759231 172.28.96.1 52186 typ prflx)
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.WAITING -> State.IN_PROGRESS
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.IN_PROGRESS -> State.SUCCEEDED
INFO:aioice.ice:Connection(0) ICE completed
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
INFO: 172.28.96.1:8927 - "POST /chat/text HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO:aioice.ice:Connection(0) Consent to send expired
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-26' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
Traceback (most recent call last):
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
frame = await track.recv()
^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
raise MediaStreamError
aiortc.mediastreams.MediaStreamError
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [404442]
.run/start-test.log
+341
View File
@@ -0,0 +1,341 @@
nohup: ignoring input
repo: /home/xsl/code/product
python: /home/xsl/code/product/.venv/bin/python
mode: auto
bind: 0.0.0.0:8018
listen: https://0.0.0.0:8018
browser test page: https://<server-ip>:8018/
INFO: Started server process [20276]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
INFO: 127.0.0.1:35988 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 127.0.0.1:36002 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36022 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 127.0.0.1:36036 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36050 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36072 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36078 - "GET /health HTTP/1.1" 200 OK
INFO: 127.0.0.1:36104 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36118 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36148 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36168 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36194 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36206 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:36224 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54498 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54524 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54542 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54562 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54584 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54596 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54608 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54628 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54642 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54650 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54664 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54684 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54706 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54718 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54748 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54756 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:54788 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59782 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59802 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59812 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59832 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59860 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59874 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59888 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59902 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59906 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59910 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59922 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59928 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59944 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59962 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59976 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:60002 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58794 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58800 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58830 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58854 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58884 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58906 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58930 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58950 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58974 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:58986 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59002 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59026 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59046 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO: 127.0.0.1:59046 - "GET /web/app.js?v=20260329m HTTP/1.1" 200 OK
INFO: 127.0.0.1:59046 - "GET /web/app.js?v=20260329q HTTP/1.1" 200 OK
INFO: 127.0.0.1:59046 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK
INFO: 127.0.0.1:59114 - "GET /health HTTP/1.1" 200 OK
INFO: 127.0.0.1:59124 - "GET /events HTTP/1.1" 200 OK
INFO: 127.0.0.1:59114 - "GET /events HTTP/1.1" 200 OK
INFO: 127.0.0.1:59150 - "GET /health HTTP/1.1" 200 OK
INFO: 127.0.0.1:59174 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 127.0.0.1:59184 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 127.0.0.1:59186 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:59190 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:44402 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:44446 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: 127.0.0.1:44450 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: 127.0.0.1:44472 - "WebSocket /ws/audio" [accepted]
INFO: connection closed
INFO: connection closed
INFO: connection open
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 329, in asgi_send
await self.send(data) # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/websockets/legacy/protocol.py", line 618, in send
await self.ensure_open()
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/websockets/legacy/protocol.py", line 916, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedOK: received 1000 (OK) stale-open; then sent 1000 (OK) stale-open
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 86, in send
await self._send(message)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 39, in sender
await send(message)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 342, in asgi_send
raise ClientDisconnected from exc
uvicorn.protocols.utils.ClientDisconnected
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 244, in run_asgi
result = await self.app(self.scope, self.asgi_receive, self.asgi_send) # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/applications.py", line 1159, in __call__
await super().__call__(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/applications.py", line 90, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 151, in __call__
await self.app(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/cors.py", line 80, in __call__
await self.app(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 660, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 680, in app
await route.handle(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/routing.py", line 350, in handle
await self.app(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 160, in app
await wrap_app_handling_exceptions(app, session)(scope, receive, send)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 157, in app
await func(session)
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 764, in app
await dependant.call(**solved_result.values)
File "/home/xsl/code/product/main.py", line 446, in ws_audio
await websocket.send_text(
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 166, in send_text
await self.send({"type": "websocket.send", "text": data})
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/starlette/websockets.py", line 89, in send
raise WebSocketDisconnect(code=1006)
starlette.websockets.WebSocketDisconnect
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
return torch._C._cuda_getDeviceCount() > 0
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK"
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
super().__init__("LSTM", *args, **kwargs)
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
WeightNorm.apply(module, name, dim)
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found"
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
import pkg_resources
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found"
Building prefix dict from the default dictionary ...
DEBUG:jieba:Building prefix dict from the default dictionary ...
Loading model from cache /tmp/jieba.cache
DEBUG:jieba:Loading model from cache /tmp/jieba.cache
Loading model cost 0.356 seconds.
DEBUG:jieba:Loading model cost 0.356 seconds.
Prefix dict has been built successfully.
DEBUG:jieba:Prefix dict has been built successfully.
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
INFO: 127.0.0.1:44470 - "POST /chat/text HTTP/1.1" 200 OK
INFO: 127.0.0.1:44470 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO:root:download models from model hub: hf
INFO:httpx:HTTP Request: GET https://huggingface.co/api/models/FunAudioLLM/SenseVoiceSmall/revision/main "HTTP/1.1 200 OK"
funasr version: 1.3.1.
Check update of funasr, and it would cost few times. You may disable it by set `disable_update=True` in AutoModel
You are using the latest version of funasr-1.3.1
Fetching 29 files: 0%| | 0/29 [00:00<?, ?it/s]
Fetching 29 files: 100%|██████████| 29/29 [00:00<00:00, 50935.85it/s]
WARNING:root:trust_remote_code: False
INFO:root:Loading pretrained params from /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt
INFO:root:ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt
INFO:root:scope_map: ['module.', 'None']
INFO:root:excludes: None
INFO:root:Loading ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt, status: <All keys matched successfully>
INFO:root:download models from model hub: hf
INFO:httpx:HTTP Request: GET https://huggingface.co/api/models/FunAudioLLM/SenseVoiceSmall/revision/main "HTTP/1.1 200 OK"
funasr version: 1.3.1.
Check update of funasr, and it would cost few times. You may disable it by set `disable_update=True` in AutoModel
You are using the latest version of funasr-1.3.1
Fetching 29 files: 0%| | 0/29 [00:00<?, ?it/s]
Fetching 29 files: 100%|██████████| 29/29 [00:00<00:00, 37987.14it/s]
WARNING:root:trust_remote_code: False
INFO:root:Loading pretrained params from /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt
INFO:root:ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt
INFO:root:scope_map: ['module.', 'None']
INFO:root:excludes: None
INFO:root:Loading ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt, status: <All keys matched successfully>
WARNING:services.asr:ASR running in degraded mode on cpu
.run/start-test.pid
+1
View File
@@ -0,0 +1 @@
20276
.run/visual-chat-8018.log
+39
View File
@@ -0,0 +1,39 @@
INFO: Started server process [3446]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
INFO: 127.0.0.1:59440 - "HEAD /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:59450 - "HEAD /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42148 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42162 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42164 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/app.js?v=20260330a HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.module.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/examples/jsm/controls/OrbitControls.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/loaders/GLTFLoader.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51616 - "GET /web/vendor/%40pixiv/three-vrm/three-vrm.module.min.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /health HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK
INFO: 127.0.0.1:51616 - "GET /events HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 127.0.0.1:51680 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 127.0.0.1:51682 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 127.0.0.1:53334 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:53342 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: connection closed
INFO: connection closed
INFO: connection closed
INFO: Shutting down
INFO: Waiting for connections to close. (CTRL+C to force quit)
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [3446]
DEPLOYMENT_GUIDE.md
-242
View File
@@ -1,242 +0,0 @@
# 可视化语音聊天系统（初版）说明文档
本文档覆盖：
- 资源位置（代码、模型、环境）
- 本机启动与部署
- 局域网/远程访问
- 常用运维命令
- 验收与排障
---
## 1. 项目概览
当前项目实现的是单机部署的可视化聊天系统（WebRTC）：
- 用户语音上行（VAD + ASR）
- 文本调用在线 LLM
- 本地 TTS 合成语音
- 数字人视频下行（MuseTalk 优先，失败回退写实底片）
- 前端字幕与指标面板（SSE + WebSocket）
- 单工仲裁（支持 barge-in 打断）
代码根目录：
- `/home/xsl/product`
---
## 2. 代码结构（关键目录）
- `main.py`：后端入口（FastAPI + WebRTC + 全链路编排）
- `config.py`：配置加载（读取 `.env`）
- `core/`
- `state_machine.py`：会话状态机（单工仲裁）
- `pipeline.py`：VAD/ASR/LLM/TTS/字幕/时延逻辑
- `services/`
- `asr.py`：SenseVoice
- `llm.py`：在线模型 + fallback
- `tts.py`：Kokoro
- `vad.py`：Silero VAD
- `avatar.py`：MuseTalk 推理调度 + 写实回退帧
- `webrtc/`
- `tracks.py`：实际音视频轨道实现
- `gateway.py`、`video_track.py`、`audio_track.py`：结构化封装
- `web/`
- `index.html`、`app.js`、`style.css`
- `scripts/`
- 启停：`service.sh`、`start-public.sh`
- 验收：`run_all_checks.sh`、`qa_check.py`、`smoke_test.py`、`model_probe.py`、`vad_check.py`、`tts_stress.py`、`llm_fallback_check.py`
- 监控：`gpu_monitor.sh`
---
## 3. 环境与模型资源位置
### 3.1 Conda 环境
当前实际运行环境：
- `/home/xsl/miniconda3/envs/MuseTalk`
系统启动脚本固定使用该 Python：
- `scripts/start-public.sh` 中 `PY=/home/xsl/miniconda3/envs/MuseTalk/bin/python`
### 3.2 主要模型路径
MuseTalk 模型目录：
- `/home/xsl/work/MuseTalk/models/`
关键文件：
- UNet：`/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth`
- 配置：`/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json`
- Whisper：`/home/xsl/work/MuseTalk/models/whisper`
数字人源视频（写实底片 + MuseTalk 驱动源）：
- `/home/xsl/work/MuseTalk/data/video/yongen.mp4`
MuseTalk 推理脚本：
- `/home/xsl/work/MuseTalk/scripts/inference.py`
### 3.3 其他环境记录
完整环境与模型清单参考：
- `environments.md`
---
## 4. 配置文件
主配置文件：
- `/home/xsl/product/.env`
当前关键项（示例）：
- `LLM_API_KEY`
- `LLM_BASE_URL=https://api.deepseek.com`
- `WEBRTC_HOST=0.0.0.0`
- `WEBRTC_PORT=8080`
- `STUN_URL=stun:stun.l.google.com:19302`
- `SSL_CERTFILE=/home/xsl/product/certs/dev-cert.pem`
- `SSL_KEYFILE=/home/xsl/product/certs/dev-key.pem`
说明：
- `LLM_MODEL` 为空时会自动按 `base_url` 推断（DeepSeek 默认 `deepseek-chat`）。
---
## 5. 启动与部署
### 5.1 本机启动（推荐）
```bash
bash /home/xsl/product/scripts/service.sh start
bash /home/xsl/product/scripts/service.sh status
```
重启：
```bash
bash /home/xsl/product/scripts/service.sh restart
```
停止：
```bash
bash /home/xsl/product/scripts/service.sh stop
```
日志：
```bash
bash /home/xsl/product/scripts/service.sh logs
```
### 5.2 访问地址
- HTTPS（推荐）：`https://<服务器IP>:8080/`
- 本机：`https://127.0.0.1:8080/`
如果你使用自签证书，浏览器首次需要手动信任。
### 5.3 远程/跨机器访问
见：
- `REMOTE_ACCESS.md`
重点：
- 跨机器麦克风通常要求 HTTPS
- WSL2 场景可能需要 Windows 端口转发/防火墙放行
---
## 6. 一键验收与调试命令
### 6.1 一键全检查
```bash
bash /home/xsl/product/scripts/run_all_checks.sh
```
覆盖：
- 服务状态
- 文本链路压测
- LLM fallback
- VAD 检查
- TTS 20 句稳定性
- smoke 媒体输出（`wav + mp4`）
### 6.2 单项脚本
```bash
conda run -n MuseTalk python /home/xsl/product/scripts/model_probe.py
conda run -n MuseTalk python /home/xsl/product/scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 20
conda run -n MuseTalk python /home/xsl/product/scripts/llm_fallback_check.py
conda run -n MuseTalk python /home/xsl/product/scripts/vad_check.py
conda run -n MuseTalk python /home/xsl/product/scripts/tts_stress.py
conda run -n MuseTalk python /home/xsl/product/scripts/smoke_test.py --text "全链路验收"
```
### 6.3 长稳与GPU监控
1小时长稳：
```bash
conda run -n MuseTalk python /home/xsl/product/scripts/longrun_test.py --base-url "https://127.0.0.1:8080" --minutes 60 --interval-sec 30
```
GPU采样（CSV）：
```bash
bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.csv 3600 5
```
---
## 7. 前端功能说明
网页包含：
- 视频窗口（数字人）
- 文本输入框
- 字幕消息区（WebSocket）
- 指标面板（SSE）
当前已支持：
- 页面刷新自动重连
- 单连接策略（新连接踢掉旧连接）
- 句级字幕流式下发
- 打断耗时等指标展示
---
## 8. 当前已知边界
- MuseTalk 结果生成可能有一定延迟，期间使用写实底片回退。
- 部分“主观体验类”验收项（如口型同步主观评分）需要人工实测判定。
- Qwen3-TTS 对比方案尚未并入主链路（可后续做 A/B 开关）。
---
## 9. 快速排障
1) 页面无声/无画面
- 先看 `service.sh status`
- 看 `/health` 是否正常
- 浏览器是否授权麦克风、是否信任 HTTPS 证书
2) 连接冲突
- 系统是单连接模式，新连接会断开旧连接（预期）
3) 口型不动
- 查看 `/health` 中 `avatar.queued_frames` 和 `avatar.last_error`
- 若 MuseTalk 未产帧，会回退写实底片 + 口型叠加
4) 模型异常
- 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD
---
## 10. 版本建议
如果要做“生产化下一步”，建议优先：
- 增加 TURN（公网复杂 NAT）
- 增加多会话隔离（当前默认单会话）
- 增加 Prometheus/结构化日志
- 增加模型切换开关（Kokoro/Qwen3-TTS）
README.md
+168
View File
@@ -0,0 +1,168 @@
# 3D 数字人语音聊天系统
这个项目提供一个浏览器侧渲染的 3D 数字人语音交互原型：
- 浏览器通过 WebSocket 上传麦克风 PCM 音频，并接收服务端回传的 AI 语音片段。
- 浏览器通过 WebSocket 订阅字幕和口型/头部控制数据。
- 服务端执行 VAD -> ASR -> DeepSeek LLM -> TTS，并把音频特征转换为驱动 3D 数字人的控制帧。
当前形态更接近单机演示和研发验证环境，不是已经拆分完毕的生产化架构。
当前运行约束：
- 服务端按“单客户端接管”模式工作。新的浏览器页面连上后，会主动断开旧页面的音频、字幕、动画 WebSocket。
- 浏览器刷新后，语音通道可能仍受浏览器媒体安全策略限制，需要一次页面点击或点“连接语音通道”来恢复麦克风与播放上下文。
- 前端目前默认保留随机眨眼；更激进的 idle body motion 已回退，避免部分 VRM 模型进入 T pose。
## 当前架构
语音链路：
```text
Browser Mic
-> /ws/audio
-> FastAPI / websocket audio
-> VAD
-> ASR
-> LLM
-> TTS
-> /ws/audio
-> browser audio playback
```
控制链路：
```text
LLM reply text / TTS audio
-> AvatarService
-> /ws/animation
-> browser animation buffer
-> VRM or GLTF morph targets / head bones
```
字幕链路：
```text
ASR text / LLM segmented reply
-> /ws/subtitles
-> browser message panel
```
## 关键目录
- [main.py](main.py): FastAPI 入口，聚合音频 WebSocket、字幕/动画 WebSocket、健康状态和文本对话接口。
- [core/pipeline.py](core/pipeline.py): 语音/文本对话流水线，负责 ASR、LLM、TTS 编排。
- [core/state_machine.py](core/state_machine.py): 简单会话状态机，处理用户说话、思考、数字人说话之间的切换。
- [services/asr.py](services/asr.py): ASR 封装，优先使用 FunASR，失败时进入降级模式。
- [services/llm.py](services/llm.py): 在线 LLM 封装，默认兼容 DeepSeek OpenAI-style API。
- [services/tts.py](services/tts.py): TTS 封装，优先使用 Kokoro，失败时返回静音兜底。
- [services/vad.py](services/vad.py): VAD 封装，基于 silero-vad。
- [services/avatar.py](services/avatar.py): 把音频转换成 blendshape 和头部控制帧。
- [web/app.js](web/app.js): 浏览器端 3D 舞台、音频 WebSocket、字幕和动画消费逻辑。
- [scripts](scripts): 启动、证书生成、巡检和服务管理脚本。
## 快速启动
安装依赖：
```bash
pip install -r requirements.txt
```
准备环境变量，至少包括：
```bash
export LLM_API_KEY="..."
export LLM_BASE_URL="https://api.deepseek.com/v1"
export LLM_MODEL="deepseek-chat"
```
复制 [.env.example](.env.example) 为 `.env`，填入有效的 `LLM_API_KEY`。
**本地权重（除 LLM 外全部在仓库内 `models/`）**：首次可用联网环境执行 `python scripts/vendor_hf_models.py`；其中 VAD 的 `silero_vad.jit` 会从已安装的 `silero-vad` 包复制到 `models/vad/`。ASR 使用 `models/asr/SenseVoiceSmall/`，TTS 使用 `models/tts/Kokoro-82M/`。
完整部署步骤（新机器、模型打包、HTTPS、防火墙、排错）见 **[docs/deployment.md](docs/deployment.md)**。
语音如果偏快、偏硬，可以优先调这几个参数：
- `LLM_SYSTEM_PROMPT`：先把回答风格约束成口语化、非书面化，并明确禁止客服套话，再去调 TTS，收益更大。
- `LLM_TEMPERATURE`：默认 `0.55`，能减少奇怪人设和跑偏表达。
- `TTS_SPEED`：默认 `0.90`，比 `1.0` 更像真人正常说话速度。
- `TTS_SENTENCE_MAX_LEN`：默认 `64`，避免回复被切得太碎。
- `TTS_SEGMENT_PAUSE_MS`：默认 `90`，给分段之间留一点停顿，但不要太拖。
- `TTS_FADE_MS`：默认 `12`，减轻段间拼接的突兀感。
部署到任意机器时：在项目根目录运行（保证 `models/`、`web/`、`certs/` 等相对路径有效）。证书可在 `.env` 里写相对路径（例如 `certs/dev-cert.pem`），会按项目根目录解析。也可用环境变量 `VISUAL_CHAT_PYTHON` 指定解释器。
本地启动：
```bash
python main.py
```
或使用脚本：
```bash
bash scripts/start-test.sh auto
```
如果你明确要固定走当前 shell/虚拟环境里的 Python，也可以用：
```bash
bash scripts/start-public.sh
```
基础自检：
```bash
python scripts/model_probe.py
python scripts/qa_check.py --base-url http://127.0.0.1:8080 --rounds 3
python scripts/smoke_test.py --text "你好，请做一个简短自我介绍。"
```
启动后访问：
- `http://127.0.0.1:8080/`
- `http://127.0.0.1:8080/health`
如果 `.env` 里已经配置了 `HTTP_PORT=8018` 与 `SSL_CERTFILE` / `SSL_KEYFILE`，实际访问地址通常会变成：
- `https://<server-ip>:8018/`
- `https://<server-ip>:8018/health`
如果浏览器与服务端跨机器访问，麦克风采集通常需要 HTTPS。可以先生成自签证书：
```bash
bash scripts/gen-self-signed-cert.sh
```
再在环境变量或 `.env` 中配置 `SSL_CERTFILE` 和 `SSL_KEYFILE`。
监听规则：
- 服务监听地址固定按 `0.0.0.0` 规则执行。
- 即使把 `HTTP_HOST` 写成 `127.0.0.1`、`localhost` 或 `::1`，启动时也会自动归一化成 `0.0.0.0`，避免局域网访问被误伤。
## 核心接口
- `GET /`: 前端页面。
- `POST /chat/text`: 文本输入链路，跳过 ASR/VAD。
- `POST /chat/reset`: 清空上下文和运行时状态。
- `GET /health`: 返回后端运行指标和组件健康状态。
- `GET /meta`: 返回 HTTPS、监听端口和动画协议配置。
- `GET /events`: Server-Sent Events 运行状态流。
- `WS /ws/audio`: 双向语音通道，上传麦克风 PCM 并接收回复语音。
- `WS /ws/subtitles`: 字幕流。
- `WS /ws/animation`: 数字人控制流。
- `GET /avatar/schema`: 当前控制协议 schema。
说明：三个 WebSocket 通道当前都只保留一个活动客户端。新页面接入后，旧页面会收到 `1012 replaced` 并停止接收数据。
## 后续开发建议
优先阅读 [docs/development-guide.md](docs/development-guide.md)。这个文档说明了：
- 语音与文本两条链路的完整时序。
- 前后端之间的协议约定。
- 当前实现的边界和适合继续扩展的位置。
- 已知的工程化约束。
REMOTE_ACCESS.md
-72
View File
@@ -1,72 +0,0 @@
# 跨机器访问说明
## 1) 服务监听
已默认监听 `0.0.0.0:8080`，同网段机器可访问：
- `http://<你的机器IP>:8080/`
推荐用服务脚本管理（避免旧进程占端口）：
```bash
bash /home/xsl/product/scripts/service.sh start
bash /home/xsl/product/scripts/service.sh status
bash /home/xsl/product/scripts/service.sh logs
bash /home/xsl/product/scripts/service.sh restart
bash /home/xsl/product/scripts/service.sh stop
```
查询本机 IP:
```bash
ip addr
```
## 2) 浏览器麦克风权限（关键）
跨机器访问时，浏览器通常要求 **HTTPS** 才允许 `getUserMedia`。
### 方案 A（推荐）：启用 HTTPS
```bash
bash /home/xsl/product/scripts/gen-self-signed-cert.sh
```
然后编辑 `.env`:
```env
SSL_CERTFILE=/home/xsl/product/certs/dev-cert.pem
SSL_KEYFILE=/home/xsl/product/certs/dev-key.pem
```
启动：
```bash
bash /home/xsl/product/scripts/service.sh restart
```
访问：
- `https://<你的机器IP>:8080/`
首次需要在浏览器信任自签证书。
### 方案 B：HTTP 测试
如果仅做 UI 验证，可先用 HTTP；但麦克风可能被浏览器拒绝。
## 3) WSL2 网络注意事项
如果服务跑在 WSL2，其他机器无法直接访问时，需要在 Windows 做端口转发与防火墙放行。
最简单做法：在 Windows 上运行同样服务，或配置端口代理到 WSL IP。
## 4) WebRTC 联通
系统已配置 STUN（`.env` 里的 `STUN_URL`），用于获取可达候选地址：
```env
STUN_URL=stun:stun.l.google.com:19302
```
若跨公网且 NAT 复杂，需增加 TURN（后续可加）。
__pycache__/config.cpython-312.pyc
BIN
View File
Binary file not shown.
__pycache__/main.cpython-312.pyc
BIN
View File
Binary file not shown.
config.py
+68 -17
View File
@@ -1,32 +1,83 @@
from pathlib import Path
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
PROJECT_ROOT = Path(__file__).resolve().parent
def asr_bundle_dir() -> Path:
return PROJECT_ROOT / "models" / "asr" / "SenseVoiceSmall"
def kokoro_bundle_dir() -> Path:
return PROJECT_ROOT / "models" / "tts" / "Kokoro-82M"
def vad_bundle_path() -> Path:
return PROJECT_ROOT / "models" / "vad" / "silero_vad.jit"
class Settings(BaseSettings): class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
hf_token: str = "" @field_validator("llm_api_key", "llm_base_url", "llm_model", mode="before")
llm_api_key: str = "" @classmethod
llm_base_url: str = "https://api.openai.com/v1" def llm_required_nonempty(cls, value: str) -> str:
llm_model: str = "gpt-4o-mini" text = str(value or "").strip()
if not text:
raise ValueError("LLM_API_KEY, LLM_BASE_URL and LLM_MODEL must be set (non-empty).")
return text
@field_validator("http_host", mode="before")
@classmethod
def normalize_listen_host(cls, value: str) -> str:
host = str(value or "0.0.0.0").strip().lower()
if host in {"127.0.0.1", "localhost", "::1"}:
return "0.0.0.0"
return host or "0.0.0.0"
@field_validator("ssl_certfile", "ssl_keyfile", mode="after")
@classmethod
def resolve_ssl_path(cls, value: str) -> str:
text = (value or "").strip()
if not text:
return ""
p = Path(text)
if not p.is_absolute():
p = PROJECT_ROOT / p
return str(p.resolve())
llm_api_key: str = Field(..., validation_alias="LLM_API_KEY")
llm_base_url: str = Field(..., validation_alias="LLM_BASE_URL")
llm_model: str = Field(..., validation_alias="LLM_MODEL")
llm_timeout: int = 8 llm_timeout: int = 8
llm_max_tokens: int = 150 llm_max_tokens: int = 150
llm_temperature: float = 0.55
llm_system_prompt: str = (
"你是一个自然、友好的中文语音助手。"
"请像真人当面聊天一样回答，口语化、简洁、自然，通常 1 到 2 句。"
"不要使用书面公告腔，不要分点，不要长段解释，不要夸张语气词。"
"不要虚构身份关系，不要自称家人、恋人或其他私人身份。"
"避免客服套话，比如‘随时为你服务’、‘很高兴为你服务’、‘请问有什么可以帮您’。"
"优先使用短句和自然停顿，适合直接语音播报。"
)
webrtc_host: str = "0.0.0.0" tts_voice: str = "zf_xiaoxiao"
webrtc_port: int = 8080 tts_speed: float = 0.9
tts_sentence_max_len: int = 64
tts_segment_pause_ms: int = 90
tts_fade_ms: int = 12
http_host: str = Field(default="0.0.0.0", validation_alias="HTTP_HOST")
http_port: int = Field(default=8080, validation_alias="HTTP_PORT")
avatar_fps: int = 25 avatar_fps: int = 25
stun_url: str = "stun:stun.l.google.com:19302" avatar_driver_mode: str = "blendshape_stream"
avatar_control_protocol: str = "ws"
avatar_blendshape_schema: str = "arkit"
cors_origins: str = "*" cors_origins: str = "*"
ssl_certfile: str = "" ssl_certfile: str = ""
ssl_keyfile: str = "" ssl_keyfile: str = ""
musetalk_enabled: bool = True
musetalk_repo_dir: str = "/home/xsl/work/MuseTalk"
musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py"
musetalk_source_video: str = "/home/xsl/work/MuseTalk/data/video/yongen.mp4"
musetalk_unet_model: str = "/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth"
musetalk_unet_config: str = "/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json"
musetalk_whisper_dir: str = "/home/xsl/work/MuseTalk/models/whisper"
musetalk_result_dir: str = "/home/xsl/work/MuseTalk/results/visual-chat"
settings = Settings() settings = Settings()
core/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
core/__pycache__/pipeline.cpython-312.pyc
BIN
View File
Binary file not shown.
core/__pycache__/state_machine.cpython-312.pyc
BIN
View File
Binary file not shown.
core/pipeline.py
+66 -25
View File
@@ -5,12 +5,16 @@ import time
from typing import Awaitable, Callable, Optional from typing import Awaitable, Callable, Optional
import numpy as np import numpy as np
from config import settings
from core.state_machine import Arbitrator from core.state_machine import Arbitrator
from services.asr import ASRService from services.asr import ASRService
from services.llm import LLMService from services.llm import LLMService
from services.tts import TTSService from services.tts import TTSService
AudioChunkCallback = Callable[[np.ndarray, int, str, int, int], Awaitable[None]]
class ChatPipeline: class ChatPipeline:
def __init__( def __init__(
self, self,
@@ -33,12 +37,10 @@ class ChatPipeline:
await self.arbitrator.on_avatar_done() await self.arbitrator.on_avatar_done()
@staticmethod @staticmethod
def _split_sentences(text: str, max_len: int = 24) -> list[str]: def _split_sentences(text: str, max_len: Optional[int] = None) -> list[str]:
# Two-stage split: max_len = max(16, int(max_len or settings.tts_sentence_max_len))
# 1) hard sentence boundaries
# 2) soft commas / max length chunks for lower first-chunk latency
hard_seps = "。！？!?；;\n" hard_seps = "。！？!?；;\n"
soft_seps = "，、, " soft_seps = "，、,：:"
stage1 = [] stage1 = []
buf = [] buf = []
@@ -57,25 +59,47 @@ class ChatPipeline:
out = [] out = []
for seg in stage1: for seg in stage1:
cur = [] seg = seg.strip()
for ch in seg: if len(seg) <= max_len:
cur.append(ch) out.append(seg)
cur_len = len(cur) continue
if ch in soft_seps and cur_len >= 8:
out.append("".join(cur).strip()) start = 0
cur = [] while start < len(seg):
elif cur_len >= max_len: end = min(len(seg), start + max_len)
out.append("".join(cur).strip()) if end >= len(seg):
cur = [] out.append(seg[start:].strip())
if cur: break
out.append("".join(cur).strip())
cut = end
probe = seg[start:end]
soft_cut = max(18, max_len // 2)
for idx in range(len(probe) - 1, soft_cut - 1, -1):
if probe[idx] in soft_seps:
cut = start + idx + 1
break
out.append(seg[start:cut].strip())
start = cut
return [s for s in out if s] return [s for s in out if s]
@staticmethod
def _append_segment_pause(audio: np.ndarray, sr: int, is_last: bool) -> np.ndarray:
if is_last or audio.size == 0:
return audio
pause_ms = max(0, int(settings.tts_segment_pause_ms))
if pause_ms <= 0:
return audio
pad = np.zeros(int(sr * pause_ms / 1000.0), dtype=np.float32)
if pad.size == 0:
return audio
return np.concatenate([audio, pad])
async def _synthesize_sentence_first( async def _synthesize_sentence_first(
self, self,
reply_text: str, reply_text: str,
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, on_audio_chunk: Optional[AudioChunkCallback] = None,
on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
should_interrupt: Optional[Callable[[], bool]] = None, should_interrupt: Optional[Callable[[], bool]] = None,
) -> tuple[np.ndarray, int, int, int]: ) -> tuple[np.ndarray, int, int, int]:
@@ -87,14 +111,17 @@ class ChatPipeline:
for i, seg in enumerate(segments): for i, seg in enumerate(segments):
if should_interrupt and should_interrupt(): if should_interrupt and should_interrupt():
break break
if on_text_segment is not None:
await on_text_segment(seg, i, len(segments))
seg_audio, seg_sr = await self.tts.synthesize(seg) seg_audio, seg_sr = await self.tts.synthesize(seg)
if should_interrupt and should_interrupt():
break
sr = seg_sr sr = seg_sr
seg_audio = self._append_segment_pause(seg_audio, seg_sr, is_last=(i == len(segments) - 1))
if i == 0: if i == 0:
first_chunk_ms = int((time.perf_counter() - t0) * 1000) first_chunk_ms = int((time.perf_counter() - t0) * 1000)
if on_text_segment is not None:
await on_text_segment(seg, i, len(segments))
if on_audio_chunk is not None: if on_audio_chunk is not None:
await on_audio_chunk(seg_audio, seg_sr) await on_audio_chunk(seg_audio, seg_sr, seg, i, len(segments))
audio_chunks.append(seg_audio) audio_chunks.append(seg_audio)
tts_total_ms = int((time.perf_counter() - t0) * 1000) tts_total_ms = int((time.perf_counter() - t0) * 1000)
audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32) audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32)
@@ -103,7 +130,7 @@ class ChatPipeline:
async def process_turn( async def process_turn(
self, self,
audio_16k: np.ndarray, audio_16k: np.ndarray,
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, on_audio_chunk: Optional[AudioChunkCallback] = None,
on_user_text: Optional[Callable[[str], Awaitable[None]]] = None, on_user_text: Optional[Callable[[str], Awaitable[None]]] = None,
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
@@ -112,6 +139,20 @@ class ChatPipeline:
asr_t0 = time.perf_counter() asr_t0 = time.perf_counter()
user_text = await self.asr.transcribe(audio_16k) user_text = await self.asr.transcribe(audio_16k)
asr_latency_ms = int((time.perf_counter() - asr_t0) * 1000) asr_latency_ms = int((time.perf_counter() - asr_t0) * 1000)
if not user_text.strip():
await self.arbitrator.on_avatar_done()
payload = {
"user_text": "",
"reply_text": "",
"audio_samples": 0,
"sample_rate": 24000,
"asr_latency_ms": asr_latency_ms,
"llm_latency_ms": 0,
"llm_source": "skipped",
"tts_latency_ms": 0,
"tts_first_chunk_ms": 0,
}
return payload, np.zeros(1, dtype=np.float32), 24000
if on_user_text is not None: if on_user_text is not None:
await on_user_text(user_text) await on_user_text(user_text)
llm_t0 = time.perf_counter() llm_t0 = time.perf_counter()
@@ -119,13 +160,13 @@ class ChatPipeline:
if on_reply_text is not None: if on_reply_text is not None:
await on_reply_text(reply_text) await on_reply_text(reply_text)
llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000) llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000)
await self.arbitrator.on_avatar_start()
audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first( audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first(
reply_text, reply_text,
on_audio_chunk=on_audio_chunk, on_audio_chunk=on_audio_chunk,
on_text_segment=on_reply_segment, on_text_segment=on_reply_segment,
should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(), should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(),
) )
await self.arbitrator.on_avatar_start()
playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr))) playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr)))
# Do not block request completion on playback duration. # Do not block request completion on playback duration.
asyncio.create_task(self._finish_avatar_after_playback(playback_s)) asyncio.create_task(self._finish_avatar_after_playback(playback_s))
@@ -145,7 +186,7 @@ class ChatPipeline:
async def process_text_turn( async def process_text_turn(
self, self,
user_text: str, user_text: str,
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, on_audio_chunk: Optional[AudioChunkCallback] = None,
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
) -> tuple[dict, np.ndarray, int]: ) -> tuple[dict, np.ndarray, int]:
@@ -154,13 +195,13 @@ class ChatPipeline:
if on_reply_text is not None: if on_reply_text is not None:
await on_reply_text(reply_text) await on_reply_text(reply_text)
llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000) llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000)
await self.arbitrator.on_avatar_start()
audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first( audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first(
reply_text, reply_text,
on_audio_chunk=on_audio_chunk, on_audio_chunk=on_audio_chunk,
on_text_segment=on_reply_segment, on_text_segment=on_reply_segment,
should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(), should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(),
) )
await self.arbitrator.on_avatar_start()
playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr))) playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr)))
asyncio.create_task(self._finish_avatar_after_playback(playback_s)) asyncio.create_task(self._finish_avatar_after_playback(playback_s))
payload = { payload = {
core/state_machine.py
+1
View File
@@ -32,4 +32,5 @@ class Arbitrator:
async def on_avatar_done(self) -> None: async def on_avatar_done(self) -> None:
async with self._lock: async with self._lock:
if self.state == SessionState.AVATAR_SPEAKING:
self.state = SessionState.IDLE self.state = SessionState.IDLE
docs/deployment.md
+146
View File
@@ -0,0 +1,146 @@
# 部署说明
本文说明如何把「3D 数字人语音聊天」服务端部署到一台新机器（Linux / WSL / macOS 等类 Unix 环境）。**唯一需要能访问公网的依赖是 LLM API**；语音相关权重应放在项目内 `models/` 目录，不在运行时从公网拉取。
## 1. 环境与依赖
- **Python**：建议 3.12（与当前 `requirements.txt` 一致）。
- **系统工具**：`openssl`（生成自签证书）；可选 `ffmpeg`（若你后续扩展音视频流程）。
- **硬件**：CPU 可运行；有 NVIDIA GPU 时 ASR 等会优先用 CUDA（无卡则回退 CPU，可能较慢）。
- **磁盘**：`models/` 中 ASR + TTS 权重合计约 **1.2GB+**，请预留空间。
## 2. 获取代码与虚拟环境
```bash
git clone <你的仓库地址> product
cd product
python3 -m venv .venv
source .venv/bin/activate # Windows: .venv\Scripts\activate
pip install -U pip
pip install -r requirements.txt
```
说明：依赖里已包含 `huggingface_hub`（由其它包装带动）。若单独只为下载权重，也可 `pip install huggingface_hub`。
## 3. 准备本地模型（`models/`）
首次在一台**能访问 Hugging Face**的机器上执行（仅需一次，或把整份 `models/` 目录打包拷贝到离线机）：
```bash
source .venv/bin/activate
python scripts/vendor_hf_models.py
```
该脚本会：
- 从已安装的 `silero-vad` 包复制 **`models/vad/silero_vad.jit`**
- 拉取 **ASR**：`models/asr/SenseVoiceSmall/`
- 拉取 **TTS（Kokoro）**：`models/tts/Kokoro-82M/`（含 `voices/` 下各 `.pt` 音色）
若目标机无外网，可在有网机器上跑完上述命令后，**整体复制 `models/`** 到部署机同一路径（相对于项目根）。
仅补全 VAD、不拉 HF（需已 `pip install silero-vad`）：
```bash
python scripts/vendor_hf_models.py --skip-hf
```
## 4. 配置环境变量（`.env`）
```bash
cp .env.example .env
```
**必须非空**（见 `config.py` 校验）：
| 变量 | 含义 |
|------|------|
| `LLM_API_KEY` | OpenAI 兼容接口的 API Key |
| `LLM_BASE_URL` | 例如 `https://api.deepseek.com/v1` |
| `LLM_MODEL` | 例如 `deepseek-chat` |
**HTTP 服务**（不要使用已废弃的 `WEBRTC_*`）：
| 变量 | 含义 |
|------|------|
| `HTTP_HOST` | 监听地址，一般 `0.0.0.0` |
| `HTTP_PORT` | 端口，例如 `8080` 或 `8018` |
**HTTPS（推荐用于跨设备访问页面与麦克风）**：
- 证书路径建议写**相对项目根**，便于迁移，例如：
- `SSL_CERTFILE=certs/dev-cert.pem`
- `SSL_KEYFILE=certs/dev-key.pem`
- 程序会把相对路径解析为项目根下的绝对路径。
**TTS 音色**：`TTS_VOICE` 必须与 `models/tts/Kokoro-82M/voices/<名称>.pt` 一致（默认示例为 `zf_xiaoxiao`）。
完整字段说明见仓库根目录 [`.env.example`](../.env.example)。
## 5. 生成自签证书（可选但常需要）
在**项目根目录**执行：
```bash
bash scripts/gen-self-signed-cert.sh
```
生成 `certs/dev-cert.pem` 与 `certs/dev-key.pem`，并在 `.env` 中配置上一节的 `SSL_CERTFILE`、`SSL_KEYFILE`。
默认证书主题为 `CN=localhost`。用局域网 IP 访问时浏览器可能提示证书与地址不符，属自签名常见情况，可在浏览器中选择继续访问；生产环境请改用正规 CA 或内网 PKI 签发的证书。
## 6. 启动服务
**开发 / 前台运行：**
```bash
source .venv/bin/activate
python main.py
```
或使用脚本（会读 `.env` 中的 `HTTP_HOST`、`HTTP_PORT`）：
```bash
bash scripts/start-public.sh
```
**HTTPS**：当 `.env` 中 `SSL_CERTFILE`、`SSL_KEYFILE` 均有效且文件存在时，`main.py` 会通过 uvicorn 加载证书（与 `config` 中解析后的路径一致）。
启动后在本机浏览器访问：
- `http(s)://127.0.0.1:<HTTP_PORT>/`
- `http(s)://<服务器局域网IP>:<HTTP_PORT>/`（需防火墙放行该端口）
## 7. 防火墙与端口
确保部署机对客户端开放 **`HTTP_PORT`（TCP）**。若前面有云厂商安全组，需同步放行。
## 8. 目录与可移植性约定
- **项目根**：所有相对路径（SSL、`models/` 布局、`web/` 静态资源）均以**含有 `main.py` 与 `config.py` 的目录**为基准；启动时工作目录应为此目录。
- **可选**：环境变量 `VISUAL_CHAT_PYTHON` 可指向指定 Python 解释器（部分 `scripts/*.sh` 会优先使用）。
## 9. 验证
```bash
curl -s "http://127.0.0.1:${HTTP_PORT:-8080}/health" | head
```
或用仓库内脚本（需根据实际 URL 调整）：
```bash
python scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 3
```
（若仅 HTTP，把 `base-url` 改成 `http://...`。）
## 10. 常见问题
- **`.env` 里 `LLM_*` 为空**：应用在加载 `config` 时会校验失败，请务必填写有效值。
- **ASR/TTS 不工作**：检查 `models/asr/SenseVoiceSmall/` 是否含 `configuration.json`；`models/tts/Kokoro-82M/` 是否含 `config.json`、权重 `.pth` 与 `voices/<TTS_VOICE>.pt`。
- **麦克风在别的设备上不可用**：多数浏览器要求 **HTTPS** 与用户手势；请启用 HTTPS 并在页面内点击「连接语音通道」等操作。
- **旧变量名**：`WEBRTC_HOST`、`WEBRTC_PORT`、`STUN_URL` 已废弃，请只使用 `HTTP_HOST`、`HTTP_PORT`。
更细的语音/文本流水线时序见 [development-guide.md](development-guide.md)。
docs/development-guide.md
+232
View File
@@ -0,0 +1,232 @@
# 开发参考
## 1. 系统边界
这个项目当前采用单进程后端 + 浏览器本地渲染 3D 数字人的模式。
明确边界如下：
- 服务端不再输出视频流，只输出音频流和动画控制帧。
- 浏览器负责 Three.js/VRM 渲染、模型加载、动画缓冲和最终口型展示。
- 浏览器和服务端之间的语音收发都通过 `WS /ws/audio` 完成。
- 字幕、状态和动画控制通过 HTTP/SSE/WebSocket 传输。
这意味着后续如果你要替换数字人渲染方案，优先改的是浏览器侧；如果你要替换识别、大模型或语音合成，优先改的是 `services/` 和 `core/`。
## 2. 运行时主链路
### 2.1 语音链路
1. 浏览器通过 `getUserMedia` 获取麦克风。
2. 浏览器连接 `WS /ws/audio`，上传 `pcm_s16le` 音频块，并接收服务端回传的回复语音片段。
3. 服务端在 [main.py](../main.py) 中接收二进制 PCM 并进入 `_process_user_audio_chunk()`。
4. 音频被重采样为 16k 单声道后送入 VAD。
5. VAD 检测到用户语音结束后，触发 `ChatPipeline.process_turn()`。
6. `ASRService` 输出用户文本。
7. `LLMService` 调用线上 DeepSeek 兼容接口生成回答。
8. `TTSService` 按句子分段合成。
9. 每段 TTS 音频一方面通过 `WS /ws/audio` 下发给浏览器播放；另一方面经 `AvatarService` 转成动画帧，通过 `/ws/animation` 发给浏览器。
10. 浏览器在播放语音的同时，从动画缓冲队列中取帧，驱动 VRM 表情、GLTF morph target 和头部骨骼。
### 2.2 文本链路
1. 浏览器调用 `POST /chat/text`。
2. 后端直接执行 `LLM -> TTS -> 动画控制`。
3. 字幕通过 `/ws/subtitles` 下发，语音通过 `WS /ws/audio` 下发。
文本链路主要用于调试 LLM/TTS/动画，不依赖麦克风、VAD 和 ASR。
## 3. 后端模块职责
### 3.1 入口层
[main.py](../main.py) 负责：
- 应用初始化。
- 管理全局单例服务。
- 维护音频、字幕、动画客户端集合。
- 维护 `RuntimeState`，给 `/health` 和 `/events` 提供实时指标。
当前实现假设一次只保留一个主音频 WebSocket 连接。新连接到来时会主动关闭旧连接。这是刻意简化，不是 bug。
现在这个约束已经扩展到三个浏览器通道：
- `WS /ws/audio`
- `WS /ws/subtitles`
- `WS /ws/animation`
也就是说，服务端当前只允许一个浏览器页面完整接管数字人会话。新页面连上后，旧页面会被主动断开，关闭码为 `1012 replaced`。
### 3.2 会话状态机
[core/state_machine.py](../core/state_machine.py) 的状态很轻：
- `idle`
- `user_speaking`
- `thinking`
- `avatar_speaking`
它目前主要服务于两个目标：
- 打断时取消数字人播报。
- 给前端和监控面板提供可观测状态。
如果后面要做更复杂的并发控制，比如排队、多会话或多路媒体编排，这里需要升级成更明确的 session controller，而不只是一个枚举状态机。
### 3.3 流水线层
[core/pipeline.py](../core/pipeline.py) 是主编排器。
当前设计特点：
- `process_turn()` 处理语音输入。
- `process_text_turn()` 处理文本输入。
- TTS 会按句或短片段切分，以降低首包时延。
- 每段合成结果都会回调到上层，用于同步推送音频和动画。
后续如果要接入流式 LLM 或更细粒度的流式 TTS，这个文件是首选改造点。
### 3.4 服务层
`services/` 下的实现都偏适配器风格：
- [services/asr.py](../services/asr.py): 本地 ASR 模型封装。
- [services/llm.py](../services/llm.py): DeepSeek/OpenAI 兼容接口封装，并维护短历史上下文。
- [services/tts.py](../services/tts.py): 本地 TTS 模型封装。
- [services/vad.py](../services/vad.py): 语音起止检测。
- [services/avatar.py](../services/avatar.py): 从音频提取能量和频谱亮度，再映射到嘴型和头部姿态。
这里最大的工程价值是“可替换性”。后续更换供应商或模型时，尽量保持 `core/pipeline.py` 的接口不变，只替换 `services/` 内部实现。
## 4. 浏览器侧实现
[web/app.js](../web/app.js) 同时承担了以下职责：
- 建立音频 WebSocket。
- 订阅 `/events`、`/ws/subtitles`、`/ws/animation`。
- 维护聊天消息面板和诊断面板。
- 用 Three.js + VRM 加载和渲染模型。
- 把动画控制帧缓存约 `120ms` 后再播放，尽量和音频保持一致。
- 处理刷新后的语音恢复：如果浏览器拦截自动恢复，会等待下一次用户手势再恢复麦克风和播放上下文。
前端当前兼容三种形态：
- VRM expression manager。
- 通用 GLTF morph target。
- 没有表情 rig 时回退到调试头像。
当前前端待机动画状态：
- 已启用随机眨眼。
- 更激进的身体 idle motion 已回退，因为部分 VRM 模型会出现 T pose 或姿态异常。
如果以后想接入真实业务数字人模型，先确认模型是否提供：
- 标准 VRM 表情。
- 或稳定可映射的 morph target 名称。
- 或至少可控制的头骨/颈骨。
## 5. 动画协议
服务端通过 `/ws/animation` 发送 JSON，关键字段包括：
- `type`: `animation_ready` / `animation_state` / `animation_reset` / `animation_chunk`
- `driver`: 当前驱动器名称。
- `mode`: 驱动模式。
- `schema`: 控制 schema，当前默认是 ARKit 风格 blendshape。
- `fps`: 帧率。
- `frame_count`: 当前消息内帧数。
- `frames`: 每帧控制数据。
`frames[*]` 典型结构：
```json
{
"seq": 101,
"time_ms": 80,
"speaking": true,
"controls": {
"jawOpen": 0.42,
"mouthClose": 0.66,
"mouthFunnel": 0.21,
"mouthPucker": 0.18,
"viseme_aa": 0.42,
"viseme_ee": 0.27,
"viseme_oh": 0.31,
"headYaw": 0.01,
"headPitch": -0.02,
"headRoll": 0.01
}
}
```
注意：当前口型驱动是“基于音频特征估算”，不是基于音素或强制对齐的精确 viseme，所以它更适合演示验证，不适合直接当作高精度唇形同步方案。
## 6. 字幕协议
`/ws/subtitles` 发送的消息结构比较简单：
- `role`: `user` 或 `ai`
- `text`: 文本内容
- `source`: `voice` 或 `text`
- `partial`: 是否是分段中间结果
- `final`: 是否已经结束
- `ts_ms`: 时间戳
前端对 AI 文本采用“局部追加”策略，所以如果以后改成 token streaming，要继续保证消息顺序和终止标记一致。
## 7. 状态与观测
后端通过 `/health` 和 `/events` 暴露运行状态。推荐排查问题时按这个顺序看：
1. `state`: 当前状态机是否符合预期。
2. `audio_clients`: 语音通道是否真正建连。
3. `vad_start_count` / `vad_end_count`: 是否检测到语音边界。
4. `last_asr_latency_ms` / `last_llm_latency_ms` / `last_tts_latency_ms`: 性能瓶颈在哪一段。
5. `llm.ready` / `asr.ready` / `tts.ready` / `vad.ready`: 组件有没有降级。
6. `last_animation_frame_count`: 是否真正产生了控制帧。
## 8. 当前约束
这些约束需要后续开发明确知晓：
- 当前是进程内单例服务，不是多租户、多 session 设计。
- 浏览器侧只允许一个页面完整接管服务端语音输入输出与字幕/动画订阅。
- 模型推理基本都直接跑在应用事件循环附近，适合研发验证，不适合高并发。
- 当前动画驱动依赖音频能量和谱质心，不具备严格唇音级别精度。
- 语音链路高度依赖 VAD；如果要做电话式长连接交互，建议补更稳的 turn management。
- 浏览器刷新后不保证零手势恢复语音；这是浏览器媒体策略约束，不是后端故障。
## 9. 推荐扩展方向
按收益排序，后续建议优先做这些：
1. 把 `RuntimeState`、音频缓冲和历史上下文下沉到真正的 session 对象，摆脱全局单例。
2. 把 ASR、TTS、动画生成从主事件循环中隔离出去，避免阻塞 FastAPI/WebSocket。
3. 为 `/ws/subtitles` 和 `/ws/animation` 增加明确版本号和协议文档，避免前后端演进时互相踩踏。
4. 如果追求更真实口型，改成音素/viseme 驱动而不是纯音频特征驱动。
5. 为关键链路补自动化回归，至少覆盖文本链路、语音链路和打断流程。
## 10. 常用调试入口
- `GET /health`: 看整体运行状态。
- `GET /events`: 看持续状态流。
- `POST /chat/text`: 快速验证 LLM/TTS/动画，不依赖麦克风。
- `WS /ws/audio`: 看语音上传和语音回放主通道。
- `GET /avatar/schema`: 看控制字段列表。
- [scripts/run_all_checks.sh](../scripts/run_all_checks.sh): 统一检查入口。
脚本使用建议：
- [scripts/model_probe.py](../scripts/model_probe.py): 快速确认 ASR/TTS/VAD 是否能加载。
- [scripts/qa_check.py](../scripts/qa_check.py): 直接压文本链路，适合看 LLM/TTS 延迟。
- [scripts/smoke_test.py](../scripts/smoke_test.py): 输出 `wav + mp4`，其中 `mp4` 是基于动画控制帧绘制的调试视频，不是最终 3D 渲染结果。
如果你刚接手这个项目，建议先按下面顺序理解：
1. [README.md](../README.md)
2. [main.py](../main.py)
3. [core/pipeline.py](../core/pipeline.py)
4. [services/avatar.py](../services/avatar.py)
5. [web/app.js](../web/app.js)
environments.md
+15 -15
View File
@@ -13,13 +13,13 @@ conda env list
| 环境名 | 路径 | 用途 | 磁盘占用 | | 环境名 | 路径 | 用途 | 磁盘占用 |
|--------|------|------|---------| |--------|------|------|---------|
| `LivePortrait` | `/home/xsl/miniconda3/envs/LivePortrait` | 头部动作驱动 | 8.6GB | | `LivePortrait` | `~/miniconda3/envs/LivePortrait` | 头部动作驱动 | 8.6GB |
| `MuseTalk` | `/home/xsl/miniconda3/envs/MuseTalk` | 唇形同步（主力） | 11GB | | `MuseTalk` | `~/miniconda3/envs/MuseTalk` | 唇形同步（主力） | 11GB |
| `latentsync` | `/home/xsl/miniconda3/envs/latentsync` | 唇形同步（备用） | 9.2GB | | `latentsync` | `~/miniconda3/envs/latentsync` | 唇形同步（备用） | 9.2GB |
| `condiff-train-hair` | `/home/xsl/miniconda3/envs/condiff-train-hair` | 其他项目（未动） | — | | `condiff-train-hair` | `~/miniconda3/envs/condiff-train-hair` | 其他项目（未动） | — |
| `onediff` | `/home/xsl/miniconda3/envs/onediff` | 其他项目（未动） | — | | `onediff` | `~/miniconda3/envs/onediff` | 其他项目（未动） | — |
| `vllm-qwen3` | `/home/xsl/miniconda3/envs/vllm-qwen3` | 其他项目（未动） | — | | `vllm-qwen3` | `~/miniconda3/envs/vllm-qwen3` | 其他项目（未动） | — |
| `py310` | `/home/xsl/miniconda3/envs/py310` | 通用 Python 3.10 | — | | `py310` | `~/miniconda3/envs/py310` | 通用 Python 3.10 | — |
--- ---
@@ -36,7 +36,7 @@ conda env list
### 已下载权重 ### 已下载权重
位置：`/home/xsl/work/LivePortrait/pretrained_weights/`（总计 **1.2GB**） 位置：`~/work/LivePortrait/pretrained_weights/`（总计 **1.2GB**）
``` ```
pretrained_weights/ pretrained_weights/
@@ -81,7 +81,7 @@ pretrained_weights/
### 已下载权重 ### 已下载权重
位置：`/home/xsl/work/MuseTalk/models/`（总计 **5.5GB**） 位置：`~/work/MuseTalk/models/`（总计 **5.5GB**）
``` ```
models/ models/
@@ -127,7 +127,7 @@ models/
### 已下载权重 ### 已下载权重
位置：`/home/xsl/work/LatentSync/checkpoints/`（总计 **5.4GB**） 位置：`~/work/LatentSync/checkpoints/`（总计 **5.4GB**）
``` ```
checkpoints/ checkpoints/
@@ -180,10 +180,10 @@ pip install --pre torch torchvision torchaudio \
--index-url https://download.pytorch.org/whl/nightly/cu128 --index-url https://download.pytorch.org/whl/nightly/cu128
# LivePortrait 依赖 # LivePortrait 依赖
pip install -r /home/xsl/work/LivePortrait/requirements.txt pip install -r ~/work/LivePortrait/requirements.txt
# MuseTalk 依赖（含 MMLab 完整链） # MuseTalk 依赖（含 MMLab 完整链）
pip install -r /home/xsl/work/MuseTalk/requirements.txt pip install -r ~/work/MuseTalk/requirements.txt
pip install --no-build-isolation chumpy pip install --no-build-isolation chumpy
pip install mmengine pip install mmengine
MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation
@@ -193,6 +193,6 @@ pip install "transformers>=4.45.0"
``` ```
**可复用的模型权重**（无需重新下载）： **可复用的模型权重**（无需重新下载）：
- LivePortrait：`/home/xsl/work/LivePortrait/pretrained_weights/` — 直接用 - LivePortrait：`~/work/LivePortrait/pretrained_weights/` — 直接用
- MuseTalk：`/home/xsl/work/MuseTalk/models/` — 直接用 - MuseTalk：`~/work/MuseTalk/models/` — 直接用
- LatentSync（如需）：`/home/xsl/work/LatentSync/checkpoints/` — 直接用 - LatentSync（如需）：`~/work/LatentSync/checkpoints/` — 直接用
main.py
+276 -162
View File
@@ -1,34 +1,31 @@
from __future__ import annotations from __future__ import annotations
import os
import logging
import asyncio import asyncio
import io
import json import json
import logging
import time import time
import wave
from dataclasses import dataclass, field from dataclasses import dataclass, field
import numpy as np import numpy as np
from aiortc import RTCPeerConnection, RTCSessionDescription
from dotenv import load_dotenv from dotenv import load_dotenv
from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel from pydantic import BaseModel
from config import settings from config import settings
from core.pipeline import ChatPipeline from core.pipeline import ChatPipeline
from core.state_machine import Arbitrator from core.state_machine import Arbitrator, SessionState
from services.asr import ASRService from services.asr import ASRService
from services.avatar import AvatarService from services.avatar import AvatarService
from services.llm import LLMService from services.llm import LLMService
from services.tts import TTSService from services.tts import TTSService
from services.vad import VADService from services.vad import VADService
from webrtc.tracks import AudioBus, AvatarAudioTrack, AvatarVideoTrack
load_dotenv() load_dotenv()
if settings.hf_token:
os.environ["HF_TOKEN"] = settings.hf_token
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
app = FastAPI(title="Visual Voice Chat") app = FastAPI(title="Visual Voice Chat")
@@ -48,9 +45,12 @@ tts_service = TTSService()
vad_service = VADService() vad_service = VADService()
avatar_service = AvatarService() avatar_service = AvatarService()
pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service) pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service)
audio_bus = AudioBus(sample_rate=48000)
pcs: set[RTCPeerConnection] = set()
subtitle_clients: set[WebSocket] = set() subtitle_clients: set[WebSocket] = set()
animation_clients: set[WebSocket] = set()
audio_clients: set[WebSocket] = set()
MAX_BUFFERED_AUDIO_SAMPLES = 16000 * 45
MIN_VOICE_TURN_SAMPLES = 16000 // 2
MIN_VOICE_RMS = 0.012
@dataclass @dataclass
@@ -69,8 +69,12 @@ class RuntimeState:
last_input_mode: str = "none" last_input_mode: str = "none"
vad_start_count: int = 0 vad_start_count: int = 0
vad_end_count: int = 0 vad_end_count: int = 0
last_animation_mode: str = settings.avatar_driver_mode
last_animation_frame_count: int = 0
busy: bool = False busy: bool = False
pending_voice_turn: bool = False
buffer_16k: list[np.ndarray] = field(default_factory=list) buffer_16k: list[np.ndarray] = field(default_factory=list)
buffered_16k_samples: int = 0
runtime = RuntimeState() runtime = RuntimeState()
@@ -80,6 +84,29 @@ class TextChatRequest(BaseModel):
text: str text: str
async def _broadcast_json(clients: set[WebSocket], payload: dict) -> None:
stale: list[WebSocket] = []
message = json.dumps(payload, ensure_ascii=False)
for ws in list(clients):
try:
await ws.send_text(message)
except Exception:
stale.append(ws)
for ws in stale:
clients.discard(ws)
async def _replace_existing_clients(clients: set[WebSocket]) -> bool:
replaced_previous = len(clients) > 0
if not replaced_previous:
return False
old_clients = list(clients)
await asyncio.gather(*(client.close(code=1012, reason="replaced") for client in old_clients), return_exceptions=True)
for client in old_clients:
clients.discard(client)
return True
async def _broadcast_subtitle( async def _broadcast_subtitle(
role: str, role: str,
text: str, text: str,
@@ -90,22 +117,67 @@ async def _broadcast_subtitle(
) -> None: ) -> None:
if not text: if not text:
return return
payload = { await _broadcast_json(
subtitle_clients,
{
"role": role, "role": role,
"text": text, "text": text,
"source": source, "source": source,
"partial": partial, "partial": partial,
"final": final, "final": final,
"ts_ms": int(time.time() * 1000), "ts_ms": int(time.time() * 1000),
},
)
async def _broadcast_animation(payload: dict) -> None:
runtime.last_animation_mode = str(payload.get("driver", settings.avatar_driver_mode))
runtime.last_animation_frame_count = int(payload.get("frame_count", 0))
await _broadcast_json(animation_clients, payload)
def _runtime_payload() -> dict:
return {
"state": str(arbitrator.state),
"audio_clients": len(audio_clients),
"subtitle_clients": len(subtitle_clients),
"animation_clients": len(animation_clients),
"pipeline_runs": runtime.pipeline_runs,
"last_latency_ms": runtime.last_latency_ms,
"last_asr_latency_ms": runtime.last_asr_latency_ms,
"last_llm_latency_ms": runtime.last_llm_latency_ms,
"last_llm_source": runtime.last_llm_source,
"last_tts_latency_ms": runtime.last_tts_latency_ms,
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
"last_barge_in_ms": runtime.last_barge_in_ms,
"barge_in_count": runtime.barge_in_count,
"last_input_mode": runtime.last_input_mode,
"vad_start_count": runtime.vad_start_count,
"vad_end_count": runtime.vad_end_count,
"last_user_text": runtime.last_user_text,
"last_reply_text": runtime.last_reply_text,
"last_animation_mode": runtime.last_animation_mode,
"last_animation_frame_count": runtime.last_animation_frame_count,
"pipeline_busy": runtime.busy,
"llm": llm_service.health,
"asr": asr_service.health,
"tts": tts_service.health,
"vad": vad_service.health,
"avatar": avatar_service.health,
} }
stale: list[WebSocket] = []
for ws in list(subtitle_clients):
try: def _clear_buffered_audio() -> None:
await ws.send_text(json.dumps(payload, ensure_ascii=False)) runtime.buffer_16k.clear()
except Exception: runtime.buffered_16k_samples = 0
stale.append(ws)
for ws in stale:
subtitle_clients.discard(ws) def _append_buffered_audio(chunk: np.ndarray) -> None:
runtime.buffer_16k.append(chunk)
runtime.buffered_16k_samples += int(chunk.shape[0])
while runtime.buffered_16k_samples > MAX_BUFFERED_AUDIO_SAMPLES and runtime.buffer_16k:
dropped = runtime.buffer_16k.pop(0)
runtime.buffered_16k_samples -= int(dropped.shape[0])
def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray:
@@ -119,7 +191,6 @@ def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray:
def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray: def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray:
# aiortc audio ndarray is typically (channels, samples)
if pcm.ndim == 2: if pcm.ndim == 2:
mono = pcm.mean(axis=0) mono = pcm.mean(axis=0)
else: else:
@@ -135,22 +206,79 @@ def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> n
return np.interp(x_new, x_old, mono).astype(np.float32) return np.interp(x_new, x_old, mono).astype(np.float32)
async def _run_pipeline_from_buffer() -> None: def _encode_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes:
if runtime.busy: pcm = np.clip(audio.astype(np.float32, copy=False) * 32767.0, -32768, 32767).astype(np.int16)
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(sample_rate)
wav_file.writeframes(pcm.tobytes())
return buffer.getvalue()
async def _broadcast_audio_chunk(audio: np.ndarray, sample_rate: int) -> None:
if not audio_clients:
return return
if not runtime.buffer_16k: stale: list[WebSocket] = []
payload = _encode_wav_bytes(audio, sample_rate)
for ws in list(audio_clients):
try:
await ws.send_bytes(payload)
except Exception:
stale.append(ws)
for ws in stale:
audio_clients.discard(ws)
async def _broadcast_audio_reset(reason: str) -> None:
await _broadcast_json(
audio_clients,
{
"type": "audio_reset",
"reason": reason,
"ts_ms": int(time.time() * 1000),
},
)
async def _enqueue_audio_and_animation(
seg_audio: np.ndarray,
seg_sr: int,
seg_text: str,
idx: int,
total: int,
) -> None:
await _broadcast_audio_chunk(seg_audio, seg_sr)
animation_payload = await avatar_service.build_controls_from_audio(
seg_audio,
seg_sr,
text=seg_text,
chunk_index=idx,
total_chunks=total,
)
await _broadcast_animation(animation_payload)
async def _run_pipeline_from_buffer() -> None:
if runtime.busy or not runtime.buffer_16k or not runtime.pending_voice_turn:
return return
runtime.busy = True runtime.busy = True
runtime.pending_voice_turn = False
start_t = time.perf_counter() start_t = time.perf_counter()
try: try:
runtime.last_input_mode = "voice"
audio_16k = np.concatenate(runtime.buffer_16k, axis=0) audio_16k = np.concatenate(runtime.buffer_16k, axis=0)
runtime.buffer_16k.clear() _clear_buffered_audio()
async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None:
audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) if audio_16k.shape[0] < MIN_VOICE_TURN_SAMPLES:
pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) return
await audio_bus.enqueue(pcm_seg)
rms = float(np.sqrt(np.mean(audio_16k.astype(np.float32) ** 2))) if audio_16k.size else 0.0
if rms < MIN_VOICE_RMS:
return
runtime.last_input_mode = "voice"
async def on_user_text_now(user_text: str) -> None: async def on_user_text_now(user_text: str) -> None:
runtime.last_user_text = user_text runtime.last_user_text = user_text
@@ -164,12 +292,13 @@ async def _run_pipeline_from_buffer() -> None:
result, audio, sr = await pipeline.process_turn( result, audio, sr = await pipeline.process_turn(
audio_16k, audio_16k,
on_audio_chunk=on_audio_chunk, on_audio_chunk=_enqueue_audio_and_animation,
on_user_text=on_user_text_now, on_user_text=on_user_text_now,
on_reply_text=on_reply_text_now, on_reply_text=on_reply_text_now,
on_reply_segment=on_reply_segment_now, on_reply_segment=on_reply_segment_now,
) )
asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) if not result["user_text"]:
return
runtime.last_user_text = result["user_text"] runtime.last_user_text = result["user_text"]
runtime.last_reply_text = result["reply_text"] runtime.last_reply_text = result["reply_text"]
runtime.pipeline_runs += 1 runtime.pipeline_runs += 1
@@ -179,28 +308,19 @@ async def _run_pipeline_from_buffer() -> None:
runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_llm_source = str(result.get("llm_source", "unknown"))
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
# Fallback: if chunk callback path produced nothing, enqueue full audio.
if result.get("audio_samples", 0) <= 1: if result.get("audio_samples", 0) <= 1:
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) await _broadcast_audio_chunk(audio, sr)
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"]))
await audio_bus.enqueue(pcm_int16)
finally: finally:
runtime.busy = False runtime.busy = False
if runtime.pending_voice_turn and runtime.buffer_16k:
asyncio.create_task(_run_pipeline_from_buffer())
async def _consume_user_audio(track) -> None: async def _process_user_audio_chunk(mono_16k: np.ndarray, vad_buffer: np.ndarray) -> np.ndarray:
vad_buffer = np.zeros(0, dtype=np.float32) _append_buffered_audio(mono_16k)
while True:
frame = await track.recv()
pcm = frame.to_ndarray()
sample_rate = getattr(frame, "sample_rate", 48000) or 48000
layout = getattr(frame, "layout", None)
channels = len(layout.channels) if layout and layout.channels else 1
mono_16k = _resample_to_16k_mono(pcm, sample_rate, channels)
runtime.buffer_16k.append(mono_16k)
vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0) vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0)
# Silero VAD requires exact 512 samples at 16k.
while vad_buffer.shape[0] >= 512: while vad_buffer.shape[0] >= 512:
chunk = vad_buffer[:512] chunk = vad_buffer[:512]
vad_buffer = vad_buffer[512:] vad_buffer = vad_buffer[512:]
@@ -211,9 +331,10 @@ async def _consume_user_audio(track) -> None:
event = None event = None
if event and "start" in event: if event and "start" in event:
barge_t0 = time.perf_counter() barge_t0 = time.perf_counter()
was_avatar_speaking = str(arbitrator.state) == "avatar_speaking" was_avatar_speaking = arbitrator.state == SessionState.AVATAR_SPEAKING
await arbitrator.on_speech_start() await arbitrator.on_speech_start()
await audio_bus.clear() await _broadcast_audio_reset("speech-start")
await _broadcast_animation(await avatar_service.build_reset_payload(reason="speech-start"))
if was_avatar_speaking: if was_avatar_speaking:
runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000) runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000)
runtime.barge_in_count += 1 runtime.barge_in_count += 1
@@ -221,72 +342,37 @@ async def _consume_user_audio(track) -> None:
if event and "end" in event: if event and "end" in event:
await arbitrator.on_speech_end() await arbitrator.on_speech_end()
runtime.vad_end_count += 1 runtime.vad_end_count += 1
runtime.pending_voice_turn = True
asyncio.create_task(_run_pipeline_from_buffer()) asyncio.create_task(_run_pipeline_from_buffer())
return vad_buffer
@app.get("/health") @app.get("/health")
async def health(): async def health():
return { return _runtime_payload()
"state": arbitrator.state,
"peers": len(pcs),
"pipeline_runs": runtime.pipeline_runs,
"last_latency_ms": runtime.last_latency_ms,
"last_asr_latency_ms": runtime.last_asr_latency_ms,
"last_llm_latency_ms": runtime.last_llm_latency_ms,
"last_llm_source": runtime.last_llm_source,
"last_tts_latency_ms": runtime.last_tts_latency_ms,
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
"last_barge_in_ms": runtime.last_barge_in_ms,
"barge_in_count": runtime.barge_in_count,
"last_input_mode": runtime.last_input_mode,
"vad_start_count": runtime.vad_start_count,
"vad_end_count": runtime.vad_end_count,
"last_user_text": runtime.last_user_text,
"last_reply_text": runtime.last_reply_text,
"pipeline_busy": runtime.busy,
"llm": llm_service.health,
"asr": asr_service.health,
"tts": tts_service.health,
"vad": vad_service.health,
"avatar": avatar_service.health,
}
@app.get("/meta") @app.get("/meta")
async def meta(): async def meta():
return { return {
"stun_url": settings.stun_url,
"https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile), "https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile),
"host": settings.webrtc_host, "host": settings.http_host,
"port": settings.webrtc_port, "port": settings.http_port,
"avatar_protocol": settings.avatar_control_protocol,
} }
@app.get("/avatar/schema")
async def avatar_schema():
return avatar_service.schema
@app.get("/events") @app.get("/events")
async def events(): async def events():
async def gen(): async def gen():
while True: while True:
payload = { yield f"data: {json.dumps(_runtime_payload(), ensure_ascii=False)}\n\n"
"state": str(arbitrator.state),
"peers": len(pcs),
"pipeline_runs": runtime.pipeline_runs,
"last_latency_ms": runtime.last_latency_ms,
"last_asr_latency_ms": runtime.last_asr_latency_ms,
"last_llm_latency_ms": runtime.last_llm_latency_ms,
"last_llm_source": runtime.last_llm_source,
"last_tts_latency_ms": runtime.last_tts_latency_ms,
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
"last_barge_in_ms": runtime.last_barge_in_ms,
"barge_in_count": runtime.barge_in_count,
"last_input_mode": runtime.last_input_mode,
"vad_start_count": runtime.vad_start_count,
"vad_end_count": runtime.vad_end_count,
"last_user_text": runtime.last_user_text,
"last_reply_text": runtime.last_reply_text,
"pipeline_busy": runtime.busy,
"avatar": avatar_service.health,
}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
await asyncio.sleep(1.0) await asyncio.sleep(1.0)
return StreamingResponse(gen(), media_type="text/event-stream") return StreamingResponse(gen(), media_type="text/event-stream")
@@ -295,6 +381,7 @@ async def events():
@app.websocket("/ws/subtitles") @app.websocket("/ws/subtitles")
async def ws_subtitles(websocket: WebSocket): async def ws_subtitles(websocket: WebSocket):
await websocket.accept() await websocket.accept()
await _replace_existing_clients(subtitle_clients)
subtitle_clients.add(websocket) subtitle_clients.add(websocket)
try: try:
while True: while True:
@@ -306,9 +393,87 @@ async def ws_subtitles(websocket: WebSocket):
subtitle_clients.discard(websocket) subtitle_clients.discard(websocket)
@app.websocket("/ws/animation")
async def ws_animation(websocket: WebSocket):
await websocket.accept()
await _replace_existing_clients(animation_clients)
animation_clients.add(websocket)
try:
await websocket.send_text(json.dumps({"type": "animation_ready", **avatar_service.schema}, ensure_ascii=False))
await websocket.send_text(json.dumps(await avatar_service.build_idle_payload(reason="client-connected"), ensure_ascii=False))
while True:
await websocket.send_text(json.dumps({"type": "ping"}, ensure_ascii=False))
await asyncio.sleep(15)
except WebSocketDisconnect:
animation_clients.discard(websocket)
except Exception:
animation_clients.discard(websocket)
@app.websocket("/ws/audio")
async def ws_audio(websocket: WebSocket):
await websocket.accept()
replaced_previous = await _replace_existing_clients(audio_clients)
audio_clients.add(websocket)
sample_rate = 48000
channels = 1
vad_buffer = np.zeros(0, dtype=np.float32)
await websocket.send_text(
json.dumps(
{
"type": "audio_ready",
"transport": "websocket_audio",
"replaced_previous": replaced_previous,
},
ensure_ascii=False,
)
)
try:
while True:
message = await websocket.receive()
message_type = message.get("type")
if message_type == "websocket.disconnect":
break
text = message.get("text")
if text is not None:
try:
payload = json.loads(text)
except json.JSONDecodeError:
payload = {}
event_type = str(payload.get("type", ""))
if event_type == "audio_config":
sample_rate = max(8000, int(payload.get("sample_rate", sample_rate) or sample_rate))
channels = max(1, int(payload.get("channels", channels) or channels))
elif event_type == "audio_reset":
_clear_buffered_audio()
runtime.pending_voice_turn = False
await _broadcast_audio_reset("client-reset")
elif event_type == "ping":
await websocket.send_text(json.dumps({"type": "pong"}, ensure_ascii=False))
continue
data = message.get("bytes")
if not data:
continue
pcm = np.frombuffer(data, dtype=np.int16)
if pcm.size == 0:
continue
mono_16k = _resample_to_16k_mono(pcm, sample_rate, channels)
vad_buffer = await _process_user_audio_chunk(mono_16k, vad_buffer)
except WebSocketDisconnect:
pass
finally:
audio_clients.discard(websocket)
@app.post("/demo/run-once") @app.post("/demo/run-once")
async def run_once(): async def run_once():
# 3 seconds of silence placeholder; replace with real microphone stream path.
fake_audio = np.zeros(16000 * 3, dtype=np.float32) fake_audio = np.zeros(16000 * 3, dtype=np.float32)
await arbitrator.on_speech_start() await arbitrator.on_speech_start()
result = await pipeline.run_once(fake_audio) result = await pipeline.run_once(fake_audio)
@@ -326,11 +491,6 @@ async def chat_text(req: TextChatRequest):
runtime.busy = True runtime.busy = True
start_t = time.perf_counter() start_t = time.perf_counter()
try: try:
async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None:
audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000)
pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16)
await audio_bus.enqueue(pcm_seg)
async def on_reply_text_now(reply_text: str) -> None: async def on_reply_text_now(reply_text: str) -> None:
runtime.last_reply_text = reply_text runtime.last_reply_text = reply_text
@@ -338,13 +498,14 @@ async def chat_text(req: TextChatRequest):
await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1)) await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1))
await _broadcast_subtitle("user", text, "text") await _broadcast_subtitle("user", text, "text")
await _broadcast_audio_reset("text-turn-start")
await _broadcast_animation(await avatar_service.build_reset_payload(reason="text-turn-start"))
result, audio, sr = await pipeline.process_text_turn( result, audio, sr = await pipeline.process_text_turn(
text, text,
on_audio_chunk=on_audio_chunk, on_audio_chunk=_enqueue_audio_and_animation,
on_reply_text=on_reply_text_now, on_reply_text=on_reply_text_now,
on_reply_segment=on_reply_segment_now, on_reply_segment=on_reply_segment_now,
) )
asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr))
runtime.last_input_mode = "text" runtime.last_input_mode = "text"
runtime.last_user_text = result["user_text"] runtime.last_user_text = result["user_text"]
runtime.last_reply_text = result["reply_text"] runtime.last_reply_text = result["reply_text"]
@@ -355,11 +516,9 @@ async def chat_text(req: TextChatRequest):
runtime.last_llm_source = str(result.get("llm_source", "unknown")) runtime.last_llm_source = str(result.get("llm_source", "unknown"))
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
# Fallback: if no chunk was enqueued for some reason, enqueue full audio.
if result.get("audio_samples", 0) <= 1: if result.get("audio_samples", 0) <= 1:
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) await _broadcast_audio_chunk(audio, sr)
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"]))
await audio_bus.enqueue(pcm_int16)
return {"ok": True, **result} return {"ok": True, **result}
finally: finally:
runtime.busy = False runtime.busy = False
@@ -368,6 +527,8 @@ async def chat_text(req: TextChatRequest):
@app.post("/chat/reset") @app.post("/chat/reset")
async def chat_reset(): async def chat_reset():
llm_service.clear_history() llm_service.clear_history()
_clear_buffered_audio()
runtime.pending_voice_turn = False
runtime.last_user_text = "" runtime.last_user_text = ""
runtime.last_reply_text = "" runtime.last_reply_text = ""
runtime.pipeline_runs = 0 runtime.pipeline_runs = 0
@@ -382,13 +543,15 @@ async def chat_reset():
runtime.last_input_mode = "none" runtime.last_input_mode = "none"
runtime.vad_start_count = 0 runtime.vad_start_count = 0
runtime.vad_end_count = 0 runtime.vad_end_count = 0
runtime.last_animation_frame_count = 0
await _broadcast_audio_reset("chat-reset")
await _broadcast_animation(await avatar_service.build_reset_payload(reason="chat-reset"))
return {"ok": True} return {"ok": True}
@app.get("/demo/frame-idle-shape") @app.get("/demo/frame-idle-shape")
async def frame_idle_shape(): async def frame_idle_shape():
frame = await avatar_service.idle_frame() return await avatar_service.build_idle_payload(reason="demo")
return {"shape": list(frame.shape)}
@app.get("/") @app.get("/")
@@ -396,62 +559,13 @@ async def root():
return FileResponse("web/index.html") return FileResponse("web/index.html")
@app.post("/webrtc/offer")
async def webrtc_offer(request: Request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
# Single-connection mode: kick existing peers before accepting new one.
replaced_previous = len(pcs) > 0
if replaced_previous:
old_peers = list(pcs)
await asyncio.gather(*(peer.close() for peer in old_peers), return_exceptions=True)
for peer in old_peers:
pcs.discard(peer)
pc = RTCPeerConnection()
pcs.add(pc)
@pc.on("connectionstatechange")
async def on_connectionstatechange():
if pc.connectionState in {"failed", "closed", "disconnected"}:
await pc.close()
pcs.discard(pc)
@pc.on("track")
def on_track(track):
if track.kind == "audio":
asyncio.create_task(_consume_user_audio(track))
pc.addTrack(AvatarVideoTrack(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus))
await pc.setRemoteDescription(offer)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return JSONResponse(
{
"sdp": pc.localDescription.sdp,
"type": pc.localDescription.type,
"replaced_previous": replaced_previous,
}
)
@app.on_event("shutdown")
async def on_shutdown():
await asyncio.gather(*(pc.close() for pc in list(pcs)), return_exceptions=True)
pcs.clear()
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn
uvicorn_kwargs = { uvicorn_kwargs = {
"app": app, "app": app,
"host": settings.webrtc_host, "host": settings.http_host,
"port": settings.webrtc_port, "port": settings.http_port,
} }
if settings.ssl_certfile and settings.ssl_keyfile: if settings.ssl_certfile and settings.ssl_keyfile:
uvicorn_kwargs["ssl_certfile"] = settings.ssl_certfile uvicorn_kwargs["ssl_certfile"] = settings.ssl_certfile
requirements.txt
-1
View File
@@ -1,6 +1,5 @@
fastapi fastapi
uvicorn[standard] uvicorn[standard]
aiortc
numpy numpy
soundfile soundfile
python-dotenv python-dotenv
research-notes.md
-517
View File
@@ -1,517 +0,0 @@
# 数字人视频生成 — 研究与验证记录
> 本文档记录了从零开始探索「单图 + 音频 → 说话视频」的完整过程，
> 包括工具选型、环境踩坑、最终跑通的完整流程，以及对产品化的建议。
>
> 日期：2026-03-25
> 硬件：RTX 5090 (Blackwell, sm_120), CUDA 12.9, WSL2
---
## 一、目标
给定一张人物图片和一段音频，生成该人物"说话"的视频：
- 嘴型与音频内容同步
- 头部有自然动作（非僵硬的照片）
- 可扩展至实时/直播场景
---
## 二、工具选型过程
### 2.1 最初方向：LivePortrait
- **项目**：https://github.com/KlingTeam/LivePortrait（快手）
- **本质**：视频驱动的人脸动画，需要一个"驱动视频"来控制人物表情和头部运动
- **误区澄清**：网上有描述称 LivePortrait 支持"音频驱动"，这是**不准确的**。它只能从驱动视频中提取音频附加到输出，无法用音频生成嘴型
- **结论**：LivePortrait 适合生成头部动作底片，不能单独完成任务
### 2.2 调研其他工具
| 工具 | 厂商 | 核心能力 | 实时性 | 结论 |
|------|------|---------|--------|------|
| LatentSync 1.6 | 字节跳动 | 扩散模型唇形同步，512x512 | ❌ 约4fps | 质量好，但太慢 |
| MuseTalk 1.5 | 腾讯 | 单步推理唇形同步 | ⚠️ 约7fps | 速度可接受，质量好 |
| Live Avatar | 阿里 | 140B 扩散模型 | ❌ 需集群 | 单卡不可行 |
| SadTalker | 开源 | 音频驱动全脸动画 | ❌ | 未测试 |
| Sonic | 腾讯 | 音频驱动 | 未知 | 项目在 `/home/xsl/work/Sonic`，未测试 |
### 2.3 最终方案
**两阶段流水线：**
```
hairstyle-result.jpg
↓
[LivePortrait] ← 驱动视频 (d3.mp4, 自然头部动作)
↓
头动底片视频 (hairstyle-result--d3.mp4, 11.8s)
↓
[MuseTalk] ← 音频 (test_speech_10s.wav)
↓
最终视频 (嘴型同步 + 头部运动)
```
---
## 三、环境配置（重要踩坑记录）
### 3.1 RTX 5090 的核心问题
RTX 5090 是 Blackwell 架构（sm_120），**PyTorch 稳定版（2.6 及以下）不支持**。
所有项目都必须使用 PyTorch nightly cu128：
```bash
pip install --pre torch torchvision torchaudio \
--index-url https://download.pytorch.org/whl/nightly/cu128
# 已验证版本：torch-2.12.0.dev20260324+cu128
```
**这是最先要做的事，否则所有 CUDA 推理都会报错：**
```
CUDA error: no kernel image is available for execution on the device
```
### 3.2 HuggingFace CLI 命令名
在某些 conda 环境中，命令名是 `hf` 而不是 `huggingface-cli`：
```bash
which hf # /home/xsl/miniconda3/envs/MuseTalk/bin/hf
hf --version # 1.7.2
```
下载需要 Token：
```bash
export HF_TOKEN=<your_token>
hf download <repo> <file> --local-dir <dir>
```
---
## 四、LivePortrait 部署
### 4.1 环境
```bash
conda create -n LivePortrait python=3.10 -y
conda activate LivePortrait
pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128
cd /home/xsl/work/LivePortrait
pip install -r requirements.txt
```
### 4.2 权重
```bash
# 约 1.1GB
huggingface-cli download KlingTeam/LivePortrait \
--local-dir pretrained_weights \
--exclude "*.git*" "README.md" "docs"
```
目录结构：
```
pretrained_weights/
├── insightface/ (~21MB)
├── liveportrait/ (~608MB)
└── liveportrait_animals/ (~500MB)
```
### 4.3 生成头动底片
```bash
conda activate LivePortrait
cd /home/xsl/work/LivePortrait
python inference.py \
-s /path/to/portrait.jpg \
-d assets/examples/driving/d3.mp4 \
--flag_crop_driving_video \
-o /home/xsl/work/head_motion_base
```
**驱动视频参考**（位于 `assets/examples/driving/`）：
| 文件 | 时长 | 特点 |
|------|------|------|
| d3.mp4 | 11.8s | 自然说话动作，与10s音频匹配 |
| d6.mp4 | 33.6s | 长片段，适合循环 |
| d10.mp4 | 15.0s | 较长 |
**技巧**：选择时长接近音频时长的驱动视频；`--flag_crop_driving_video` 会自动裁剪驱动视频。
### 4.4 已知警告（可忽略）
```
[E:onnxruntime] Failed to load library libonnxruntime_providers_cuda.so
```
onnxruntime-gpu 依赖 CUDA 11.x，与系统 CUDA 12.9 不兼容，人脸检测自动回退 CPU，不影响结果。
---
## 五、LatentSync 部署（已验证，非推荐方案）
> **结论**：质量好，但速度太慢（4fps），不适合产品化。作为备选方案保留。
### 5.1 环境
```bash
conda create -y -n latentsync python=3.10.13
conda activate latentsync
conda install -y -c conda-forge ffmpeg
pip install -r /home/xsl/work/LatentSync/requirements.txt
# RTX 5090 必须卸载稳定版再装 nightly
pip uninstall torch torchvision torchaudio -y
pip install --pre torch torchvision torchaudio \
--index-url https://download.pytorch.org/whl/nightly/cu128
```
### 5.2 权重（约 4.9GB）
```bash
cd /home/xsl/work/LatentSync
huggingface-cli download ByteDance/LatentSync-1.6 whisper/tiny.pt --local-dir checkpoints
huggingface-cli download ByteDance/LatentSync-1.6 latentsync_unet.pt --local-dir checkpoints
```
### 5.3 输入准备
LatentSync 需要**视频**（非图片）+ **WAV 音频**：
```bash
# 图片 → 视频
ffmpeg -loop 1 -i portrait.jpg \
-t <duration> \
-vf "scale=512:512:force_original_aspect_ratio=decrease,pad=512:512:(ow-iw)/2:(oh-ih)/2" \
-r 25 -c:v libx264 -pix_fmt yuv420p input_video.mp4 -y
# mp3 → wav
ffmpeg -i audio.mp3 -ar 16000 -ac 1 input_audio.wav -y
```
### 5.4 推理
```bash
conda activate latentsync
cd /home/xsl/work/LatentSync
python -m scripts.inference \
--unet_config_path "configs/unet/stage2_512.yaml" \
--inference_ckpt_path "checkpoints/latentsync_unet.pt" \
--inference_steps 20 \
--guidance_scale 1.5 \
--enable_deepcache \
--video_path "assets/input_video.mp4" \
--audio_path "assets/input_audio.wav" \
--video_out_path "video_out.mp4"
```
---
## 六、MuseTalk 部署（推荐方案）
### 6.1 项目路径
```
/home/xsl/work/MuseTalk
```
### 6.2 环境搭建（完整步骤，含所有踩坑修复）
```bash
conda create -y -n MuseTalk python=3.10
conda activate MuseTalk
conda install -y -c conda-forge ffmpeg
# 1. PyTorch nightly（RTX 5090 必须）
pip install --pre torch torchvision torchaudio \
--index-url https://download.pytorch.org/whl/nightly/cu128
# 2. 项目依赖
cd /home/xsl/work/MuseTalk
pip install -r requirements.txt
# 3. 修复 chumpy 构建问题
pip install --no-build-isolation chumpy
# 4. MMLab（版本必须严格匹配，不可随意升降）
pip install mmengine
MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation # 需要系统 nvcc
pip install "mmdet>=3.3.0" # ≥3.3.0 才兼容 mmcv 2.1.0
pip install "mmpose>=1.3.0" # ≥1.3.0 才兼容 mmcv 2.1.0
pip install xtcocotools json-tricks munkres
# 5. 更新 transformers（旧版与 huggingface-hub 1.x 冲突）
pip install "transformers>=4.45.0"
```
### 6.3 MMLab 版本兼容矩阵（关键）
```
mmcv 2.1.0 ←→ mmdet ≥3.3.0 ←→ mmpose ≥1.3.0
```
| 踩坑 | 原因 | 修复 |
|------|------|------|
| mmcv 构建失败 | nightly PyTorch 没有预编译轮子 | `MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation`（需系统 nvcc） |
| `MMCV==2.1.0 is used but incompatible` | mmdet 3.1.0 要求 mmcv < 2.1.0 | 升级到 mmdet ≥3.3.0 |
| `mmpose 1.1.0 incompatible` | 同上 | 升级到 mmpose ≥1.3.0 |
| `transformers ImportError` | huggingface-hub 1.x 与旧 transformers 冲突 | 升级 transformers ≥4.45.0 |
### 6.4 权重下载
```bash
cd /home/xsl/work/MuseTalk
export HF_TOKEN=<your_token>
# 注意：此环境中命令是 hf，不是 huggingface-cli
hf download TMElyralab/MuseTalk \
"musetalkV15/musetalk.json" "musetalkV15/unet.pth" \
--local-dir models
hf download stabilityai/sd-vae-ft-mse \
config.json diffusion_pytorch_model.bin \
--local-dir models/sd-vae
hf download yzd-v/DWPose \
dw-ll_ucoco_384.pth \
--local-dir models/dwpose
hf download openai/whisper-tiny \
config.json pytorch_model.bin preprocessor_config.json \
--local-dir models/whisper
hf download ByteDance/LatentSync \
latentsync_syncnet.pt \
--local-dir models/syncnet
# face-parse 权重（不在 HuggingFace）
pip install gdown
gdown --id 154JgKpzCPW82qINcVieuPH3fZ2e0P812 -O models/face-parse-bisent/79999_iter.pth
curl -L https://download.pytorch.org/models/resnet18-5c106cde.pth \
-o models/face-parse-bisent/resnet18-5c106cde.pth
```
权重总大小约 **5GB**：
```
models/
├── musetalkV15/
│ ├── unet.pth (3.2GB)
│ └── musetalk.json
├── sd-vae/
│ ├── diffusion_pytorch_model.bin (320MB)
│ └── config.json
├── dwpose/
│ └── dw-ll_ucoco_384.pth (389MB)
├── whisper/
│ └── pytorch_model.bin (145MB)
├── syncnet/
│ └── latentsync_syncnet.pt (1.4GB)
└── face-parse-bisent/
├── 79999_iter.pth (51MB)
└── resnet18-5c106cde.pth (45MB)
```
### 6.5 PyTorch 2.6+ 兼容性补丁（必须）
PyTorch 2.6+ 将 `torch.load` 的 `weights_only` 默认值改为 `True`，导致 MuseTalk 加载旧格式权重失败。
已在 `scripts/inference.py` 中加入 monkey-patch：
```python
# 在 inference.py 开头的 import torch 后面加入：
_orig_torch_load = torch.load
def _patched_torch_load(f, *args, **kwargs):
kwargs.setdefault('weights_only', False)
return _orig_torch_load(f, *args, **kwargs)
torch.load = _patched_torch_load
```
同样的补丁也需要加到 `scripts/realtime_inference.py`（如果使用实时模式）。
### 6.6 推理配置文件
创建 YAML 配置文件（`configs/inference/my_task.yaml`）：
```yaml
task_0:
video_path: "/path/to/head_motion_video.mp4"
audio_path: "/path/to/audio.wav"
bbox_shift: 0 # 可选：调整嘴部区域位置，负值下移，正值上移
```
### 6.7 运行推理
```bash
conda activate MuseTalk
cd /home/xsl/work/MuseTalk
python -m scripts.inference \
--inference_config configs/inference/my_task.yaml \
--result_dir results/output \
--unet_model_path models/musetalkV15/unet.pth \
--unet_config models/musetalkV15/musetalk.json \
--version v15
```
输出路径：`results/output/v15/<video_name>.mp4`
---
## 七、完整流水线（已验证示例）
### 输入
- 图片：`/home/xsl/work/LivePortrait/hairstyle-result.jpg`（1080x1920，正面人像）
- 音频：`/home/xsl/work/LivePortrait/test_speech_10s.mp3`（11.8秒，中文语音）
### Step 1：生成头动底片
```bash
conda activate LivePortrait
cd /home/xsl/work/LivePortrait
python inference.py \
-s /home/xsl/work/LivePortrait/hairstyle-result.jpg \
-d assets/examples/driving/d3.mp4 \
--flag_crop_driving_video \
-o /home/xsl/work/head_motion_base
```
输出：`/home/xsl/work/head_motion_base/hairstyle-result--d3.mp4`（11.8s, 2.1MB）
### Step 2：音频转 WAV
```bash
ffmpeg -i /home/xsl/work/LivePortrait/test_speech_10s.mp3 \
-ar 16000 -ac 1 /tmp/input_audio.wav -y
```
### Step 3：MuseTalk 嘴型同步
```bash
conda activate MuseTalk
cd /home/xsl/work/MuseTalk
cat > configs/inference/hairstyle.yaml << 'EOF'
task_0:
video_path: "/home/xsl/work/head_motion_base/hairstyle-result--d3.mp4"
audio_path: "/tmp/input_audio.wav"
EOF
python -m scripts.inference \
--inference_config configs/inference/hairstyle.yaml \
--result_dir results/hairstyle \
--unet_model_path models/musetalkV15/unet.pth \
--unet_config models/musetalkV15/musetalk.json \
--version v15
```
输出：`/home/xsl/work/MuseTalk/results/hairstyle/v15/hairstyle-result--d3_input_audio.mp4`
### 性能数据
| 阶段 | 耗时 | 帧数 | 等效速度 |
|------|------|------|---------|
| LivePortrait 头动生成 | ~30s | 295帧 | ~10fps |
| MuseTalk 嘴型合成 | ~50s | 354帧 | ~7fps |
| 总计 | ~80s | — | 生成12s视频 |
---
## 八、产品化建议
### 8.1 当前方案的局限
1. **非实时**：总体约 7fps 生成速度，12秒视频需要 80 秒处理
2. **两套环境**：LivePortrait 和 MuseTalk 分别用不同 conda 环境，pipeline 不够整洁
3. **头动单调**：驱动视频固定，长时间使用会重复
4. **图片分辨率**：MuseTalk 处理区域限定在 256x256 嘴部区域
### 8.2 提速方向
- **MuseTalk 实时模式**：`scripts/realtime_inference.py` 支持流式处理，延迟更低
- **头动循环库**：预先用 LivePortrait 生成多种头动视频（点头、摇头、思考等），运行时随机选取
- **TensorRT 量化**：对 MuseTalk UNet 做 TRT 优化，预计可提速 2-3x
### 8.3 架构建议（产品级）
```
音频输入
↓
[ASR/VAD] → 静音检测、分段
↓
[MuseTalk 实时流] ← 头动视频循环
↓
[后处理：美颜/超分]（可选）
↓
视频输出流
```
### 8.4 替代方案调研建议
在正式产品开发前，建议调研以下工具：
- **Sonic**（`/home/xsl/work/Sonic`）：已有项目，未测试，可能支持更好的实时性
- **EchoMimic**：阿里的音频驱动方案，支持半身动作
- **AniPortrait**：同时驱动头部和嘴型
---
## 九、目录结构总览
```
/home/xsl/work/
├── LivePortrait/ # 头部动作驱动
│ ├── pretrained_weights/ (~1.1GB)
│ ├── hairstyle-result.jpg # 测试图片
│ ├── test_speech_10s.mp3 # 测试音频
│ └── DEPLOYMENT.md # 详细部署笔记
│
├── LatentSync/ # 扩散模型唇形同步（备用）
│ ├── checkpoints/ (~4.9GB)
│ └── video_out.mp4 # 测试输出
│
├── MuseTalk/ # 单步推理唇形同步（推荐）
│ ├── models/ (~5GB)
│ └── results/hairstyle/v15/hairstyle-result--d3_input_audio.mp4
│
├── head_motion_base/ # LivePortrait 生成的头动底片
│ └── hairstyle-result--d3.mp4
│
└── Sonic/ # 待调研
```
### Conda 环境
```bash
conda env list
# LivePortrait /home/xsl/miniconda3/envs/LivePortrait
# latentsync /home/xsl/miniconda3/envs/latentsync
# MuseTalk /home/xsl/miniconda3/envs/MuseTalk
```
---
## 十、快速复现命令（Cheat Sheet）
```bash
# === 生成一个说话视频（完整流程）===
# 1. 生成头动底片
conda activate LivePortrait && cd /home/xsl/work/LivePortrait
python inference.py -s portrait.jpg -d assets/examples/driving/d3.mp4 \
--flag_crop_driving_video -o /tmp/head_motion
# 2. 音频转 WAV
ffmpeg -i input.mp3 -ar 16000 -ac 1 /tmp/audio.wav -y
# 3. 嘴型同步
conda activate MuseTalk && cd /home/xsl/work/MuseTalk
python -m scripts.inference \
--inference_config <(echo "task_0:\n video_path: /tmp/head_motion/*.mp4\n audio_path: /tmp/audio.wav") \
--result_dir /tmp/output \
--unet_model_path models/musetalkV15/unet.pth \
--unet_config models/musetalkV15/musetalk.json \
--version v15
```
scripts/gen-self-signed-cert.sh
+5 -3
View File
@@ -1,7 +1,9 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
cd /home/xsl/product ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
mkdir -p certs mkdir -p certs
openssl req -x509 -nodes -newkey rsa:2048 -days 365 \ openssl req -x509 -nodes -newkey rsa:2048 -days 365 \
@@ -10,6 +12,6 @@ openssl req -x509 -nodes -newkey rsa:2048 -days 365 \
-subj "/C=CN/ST=Local/L=Local/O=VisualChat/OU=Dev/CN=localhost" -subj "/C=CN/ST=Local/L=Local/O=VisualChat/OU=Dev/CN=localhost"
echo "Generated:" echo "Generated:"
echo " /home/xsl/product/certs/dev-cert.pem" echo " $ROOT/certs/dev-cert.pem"
echo " /home/xsl/product/certs/dev-key.pem" echo " $ROOT/certs/dev-key.pem"
echo "Set SSL_CERTFILE and SSL_KEYFILE in .env to enable HTTPS." echo "Set SSL_CERTFILE and SSL_KEYFILE in .env to enable HTTPS."
scripts/gpu_monitor.sh
+2 -1
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
OUT="${1:-/home/xsl/product/outputs/gpu-metrics.csv}" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
OUT="${1:-$ROOT/outputs/gpu-metrics.csv}"
SECONDS_TOTAL="${2:-3600}" SECONDS_TOTAL="${2:-3600}"
INTERVAL="${3:-5}" INTERVAL="${3:-5}"
scripts/qa_check.py
+1 -1
View File
@@ -20,7 +20,7 @@ def _json_request(url: str, method: str = "GET", payload: dict | None = None, ti
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Quick QA for text pipeline and metrics") parser = argparse.ArgumentParser(description="Quick QA for text pipeline and metrics")
parser.add_argument("--base-url", default="https://127.0.0.1:8080") parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--rounds", type=int, default=5) parser.add_argument("--rounds", type=int, default=5)
args = parser.parse_args() args = parser.parse_args()
scripts/run_all_checks.sh
+39 -3
View File
@@ -1,14 +1,50 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
cd /home/xsl/product ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python"
cd "$ROOT"
pick_python() {
if [[ -n "${VISUAL_CHAT_PYTHON:-}" && -x "${VISUAL_CHAT_PYTHON}" ]]; then
echo "${VISUAL_CHAT_PYTHON}"
return 0
fi
if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then
echo "${VIRTUAL_ENV}/bin/python"
return 0
fi
if [[ -x "$ROOT/.venv/bin/python" ]]; then
echo "$ROOT/.venv/bin/python"
return 0
fi
if command -v python3 >/dev/null 2>&1; then
command -v python3
return 0
fi
command -v python
}
PY="$(pick_python)"
if [[ -f ".env" ]]; then
set -a
# shellcheck disable=SC1091
source ./.env
set +a
fi
SCHEME="http"
if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then
SCHEME="https"
fi
BASE_URL="${BASE_URL:-${SCHEME}://127.0.0.1:${HTTP_PORT:-8080}}"
echo "[1/6] service status" echo "[1/6] service status"
bash scripts/service.sh status || true bash scripts/service.sh status || true
echo "[2/6] quick pipeline qa" echo "[2/6] quick pipeline qa"
"$PY" scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 5 "$PY" scripts/qa_check.py --base-url "$BASE_URL" --rounds 5
echo "[3/6] llm fallback" echo "[3/6] llm fallback"
"$PY" scripts/llm_fallback_check.py "$PY" scripts/llm_fallback_check.py
scripts/service-8018.sh
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_DIR="$ROOT_DIR/.run"
PID_FILE="$RUN_DIR/visual-chat-8018.pid"
LOG_FILE="$RUN_DIR/visual-chat-8018.log"
ENV_FILE="$ROOT_DIR/.env"
mkdir -p "$RUN_DIR"
if [[ -f "$ENV_FILE" ]]; then
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a
fi
normalize_host() {
local host="${1:-0.0.0.0}"
case "$host" in
127.0.0.1|localhost|::1)
echo "0.0.0.0"
;;
*)
echo "$host"
;;
esac
}
HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")"
PORT="${HTTP_PORT:-8018}"
SSL_CERTFILE="${SSL_CERTFILE:-$ROOT_DIR/certs/dev-cert.pem}"
SSL_KEYFILE="${SSL_KEYFILE:-$ROOT_DIR/certs/dev-key.pem}"
if [[ -x "$ROOT_DIR/.venv/bin/python" ]]; then
PYTHON_BIN="$ROOT_DIR/.venv/bin/python"
else
PYTHON_BIN="$(command -v python3 || true)"
fi
if [[ -z "$PYTHON_BIN" ]]; then
echo "python3 not found"
exit 1
fi
port_pids() {
ss -ltnp 2>/dev/null \
| sed -n "s/.*:${PORT} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" \
| tr '\n' ' '
}
is_running() {
if [[ ! -f "$PID_FILE" ]]; then
return 1
fi
local pid
pid="$(cat "$PID_FILE")"
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
}
start() {
if is_running; then
echo "already running: pid=$(cat "$PID_FILE")"
return 0
fi
local stale
stale="$(port_pids)"
if [[ -n "${stale// /}" ]]; then
echo "killing stale python on port ${PORT}: $stale"
# shellcheck disable=SC2086
kill $stale || true
sleep 1
fi
local args
args=(main:app --host "$HOST" --port "$PORT")
if [[ -n "$SSL_CERTFILE" && -n "$SSL_KEYFILE" ]]; then
args+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE")
fi
nohup "$PYTHON_BIN" -m uvicorn "${args[@]}" >>"$LOG_FILE" 2>&1 &
local pid=$!
echo "$pid" >"$PID_FILE"
sleep 1
if kill -0 "$pid" 2>/dev/null; then
echo "started: pid=$pid"
echo "url: https://$HOST:$PORT"
echo "log: $LOG_FILE"
else
echo "failed to start, see log: $LOG_FILE"
rm -f "$PID_FILE"
return 1
fi
}
stop() {
if ! is_running; then
echo "not running"
rm -f "$PID_FILE"
return 0
fi
local pid
pid="$(cat "$PID_FILE")"
kill "$pid" || true
for _ in {1..20}; do
if ! kill -0 "$pid" 2>/dev/null; then
break
fi
sleep 0.2
done
if kill -0 "$pid" 2>/dev/null; then
kill -9 "$pid" || true
fi
rm -f "$PID_FILE"
echo "stopped"
}
status() {
if is_running; then
echo "running: pid=$(cat "$PID_FILE")"
return 0
fi
echo "not running"
return 1
}
logs() {
if [[ -f "$LOG_FILE" ]]; then
tail -n 120 "$LOG_FILE"
else
echo "no log file yet"
fi
}
restart() {
stop || true
start
}
case "${1:-}" in
start) start ;;
stop) stop ;;
status) status ;;
restart) restart ;;
logs) logs ;;
*)
echo "usage: $0 {start|stop|status|restart|logs}"
exit 2
;;
esac
scripts/service.sh
+2 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
ROOT="/home/xsl/product" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PID_FILE="$ROOT/.run/visual-chat.pid" PID_FILE="$ROOT/.run/visual-chat.pid"
LOG_FILE="$ROOT/.run/visual-chat.log" LOG_FILE="$ROOT/.run/visual-chat.log"
START_CMD="$ROOT/scripts/start-public.sh" START_CMD="$ROOT/scripts/start-public.sh"
@@ -28,7 +28,7 @@ start() {
# If stale process still binds configured port, stop it first. # If stale process still binds configured port, stop it first.
local port local port
port="$(awk -F= '/^WEBRTC_PORT=/{print $2}' "$ROOT/.env" 2>/dev/null || true)" port="$(awk -F= '/^HTTP_PORT=/{print $2}' "$ROOT/.env" 2>/dev/null || true)"
port="${port:-8080}" port="${port:-8080}"
local pids local pids
pids="$(ss -ltnp 2>/dev/null | sed -n "s/.*:${port} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" | tr '\n' ' ')" pids="$(ss -ltnp 2>/dev/null | sed -n "s/.*:${port} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" | tr '\n' ' ')"
scripts/smoke_test.py
+64 -16
View File
@@ -35,34 +35,82 @@ async def _write_avatar_video(
avatar: AvatarService, audio: np.ndarray, sr: int, out_path: str, fps: int avatar: AvatarService, audio: np.ndarray, sr: int, out_path: str, fps: int
) -> tuple[int, float]: ) -> tuple[int, float]:
fourcc = cv2.VideoWriter_fourcc(*"mp4v") fourcc = cv2.VideoWriter_fourcc(*"mp4v")
writer = cv2.VideoWriter(out_path, fourcc, float(fps), (avatar.width, avatar.height)) width = 720
height = 720
writer = cv2.VideoWriter(out_path, fourcc, float(fps), (width, height))
if not writer.isOpened(): if not writer.isOpened():
raise RuntimeError(f"failed to open video writer: {out_path}") raise RuntimeError(f"failed to open video writer: {out_path}")
duration_s = float(audio.shape[0]) / float(sr) if sr > 0 else 0.0 payload = await avatar.build_controls_from_audio(audio, sr, text="smoke-test")
total_frames = max(1, int(math.ceil(duration_s * fps))) frames = payload.get("frames", [])
samples_per_frame = max(1, int(sr / fps)) duration_ms = int(payload.get("duration_ms", 0))
for i in range(total_frames): for frame_payload in frames:
s = i * samples_per_frame controls = frame_payload.get("controls", {})
e = min(audio.shape[0], (i + 1) * samples_per_frame) frame = _render_debug_frame(width, height, controls, speaking=bool(frame_payload.get("speaking", False)))
seg = audio[s:e]
if seg.size == 0:
level = 0.0
else:
rms = float(np.sqrt(np.mean(seg.astype(np.float32) ** 2)))
level = float(np.clip(rms * 10.0, 0.0, 1.0))
frame = await avatar.render_frame(mouth_open=level, speaking=True)
writer.write(frame) writer.write(frame)
# add 0.4s idle tail for easy visual check # add 0.4s idle tail for easy visual check
tail_frames = max(1, int(0.4 * fps)) tail_frames = max(1, int(0.4 * fps))
for _ in range(tail_frames): for _ in range(tail_frames):
frame = await avatar.render_frame(mouth_open=0.0, speaking=False) frame = _render_debug_frame(width, height, {}, speaking=False)
writer.write(frame) writer.write(frame)
writer.release() writer.release()
return total_frames + tail_frames, duration_s + 0.4 return len(frames) + tail_frames, (duration_ms / 1000.0) + 0.4
def _render_debug_frame(width: int, height: int, controls: dict, speaking: bool) -> np.ndarray:
jaw_open = float(np.clip(controls.get("jawOpen", 0.0), 0.0, 1.0))
mouth_pucker = float(np.clip(controls.get("mouthPucker", 0.0), 0.0, 1.0))
viseme_aa = float(np.clip(controls.get("viseme_aa", jaw_open), 0.0, 1.0))
viseme_ee = float(np.clip(controls.get("viseme_ee", 0.0), 0.0, 1.0))
viseme_oh = float(np.clip(controls.get("viseme_oh", 0.0), 0.0, 1.0))
head_yaw = float(np.clip(controls.get("headYaw", 0.0), -1.0, 1.0))
head_pitch = float(np.clip(controls.get("headPitch", 0.0), -1.0, 1.0))
head_roll = float(np.clip(controls.get("headRoll", 0.0), -1.0, 1.0))
frame = np.zeros((height, width, 3), dtype=np.uint8)
frame[:] = (16, 12, 10)
center_x = width // 2 + int(head_yaw * 48)
center_y = height // 2 + int(head_pitch * 42) - 24
head_radius = 190
cv2.circle(frame, (center_x, center_y), head_radius + 26, (42, 28, 18), -1, lineType=cv2.LINE_AA)
cv2.circle(frame, (center_x, center_y), head_radius, (198, 214, 238), -1, lineType=cv2.LINE_AA)
eye_y = center_y - 44
eye_offset = 64
cv2.circle(frame, (center_x - eye_offset, eye_y), 12, (26, 31, 43), -1, lineType=cv2.LINE_AA)
cv2.circle(frame, (center_x + eye_offset, eye_y), 12, (26, 31, 43), -1, lineType=cv2.LINE_AA)
brow_dx = int(head_roll * 28)
cv2.line(frame, (center_x - 100 - brow_dx, eye_y - 34), (center_x - 42 - brow_dx, eye_y - 40), (48, 58, 74), 6, lineType=cv2.LINE_AA)
cv2.line(frame, (center_x + 42 - brow_dx, eye_y - 40), (center_x + 100 - brow_dx, eye_y - 34), (48, 58, 74), 6, lineType=cv2.LINE_AA)
mouth_center = (center_x, center_y + 86)
mouth_width = int(64 + viseme_oh * 36 + mouth_pucker * 18)
mouth_height = int(12 + jaw_open * 98 + viseme_aa * 18)
lip_color = (96, 54, 130) if speaking else (88, 74, 108)
inner_color = (36, 22, 54)
cv2.ellipse(frame, mouth_center, (mouth_width, max(6, mouth_height)), 0, 0, 360, lip_color, -1, lineType=cv2.LINE_AA)
cv2.ellipse(frame, mouth_center, (max(12, mouth_width - 16), max(4, mouth_height - 8)), 0, 0, 360, inner_color, -1, lineType=cv2.LINE_AA)
ee_width = int(max(20, 86 - viseme_ee * 46))
cv2.line(frame, (mouth_center[0] - ee_width, mouth_center[1]), (mouth_center[0] + ee_width, mouth_center[1]), (220, 224, 232), 3, lineType=cv2.LINE_AA)
metrics = [
f"jawOpen={jaw_open:.2f}",
f"viseme_aa={viseme_aa:.2f}",
f"viseme_ee={viseme_ee:.2f}",
f"viseme_oh={viseme_oh:.2f}",
f"headYaw={head_yaw:.2f}",
]
for index, text in enumerate(metrics):
cv2.putText(frame, text, (28, 42 + index * 28), cv2.FONT_HERSHEY_SIMPLEX, 0.72, (210, 220, 235), 2, lineType=cv2.LINE_AA)
return frame
async def main() -> None: async def main() -> None:
scripts/start-public.sh
+52 -5
View File
@@ -1,19 +1,66 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
cd /home/xsl/product ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python" cd "$ROOT"
if [[ -f ".env" ]]; then if [[ -f ".env" ]]; then
# shellcheck disable=SC2046 set -a
export $(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' .env | xargs) # shellcheck disable=SC1091
source ./.env
set +a
fi fi
ARGS=(main:app --host "${WEBRTC_HOST:-0.0.0.0}" --port "${WEBRTC_PORT:-8080}") pick_python() {
if [[ -n "${VISUAL_CHAT_PYTHON:-}" && -x "${VISUAL_CHAT_PYTHON}" ]]; then
echo "$VISUAL_CHAT_PYTHON"
return 0
fi
if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then
echo "${VIRTUAL_ENV}/bin/python"
return 0
fi
if [[ -x "$ROOT/.venv/bin/python" ]]; then
echo "$ROOT/.venv/bin/python"
return 0
fi
if command -v python3 >/dev/null 2>&1; then
command -v python3
return 0
fi
command -v python
}
normalize_host() {
local host="${1:-0.0.0.0}"
case "$host" in
127.0.0.1|localhost|::1)
echo "0.0.0.0"
;;
*)
echo "$host"
;;
esac
}
HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")"
PORT="${HTTP_PORT:-8080}"
PY="$(pick_python)"
SCHEME="http"
ARGS=(main:app --host "$HOST" --port "$PORT")
if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then
SCHEME="https"
ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE") ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE")
fi fi
echo "repo: $ROOT"
echo "python: $PY"
echo "bind: ${HOST}:${PORT}"
echo "listen: ${SCHEME}://${HOST}:${PORT}"
echo "browser test page: ${SCHEME}://<server-ip>:${PORT}/"
echo "note: only one browser page is allowed to own the session; a newer client replaces the older one."
echo "note: after browser refresh, one click may still be required to resume mic/audio due to browser media policy."
exec "$PY" -m uvicorn "${ARGS[@]}" exec "$PY" -m uvicorn "${ARGS[@]}"
scripts/start-test.sh
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
MODE="${1:-auto}"
if [[ -f ".env" ]]; then
set -a
# shellcheck disable=SC1091
source ./.env
set +a
fi
normalize_host() {
local host="${1:-0.0.0.0}"
case "$host" in
127.0.0.1|localhost|::1)
echo "0.0.0.0"
;;
*)
echo "$host"
;;
esac
}
pick_python() {
if [[ -n "${VISUAL_CHAT_PYTHON:-}" && -x "${VISUAL_CHAT_PYTHON}" ]]; then
echo "$VISUAL_CHAT_PYTHON"
return 0
fi
if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then
echo "${VIRTUAL_ENV}/bin/python"
return 0
fi
if [[ -x "$ROOT/.venv/bin/python" ]]; then
echo "$ROOT/.venv/bin/python"
return 0
fi
if command -v python3 >/dev/null 2>&1; then
command -v python3
return 0
fi
command -v python
}
PY="$(pick_python)"
HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")"
PORT="${HTTP_PORT:-8080}"
SCHEME="http"
ARGS=(main:app --host "$HOST" --port "$PORT")
case "$MODE" in
auto)
if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then
SCHEME="https"
ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE")
fi
;;
https)
if [[ -z "${SSL_CERTFILE:-}" || -z "${SSL_KEYFILE:-}" ]]; then
echo "SSL_CERTFILE / SSL_KEYFILE 未配置，无法以 https 模式启动" >&2
exit 1
fi
SCHEME="https"
ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE")
;;
http)
;;
*)
echo "usage: bash scripts/start-test.sh [auto|http|https]" >&2
exit 2
;;
esac
echo "repo: $ROOT"
echo "python: $PY"
echo "mode: $MODE"
echo "bind: ${HOST}:${PORT}"
echo "listen: ${SCHEME}://${HOST}:${PORT}"
echo "browser test page: ${SCHEME}://<server-ip>:${PORT}/"
echo "note: only one browser page is allowed to own the session; a newer client replaces the older one."
echo "note: after browser refresh, one click may still be required to resume mic/audio due to browser media policy."
exec "$PY" -m uvicorn "${ARGS[@]}"
scripts/start.sh
+3 -1
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
cd /home/xsl/product ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
python -m uvicorn main:app --host 0.0.0.0 --port 8080 python -m uvicorn main:app --host 0.0.0.0 --port 8080
scripts/vendor_hf_models.py
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env python3
"""Populate ./models from local silero wheel + Hugging Face snapshots (ASR/TTS only; needs network for HF)."""
from __future__ import annotations
import argparse
import importlib.resources as ir
import sys
from pathlib import Path
_ROOT = Path(__file__).resolve().parent.parent
_DEFAULT_ASR = _ROOT / "models" / "asr" / "SenseVoiceSmall"
_DEFAULT_KOKORO = _ROOT / "models" / "tts" / "Kokoro-82M"
_VAD_JIT = _ROOT / "models" / "vad" / "silero_vad.jit"
def _copy_silero_vad() -> None:
_VAD_JIT.parent.mkdir(parents=True, exist_ok=True)
src = ir.files("silero_vad.data").joinpath("silero_vad.jit")
_VAD_JIT.write_bytes(src.read_bytes())
print(f"VAD -> {_VAD_JIT}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--asr-repo", default="FunAudioLLM/SenseVoiceSmall")
parser.add_argument("--kokoro-repo", default="hexgrad/Kokoro-82M")
parser.add_argument("--asr-dir", type=Path, default=_DEFAULT_ASR)
parser.add_argument("--kokoro-dir", type=Path, default=_DEFAULT_KOKORO)
parser.add_argument("--asr-only", action="store_true")
parser.add_argument("--kokoro-only", action="store_true")
parser.add_argument("--skip-hf", action="store_true", help="Only copy Silero VAD from installed wheel")
args = parser.parse_args()
try:
_copy_silero_vad()
except Exception as exc:
print(f"Silero VAD copy failed (pip install silero-vad): {exc}", file=sys.stderr)
return 1
if args.skip_hf:
print("Done (HF skipped).")
return 0
try:
from huggingface_hub import snapshot_download
except ImportError:
print("Install huggingface_hub for ASR/TTS downloads.", file=sys.stderr)
return 1
if not args.kokoro_only:
args.asr_dir.mkdir(parents=True, exist_ok=True)
print(f"ASR -> {args.asr_dir}")
snapshot_download(repo_id=args.asr_repo, local_dir=str(args.asr_dir))
if not args.asr_only:
args.kokoro_dir.mkdir(parents=True, exist_ok=True)
print(f"Kokoro -> {args.kokoro_dir}")
snapshot_download(repo_id=args.kokoro_repo, local_dir=str(args.kokoro_dir))
print("Done.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
services/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
services/__pycache__/asr.cpython-312.pyc
BIN
View File
Binary file not shown.
services/__pycache__/avatar.cpython-312.pyc
BIN
View File
Binary file not shown.
services/__pycache__/llm.cpython-312.pyc
BIN
View File
Binary file not shown.
services/__pycache__/tts.cpython-312.pyc
BIN
View File
Binary file not shown.
services/__pycache__/vad.cpython-312.pyc
BIN
View File
Binary file not shown.
services/asr.py
+39 -8
View File
@@ -1,13 +1,25 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
from pathlib import Path
from typing import Optional from typing import Optional
import numpy as np import numpy as np
from config import asr_bundle_dir
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _asr_model_path() -> Path:
p = asr_bundle_dir().resolve()
if not p.is_dir():
raise FileNotFoundError(f"ASR model directory missing: {p}")
if not (p / "configuration.json").is_file():
raise FileNotFoundError(f"ASR bundle incomplete (no configuration.json): {p}")
return p
class ASRService: class ASRService:
def __init__(self) -> None: def __init__(self) -> None:
self._model = None self._model = None
@@ -15,25 +27,46 @@ class ASRService:
self._ready = False self._ready = False
self._attempted = False self._attempted = False
self._init_error: Optional[str] = None self._init_error: Optional[str] = None
self._device: str = "uninitialized"
def _ensure_loaded(self) -> None: def _ensure_loaded(self) -> None:
if self._ready or self._attempted: if self._ready or self._attempted:
return return
self._attempted = True self._attempted = True
try: try:
model_dir = _asr_model_path()
except FileNotFoundError as exc:
self._init_error = str(exc)
self._device = "unavailable"
logger.error("%s", exc)
return
from funasr import AutoModel from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess from funasr.utils.postprocess_utils import rich_transcription_postprocess
model_ref = str(model_dir)
errors: list[str] = []
for device in ("cuda:0", "cpu"):
try:
self._model = AutoModel( self._model = AutoModel(
model="FunAudioLLM/SenseVoiceSmall", model=model_ref,
device="cuda:0", device=device,
hub="hf", hub="hf",
disable_update=True,
) )
self._post = rich_transcription_postprocess self._post = rich_transcription_postprocess
self._ready = True self._ready = True
self._init_error = None
self._device = device
if device != "cuda:0":
logger.warning("ASR running in degraded mode on %s", device)
return
except Exception as exc: # pragma: no cover except Exception as exc: # pragma: no cover
self._init_error = str(exc) errors.append(f"{device}: {exc}")
logger.warning("ASR fallback mode: %s", exc)
self._init_error = " | ".join(errors)
self._device = "unavailable"
logger.warning("ASR unavailable: %s", self._init_error)
async def transcribe(self, audio_16k: np.ndarray) -> str: async def transcribe(self, audio_16k: np.ndarray) -> str:
self._ensure_loaded() self._ensure_loaded()
@@ -45,10 +78,8 @@ class ASRService:
use_itn=True, use_itn=True,
) )
return self._post(res[0]["text"]) return self._post(res[0]["text"])
return ""
# Fallback for initial bring-up
return "（ASR降级）语音已接收"
@property @property
def health(self) -> dict: def health(self) -> dict:
return {"ready": self._ready, "attempted": self._attempted, "error": self._init_error} return {"ready": self._ready, "attempted": self._attempted, "device": self._device, "error": self._init_error}
services/avatar.py
+190 -170
View File
@@ -1,193 +1,213 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import os import math
import sys import time
import uuid from typing import Any
import wave
from collections import deque
from pathlib import Path
import cv2
import numpy as np import numpy as np
from config import settings from config import settings
class AvatarService: class AvatarService:
def __init__(self, width: int = 640, height: int = 360) -> None: def __init__(self) -> None:
self.width = width self.driver_name = "arkit-blendshape-v1"
self.height = height
self._frames: deque[np.ndarray] = deque()
self._lock = asyncio.Lock() self._lock = asyncio.Lock()
self._job_lock = asyncio.Lock() self._sequence = 0
self._last_error: str | None = None self._last_error: str | None = None
self._base_frame = self._load_base_frame() self._last_frame_count = 0
self._last_chunk_duration_ms = 0
async def render_frame(self, mouth_open: float = 0.0, speaking: bool = False) -> np.ndarray: async def build_idle_payload(self, reason: str = "idle") -> dict[str, Any]:
frame = self._base_frame.copy() frame = self._build_frame(seq=await self._next_sequence(), time_ms=0, controls=self._neutral_controls())
if speaking: self._last_frame_count = 1
alpha = float(np.clip(mouth_open, 0.0, 1.0)) self._last_chunk_duration_ms = 0
# Slightly boost contrast/brightness while speaking to keep a "live" feeling.
frame = cv2.convertScaleAbs(frame, alpha=1.0 + 0.10 * alpha, beta=2 + int(8 * alpha))
# Fallback mouth animation on realistic base frame.
h, w = frame.shape[:2]
cx = w // 2
cy = int(h * 0.70)
half_w = max(28, int(w * 0.055))
half_h = max(4, int(4 + alpha * h * 0.035))
lip_color = (26, 26, 92)
lip_border = (70, 70, 160)
cv2.ellipse(frame, (cx, cy), (half_w + 2, half_h + 2), 0, 0, 360, lip_border, -1)
cv2.ellipse(frame, (cx, cy), (half_w, half_h), 0, 0, 360, lip_color, -1)
return frame
async def idle_frame(self) -> np.ndarray:
return await self.render_frame(mouth_open=0.0, speaking=False)
async def speaking_frame(self, mouth_open: float = 0.5) -> np.ndarray:
return await self.render_frame(mouth_open=mouth_open, speaking=True)
async def pop_generated_frame(self) -> np.ndarray | None:
async with self._lock:
if self._frames:
return self._frames.popleft()
return None
async def enqueue_musetalk_from_audio(self, audio: np.ndarray, sr: int) -> None:
if not settings.musetalk_enabled:
return
if audio.size <= 1 or sr <= 0:
return
async with self._job_lock:
try:
job_id = f"job_{uuid.uuid4().hex[:10]}"
repo_dir = Path(settings.musetalk_repo_dir)
work_root = repo_dir / "results" / "visual-chat-jobs" / job_id
work_root.mkdir(parents=True, exist_ok=True)
wav_path = work_root / "input.wav"
cfg_path = work_root / "inference.yaml"
output_name = f"{job_id}.mp4"
output_path = Path(settings.musetalk_result_dir) / "v15" / output_name
self._write_wav(wav_path, audio.astype(np.float32, copy=False), sr)
cfg_path.write_text(
(
"task_0:\n"
f" video_path: \"{settings.musetalk_source_video}\"\n"
f" audio_path: \"{wav_path}\"\n"
f" result_name: \"{output_name}\"\n"
),
encoding="utf-8",
)
env = os.environ.copy()
env.setdefault("HF_HOME", str(repo_dir / "models"))
env["PYTHONPATH"] = (
f"{repo_dir}:{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(repo_dir)
)
proc = await asyncio.create_subprocess_exec(
sys.executable,
str(settings.musetalk_infer_script),
"--version",
"v15",
"--inference_config",
str(cfg_path),
"--result_dir",
str(settings.musetalk_result_dir),
"--unet_model_path",
str(settings.musetalk_unet_model),
"--unet_config",
str(settings.musetalk_unet_config),
"--whisper_dir",
str(settings.musetalk_whisper_dir),
"--batch_size",
"8",
"--use_float16",
cwd=str(repo_dir),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
self._last_error = stderr.decode("utf-8", errors="ignore")[-500:]
return
frames: list[np.ndarray] = []
if output_path.exists():
frames = self._read_video_frames(str(output_path))
if not frames:
# Some MuseTalk runs leave image frames instead of final mp4.
fallback_dir = Path(settings.musetalk_result_dir) / "v15" / "yongen"
if fallback_dir.exists():
frames = self._read_image_frames(str(fallback_dir))
if not frames:
self._last_error = f"MuseTalk produced no readable frames: {output_path}"
return
async with self._lock:
self._frames.clear()
self._frames.extend(frames)
self._last_error = None self._last_error = None
except Exception as exc: # pragma: no cover return {
self._last_error = str(exc) "type": "animation_state",
"driver": self.driver_name,
"mode": settings.avatar_driver_mode,
"schema": settings.avatar_blendshape_schema,
"fps": settings.avatar_fps,
"reason": reason,
"frame_count": 1,
"frames": [frame],
"ts_ms": int(time.time() * 1000),
}
def _read_video_frames(self, path: str) -> list[np.ndarray]: async def build_reset_payload(self, reason: str = "reset") -> dict[str, Any]:
cap = cv2.VideoCapture(path) payload = await self.build_idle_payload(reason=reason)
frames: list[np.ndarray] = [] payload["type"] = "animation_reset"
while True: return payload
ok, frame = cap.read()
if not ok or frame is None:
break
if frame.shape[1] != self.width or frame.shape[0] != self.height:
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR)
frames.append(frame)
cap.release()
return frames
def _read_image_frames(self, dir_path: str) -> list[np.ndarray]: async def build_controls_from_audio(
frames: list[np.ndarray] = [] self,
for name in sorted(os.listdir(dir_path)): audio: np.ndarray,
if not name.lower().endswith(".png"): sr: int,
continue *,
frame = cv2.imread(os.path.join(dir_path, name)) text: str = "",
if frame is None: chunk_index: int = 0,
continue total_chunks: int = 1,
if frame.shape[1] != self.width or frame.shape[0] != self.height: ) -> dict[str, Any]:
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) if sr <= 0:
frames.append(frame) self._last_error = f"invalid sample rate: {sr}"
return frames return await self.build_idle_payload(reason="invalid-sample-rate")
def _load_base_frame(self) -> np.ndarray: mono = np.asarray(audio, dtype=np.float32).reshape(-1)
src = settings.musetalk_source_video if mono.size <= 1:
if src and os.path.exists(src): return await self.build_idle_payload(reason="empty-audio")
cap = cv2.VideoCapture(src)
ok, frame = cap.read()
cap.release()
if ok and frame is not None:
if frame.shape[1] != self.width or frame.shape[0] != self.height:
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR)
return frame
# Last-resort dark background (should rarely happen).
fallback = np.zeros((self.height, self.width, 3), dtype=np.uint8)
fallback[:, :, :] = (20, 20, 20)
return fallback
@staticmethod mono = np.clip(mono, -1.0, 1.0)
def _write_wav(path: Path, audio: np.ndarray, sr: int) -> None: frame_count = max(1, int(math.ceil((mono.shape[0] / float(sr)) * settings.avatar_fps)))
pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16) seqs = await self._reserve_sequences(frame_count)
with wave.open(str(path), "wb") as wf: frames = self._audio_to_frames(mono, sr, seqs)
wf.setnchannels(1)
wf.setsampwidth(2) self._last_frame_count = len(frames)
wf.setframerate(sr) self._last_chunk_duration_ms = int(round(1000.0 * mono.shape[0] / float(sr)))
wf.writeframes(pcm.tobytes()) self._last_error = None
return {
"type": "animation_chunk",
"driver": self.driver_name,
"mode": settings.avatar_driver_mode,
"schema": settings.avatar_blendshape_schema,
"chunk_index": chunk_index,
"total_chunks": total_chunks,
"sample_rate": sr,
"fps": settings.avatar_fps,
"text": text,
"frame_count": len(frames),
"duration_ms": self._last_chunk_duration_ms,
"frames": frames,
"ts_ms": int(time.time() * 1000),
}
@property @property
def health(self) -> dict: def schema(self) -> dict[str, Any]:
return { return {
"enabled": settings.musetalk_enabled, "driver": self.driver_name,
"queued_frames": len(self._frames), "mode": settings.avatar_driver_mode,
"protocol": settings.avatar_control_protocol,
"schema": settings.avatar_blendshape_schema,
"controls": [
"jawOpen",
"mouthClose",
"mouthFunnel",
"mouthPucker",
"viseme_aa",
"viseme_ee",
"viseme_oh",
"headYaw",
"headPitch",
"headRoll",
],
}
def _audio_to_frames(self, audio: np.ndarray, sr: int, sequences: range) -> list[dict[str, Any]]:
samples_per_frame = max(1, int(round(sr / max(1, settings.avatar_fps))))
frames: list[dict[str, Any]] = []
jaw_prev = 0.0
for index, seq in enumerate(sequences):
start = index * samples_per_frame
end = min(audio.shape[0], start + samples_per_frame)
window = audio[start:end]
if window.size == 0:
window = np.zeros(1, dtype=np.float32)
energy, brightness = self._extract_features(window, sr)
jaw_open = 0.70 * jaw_prev + 0.30 * energy
jaw_prev = jaw_open
phase = index / max(1, len(sequences) - 1)
frames.append(
self._build_frame(
seq=seq,
time_ms=int(round(index * 1000.0 / settings.avatar_fps)),
controls=self._controls_from_features(jaw_open, brightness, phase),
)
)
return frames
def _extract_features(self, window: np.ndarray, sr: int) -> tuple[float, float]:
rms = float(np.sqrt(np.mean(window**2))) if window.size else 0.0
energy = float(np.clip((rms - 0.01) * 10.0, 0.0, 1.0))
if window.size < 16:
return energy, 0.5
spectrum = np.abs(np.fft.rfft(window * np.hanning(window.size)))
if spectrum.size <= 1:
return energy, 0.5
freqs = np.fft.rfftfreq(window.size, d=1.0 / float(sr))
weights = spectrum[1:]
freq_values = freqs[1:]
denom = float(np.sum(weights)) + 1e-6
centroid = float(np.sum(freq_values * weights) / denom)
brightness = float(np.clip((centroid - 500.0) / 2500.0, 0.0, 1.0))
return energy, brightness
def _controls_from_features(self, energy: float, brightness: float, phase: float) -> dict[str, float]:
jaw_open = float(np.clip(energy, 0.0, 1.0))
mouth_close = float(np.clip(1.0 - jaw_open * 0.82, 0.0, 1.0))
mouth_funnel = float(np.clip(0.16 + jaw_open * (0.40 - brightness * 0.12), 0.0, 1.0))
mouth_pucker = float(np.clip(0.06 + jaw_open * (0.20 + (1.0 - brightness) * 0.30), 0.0, 1.0))
viseme_aa = jaw_open
viseme_ee = float(np.clip(jaw_open * (0.25 + brightness * 0.95), 0.0, 1.0))
viseme_oh = float(np.clip(jaw_open * (0.30 + (1.0 - brightness) * 0.90), 0.0, 1.0))
head_yaw = float(np.clip(math.sin(phase * math.pi * 1.7) * 0.02 * max(jaw_open, 0.18), -0.25, 0.25))
head_pitch = float(np.clip(math.cos(phase * math.pi * 2.0) * 0.03 * max(jaw_open, 0.15), -0.25, 0.25))
head_roll = float(np.clip(math.sin(phase * math.pi) * 0.015, -0.15, 0.15))
return {
"jawOpen": round(jaw_open, 4),
"mouthClose": round(mouth_close, 4),
"mouthFunnel": round(mouth_funnel, 4),
"mouthPucker": round(mouth_pucker, 4),
"viseme_aa": round(viseme_aa, 4),
"viseme_ee": round(viseme_ee, 4),
"viseme_oh": round(viseme_oh, 4),
"headYaw": round(head_yaw, 4),
"headPitch": round(head_pitch, 4),
"headRoll": round(head_roll, 4),
}
def _build_frame(self, seq: int, time_ms: int, controls: dict[str, float]) -> dict[str, Any]:
return {
"seq": seq,
"time_ms": time_ms,
"speaking": controls["jawOpen"] > 0.04,
"controls": controls,
}
def _neutral_controls(self) -> dict[str, float]:
return {
"jawOpen": 0.0,
"mouthClose": 1.0,
"mouthFunnel": 0.0,
"mouthPucker": 0.0,
"viseme_aa": 0.0,
"viseme_ee": 0.0,
"viseme_oh": 0.0,
"headYaw": 0.0,
"headPitch": 0.0,
"headRoll": 0.0,
}
async def _next_sequence(self) -> int:
async with self._lock:
seq = self._sequence
self._sequence += 1
return seq
async def _reserve_sequences(self, count: int) -> range:
async with self._lock:
start = self._sequence
self._sequence += count
return range(start, start + count)
@property
def health(self) -> dict[str, Any]:
return {
"driver": self.driver_name,
"mode": settings.avatar_driver_mode,
"protocol": settings.avatar_control_protocol,
"schema": settings.avatar_blendshape_schema,
"last_frame_count": self._last_frame_count,
"last_chunk_duration_ms": self._last_chunk_duration_ms,
"last_error": self._last_error, "last_error": self._last_error,
} }
services/llm.py
+28 -13
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import re
from typing import Optional from typing import Optional
from collections import deque from collections import deque
@@ -15,48 +16,61 @@ class LLMService:
self._error: Optional[str] = None self._error: Optional[str] = None
self._client = None self._client = None
self._model = (settings.llm_model or "").strip() self._model = (settings.llm_model or "").strip()
self._base_url = (settings.llm_base_url or "").strip()
self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant) self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant)
self._system_prompt = "请用中文口语化简短回复，1-2句。" self._system_prompt = (settings.llm_system_prompt or "").strip() or "请用中文口语化简短回复，1-2句。"
if not settings.llm_api_key:
return
if not self._model:
base = (settings.llm_base_url or "").lower()
self._model = "deepseek-chat" if "deepseek.com" in base else "gpt-4o-mini"
try: try:
from openai import AsyncOpenAI from openai import AsyncOpenAI
self._client = AsyncOpenAI( self._client = AsyncOpenAI(
api_key=settings.llm_api_key, api_key=settings.llm_api_key,
base_url=settings.llm_base_url, base_url=self._base_url,
timeout=settings.llm_timeout, timeout=settings.llm_timeout,
) )
self._ready = True self._ready = True
self._error = None
except Exception as exc: # pragma: no cover except Exception as exc: # pragma: no cover
self._error = str(exc) self._error = str(exc)
logger.warning("LLM online mode unavailable: %s", exc) logger.warning("LLM client init failed: %s", exc)
@staticmethod
def _normalize_reply_text(text: str) -> str:
cleaned = (text or "").strip()
if not cleaned:
return ""
cleaned = cleaned.replace("\r", "\n")
cleaned = re.sub(r"```.*?```", "", cleaned, flags=re.S)
cleaned = re.sub(r"`([^`]*)`", r"\1", cleaned)
cleaned = re.sub(r"\*\*(.*?)\*\*", r"\1", cleaned)
cleaned = re.sub(r"\*(.*?)\*", r"\1", cleaned)
cleaned = re.sub(r"\s*([，。！？；：、,.!?;:])\s*", r"\1", cleaned)
cleaned = re.sub(r"(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])", "", cleaned)
cleaned = re.sub(r"\n+", " ", cleaned)
cleaned = re.sub(r"\s{2,}", " ", cleaned)
return cleaned.strip()
async def reply_with_meta(self, text: str) -> tuple[str, str]: async def reply_with_meta(self, text: str) -> tuple[str, str]:
if not self._ready or self._client is None: if not self._ready or self._client is None:
return f"收到：{text}。这是本地兜底回复。", "fallback" return "", "unavailable"
try: try:
resp = await self._client.chat.completions.create( resp = await self._client.chat.completions.create(
model=self._model, model=self._model,
messages=[{"role": "system", "content": self._system_prompt}, *list(self._history), {"role": "user", "content": text}], messages=[{"role": "system", "content": self._system_prompt}, *list(self._history), {"role": "user", "content": text}],
max_tokens=settings.llm_max_tokens, max_tokens=settings.llm_max_tokens,
temperature=0.7, temperature=max(0.1, min(1.2, settings.llm_temperature)),
) )
content = (resp.choices[0].message.content or "").strip() content = self._normalize_reply_text(resp.choices[0].message.content or "")
if content: if content:
self._history.append({"role": "user", "content": text}) self._history.append({"role": "user", "content": text})
self._history.append({"role": "assistant", "content": content}) self._history.append({"role": "assistant", "content": content})
return content, "online" return content, "online"
except Exception as exc: # pragma: no cover except Exception as exc: # pragma: no cover
self._error = str(exc) self._error = str(exc)
logger.warning("LLM request failed, fallback used: %s", exc) logger.warning("LLM request failed: %s", exc)
return f"收到：{text}。这是本地兜底回复。", "fallback" return "", "error"
async def reply(self, text: str) -> str: async def reply(self, text: str) -> str:
content, _ = await self.reply_with_meta(text) content, _ = await self.reply_with_meta(text)
@@ -67,6 +81,7 @@ class LLMService:
return { return {
"ready": self._ready, "ready": self._ready,
"model": self._model, "model": self._model,
"base_url": self._base_url,
"history_items": len(self._history), "history_items": len(self._history),
"error": self._error, "error": self._error,
} }
services/tts.py
+89 -8
View File
@@ -1,20 +1,50 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import re
from pathlib import Path
from typing import Optional from typing import Optional
import numpy as np import numpy as np
from config import kokoro_bundle_dir, settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _kokoro_root() -> Path:
p = kokoro_bundle_dir().resolve()
if not p.is_dir():
raise FileNotFoundError(f"Kokoro model directory missing: {p}")
cfg = p / "config.json"
w0 = p / "kokoro-v1_0.pth"
w1 = p / "kokoro-v1_1-zh.pth"
weight = w0 if w0.is_file() else (w1 if w1.is_file() else None)
if not cfg.is_file() or weight is None:
raise FileNotFoundError(f"Kokoro bundle incomplete (need config.json and weight .pth): {p}")
return p
class TTSService: class TTSService:
def __init__(self) -> None: def __init__(self) -> None:
self._pipeline = None self._pipeline = None
self._kokoro_root: Optional[Path] = None
self._ready = False self._ready = False
self._attempted = False self._attempted = False
self._init_error: Optional[str] = None self._init_error: Optional[str] = None
def _voice_pt_path(self) -> Path:
root = self._kokoro_root
if root is None:
raise RuntimeError("Kokoro pipeline not initialized")
name = (settings.tts_voice or "").strip()
if not name:
raise ValueError("TTS_VOICE must be set (voice name, e.g. zf_xiaoxiao)")
p = root / "voices" / f"{name}.pt"
if not p.is_file():
raise FileNotFoundError(f"Voice file missing: {p}")
return p
def _ensure_loaded(self) -> None: def _ensure_loaded(self) -> None:
if self._ready or self._attempted: if self._ready or self._attempted:
return return
@@ -22,23 +52,74 @@ class TTSService:
try: try:
import kokoro import kokoro
self._pipeline = kokoro.KPipeline(lang_code="z") root = _kokoro_root()
w0 = root / "kokoro-v1_0.pth"
w1 = root / "kokoro-v1_1-zh.pth"
weight = w0 if w0.is_file() else w1
hf_repo_id = (
"hexgrad/Kokoro-82M-v1.1-zh" if weight.name.startswith("kokoro-v1_1") else "hexgrad/Kokoro-82M"
)
kmodel = kokoro.KModel(
repo_id=hf_repo_id,
config=str(root / "config.json"),
model=str(weight),
)
self._pipeline = kokoro.KPipeline(lang_code="z", model=kmodel, repo_id=str(root))
self._kokoro_root = root
self._ready = True self._ready = True
except Exception as exc: # pragma: no cover except Exception as exc: # pragma: no cover
self._init_error = str(exc) self._init_error = str(exc)
logger.warning("TTS fallback mode: %s", exc) logger.error("TTS init failed: %s", exc)
@staticmethod
def _normalize_text(text: str) -> str:
cleaned = (text or "").strip()
if not cleaned:
return ""
cleaned = cleaned.replace("\r", "\n")
cleaned = re.sub(r"[\t\f\v]+", " ", cleaned)
cleaned = re.sub(r"\s*([，。！？；：、,.!?;:])\s*", r"\1", cleaned)
cleaned = re.sub(r"(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])", "", cleaned)
cleaned = re.sub(r"\n{2,}", "\n", cleaned)
cleaned = re.sub(r"\s{2,}", " ", cleaned)
return cleaned.strip()
@staticmethod
def _apply_edge_fade(audio: np.ndarray, sr: int) -> np.ndarray:
if audio.size == 0:
return audio
fade_samples = max(1, min(int(sr * max(settings.tts_fade_ms, 0) / 1000.0), audio.shape[0] // 8))
if fade_samples <= 1:
return audio
out = np.array(audio, copy=True)
ramp = np.linspace(0.0, 1.0, fade_samples, dtype=np.float32)
out[:fade_samples] *= ramp
out[-fade_samples:] *= ramp[::-1]
return out
async def synthesize(self, text: str) -> tuple[np.ndarray, int]: async def synthesize(self, text: str) -> tuple[np.ndarray, int]:
text = self._normalize_text(text)
if not text:
return np.zeros(1, dtype=np.float32), 24000
self._ensure_loaded() self._ensure_loaded()
if self._ready and self._pipeline is not None: if not self._ready or self._pipeline is None:
return np.zeros(1, dtype=np.float32), 24000
try:
voice_path = str(self._voice_pt_path())
except (OSError, ValueError, RuntimeError) as exc:
logger.error("TTS voice: %s", exc)
return np.zeros(1, dtype=np.float32), 24000
chunks = [] chunks = []
for _, _, audio in self._pipeline(text, voice="zf_xiaoxiao"): for _, _, audio in self._pipeline(
chunks.append(audio) text,
voice=voice_path,
speed=max(0.8, min(1.1, settings.tts_speed)),
split_pattern=r"\n+",
):
chunks.append(self._apply_edge_fade(np.asarray(audio, dtype=np.float32), 24000))
if chunks: if chunks:
return np.concatenate(chunks), 24000 return np.concatenate(chunks), 24000
return np.zeros(1, dtype=np.float32), 24000
# 0.5s silence fallback for flow verification
return np.zeros(12000, dtype=np.float32), 24000
@property @property
def health(self) -> dict: def health(self) -> dict:
services/vad.py
+12 -5
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import Optional from typing import Optional
from config import vad_bundle_path
class VADService: class VADService:
def __init__(self) -> None: def __init__(self) -> None:
@@ -15,16 +17,21 @@ class VADService:
if self._ready or self._attempted: if self._ready or self._attempted:
return return
self._attempted = True self._attempted = True
path = vad_bundle_path().resolve()
if not path.is_file():
self._error = f"missing Silero VAD weights: {path}"
return
try: try:
from silero_vad import VADIterator, load_silero_vad from silero_vad import VADIterator
from silero_vad.utils_vad import init_jit_model
self._model = load_silero_vad() self._model = init_jit_model(str(path))
self._iterator = VADIterator( self._iterator = VADIterator(
self._model, self._model,
threshold=0.5, threshold=0.65,
sampling_rate=16000, sampling_rate=16000,
min_silence_duration_ms=300, min_silence_duration_ms=450,
speech_pad_ms=30, speech_pad_ms=80,
) )
self._ready = True self._ready = True
except Exception as exc: # pragma: no cover except Exception as exc: # pragma: no cover
visual-chat-task-list.md
-724
View File
@@ -1,724 +0,0 @@
# 智能可视化语音聊天系统 — 详细任务清单（本机实时版）
> **目标**：在本机（RTX 5090 / WSL2）实现"浏览器访问 → WebRTC 传输 → 单工语音对话 → 数字人口型同步"的**实时**可用系统。
> **基础**：`environments.md` 与 `research-notes.md` 中已验证的环境、模型和踩坑修复全部复用。
> **最后更新**：2026-03-25 — 经过逐组件可行性硬核验证后修订（第三版）。
---
## 0. 逐组件可行性验证报告（执行前必读）
### 0.1 组件全景审计表
| 组件 | 选型 | 许可证 | 纯本地 | RTX 5090 验证 | 实时性证据 | 风险等级 |
|------|------|--------|--------|---------------|-----------|---------|
| **VAD** | Silero VAD 6.x | MIT | 是(CPU) | N/A(纯CPU) | 流式 VADIterator，32ms 帧级延迟 | 无 |
| **ASR** | SenseVoice-Small (FunASR) | Apache 2.0 | 是(GPU) | 已有用户确认 cu128 可跑 | 中文 CER 4.2%，CPU 14x实时，GPU 更快 | 低 |
| **LLM** | 在线 API (OpenAI 兼容) | N/A | 否(设计如此) | N/A | 流式，首 token ~0.5-1.5s | 中(网络) |
| **TTS** | Kokoro-82M-v1.1-zh | Apache 2.0 | 是(GPU) | 确认支持 CUDA 12.8 RTX 50 系 | 210x实时(4090)，<2GB VRAM | 低 |
| **TTS 升级** | Qwen3-TTS 0.6B | Apache 2.0 | 是(GPU) | 确认可跑，需 torch.compile 优化 | 97ms 首音频，RTF<0.25(优化后) | 中 |
| **口型** | MuseTalk v1.5 | MIT | 是(GPU) | 已在你机器验证通过 | 30fps+(V100)，RTX 5090 更快 | 中(需改流式) |
| **头动底片** | LivePortrait | MIT | 是(离线预处理) | 已在你机器验证通过 | 离线，无实时要求 | 无 |
| **WebRTC** | aiortc | BSD-3-Clause | 是 | N/A(纯Python) | asyncio 原生，H.264 已优化 | 低 |
**结论：全部组件均为开源许可（MIT / Apache 2.0 / BSD-3），全部可本地运行，全部有 RTX 5090 兼容路径。**
### 0.2 两个致命问题及修正
#### 问题一：faster-whisper 中文识别率灾难
| | faster-whisper large-v3-turbo | SenseVoice-Small |
|---|---|---|
| 中文 WER/CER | **WER 77%**（几乎不可用） | **CER 4.2%**（优秀） |
| 中文方言 | 不支持 | 7 大方言 + 26 种口音 |
| 推理速度(GPU) | ~50x 实时 | RTF 0.0076（130x 实时） |
| 模型大小 | ~1.5GB | ~234MB |
| 许可证 | MIT | Apache 2.0 |
**决定：ASR 从 faster-whisper 改为 SenseVoice-Small。** faster-whisper 的 turbo 模型砍掉了 28/32 层 decoder，导致中文（需要更复杂解码的语言）准确率暴跌，完全不可接受。
#### 问题二：CosyVoice2 实际延迟远超宣传
| | 宣传值 | 开源版实测值(社区) |
|---|---|---|
| 流式首块延迟 | 150ms | **1 - 4.5 秒** |
| 原因 | 阿里云预训练音色，无特征提取 | 开源 zero-shot 需提取 token/embedding |
| 流式实现 | 优化的 chunk 流 | 累积式 chunk（1A, 2A, 3A...），无 KV cache |
**决定：TTS 从 CosyVoice2 改为 Kokoro-82M（快速启动） + Qwen3-TTS 0.6B（质量升级路径）。**
### 0.3 修正后的端到端延迟预算
| 阶段 | 选型 | 预估延迟 | 数据来源 |
|------|------|----------|---------|
| VAD 尾部检测 | Silero VAD | ~300ms | 默认 silence_duration_ms=300 |
| ASR 转写(3s) | SenseVoice-Small GPU | ~50-150ms | RTF 0.0076，3s × 0.0076 ≈ 23ms + 开销 |
| LLM 首 token | 在线流式 | ~500-1500ms | 取决于模型/网络 |
| TTS 首音频块 | Kokoro-82M | ~20-50ms | 210x 实时 on 4090 |
| MuseTalk 首帧批 | MuseTalk batch=4 | ~100-200ms | 30fps+ → 4帧 ≈ 130ms |
| WebRTC 编码传输 | aiortc H.264 | ~50-100ms | 软编码 |
| **合计** | | **~1.0 - 2.3s** | |
> 比上版预算更优，因为 SenseVoice 比 faster-whisper 更快，Kokoro 比 CosyVoice2 快一个数量级。
### 0.4 GPU 显存预算（RTX 5090 = 32GB）
| 模型 | 精度 | VRAM | 常驻 |
|------|------|------|------|
| MuseTalk (UNet + VAE + Whisper) | fp16 | ~3-4GB | 是 |
| SenseVoice-Small | fp16 | ~0.8GB | 是 |
| Kokoro-82M | fp16 | ~1.5GB | 是 |
| PyTorch CUDA 上下文 | — | ~1GB | 固定 |
| **合计** | | **~6-7GB** | |
> 32GB VRAM 用不到 1/4，若后续升级 Qwen3-TTS 0.6B（+3GB）仍有充裕余量。
### 0.5 "开源可控"确认清单
| 组件 | 许可证 | 可商用 | 代码仓库 | 离线可用 |
|------|--------|--------|----------|---------|
| Silero VAD | MIT | 是 | github.com/snakers4/silero-vad | 是 |
| SenseVoice-Small | Apache 2.0 | 是 | github.com/FunAudioLLM/SenseVoice | 是 |
| Kokoro-82M | Apache 2.0 | 是 | github.com/hexgrad/kokoro | 是 |
| Qwen3-TTS | Apache 2.0 | 是 | github.com/QwenLM/Qwen3-TTS | 是 |
| MuseTalk v1.5 | MIT | 是 | github.com/TMElyralab/MuseTalk | 是 |
| LivePortrait | MIT | 是 | github.com/KlingTeam/LivePortrait | 是 |
| aiortc | BSD-3 | 是 | github.com/aiortc/aiortc | 是 |
**零云端依赖（LLM 除外，且 LLM 有本地兜底），零非开源组件。**
### 0.6 "稳健落地"保障策略
| 风险 | 发生时 | 保底方案 | 切换成本 |
|------|--------|----------|---------|
| SenseVoice 在 nightly PyTorch 编译失败 | M0 | 用 MuseTalk 自带 whisper-tiny 做 ASR（精度低但能跑） | 改一行 import |
| Kokoro 中文质量不达标 | M1 | 切 Qwen3-TTS 0.6B（已验证 RTX 5090 可跑） | 换一个 TTS 服务封装 |
| Qwen3-TTS RTX 5090 推理慢 | M4 | torch.compile + CUDA graphs 优化（社区已有方案，RTF<0.25） | 加 2 行代码 |
| MuseTalk 流式改造困难 | M1 | 先用"分段文件处理"：TTS 输出完整 WAV → MuseTalk 批量合成 → 整段播放 | 延迟增加 2-3s 但可用 |
| 在线 LLM 不可用 | 运行时 | 本地预设话术兜底，8s 超时自动触发 | 已内建 |
| aiortc 软编码瓶颈 | M2 | 改用 ffmpeg + NVENC 硬编码管道 | 约 1 天工作量 |
---
## 1. 总体架构
### 1.1 系统拓扑
```
浏览器 (Chrome)
│
│ WebRTC (ICE/DTLS/SRTP)
│
▼
┌─────────────────────────────────────────────────────┐
│ webrtc-gateway (aiortc + FastAPI) │
│ ┌──────────┐ ┌──────────────────────┐ │
│ │ 上行音频 │ │ 下行视频 + 音频轨 │ │
│ └────┬─────┘ └──────────▲───────────┘ │
└───────┼───────────────────────────────┼─────────────┘
│ │
▼ │
┌──────────────┐ ┌──────┴──────┐
│ Silero VAD │ │ mixer │
│ (CPU,MIT) │ │ (音视频合流) │
└──────┬───────┘ └──▲───────▲──┘
│ speech.end │ │
▼ │ │
┌──────────────┐ ┌──────┴──┐ ┌──┴──────────┐
│ SenseVoice │ │MuseTalk │ │ TTS 音频 │
│ ASR (GPU) │ │ 口型渲染│ │ (Kokoro) │
│ Apache 2.0 │ │ (GPU) │ │ Apache 2.0 │
└──────┬───────┘ │ MIT │ └──────▲──────┘
│ text └────▲────┘ │
▼ │ wav │ text
┌──────────────┐ ┌─────┴──────┐ │
│ LLM Client │──text──▶│ Kokoro TTS ├───────┘
│ (在线 API) │ │ (GPU) │
└──────────────┘ └────────────┘
```
### 1.2 单工状态机
```
speech.start speech.end
┌────┐ ──────────▶ ┌──────────────┐ ────────▶ ┌──────────┐
│IDLE│ │USER_SPEAKING │ │THINKING │
└──▲─┘ ◀────────── └──────────────┘ └────┬─────┘
│ barge-in │ tts.ready
│ (用户抢话) ▼
│ ┌─────────────────┐ ┌────────────────┐
└───────────────│ │◀────────│AVATAR_SPEAKING │
playback.done │ (回到 IDLE) │ └────────────────┘
└─────────────────┘
```
**状态规则**：
- `IDLE`：数字人播放闭嘴循环视频，下行静音
- `USER_SPEAKING`：数字人立即切闭嘴循环（如正在说话则中断），录音开始
- `THINKING`：录音结束，ASR → LLM → TTS 管线启动，数字人继续闭嘴循环
- `AVATAR_SPEAKING`：MuseTalk 输出帧 + TTS 音频同步下发
- 任何状态下检测到 `speech.start` → 立即跳转 `USER_SPEAKING`（barge-in）
### 1.3 进程内通信（不用消息中间件）
单机单进程，所有模块通过 `asyncio.Queue` 和 `asyncio.Event` 通信，延迟最低：
```python
audio_chunk_queue: asyncio.Queue[bytes] # VAD → ASR
text_queue: asyncio.Queue[str] # ASR → LLM
tts_chunk_queue: asyncio.Queue[bytes] # LLM→TTS → mixer
avatar_frame_queue: asyncio.Queue[np.ndarray] # TTS→MuseTalk → mixer
state_event: asyncio.Event # 状态变更通知
```
---
## 2. 环境与依赖
### 2.1 创建统一 conda 环境
```bash
conda create -n digital-human python=3.10 -y
conda activate digital-human
conda install -y -c conda-forge ffmpeg
# PyTorch nightly（RTX 5090 必须，已验证版本）
pip install --pre torch torchvision torchaudio \
--index-url https://download.pytorch.org/whl/nightly/cu128
```
### 2.2 安装各模块依赖
```bash
# MuseTalk 依赖链（按 research-notes.md 已验证）
cd /home/xsl/work/MuseTalk
pip install -r requirements.txt
pip install --no-build-isolation chumpy
pip install mmengine
MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation
pip install "mmdet>=3.3.0" "mmpose>=1.3.0"
pip install xtcocotools json-tricks munkres
pip install "transformers>=4.45.0"
# ASR - SenseVoice
pip install funasr
# TTS - Kokoro
pip install kokoro soundfile
# TTS 升级备选 - Qwen3-TTS（M4 阶段再装）
# pip install qwen3-tts
# VAD
pip install silero-vad
# WebRTC + Web 服务
pip install aiortc aiohttp uvicorn fastapi
# 工具库
pip install pydantic python-dotenv tenacity numpy opencv-python-headless
```
### 2.3 模型权重
| 模型 | 路径 | 状态 | 大小 |
|------|------|------|------|
| MuseTalk v1.5 全套 | `/home/xsl/work/MuseTalk/models/` | 已有 | 5.5GB |
| LivePortrait 权重 | `/home/xsl/work/LivePortrait/pretrained_weights/` | 已有 | 1.2GB |
| SenseVoice-Small | 首次调用自动下载 | **新增** | ~234MB |
| Kokoro-82M-v1.1-zh | 首次调用自动下载 | **新增** | ~180MB |
| Silero VAD | 首次调用自动下载 | **自动** | ~2MB |
```bash
# SenseVoice 模型会在首次 AutoModel() 调用时自动下载
# Kokoro 模型会在首次 import kokoro 时自动下载
# 也可以手动预下载：
python -c "from funasr import AutoModel; AutoModel(model='FunAudioLLM/SenseVoiceSmall', device='cuda:0')"
python -c "import kokoro; kokoro.KPipeline(lang_code='z')"
```
### 2.4 MuseTalk 补丁（必须，已验证）
`torch.load(weights_only=False)` monkey patch — 保持 research-notes.md 中已验证的方案。
### 2.5 验收清单
- [x] `python -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name())"` → True, NVIDIA GeForce RTX 5090
- [x] SenseVoice 加载模型 + 转写一段中文音频成功
- [x] Kokoro 合成一句中文语音成功
- [x] MuseTalk 单样例推理输出视频成功
---
## 3. VAD 语音活动检测（Silero VAD, MIT）
### 3.1 实现
```python
from silero_vad import load_silero_vad, VADIterator
model = load_silero_vad()
vad_iterator = VADIterator(
model,
threshold=0.5,
sampling_rate=16000,
min_silence_duration_ms=300, # 尾部静音判定
speech_pad_ms=30,
)
# 流式处理 WebRTC 上行音频帧
for audio_chunk in webrtc_audio_stream:
speech_dict = vad_iterator(audio_chunk)
if speech_dict:
if 'start' in speech_dict:
arbitrator.on_speech_start()
if 'end' in speech_dict:
arbitrator.on_speech_end(recorded_audio)
```
### 3.2 验收
- [x] 静音环境不误触发
- [x] 正常说话稳定触发
- [x] 说话结束后 ~300ms 内检测到 speech.end
---
## 4. ASR 语音识别（SenseVoice-Small, Apache 2.0, 本地 GPU）
### 4.1 模型加载
```python
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
asr_model = AutoModel(
model="FunAudioLLM/SenseVoiceSmall",
device="cuda:0",
hub="hf",
)
```
### 4.2 推理接口
```python
async def transcribe(audio_array: np.ndarray) -> str:
"""16kHz mono float32 numpy array → 文本"""
res = asr_model.generate(
input=audio_array,
cache={},
language="zh",
use_itn=True,
)
return rich_transcription_postprocess(res[0]["text"])
```
### 4.3 性能预期
- AISHELL-1 中文 CER: 4.2%（比 whisper-turbo 的 77% WER 好一个量级）
- 3 秒语音推理延迟: ~50-150ms (GPU)
- VRAM: ~0.8GB
### 4.4 验收
- [ ] 10 条中文测试语音，转写准确率主观 > 90%
- [ ] 单段 3 秒语音延迟 < 300ms
---
## 5. LLM 对话（在线 API，流式输出）
### 5.1 客户端实现
```python
import openai
client = openai.AsyncOpenAI(
api_key=os.getenv("LLM_API_KEY"),
base_url=os.getenv("LLM_BASE_URL"),
)
async def chat_stream(messages: list[dict]) -> AsyncIterator[str]:
response = await client.chat.completions.create(
model=os.getenv("LLM_MODEL", "gpt-4o-mini"),
messages=messages,
stream=True,
max_tokens=150,
temperature=0.7,
)
async for chunk in response:
if chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
```
### 5.2 流式分句送 TTS
- LLM 流式输出 token → 累积到句子边界（。！？，\n）→ 立即送 TTS
- 系统提示词要求"用简短口语回复，每句不超过 30 字"
### 5.3 降级策略
- 超时 8s 或网络错误 → 返回预设兜底话术
- 用 `tenacity` 做 1 次快速重试（2s 超时）
### 5.4 验收
- [x] 正常首 token < 2 秒
- [x] 断网 3 秒内返回兜底话术
- [x] 20 轮对话上下文连贯
---
## 6. TTS 语音合成（Kokoro-82M, Apache 2.0, 本地 GPU）
### 6.1 模型加载
```python
import kokoro
pipeline = kokoro.KPipeline(lang_code='z') # z = 中文
```
### 6.2 合成接口
```python
async def synthesize(text: str) -> tuple[np.ndarray, int]:
"""文本 → (audio_array, sample_rate)"""
generator = pipeline(text, voice='zf_xiaoxiao') # 中文女声
audio_chunks = []
for _, _, chunk in generator:
audio_chunks.append(chunk)
audio = np.concatenate(audio_chunks)
return audio, 24000 # Kokoro 输出 24kHz
```
### 6.3 音频格式对齐
- Kokoro 输出 24kHz → 需重采样到 16kHz 供 MuseTalk whisper
- 下行音频可保持 24kHz（WebRTC 支持）
### 6.4 升级路径（Qwen3-TTS）
M4 阶段如果 Kokoro 中文质量不满意，切换到 Qwen3-TTS 0.6B：
- 97ms 首音频延迟
- 更好的中文自然度
- 需额外 ~3GB VRAM（总计仍 <10GB）
- RTX 5090 需 `torch.compile` + CUDA graphs 优化（社区已验证方案）
### 6.5 验收
- [ ] 40 字中文文本合成延迟 < 500ms
- [ ] 音质清晰，无明显机器感
- [x] 连续合成 20 句无崩溃
---
## 7. 数字人渲染（MuseTalk v1.5, MIT, 本地 GPU）
### 7.1 离线预处理：底片库（一次性）
**Step 1**: 用 LivePortrait 生成闭嘴自然动作循环视频
```bash
conda activate digital-human
cd /home/xsl/work/LivePortrait
for drv in d3.mp4 d6.mp4 d10.mp4; do
python inference.py \
-s /path/to/avatar_photo.jpg \
-d assets/examples/driving/$drv \
--flag_crop_driving_video \
-o /home/xsl/product/assets/avatar_loops/
done
```
**Step 2**: 用 MuseTalk 预处理（提取 latents/coords/masks → 缓存）
```bash
cd /home/xsl/work/MuseTalk
python -m scripts.realtime_inference \
--inference_config configs/inference/realtime.yaml \
--version v15 \
--unet_model_path models/musetalkV15/unet.pth \
--unet_config models/musetalkV15/musetalk.json
```
预处理产物缓存后，运行时直接加载，不再需要人脸检测和解析。
### 7.2 实时口型合成：改造 MuseTalk
**核心**：将 `Avatar.inference()` 从完整音频文件改为流式音频 chunk。
```python
class RealtimeAvatar:
def __init__(self, prepared_data_path):
# 加载预处理缓存
self.latents = torch.load(f"{prepared_data_path}/latents.pt")
self.coords = pickle.load(open(f"{prepared_data_path}/coords.pkl", 'rb'))
self.frames = [...] # 预加载原始帧
self.masks = [...] # 预加载 mask
self.frame_idx = 0
async def process_audio_chunk(self, pcm_16k: np.ndarray) -> list[np.ndarray]:
"""接收 TTS PCM 块 → 返回口型帧列表"""
whisper_features = audio_processor.get_audio_feature(pcm_16k)
whisper_chunks = audio_processor.get_whisper_chunk(whisper_features, ...)
output_frames = []
for batch in datagen(whisper_chunks, self.latents, batch_size=4):
pred = unet.model(batch_latent, timesteps, encoder_hidden_states=audio_feat)
recon = vae.decode_latents(pred)
for frame in recon:
blended = blend_frame(frame, self.frame_idx)
output_frames.append(blended)
self.frame_idx = (self.frame_idx + 1) % len(self.frames)
return output_frames
def get_idle_frame(self) -> np.ndarray:
"""闭嘴循环帧（不过 UNet）"""
frame = self.frames[self.frame_idx % len(self.frames)]
self.frame_idx = (self.frame_idx + 1) % len(self.frames)
return frame
```
### 7.3 帧率与同步
- 25fps（与底片一致），每帧 40ms
- TTS 音频时长 → 计算帧数 → MuseTalk 生成对应数量帧
- asyncio 定时器控制帧发送节奏
### 7.4 验收
- [ ] 闭嘴循环 60 秒无卡顿
- [ ] 5 秒 TTS 音频口型帧 200ms 内开始输出
- [ ] 口型同步主观无 >200ms 错位
---
## 8. WebRTC 网关与前端
### 8.1 后端（aiortc + FastAPI, BSD-3）
```python
@app.post("/webrtc/offer")
async def webrtc_offer(request: Request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
video_track = AvatarVideoTrack() # 从 MuseTalk 帧队列读取
audio_track = AvatarAudioTrack() # 从 TTS 音频队列读取
pc.addTrack(video_track)
pc.addTrack(audio_track)
@pc.on("track")
def on_track(track):
if track.kind == "audio":
asyncio.ensure_future(process_incoming_audio(track))
await pc.setRemoteDescription(offer)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
```
### 8.2 前端页面（纯 HTML/JS，无框架依赖）
```
┌──────────────────────────────┐
│ 数字人视频 │
│ (WebRTC video 标签) │
│ │
├──────────────────────────────┤
│ 🎤 正在倾听... │ ← 状态指示
├──────────────────────────────┤
│ 你: "今天天气怎么样？" │ ← ASR 字幕
│ AI: "今天天气不错呢..." │ ← LLM 回复字幕
└──────────────────────────────┘
```
- 页面加载 → 自动建 WebRTC 连接 → 请求麦克风权限
- 下行视频+音频 → `<video>` 标签
- WebSocket 接收字幕和状态更新
- 断连 3 秒自动重连
### 8.3 验收
- [ ] Chrome 打开 3 秒内看到数字人视频
- [x] 说话后字幕区显示 ASR 文本
- [x] 刷新页面自动重连
---
## 9. 单工仲裁器
```python
class SessionState(Enum):
IDLE = "idle"
USER_SPEAKING = "user_speaking"
THINKING = "thinking"
AVATAR_SPEAKING = "avatar_speaking"
class Arbitrator:
def __init__(self):
self.state = SessionState.IDLE
self._lock = asyncio.Lock()
self._cancel = asyncio.Event()
async def on_speech_start(self):
async with self._lock:
if self.state == SessionState.AVATAR_SPEAKING:
self._cancel.set()
self.state = SessionState.USER_SPEAKING
async def on_speech_end(self, audio: bytes):
async with self._lock:
self.state = SessionState.THINKING
asyncio.create_task(self._run_pipeline(audio))
async def on_playback_done(self):
async with self._lock:
if not self._cancel.is_set():
self.state = SessionState.IDLE
```
### 验收
- [ ] 数字人说话中用户插嘴 → 200ms 内停止
- [ ] 连续抢话 20 次无死锁
---
## 10. 配置
### `.env`
```env
# LLM
LLM_API_KEY=sk-xxx
LLM_BASE_URL=https://api.openai.com/v1
LLM_MODEL=gpt-4o-mini
LLM_MAX_TOKENS=150
LLM_TIMEOUT=8
# 模型路径（复用现有）
MUSETALK_DIR=/home/xsl/work/MuseTalk
MUSETALK_UNET=/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth
MUSETALK_CONFIG=/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json
# 服务
WEBRTC_PORT=8080
AVATAR_FPS=25
```
---
## 11. 目录结构
```
/home/xsl/product/
├── main.py # 入口
├── config.py # 配置
├── .env
├── requirements.txt
│
├── core/
│ ├── state_machine.py # 单工仲裁器
│ ├── session.py # 会话管理
│ └── pipeline.py # ASR→LLM→TTS→Avatar 编排
│
├── services/
│ ├── vad.py # Silero VAD
│ ├── asr.py # SenseVoice-Small
│ ├── llm.py # LLM 客户端
│ ├── tts.py # Kokoro TTS
│ └── avatar.py # MuseTalk 实时推理
│
├── webrtc/
│ ├── gateway.py # aiortc 信令
│ ├── video_track.py # 数字人视频轨
│ └── audio_track.py # 数字人音频轨
│
├── web/
│ ├── index.html
│ ├── app.js
│ └── style.css
│
├── assets/
│ ├── avatar_loops/ # 预生成闭嘴循环视频
│ └── avatar_prepared/ # MuseTalk 预处理缓存
│
├── scripts/
│ ├── prepare_avatar.sh # 预处理底片
│ ├── start.sh # 一键启动
│ └── smoke_test.py # 冒烟测试
│
└── docs/
├── visual-chat-task-list.md
├── environments.md
└── research-notes.md
```
---
## 12. 里程碑
### M0：环境就绪（0.5 天）
- [x] 创建 `digital-human` conda 环境
- [x] 安装全部依赖（PyTorch nightly + MuseTalk + SenseVoice + Kokoro + aiortc）
- [x] 验证四个 GPU 模型可分别加载并推理
- [ ] 验证四个模型可同时常驻 GPU（显存 < 10GB）
### M1：离线管线串通（2 天）
- [x] `services/asr.py`：SenseVoice 输入 WAV → 输出文本
- [x] `services/llm.py`：输入文本 → 流式输出文本
- [x] `services/tts.py`：Kokoro 输入文本 → 输出 WAV
- [x] `services/avatar.py`：MuseTalk 输入 WAV → 输出帧序列（失败自动回退简易口型）
- [x] `scripts/smoke_test.py`：一段录音走完全链路输出视频文件（当前为文本输入链路，产出 `wav + mp4`）
### M2：WebRTC + 数字人常驻（2 天）
- [x] `webrtc/gateway.py`：aiortc 信令接口
- [x] `webrtc/video_track.py`：发闭嘴循环帧
- [x] `webrtc/audio_track.py`：发静音
- [x] `web/`：前端页面
- [x] 浏览器可看到数字人循环视频
### M3：全链路联调 + 单工（2 天）
- [x] `services/vad.py`：Silero VAD 处理上行音频
- [x] `core/state_machine.py`：仲裁器
- [x] `core/pipeline.py`：完整 VAD→ASR→LLM→TTS→MuseTalk→下行
- [x] barge-in 实现
- [x] WebSocket 字幕下发
### M4：优化与稳定（2 天）
- [ ] 端到端延迟调优（目标 < 3s）
- [ ] 1 小时长时间运行测试
- [x] 显存监控（nvidia-smi 周期采样）
- [x] LLM 降级测试
- [ ] 评估是否升级 Qwen3-TTS（中文质量对比）
- [x] 20 轮连续对话压测
补充自动化脚本：
- `scripts/qa_check.py`：文本链路压测（支持 20 轮）
- `scripts/llm_fallback_check.py`：LLM 断网/超时兜底验证
- `scripts/gpu_monitor.sh`：GPU 利用率与显存采样（CSV）
- `scripts/model_probe.py`：ASR/TTS/VAD 快速加载与推理探测
---
## 13. 被排除的方案及原因（决策日志）
| 方案 | 排除原因 |
|------|---------|
| faster-whisper (大模型中文) | 中文 WER 77%，完全不可接受 |
| CosyVoice2 (TTS) | 开源版流式首包 1-4.5s，与宣传 150ms 严重不符 |
| edge-tts | 依赖微软云服务器，非本地，离线不可用 |
| Fish-Speech S2 | 许可证限制商用，需单独授权 |
| LatentSync | 仅 4fps，无法实时 |
| Redis/ZeroMQ 消息总线 | 单机单进程无需中间件，asyncio Queue 延迟更低 |
| ChatTTS | 流式推理未完全实现，首包延迟不稳定 |
web/app.js
+1234 -95
View File
File diff suppressed because it is too large Load Diff
web/index.html
+52 -7
View File
@@ -8,14 +8,47 @@
</head> </head>
<body> <body>
<main class="wrap"> <main class="wrap">
<h1>可视化语音聊天系统</h1> <h1>3D 数字人语音聊天系统</h1>
<p id="status">状态：启动中</p> <p id="status">状态：启动中</p>
<div class="video-box"> <div class="stage-box">
<video id="remoteVideo" autoplay playsinline></video> <div class="stage-copy">
<strong>Three.js + VRM 驱动舞台</strong>
<span id="animationConn">未连接</span>
</div> </div>
<div class="stage-toolbar">
<label class="file-pill" for="modelFile">载入本地 VRM/GLB</label>
<input id="modelFile" type="file" accept=".vrm,.glb,.gltf" />
<input id="modelUrl" type="text" value="/web/models/VRM1_Constraint_Twist_Sample.vrm" placeholder="输入 VRM/GLB/GLTF 地址，例如官方示例 VRM 或 /web/models/avatar.vrm" />
<button id="loadModelBtn">加载模型 URL</button>
<button id="resetViewBtn">重置视角</button>
</div>
<div class="canvas-shell">
<canvas id="avatarCanvas"></canvas>
<div class="canvas-overlay">
<div class="overlay-card">
<span>模型状态</span>
<strong id="modelStatus">使用调试头像</strong>
</div>
<div class="overlay-card">
<span>驱动器</span>
<strong id="animationDriver">--</strong>
</div>
<div class="overlay-card">
<span>最近帧数</span>
<strong id="animationFrames">0</strong>
</div>
</div>
</div>
<div class="stage-metrics">
<div><span>当前模型</span><strong id="modelName">debug-avatar</strong></div>
<div><span>动画缓冲</span><strong id="animationBuffer">0</strong></div>
</div>
<pre id="animationPreview">等待动画控制数据...</pre>
</div>
<audio id="remoteAudio" autoplay></audio>
<div class="actions"> <div class="actions">
<button id="healthBtn">检查服务状态</button> <button id="healthBtn">检查服务状态</button>
<button id="offerBtn">连接 WebRTC</button> <button id="offerBtn">连接语音通道</button>
<button id="resetBtn">清空会话</button> <button id="resetBtn">清空会话</button>
<span id="connBadge" class="badge neutral">未连接</span> <span id="connBadge" class="badge neutral">未连接</span>
</div> </div>
@@ -25,14 +58,14 @@
</div> </div>
<div id="messages" class="messages"></div> <div id="messages" class="messages"></div>
<div class="diag" id="diag"> <div class="diag" id="diag">
<div class="diag-item"><span class="dot" id="dConn"></span><span>WebRTC连接</span></div> <div class="diag-item"><span class="dot" id="dConn"></span><span>语音通道</span></div>
<div class="diag-item"><span class="dot" id="dVad"></span><span>VAD触发</span></div> <div class="diag-item"><span class="dot" id="dVad"></span><span>VAD触发</span></div>
<div class="diag-item"><span class="dot" id="dAsr"></span><span>ASR</span></div> <div class="diag-item"><span class="dot" id="dAsr"></span><span>ASR</span></div>
<div class="diag-item"><span class="dot" id="dLlm"></span><span>LLM</span></div> <div class="diag-item"><span class="dot" id="dLlm"></span><span>LLM</span></div>
<div class="diag-item"><span class="dot" id="dTts"></span><span>TTS</span></div> <div class="diag-item"><span class="dot" id="dTts"></span><span>TTS</span></div>
</div> </div>
<div class="metrics" id="metrics"> <div class="metrics" id="metrics">
<div class="metric"><span>连接数</span><strong id="mPeers">0</strong></div> <div class="metric"><span>语音通道</span><strong id="mPeers">0</strong></div>
<div class="metric"><span>处理中</span><strong id="mBusy">否</strong></div> <div class="metric"><span>处理中</span><strong id="mBusy">否</strong></div>
<div class="metric"><span>总耗时</span><strong id="mLatency">0 ms</strong></div> <div class="metric"><span>总耗时</span><strong id="mLatency">0 ms</strong></div>
<div class="metric"><span>ASR耗时</span><strong id="mAsrLatency">0 ms</strong></div> <div class="metric"><span>ASR耗时</span><strong id="mAsrLatency">0 ms</strong></div>
@@ -45,12 +78,24 @@
<div class="metric"><span>打断耗时</span><strong id="mBargeIn">0 ms</strong></div> <div class="metric"><span>打断耗时</span><strong id="mBargeIn">0 ms</strong></div>
<div class="metric"><span>输入模式</span><strong id="mInputMode">none</strong></div> <div class="metric"><span>输入模式</span><strong id="mInputMode">none</strong></div>
<div class="metric"><span>VAD触发</span><strong id="mVadCount">0/0</strong></div> <div class="metric"><span>VAD触发</span><strong id="mVadCount">0/0</strong></div>
<div class="metric"><span>动画客户端</span><strong id="mAnimClients">0</strong></div>
<div class="metric"><span>动画驱动</span><strong id="mAnimDriver">--</strong></div>
<div class="metric"><span>最近动画帧</span><strong id="mAnimFrames">0</strong></div>
</div> </div>
<div class="panel"> <div class="panel">
<h3>运行状态</h3> <h3>运行状态</h3>
<pre id="output"></pre> <pre id="output"></pre>
</div> </div>
</main> </main>
<script src="/web/app.js"></script> <script type="importmap">
{
"imports": {
"three": "/web/vendor/three/three.module.js",
"three/addons/": "/web/vendor/three/examples/jsm/",
"@pixiv/three-vrm": "/web/vendor/@pixiv/three-vrm/three-vrm.module.min.js"
}
}
</script>
<script type="module" src="/web/app.js?v=20260330a"></script>
</body> </body>
</html> </html>
web/models/VRM1_Constraint_Twist_Sample.vrm
BIN
View File
Binary file not shown.
web/style.css
+181 -24
View File
@@ -1,32 +1,167 @@
body { body {
margin: 0; margin: 0;
font-family: sans-serif; font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
background: #0f1115; background:
radial-gradient(circle at top, rgba(42, 93, 255, 0.16), transparent 36%),
radial-gradient(circle at right, rgba(15, 173, 129, 0.12), transparent 28%),
linear-gradient(180deg, #091018 0%, #111827 46%, #0b1018 100%);
color: #f2f2f2; color: #f2f2f2;
} }
.wrap { .wrap {
max-width: 900px; max-width: 1120px;
margin: 24px auto; margin: 24px auto;
padding: 0 16px; padding: 0 16px;
} }
.video-box { .stage-box {
width: 100%; width: 100%;
aspect-ratio: 16 / 9; border: 1px solid rgba(116, 139, 166, 0.22);
border: 1px solid #333; border-radius: 24px;
border-radius: 10px; background: linear-gradient(180deg, rgba(18, 25, 36, 0.94), rgba(12, 18, 28, 0.96));
background: #1a1f27;
overflow: hidden; overflow: hidden;
padding: 18px;
box-sizing: border-box;
display: flex;
flex-direction: column;
gap: 14px;
box-shadow: 0 28px 70px rgba(0, 0, 0, 0.32);
} }
video { .stage-copy {
display: flex;
justify-content: space-between;
align-items: center;
gap: 12px;
}
.stage-copy strong {
font-size: 20px;
letter-spacing: 0.01em;
}
.stage-copy span {
font-size: 13px;
color: #7dd3fc;
}
.stage-toolbar {
display: grid;
grid-template-columns: auto minmax(0, 1fr) auto auto;
gap: 12px;
align-items: center;
}
.stage-toolbar input[type="text"] {
min-width: 0;
background: rgba(10, 15, 24, 0.78);
color: #e5eef8;
border: 1px solid rgba(112, 146, 190, 0.3);
border-radius: 12px;
padding: 10px 12px;
}
.stage-toolbar input[type="file"] {
position: absolute;
width: 1px;
height: 1px;
opacity: 0;
pointer-events: none;
}
.file-pill {
display: inline-flex;
align-items: center;
justify-content: center;
min-height: 40px;
border-radius: 999px;
border: 1px solid rgba(102, 199, 255, 0.28);
background: rgba(9, 60, 86, 0.32);
color: #d7f2ff;
padding: 0 16px;
cursor: pointer;
white-space: nowrap;
}
.canvas-shell {
position: relative;
aspect-ratio: 16 / 9;
border-radius: 18px;
overflow: hidden;
background:
radial-gradient(circle at 50% 28%, rgba(148, 196, 255, 0.18), transparent 24%),
linear-gradient(180deg, #152235 0%, #0c131d 52%, #0a1017 100%);
border: 1px solid rgba(95, 128, 160, 0.18);
}
#avatarCanvas {
width: 100%; width: 100%;
height: 100%; height: 100%;
object-fit: cover;
display: block; display: block;
} }
.canvas-overlay {
position: absolute;
left: 14px;
right: 14px;
bottom: 14px;
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 10px;
pointer-events: none;
}
.overlay-card {
border: 1px solid rgba(144, 170, 197, 0.18);
background: rgba(7, 11, 18, 0.68);
backdrop-filter: blur(10px);
border-radius: 14px;
padding: 10px 12px;
display: flex;
flex-direction: column;
gap: 4px;
}
.overlay-card span {
color: #8ea6c3;
font-size: 12px;
}
.overlay-card strong {
font-size: 14px;
}
.stage-metrics {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
}
.stage-metrics div {
border: 1px solid rgba(52, 72, 96, 0.7);
border-radius: 14px;
background: rgba(14, 22, 33, 0.7);
padding: 12px;
display: flex;
flex-direction: column;
gap: 6px;
}
.stage-metrics span {
color: #94a3b8;
font-size: 12px;
}
.stage-metrics strong {
font-size: 16px;
}
#animationPreview {
flex: 1;
min-height: 180px;
max-height: 260px;
}
.actions { .actions {
margin-top: 16px; margin-top: 16px;
display: flex; display: flex;
@@ -42,10 +177,10 @@ video {
.chat-box input { .chat-box input {
flex: 1; flex: 1;
background: #111827; background: rgba(13, 19, 29, 0.9);
color: #f3f4f6; color: #f3f4f6;
border: 1px solid #374151; border: 1px solid #374151;
border-radius: 8px; border-radius: 12px;
padding: 10px; padding: 10px;
} }
@@ -55,9 +190,9 @@ video {
max-height: 360px; max-height: 360px;
overflow: auto; overflow: auto;
border: 1px solid #1f2937; border: 1px solid #1f2937;
border-radius: 10px; border-radius: 16px;
padding: 10px; padding: 10px;
background: #0b1220; background: rgba(10, 18, 32, 0.82);
display: flex; display: flex;
flex-direction: column; flex-direction: column;
gap: 8px; gap: 8px;
@@ -66,8 +201,8 @@ video {
.diag { .diag {
margin-top: 12px; margin-top: 12px;
border: 1px solid #1f2937; border: 1px solid #1f2937;
border-radius: 10px; border-radius: 16px;
background: #0b0d12; background: rgba(8, 13, 18, 0.78);
padding: 10px; padding: 10px;
display: flex; display: flex;
gap: 16px; gap: 16px;
@@ -122,12 +257,12 @@ video {
.msg.user { .msg.user {
align-self: flex-end; align-self: flex-end;
background: #1d4ed8; background: linear-gradient(135deg, #2563eb, #0ea5e9);
} }
.msg.ai { .msg.ai {
align-self: flex-start; align-self: flex-start;
background: #374151; background: linear-gradient(135deg, #334155, #475569);
} }
.panel { .panel {
@@ -143,8 +278,8 @@ video {
.metric { .metric {
border: 1px solid #1f2937; border: 1px solid #1f2937;
background: #0b0d12; background: rgba(8, 13, 18, 0.78);
border-radius: 8px; border-radius: 14px;
padding: 8px 10px; padding: 8px 10px;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
@@ -167,10 +302,10 @@ video {
} }
button { button {
background: #2563eb; background: linear-gradient(135deg, #2563eb, #0891b2);
color: white; color: white;
border: 0; border: 0;
border-radius: 8px; border-radius: 12px;
padding: 8px 12px; padding: 8px 12px;
cursor: pointer; cursor: pointer;
} }
@@ -231,9 +366,31 @@ button:disabled {
pre { pre {
margin-top: 0; margin-top: 0;
background: #0b0d12; background: rgba(8, 13, 18, 0.84);
border: 1px solid #222; border: 1px solid #222;
border-radius: 8px; border-radius: 14px;
padding: 12px; padding: 12px;
overflow: auto; overflow: auto;
} }
audio {
display: none;
}
@media (max-width: 720px) {
.metrics {
grid-template-columns: repeat(2, minmax(120px, 1fr));
}
.stage-toolbar {
grid-template-columns: 1fr;
}
.stage-metrics {
grid-template-columns: 1fr;
}
.canvas-overlay {
grid-template-columns: 1fr;
}
}
web/vendor/@pixiv/three-vrm/three-vrm.module.min.js
Vendored
+983
View File
File diff suppressed because one or more lines are too long
web/vendor/three/examples/jsm/controls/OrbitControls.js
Vendored
+1860
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/examples/jsm/loaders/GLTFLoader.js
Vendored
+4890
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js
Vendored
+1435
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/three.core.js
Vendored
+58773
View File
File diff suppressed because one or more lines are too long
web/vendor/three/three.module.js
Vendored
+18251
View File
File diff suppressed because one or more lines are too long
webrtc/__init__.py
-1
View File
@@ -1 +0,0 @@
webrtc/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
webrtc/__pycache__/tracks.cpython-310.pyc
BIN
View File
Binary file not shown.
webrtc/audio_track.py
-9
View File
@@ -1,9 +0,0 @@
from __future__ import annotations
from webrtc.tracks import AudioBus, AvatarAudioTrack
__all__ = ["AvatarAudioTrack", "build_audio_track"]
def build_audio_track(audio_bus: AudioBus, sample_rate: int = 48000, frame_ms: int = 20) -> AvatarAudioTrack:
return AvatarAudioTrack(audio_bus=audio_bus, sample_rate=sample_rate, frame_ms=frame_ms)
webrtc/gateway.py
-27
View File
@@ -1,27 +0,0 @@
from __future__ import annotations
from aiortc import RTCPeerConnection
from fastapi import FastAPI
from config import settings
from core.state_machine import Arbitrator
from services.avatar import AvatarService
from webrtc.audio_track import build_audio_track
from webrtc.tracks import AudioBus
from webrtc.video_track import build_video_track
def attach_webrtc_tracks(
pc: RTCPeerConnection,
avatar_service: AvatarService,
arbitrator: Arbitrator,
audio_bus: AudioBus,
) -> None:
pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
pc.addTrack(build_audio_track(audio_bus))
def register_gateway_routes(app: FastAPI) -> None:
# Gateway endpoints are currently implemented in `main.py`.
# This function exists to keep project structure aligned with task list.
_ = app
webrtc/tracks.py
-114
View File
@@ -1,114 +0,0 @@
from __future__ import annotations
import asyncio
from fractions import Fraction
from collections import deque
import numpy as np
from aiortc import MediaStreamTrack
from av import AudioFrame, VideoFrame
from core.state_machine import Arbitrator, SessionState
from services.avatar import AvatarService
class AudioBus:
def __init__(self, sample_rate: int = 48000) -> None:
self.sample_rate = sample_rate
self._chunks: deque[np.ndarray] = deque()
self._lock = asyncio.Lock()
self._level = 0.0
async def enqueue(self, pcm_int16_mono: np.ndarray) -> None:
async with self._lock:
if pcm_int16_mono.ndim != 1:
pcm_int16_mono = pcm_int16_mono.reshape(-1)
chunk = pcm_int16_mono.astype(np.int16, copy=False)
self._chunks.append(chunk)
if chunk.size > 0:
rms = float(np.sqrt(np.mean((chunk.astype(np.float32) / 32768.0) ** 2)))
self._level = 0.85 * self._level + 0.15 * min(1.0, rms * 10.0)
async def read(self, samples: int) -> np.ndarray:
out = np.zeros(samples, dtype=np.int16)
async with self._lock:
i = 0
while i < samples and self._chunks:
head = self._chunks[0]
n = min(samples - i, head.shape[0])
out[i : i + n] = head[:n]
i += n
if n == head.shape[0]:
self._chunks.popleft()
else:
self._chunks[0] = head[n:]
self._level = 0.92 * self._level
return out
async def level(self) -> float:
async with self._lock:
return float(self._level)
async def clear(self) -> None:
async with self._lock:
self._chunks.clear()
self._level = 0.0
class AvatarVideoTrack(MediaStreamTrack):
kind = "video"
def __init__(
self,
avatar_service: AvatarService,
arbitrator: Arbitrator,
audio_bus: AudioBus,
fps: int = 25,
) -> None:
super().__init__()
self.avatar_service = avatar_service
self.arbitrator = arbitrator
self.audio_bus = audio_bus
self.fps = fps
self._pts = 0
self._time_base = Fraction(1, 90000)
async def recv(self) -> VideoFrame:
await asyncio.sleep(1 / self.fps)
generated = await self.avatar_service.pop_generated_frame()
if generated is not None:
arr = generated
else:
level = await self.audio_bus.level()
speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
frame = VideoFrame.from_ndarray(arr, format="bgr24")
frame.pts = self._pts
frame.time_base = self._time_base
self._pts += int(90000 / self.fps)
return frame
class AvatarAudioTrack(MediaStreamTrack):
kind = "audio"
def __init__(self, audio_bus: AudioBus, sample_rate: int = 48000, frame_ms: int = 20) -> None:
super().__init__()
self.audio_bus = audio_bus
self.sample_rate = sample_rate
self.samples_per_frame = int(sample_rate * frame_ms / 1000)
self._pts = 0
self._time_base = Fraction(1, sample_rate)
async def recv(self) -> AudioFrame:
await asyncio.sleep(self.samples_per_frame / self.sample_rate)
mono = await self.audio_bus.read(self.samples_per_frame)
samples = mono.reshape(1, -1)
frame = AudioFrame(format="s16", layout="mono", samples=self.samples_per_frame)
frame.planes[0].update(samples.tobytes())
frame.sample_rate = self.sample_rate
frame.pts = self._pts
frame.time_base = self._time_base
self._pts += self.samples_per_frame
return frame
webrtc/video_track.py
-21
View File
@@ -1,21 +0,0 @@
from __future__ import annotations
from core.state_machine import Arbitrator
from services.avatar import AvatarService
from webrtc.tracks import AudioBus, AvatarVideoTrack
__all__ = ["AvatarVideoTrack", "build_video_track"]
def build_video_track(
avatar_service: AvatarService,
arbitrator: Arbitrator,
audio_bus: AudioBus,
fps: int = 25,
) -> AvatarVideoTrack:
return AvatarVideoTrack(
avatar_service=avatar_service,
arbitrator=arbitrator,
audio_bus=audio_bus,
fps=fps,
)