大功告成
This commit is contained in:
@@ -0,0 +1,22 @@
|
|||||||
|
# LLM (DeepSeek, OpenAI-compatible)
|
||||||
|
# Fill your real key below.
|
||||||
|
LLM_API_KEY=sk-aac48115300845be97a032444fc11560
|
||||||
|
LLM_BASE_URL=https://api.deepseek.com/v1
|
||||||
|
LLM_MODEL=deepseek-chat
|
||||||
|
LLM_TIMEOUT=20
|
||||||
|
LLM_MAX_TOKENS=300
|
||||||
|
|
||||||
|
# Web service
|
||||||
|
WEBRTC_HOST=0.0.0.0
|
||||||
|
WEBRTC_PORT=8018
|
||||||
|
STUN_URL=stun:stun.l.google.com:19302
|
||||||
|
CORS_ORIGINS=*
|
||||||
|
|
||||||
|
# HTTPS (for cross-device mic/WebRTC)
|
||||||
|
SSL_CERTFILE=/home/xsl/code/product/certs/dev-cert.pem
|
||||||
|
SSL_KEYFILE=/home/xsl/code/product/certs/dev-key.pem
|
||||||
|
|
||||||
|
# 3D avatar driver mode
|
||||||
|
AVATAR_DRIVER_MODE=blendshape_stream
|
||||||
|
AVATAR_CONTROL_PROTOCOL=ws
|
||||||
|
AVATAR_BLENDSHAPE_SCHEMA=arkit
|
||||||
+386
@@ -0,0 +1,386 @@
|
|||||||
|
INFO: Started server process [401194]
|
||||||
|
INFO: Waiting for application startup.
|
||||||
|
INFO: Application startup complete.
|
||||||
|
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
|
||||||
|
INFO: 127.0.0.1:43090 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
|
||||||
|
return torch._C._cuda_getDeviceCount() > 0
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK"
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
|
||||||
|
super().__init__("LSTM", *args, **kwargs)
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
|
||||||
|
WeightNorm.apply(module, name, dim)
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found"
|
||||||
|
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||||||
|
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
|
||||||
|
import pkg_resources
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found"
|
||||||
|
Building prefix dict from the default dictionary ...
|
||||||
|
DEBUG:jieba:Building prefix dict from the default dictionary ...
|
||||||
|
Loading model from cache /tmp/jieba.cache
|
||||||
|
DEBUG:jieba:Loading model from cache /tmp/jieba.cache
|
||||||
|
Loading model cost 0.358 seconds.
|
||||||
|
DEBUG:jieba:Loading model cost 0.358 seconds.
|
||||||
|
Prefix dict has been built successfully.
|
||||||
|
DEBUG:jieba:Prefix dict has been built successfully.
|
||||||
|
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
|
||||||
|
INFO: 127.0.0.1:60258 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: 127.0.0.1:40524 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2190 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2190 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2190 - "GET /web/app.js HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2190 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2190 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:12419 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:9602 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:6375 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:6375 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6375 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(5xwO9ewTsb 1 udp 1853759231 172.28.96.1 60702 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 35860) -> ('172.28.96.1', 60702)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(0) ICE completed
|
||||||
|
INFO: 172.28.96.1:9899 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(0) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-27' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:1631 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1631 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1631 - "GET /web/app.js HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1631 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1631 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:6354 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:11967 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2728 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: 172.28.96.1:2728 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13395 - "GET /web/app.js HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4941 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4941 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:5793 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:3627 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(2) Discovered peer reflexive candidate Candidate(Ly5ELFkmha 1 udp 1853759231 172.28.96.1 50062 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(2) Check CandidatePair(('172.28.107.97', 48528) -> ('172.28.96.1', 50062)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(2) ICE completed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(2) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-941' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:1723 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1723 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1723 - "GET /web/app.js HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1723 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1723 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14573 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:7534 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(3) Discovered peer reflexive candidate Candidate(sCNsxRJ8b8 1 udp 1853759231 172.28.96.1 64421 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(3) Check CandidatePair(('172.28.107.97', 48775) -> ('172.28.96.1', 64421)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(3) ICE completed
|
||||||
|
INFO: 172.28.96.1:7236 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10460 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(3) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-1283' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:14796 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14796 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14796 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14796 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14796 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2032 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:14707 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:13486 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13486 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(4) Discovered peer reflexive candidate Candidate(Qady0BDpmC 1 udp 1853759231 172.28.96.1 53980 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(4) Check CandidatePair(('172.28.107.97', 41950) -> ('172.28.96.1', 53980)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(4) ICE completed
|
||||||
|
INFO: 172.28.96.1:13486 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-2812' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:13486 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13486 - "GET /web/app.js?v=20260327b HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13486 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13486 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10285 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:5486 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:13787 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:10285 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10285 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(5) Discovered peer reflexive candidate Candidate(BLyMgNoiBf 1 udp 1853759231 172.28.96.1 55730 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(5) Check CandidatePair(('172.28.107.97', 41397) -> ('172.28.96.1', 55730)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(5) ICE completed
|
||||||
|
INFO: 172.28.96.1:8337 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(5) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-2887' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:7750 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7750 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7750 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7750 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7750 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:1059 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:14463 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:14634 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14634 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(6) Discovered peer reflexive candidate Candidate(dbehuF1olz 1 udp 1853759231 172.28.96.1 56094 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(6) Check CandidatePair(('172.28.107.97', 45460) -> ('172.28.96.1', 56094)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(6) ICE completed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: 172.28.96.1:14493 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14493 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14493 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14493 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14493 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10905 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:10117 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:6762 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-4222' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:6762 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(7) Discovered peer reflexive candidate Candidate(8K1dQt9yU0 1 udp 1853759231 172.28.96.1 49725 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(7) Check CandidatePair(('172.28.107.97', 39523) -> ('172.28.96.1', 49725)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(7) ICE completed
|
||||||
|
INFO: 172.28.96.1:14132 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-4834' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:14132 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14132 - "GET /web/app.js?v=20260327c HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14132 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14132 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2807 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:6331 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:5113 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:2807 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:2807 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(8) Discovered peer reflexive candidate Candidate(ROwaVxsM9g 1 udp 1853759231 172.28.96.1 52410 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(8) Check CandidatePair(('172.28.107.97', 60159) -> ('172.28.96.1', 52410)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(8) ICE completed
|
||||||
|
INFO: 172.28.96.1:10113 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-6923' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 172.28.96.1:10113 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:4914 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10113 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:5903 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:3565 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:10113 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:10113 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(9) Discovered peer reflexive candidate Candidate(E8HKWSMFTW 1 udp 1853759231 172.28.96.1 50992 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(9) Check CandidatePair(('172.28.107.97', 43510) -> ('172.28.96.1', 50992)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(9) ICE completed
|
||||||
|
INFO: 172.28.96.1:13074 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-15396' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: 172.28.96.1:13074 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13074 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7285 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:7285 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13646 - "GET /favicon.ico HTTP/1.1" 404 Not Found
|
||||||
|
INFO: 172.28.96.1:9569 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:2629 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:13646 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:13646 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(10) Discovered peer reflexive candidate Candidate(AbT1vCac8Y 1 udp 1853759231 172.28.96.1 62912 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(10) Check CandidatePair(('172.28.107.97', 43507) -> ('172.28.96.1', 62912)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(10) ICE completed
|
||||||
|
INFO: 172.28.96.1:6176 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(10) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-16263' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: 127.0.0.1:51958 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: Shutting down
|
||||||
|
INFO: Waiting for application shutdown.
|
||||||
|
INFO: Application shutdown complete.
|
||||||
|
INFO: Finished server process [401194]
|
||||||
|
INFO: Started server process [404442]
|
||||||
|
INFO: Waiting for application startup.
|
||||||
|
INFO: Application startup complete.
|
||||||
|
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
|
||||||
|
INFO: 127.0.0.1:34264 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 12090). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)
|
||||||
|
return torch._C._cuda_getDeviceCount() > 0
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||||||
|
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||||||
|
WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/hexgrad/Kokoro-82M/f3ff3571791e39611d31c381e3a41a3af07b4987/config.json "HTTP/1.1 200 OK"
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py:1013: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
|
||||||
|
super().__init__("LSTM", *args, **kwargs)
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
|
||||||
|
WeightNorm.apply(module, name, dim)
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/kokoro-v1_0.pth "HTTP/1.1 302 Found"
|
||||||
|
/home/xsl/code/product/.venv/lib/python3.12/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
|
||||||
|
import pkg_resources
|
||||||
|
INFO:httpx:HTTP Request: HEAD https://huggingface.co/hexgrad/Kokoro-82M/resolve/main/voices/zf_xiaoxiao.pt "HTTP/1.1 302 Found"
|
||||||
|
Building prefix dict from the default dictionary ...
|
||||||
|
DEBUG:jieba:Building prefix dict from the default dictionary ...
|
||||||
|
Loading model from cache /tmp/jieba.cache
|
||||||
|
DEBUG:jieba:Loading model from cache /tmp/jieba.cache
|
||||||
|
Loading model cost 0.384 seconds.
|
||||||
|
DEBUG:jieba:Loading model cost 0.384 seconds.
|
||||||
|
Prefix dict has been built successfully.
|
||||||
|
DEBUG:jieba:Prefix dict has been built successfully.
|
||||||
|
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
|
||||||
|
INFO: 127.0.0.1:35522 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6932 - "GET / HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6932 - "GET /web/style.css HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6932 - "GET /web/app.js?v=20260327d HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6932 - "GET /health HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:6932 - "GET /events HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:14179 - "WebSocket /ws/subtitles" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:12729 - "WebSocket /ws/animation" [accepted]
|
||||||
|
INFO: connection open
|
||||||
|
INFO: 172.28.96.1:3368 - "GET /meta HTTP/1.1" 200 OK
|
||||||
|
INFO: 172.28.96.1:3368 - "POST /webrtc/offer HTTP/1.1" 200 OK
|
||||||
|
INFO:aioice.ice:Connection(0) Discovered peer reflexive candidate Candidate(jQ7HV4xKqV 1 udp 1853759231 172.28.96.1 52186 typ prflx)
|
||||||
|
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.WAITING -> State.IN_PROGRESS
|
||||||
|
INFO:aioice.ice:Connection(0) Check CandidatePair(('172.28.107.97', 45656) -> ('172.28.96.1', 52186)) State.IN_PROGRESS -> State.SUCCEEDED
|
||||||
|
INFO:aioice.ice:Connection(0) ICE completed
|
||||||
|
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
|
||||||
|
INFO: 172.28.96.1:8927 - "POST /chat/text HTTP/1.1" 200 OK
|
||||||
|
INFO: connection closed
|
||||||
|
INFO: connection closed
|
||||||
|
INFO:aioice.ice:Connection(0) Consent to send expired
|
||||||
|
ERROR:asyncio:Task exception was never retrieved
|
||||||
|
future: <Task finished name='Task-26' coro=<_consume_user_audio() done, defined at /home/xsl/code/product/main.py:250> exception=MediaStreamError()>
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "/home/xsl/code/product/main.py", line 253, in _consume_user_audio
|
||||||
|
frame = await track.recv()
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
File "/home/xsl/code/product/.venv/lib/python3.12/site-packages/aiortc/rtcrtpreceiver.py", line 209, in recv
|
||||||
|
raise MediaStreamError
|
||||||
|
aiortc.mediastreams.MediaStreamError
|
||||||
|
INFO: Shutting down
|
||||||
|
INFO: Waiting for application shutdown.
|
||||||
|
INFO: Application shutdown complete.
|
||||||
|
INFO: Finished server process [404442]
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
# 3D 数字人驱动重构方案
|
||||||
|
|
||||||
|
## 1. 目标变更
|
||||||
|
|
||||||
|
原链路：
|
||||||
|
|
||||||
|
ASR -> LLM -> TTS -> MuseTalk/图片或视频驱动 -> WebRTC 视频轨 -> 前端播放
|
||||||
|
|
||||||
|
新链路：
|
||||||
|
|
||||||
|
ASR -> LLM -> TTS -> 音频特征/viseme/blendshape 控制数据 -> 前端 3D 渲染引擎 -> 浏览器展示
|
||||||
|
|
||||||
|
核心变化：
|
||||||
|
- 后端不再负责生成数字人视频帧
|
||||||
|
- 后端保留实时语音、时序仲裁、字幕、音频下行
|
||||||
|
- 后端新增 3D 控制流输出能力
|
||||||
|
- 3D 模型渲染放到前端或独立渲染端
|
||||||
|
|
||||||
|
## 2. 为什么要这样改
|
||||||
|
|
||||||
|
MuseTalk 这类方案适合“2D 说话视频合成”，但不适合 3D 模型系统：
|
||||||
|
- 输出形态是视频帧，不是骨骼或 blendshape 控制数据
|
||||||
|
- 一旦换 3D 模型，原本的视频生成结果无法直接复用
|
||||||
|
- 每轮都做视频推理，延迟、GPU 占用、链路复杂度都偏高
|
||||||
|
- 3D 系统通常需要的是时间序列控制参数，而不是渲染后的像素
|
||||||
|
|
||||||
|
因此，3D 化的正确边界是：
|
||||||
|
- 后端负责“说什么、怎么说、何时说”
|
||||||
|
- 渲染端负责“模型如何动、如何显示”
|
||||||
|
|
||||||
|
## 3. 本次代码重构内容
|
||||||
|
|
||||||
|
当前代码已调整为：
|
||||||
|
- WebRTC 下行仅保留音频轨
|
||||||
|
- 新增 `/ws/animation` WebSocket，下发 3D 动画控制数据
|
||||||
|
- `services/avatar.py` 改为音频到控制参数的驱动服务
|
||||||
|
- `/avatar/schema` 暴露控制协议说明
|
||||||
|
- 前端采用 Three.js + GLTFLoader + VRM loader，直接在浏览器渲染 3D 模型
|
||||||
|
- 支持加载本地 VRM/GLB 文件，也支持加载 URL 形式的 VRM/GLB/GLTF
|
||||||
|
- 若未加载模型，页面会回退到一个调试用的 3D 头像，便于联调口型和头部姿态
|
||||||
|
|
||||||
|
当前控制协议字段包括：
|
||||||
|
- `jawOpen`
|
||||||
|
- `mouthClose`
|
||||||
|
- `mouthFunnel`
|
||||||
|
- `mouthPucker`
|
||||||
|
- `viseme_aa`
|
||||||
|
- `viseme_ee`
|
||||||
|
- `viseme_oh`
|
||||||
|
- `headYaw`
|
||||||
|
- `headPitch`
|
||||||
|
- `headRoll`
|
||||||
|
|
||||||
|
输出包结构示例：
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "animation_chunk",
|
||||||
|
"driver": "arkit-blendshape-v1",
|
||||||
|
"mode": "blendshape_stream",
|
||||||
|
"schema": "arkit",
|
||||||
|
"chunk_index": 0,
|
||||||
|
"total_chunks": 3,
|
||||||
|
"sample_rate": 24000,
|
||||||
|
"fps": 25,
|
||||||
|
"text": "你好，欢迎使用系统。",
|
||||||
|
"frame_count": 18,
|
||||||
|
"duration_ms": 720,
|
||||||
|
"frames": [
|
||||||
|
{
|
||||||
|
"seq": 102,
|
||||||
|
"time_ms": 0,
|
||||||
|
"speaking": true,
|
||||||
|
"controls": {
|
||||||
|
"jawOpen": 0.41,
|
||||||
|
"mouthClose": 0.66,
|
||||||
|
"mouthFunnel": 0.27,
|
||||||
|
"mouthPucker": 0.19,
|
||||||
|
"viseme_aa": 0.41,
|
||||||
|
"viseme_ee": 0.24,
|
||||||
|
"viseme_oh": 0.31,
|
||||||
|
"headYaw": 0.004,
|
||||||
|
"headPitch": 0.009,
|
||||||
|
"headRoll": 0.001
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 当前实现的定位
|
||||||
|
|
||||||
|
当前实现是“联调级骨架”，作用是：
|
||||||
|
- 先把系统边界改对
|
||||||
|
- 让前后端可以并行开发
|
||||||
|
- 让 3D 引擎接入方尽快有稳定协议可以消费
|
||||||
|
|
||||||
|
它不是最终口型算法，原因很明确：
|
||||||
|
- 目前控制值来自音频能量和频谱特征
|
||||||
|
- 没有做 phoneme 强制对齐
|
||||||
|
- 没有做语言相关 viseme 映射
|
||||||
|
- 没有结合 3D 模型自身 rig 约束做校准
|
||||||
|
|
||||||
|
所以这版适合：
|
||||||
|
- 打通链路
|
||||||
|
- 调接口
|
||||||
|
- 验证时序
|
||||||
|
- 校验驱动字段
|
||||||
|
|
||||||
|
不适合直接作为最终高保真口型方案。
|
||||||
|
|
||||||
|
## 5. 推荐的生产化方案
|
||||||
|
|
||||||
|
### 方案 A：前端 Three.js / React Three Fiber
|
||||||
|
|
||||||
|
适合：
|
||||||
|
- 浏览器直接渲染 WebGL 角色
|
||||||
|
- glTF/VRM 模型
|
||||||
|
- ARKit blendshape 或自定义 morph target
|
||||||
|
|
||||||
|
建议做法：
|
||||||
|
- 前端加载 3D 模型
|
||||||
|
- 建立 `controlsKey -> morphTargetInfluence` 映射表
|
||||||
|
- `/ws/animation` 按时间戳缓冲一小段再播放
|
||||||
|
- 音频继续使用 WebRTC 远端音频轨
|
||||||
|
- 前端用远端音频播放时间作为动画时钟基准
|
||||||
|
|
||||||
|
当前仓库已经按这个方向落地了第一版：
|
||||||
|
- 静态页面通过 import map 从 CDN 引入 `three`、`GLTFLoader`、`@pixiv/three-vrm`
|
||||||
|
- VRM 优先使用 `expressionManager` 写入 `aa/ee/oh/ou`
|
||||||
|
- 普通 glTF 则尝试把控制字段映射到 morph target
|
||||||
|
- 头部姿态优先驱动 `head/neck` 骨骼，找不到时回退调试头像
|
||||||
|
|
||||||
|
### 方案 B：Unity/Unreal 渲染端
|
||||||
|
|
||||||
|
适合：
|
||||||
|
- 更复杂的材质、灯光、骨骼、表情系统
|
||||||
|
- 要求更强的 3D 表现力
|
||||||
|
|
||||||
|
建议做法：
|
||||||
|
- 浏览器只做 UI 和音频交互
|
||||||
|
- Unity/Unreal 作为单独渲染客户端消费 `/ws/animation`
|
||||||
|
- 浏览器与渲染端通过流媒体或共享画面集成
|
||||||
|
|
||||||
|
### 方案 C：专用音频驱动服务
|
||||||
|
|
||||||
|
适合：
|
||||||
|
- 追求更准的嘴型
|
||||||
|
- 后端可以接受增加一个模型服务
|
||||||
|
|
||||||
|
可以考虑接入：
|
||||||
|
- phoneme 对齐模型
|
||||||
|
- viseme 预测模型
|
||||||
|
- NVIDIA Audio2Face 类服务
|
||||||
|
- 自研 `audio -> blendshape` 网络
|
||||||
|
|
||||||
|
推荐协议保持不变：
|
||||||
|
- 外部模型服务可以替换 `services/avatar.py`
|
||||||
|
- 对外仍输出统一 `frames[].controls`
|
||||||
|
|
||||||
|
## 6. 推荐的下一步实施顺序
|
||||||
|
|
||||||
|
1. 先确定 3D 模型和渲染端技术栈
|
||||||
|
2. 固定一份 blendshape 映射表
|
||||||
|
3. 把当前前端调试面板替换成真实 3D 场景
|
||||||
|
4. 增加时间同步与缓冲策略
|
||||||
|
5. 再替换掉启发式控制算法，接入更强的 viseme 模型
|
||||||
|
|
||||||
|
## 7. 接口约束建议
|
||||||
|
|
||||||
|
为了后续可替换，建议保持这些约束不变：
|
||||||
|
- 下行动画通道继续使用 `/ws/animation`
|
||||||
|
- 每条消息包含 `frame_count`、`fps`、`frames`
|
||||||
|
- 每帧保留 `seq` 和 `time_ms`
|
||||||
|
- `controls` 使用稳定字段名，不轻易改动
|
||||||
|
- 新增字段时只追加，不破坏已有字段
|
||||||
|
|
||||||
|
## 8. 风险与注意事项
|
||||||
|
|
||||||
|
- 真实 3D 模型的 blendshape 命名不一定和当前字段一致，需要做一次映射层
|
||||||
|
- 如果模型是骨骼嘴型而不是 morph target，需要把 `controls` 转成骨骼驱动参数
|
||||||
|
- 浏览器渲染时要注意音频时钟和动画时钟一致，否则会出现“听起来对，但嘴型慢半拍”
|
||||||
|
- 如果后续改成 phoneme/viseme 对齐，建议在包里增加 phoneme 片段和置信度字段
|
||||||
|
|
||||||
|
## 9. 结论
|
||||||
|
|
||||||
|
这次重构的重点不是“把 2D 数字人继续硬改下去”，而是把系统边界调整到适合 3D 模型的形态。
|
||||||
|
|
||||||
|
现在的系统已经从“视频合成型数字人”切到了“控制流型数字人”，后续无论接 Three.js、Unity、Unreal，还是专门的 Audio2Face/viseme 服务，都可以在这个边界上继续演进。
|
||||||
+18
-10
@@ -15,7 +15,7 @@
|
|||||||
- 用户语音上行（VAD + ASR）
|
- 用户语音上行（VAD + ASR）
|
||||||
- 文本调用在线 LLM
|
- 文本调用在线 LLM
|
||||||
- 本地 TTS 合成语音
|
- 本地 TTS 合成语音
|
||||||
- 数字人视频下行（MuseTalk 优先，失败回退写实底片）
|
- WebRTC 音频下行 + 3D 数字人控制流下行（WebSocket）
|
||||||
- 前端字幕与指标面板（SSE + WebSocket）
|
- 前端字幕与指标面板（SSE + WebSocket）
|
||||||
- 单工仲裁（支持 barge-in 打断）
|
- 单工仲裁（支持 barge-in 打断）
|
||||||
|
|
||||||
@@ -36,10 +36,10 @@
|
|||||||
- `llm.py`：在线模型 + fallback
|
- `llm.py`：在线模型 + fallback
|
||||||
- `tts.py`：Kokoro
|
- `tts.py`：Kokoro
|
||||||
- `vad.py`：Silero VAD
|
- `vad.py`：Silero VAD
|
||||||
- `avatar.py`：MuseTalk 推理调度 + 写实回退帧
|
- `avatar.py`：音频转 3D 控制数据（blendshape / 头部姿态）
|
||||||
- `webrtc/`
|
- `webrtc/`
|
||||||
- `tracks.py`：实际音视频轨道实现
|
- `tracks.py`：实际音频轨道实现
|
||||||
- `gateway.py`、`video_track.py`、`audio_track.py`：结构化封装
|
- `gateway.py`、`audio_track.py`：结构化封装
|
||||||
- `web/`
|
- `web/`
|
||||||
- `index.html`、`app.js`、`style.css`
|
- `index.html`、`app.js`、`style.css`
|
||||||
- `scripts/`
|
- `scripts/`
|
||||||
@@ -192,11 +192,18 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c
|
|||||||
## 7. 前端功能说明
|
## 7. 前端功能说明
|
||||||
|
|
||||||
网页包含：
|
网页包含：
|
||||||
- 视频窗口（数字人）
|
- Three.js 3D 数字人舞台
|
||||||
- 文本输入框
|
- 文本输入框
|
||||||
- 字幕消息区（WebSocket）
|
- 字幕消息区（WebSocket）
|
||||||
- 指标面板（SSE）
|
- 指标面板（SSE）
|
||||||
|
|
||||||
|
当前前端能力：
|
||||||
|
- 浏览器通过 import map 直接加载 Three.js、GLTFLoader、three-vrm
|
||||||
|
- 可加载本地 `VRM`/`GLB` 文件
|
||||||
|
- 可加载 URL 形式的 `VRM`/`GLB`/`GLTF` 模型
|
||||||
|
- 页面默认会自动尝试加载官方 `three-vrm` 示例模型：`VRM1_Constraint_Twist_Sample.vrm`
|
||||||
|
- 未加载真实模型时会显示调试头像，仍可联调动画控制流
|
||||||
|
|
||||||
当前已支持：
|
当前已支持：
|
||||||
- 页面刷新自动重连
|
- 页面刷新自动重连
|
||||||
- 单连接策略（新连接踢掉旧连接）
|
- 单连接策略（新连接踢掉旧连接）
|
||||||
@@ -207,9 +214,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c
|
|||||||
|
|
||||||
## 8. 当前已知边界
|
## 8. 当前已知边界
|
||||||
|
|
||||||
- MuseTalk 结果生成可能有一定延迟，期间使用写实底片回退。
|
- 当前后端输出的是可直接对接 3D 引擎的控制数据，不负责最终 3D 渲染。
|
||||||
- 部分“主观体验类”验收项（如口型同步主观评分）需要人工实测判定。
|
- 当前控制数据基于音频能量和频谱特征做启发式推断，适合作为联调骨架，不等同于生产级口型模型。
|
||||||
- Qwen3-TTS 对比方案尚未并入主链路（可后续做 A/B 开关）。
|
- 若要达到更高口型精度，建议后续接入 phoneme/viseme 对齐模型或 Audio2Face 类推理服务。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -224,8 +231,9 @@ bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.c
|
|||||||
- 系统是单连接模式，新连接会断开旧连接（预期）
|
- 系统是单连接模式，新连接会断开旧连接（预期）
|
||||||
|
|
||||||
3) 口型不动
|
3) 口型不动
|
||||||
- 查看 `/health` 中 `avatar.queued_frames` 和 `avatar.last_error`
|
- 查看 `/health` 中 `avatar.last_frame_count` 和 `avatar.last_error`
|
||||||
- 若 MuseTalk 未产帧，会回退写实底片 + 口型叠加
|
- 查看前端 `/ws/animation` 是否已连接，控制流预览是否持续刷新
|
||||||
|
- 若模型已加载但不动，先检查该模型是否带有 VRM expression 或 morph target
|
||||||
|
|
||||||
4) 模型异常
|
4) 模型异常
|
||||||
- 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD
|
- 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD
|
||||||
|
|||||||
@@ -0,0 +1,278 @@
|
|||||||
|
# 3D Digital Human System Technical Documentation
|
||||||
|
|
||||||
|
## 1. Project Summary
|
||||||
|
|
||||||
|
This project implements a real-time conversational digital human system with these core capabilities:
|
||||||
|
|
||||||
|
- Text and voice interaction
|
||||||
|
- LLM response generation (DeepSeek online, with fallback)
|
||||||
|
- TTS synthesis for reply audio
|
||||||
|
- Audio-driven 3D animation control stream
|
||||||
|
- Browser-side 3D rendering (Three.js + VRM/GLTF)
|
||||||
|
- WebRTC audio transport and WebSocket animation transport
|
||||||
|
|
||||||
|
Current workspace root:
|
||||||
|
|
||||||
|
- `/home/xsl/code/product`
|
||||||
|
|
||||||
|
## 2. Runtime Architecture
|
||||||
|
|
||||||
|
### 2.1 High-Level Pipeline
|
||||||
|
|
||||||
|
1. User input enters through `/chat/text` or voice pipeline.
|
||||||
|
2. LLM generates reply text.
|
||||||
|
3. TTS synthesizes reply waveform.
|
||||||
|
4. Avatar service converts audio chunks to animation control frames.
|
||||||
|
5. Backend pushes:
|
||||||
|
- reply audio to WebRTC audio track
|
||||||
|
- animation frames to `/ws/animation`
|
||||||
|
6. Frontend renders 3D model and applies mouth/head/body motions.
|
||||||
|
|
||||||
|
### 2.2 Transport Split
|
||||||
|
|
||||||
|
- WebRTC: audio only
|
||||||
|
- WebSocket (`/ws/animation`): animation control data
|
||||||
|
- SSE (`/events`): system metrics and runtime status
|
||||||
|
|
||||||
|
This split removes server-side video generation and makes rendering native in the browser.
|
||||||
|
|
||||||
|
## 3. Core Modules
|
||||||
|
|
||||||
|
### 3.1 Backend Entry
|
||||||
|
|
||||||
|
- `main.py`
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
|
||||||
|
- FastAPI app and route registration
|
||||||
|
- WebRTC offer/peer management
|
||||||
|
- Animation WebSocket broadcasting
|
||||||
|
- Runtime metrics and `/health`
|
||||||
|
- Session reset and chat endpoints
|
||||||
|
|
||||||
|
### 3.2 Pipeline Orchestration
|
||||||
|
|
||||||
|
- `core/pipeline.py`
|
||||||
|
- `core/state_machine.py`
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
|
||||||
|
- Turn-based orchestration (input -> LLM -> TTS -> output)
|
||||||
|
- State transitions (idle, thinking, speaking)
|
||||||
|
- Latency accounting and result metadata
|
||||||
|
|
||||||
|
### 3.3 Services
|
||||||
|
|
||||||
|
- `services/llm.py`
|
||||||
|
- `services/tts.py`
|
||||||
|
- `services/asr.py`
|
||||||
|
- `services/vad.py`
|
||||||
|
- `services/avatar.py`
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
|
||||||
|
- `LLMService`: online model calls via OpenAI-compatible client
|
||||||
|
- `TTSService`: speech synthesis
|
||||||
|
- `AvatarService`: audio-to-controls mapping for mouth/viseme/head motion
|
||||||
|
|
||||||
|
### 3.4 Frontend
|
||||||
|
|
||||||
|
- `web/index.html`
|
||||||
|
- `web/app.js`
|
||||||
|
- `web/style.css`
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
|
||||||
|
- Three.js scene setup and camera controls
|
||||||
|
- VRM/GLTF model loading (URL or local file)
|
||||||
|
- Animation buffering and frame application
|
||||||
|
- Idle body pose + talking expressions
|
||||||
|
- WebRTC audio playback
|
||||||
|
|
||||||
|
## 4. Animation Data Contract
|
||||||
|
|
||||||
|
Backend pushes animation chunks shaped like:
|
||||||
|
|
||||||
|
- `type`: `animation_chunk` / `animation_reset` / `animation_state`
|
||||||
|
- `fps`, `duration_ms`, `frame_count`
|
||||||
|
- `frames[]` with:
|
||||||
|
- `seq`
|
||||||
|
- `time_ms`
|
||||||
|
- `controls`
|
||||||
|
|
||||||
|
Typical controls include:
|
||||||
|
|
||||||
|
- `jawOpen`
|
||||||
|
- `viseme_aa`, `viseme_ee`, `viseme_oh`
|
||||||
|
- `mouthPucker`
|
||||||
|
- `headYaw`, `headPitch`, `headRoll`
|
||||||
|
|
||||||
|
Frontend behavior:
|
||||||
|
|
||||||
|
- buffers frames for stable playback
|
||||||
|
- applies VRM expressions when available
|
||||||
|
- falls back to morph targets or bone motion
|
||||||
|
|
||||||
|
## 5. Configuration
|
||||||
|
|
||||||
|
Configuration file:
|
||||||
|
|
||||||
|
- `.env` (root)
|
||||||
|
|
||||||
|
### 5.1 LLM Settings
|
||||||
|
|
||||||
|
Supported API key aliases:
|
||||||
|
|
||||||
|
- `LLM_API_KEY`
|
||||||
|
- `DEEPSEEK_API_KEY`
|
||||||
|
- `OPENAI_API_KEY`
|
||||||
|
|
||||||
|
Recommended DeepSeek settings:
|
||||||
|
|
||||||
|
- `LLM_BASE_URL=https://api.deepseek.com/v1`
|
||||||
|
- `LLM_MODEL=deepseek-chat`
|
||||||
|
|
||||||
|
### 5.2 Server Settings
|
||||||
|
|
||||||
|
- `WEBRTC_HOST` (default `0.0.0.0`)
|
||||||
|
- `WEBRTC_PORT` (default `8018`)
|
||||||
|
- `SSL_CERTFILE`
|
||||||
|
- `SSL_KEYFILE`
|
||||||
|
|
||||||
|
### 5.3 Avatar Settings
|
||||||
|
|
||||||
|
- `AVATAR_DRIVER_MODE=blendshape_stream`
|
||||||
|
- `AVATAR_CONTROL_PROTOCOL=ws`
|
||||||
|
- `AVATAR_BLENDSHAPE_SCHEMA=arkit`
|
||||||
|
|
||||||
|
## 6. Startup and Process Control
|
||||||
|
|
||||||
|
New process script:
|
||||||
|
|
||||||
|
- `scripts/service-8018.sh`
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/service-8018.sh start
|
||||||
|
bash scripts/service-8018.sh status
|
||||||
|
bash scripts/service-8018.sh logs
|
||||||
|
bash scripts/service-8018.sh restart
|
||||||
|
bash scripts/service-8018.sh stop
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- Loads environment from `.env`
|
||||||
|
- Starts uvicorn from project venv if available
|
||||||
|
- Supports HTTPS cert/key from `.env`
|
||||||
|
- Writes pid and log to `.run/`
|
||||||
|
- Cleans stale python listeners on target port
|
||||||
|
|
||||||
|
Runtime files:
|
||||||
|
|
||||||
|
- `.run/visual-chat-8018.pid`
|
||||||
|
- `.run/visual-chat-8018.log`
|
||||||
|
|
||||||
|
## 7. API and Web Endpoints
|
||||||
|
|
||||||
|
### 7.1 Health and Metadata
|
||||||
|
|
||||||
|
- `GET /health`
|
||||||
|
- `GET /meta`
|
||||||
|
|
||||||
|
### 7.2 Chat and Session
|
||||||
|
|
||||||
|
- `POST /chat/text`
|
||||||
|
- `POST /chat/reset`
|
||||||
|
|
||||||
|
### 7.3 Streaming Channels
|
||||||
|
|
||||||
|
- `GET /events` (SSE)
|
||||||
|
- `GET /ws/subtitles` (WebSocket)
|
||||||
|
- `GET /ws/animation` (WebSocket)
|
||||||
|
|
||||||
|
### 7.4 WebRTC
|
||||||
|
|
||||||
|
- `POST /webrtc/offer`
|
||||||
|
|
||||||
|
## 8. Validation Checklist
|
||||||
|
|
||||||
|
### 8.1 Service Up
|
||||||
|
|
||||||
|
- `bash scripts/service-8018.sh status` should show running
|
||||||
|
- `GET /health` should return valid JSON
|
||||||
|
|
||||||
|
### 8.2 LLM Online
|
||||||
|
|
||||||
|
Health fields should indicate:
|
||||||
|
|
||||||
|
- `llm.ready = true`
|
||||||
|
- `llm.error = null`
|
||||||
|
- `last_llm_source = online` after at least one chat turn
|
||||||
|
|
||||||
|
### 8.3 Frontend Motion and Audio
|
||||||
|
|
||||||
|
- Model loads successfully
|
||||||
|
- Mouth motion follows replies
|
||||||
|
- Body remains in non-T-pose idle stance
|
||||||
|
- Remote audio is audible via WebRTC
|
||||||
|
|
||||||
|
## 9. Troubleshooting Guide
|
||||||
|
|
||||||
|
### 9.1 LLM stays fallback
|
||||||
|
|
||||||
|
Symptoms:
|
||||||
|
|
||||||
|
- `last_llm_source = fallback`
|
||||||
|
- `llm.ready = false`
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
1. Confirm `.env` exists in workspace root.
|
||||||
|
2. Confirm key value is set in `LLM_API_KEY` (or alias).
|
||||||
|
3. Restart service after any `.env` update.
|
||||||
|
4. Verify `llm.base_url` and `llm.model` in `/health`.
|
||||||
|
|
||||||
|
### 9.2 Model load errors
|
||||||
|
|
||||||
|
Symptoms:
|
||||||
|
|
||||||
|
- Web UI shows model fetch failures
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
1. Use reachable URL or local `.vrm` file.
|
||||||
|
2. Confirm browser can access external CDN.
|
||||||
|
3. Prefer VRM full-body models for body-pose control.
|
||||||
|
|
||||||
|
### 9.3 Mouth not moving
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
1. Confirm animation websocket connected.
|
||||||
|
2. Confirm animation frame count increases.
|
||||||
|
3. Confirm TTS is ready and not in silence fallback.
|
||||||
|
|
||||||
|
### 9.4 HTTPS/microphone issues on LAN
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
1. Use valid cert paths in `.env`.
|
||||||
|
2. Trust self-signed cert on browser/device.
|
||||||
|
3. Ensure firewall allows target port.
|
||||||
|
|
||||||
|
## 10. Security and Operations Notes
|
||||||
|
|
||||||
|
- Keep `.env` out of source control.
|
||||||
|
- Rotate API keys regularly.
|
||||||
|
- Restrict exposed port by network policy if internet-facing.
|
||||||
|
- Consider reverse proxy and managed TLS for production.
|
||||||
|
|
||||||
|
## 11. Suggested Next Production Steps
|
||||||
|
|
||||||
|
1. Add structured request tracing (request id, turn id).
|
||||||
|
2. Add persistent metrics export (Prometheus/OpenTelemetry).
|
||||||
|
3. Add automatic reconnect and health watchdog.
|
||||||
|
4. Add integration tests for LLM online/fallback branches.
|
||||||
|
5. Add model asset mirror to avoid public CDN instability.
|
||||||
Binary file not shown.
Binary file not shown.
@@ -1,3 +1,4 @@
|
|||||||
|
from pydantic import AliasChoices, Field
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
|
|
||||||
@@ -5,20 +6,33 @@ class Settings(BaseSettings):
|
|||||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
|
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
|
||||||
|
|
||||||
hf_token: str = ""
|
hf_token: str = ""
|
||||||
llm_api_key: str = ""
|
llm_api_key: str = Field(
|
||||||
llm_base_url: str = "https://api.openai.com/v1"
|
default="",
|
||||||
llm_model: str = "gpt-4o-mini"
|
validation_alias=AliasChoices("LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY"),
|
||||||
|
)
|
||||||
|
llm_base_url: str = Field(
|
||||||
|
default="https://api.deepseek.com/v1",
|
||||||
|
validation_alias=AliasChoices("LLM_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"),
|
||||||
|
)
|
||||||
|
llm_model: str = Field(
|
||||||
|
default="deepseek-chat",
|
||||||
|
validation_alias=AliasChoices("LLM_MODEL", "DEEPSEEK_MODEL", "OPENAI_MODEL"),
|
||||||
|
)
|
||||||
llm_timeout: int = 8
|
llm_timeout: int = 8
|
||||||
llm_max_tokens: int = 150
|
llm_max_tokens: int = 150
|
||||||
|
|
||||||
webrtc_host: str = "0.0.0.0"
|
webrtc_host: str = "0.0.0.0"
|
||||||
webrtc_port: int = 8080
|
webrtc_port: int = 8080
|
||||||
avatar_fps: int = 25
|
avatar_fps: int = 25
|
||||||
|
avatar_driver_mode: str = "blendshape_stream"
|
||||||
|
avatar_control_protocol: str = "ws"
|
||||||
|
avatar_blendshape_schema: str = "arkit"
|
||||||
stun_url: str = "stun:stun.l.google.com:19302"
|
stun_url: str = "stun:stun.l.google.com:19302"
|
||||||
cors_origins: str = "*"
|
cors_origins: str = "*"
|
||||||
ssl_certfile: str = ""
|
ssl_certfile: str = ""
|
||||||
ssl_keyfile: str = ""
|
ssl_keyfile: str = ""
|
||||||
|
|
||||||
|
# Legacy MuseTalk settings retained for migration reference only.
|
||||||
musetalk_enabled: bool = True
|
musetalk_enabled: bool = True
|
||||||
musetalk_repo_dir: str = "/home/xsl/work/MuseTalk"
|
musetalk_repo_dir: str = "/home/xsl/work/MuseTalk"
|
||||||
musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py"
|
musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py"
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+7
-4
@@ -11,6 +11,9 @@ from services.llm import LLMService
|
|||||||
from services.tts import TTSService
|
from services.tts import TTSService
|
||||||
|
|
||||||
|
|
||||||
|
AudioChunkCallback = Callable[[np.ndarray, int, str, int, int], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
class ChatPipeline:
|
class ChatPipeline:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -75,7 +78,7 @@ class ChatPipeline:
|
|||||||
async def _synthesize_sentence_first(
|
async def _synthesize_sentence_first(
|
||||||
self,
|
self,
|
||||||
reply_text: str,
|
reply_text: str,
|
||||||
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None,
|
on_audio_chunk: Optional[AudioChunkCallback] = None,
|
||||||
on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
||||||
should_interrupt: Optional[Callable[[], bool]] = None,
|
should_interrupt: Optional[Callable[[], bool]] = None,
|
||||||
) -> tuple[np.ndarray, int, int, int]:
|
) -> tuple[np.ndarray, int, int, int]:
|
||||||
@@ -94,7 +97,7 @@ class ChatPipeline:
|
|||||||
if i == 0:
|
if i == 0:
|
||||||
first_chunk_ms = int((time.perf_counter() - t0) * 1000)
|
first_chunk_ms = int((time.perf_counter() - t0) * 1000)
|
||||||
if on_audio_chunk is not None:
|
if on_audio_chunk is not None:
|
||||||
await on_audio_chunk(seg_audio, seg_sr)
|
await on_audio_chunk(seg_audio, seg_sr, seg, i, len(segments))
|
||||||
audio_chunks.append(seg_audio)
|
audio_chunks.append(seg_audio)
|
||||||
tts_total_ms = int((time.perf_counter() - t0) * 1000)
|
tts_total_ms = int((time.perf_counter() - t0) * 1000)
|
||||||
audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32)
|
audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32)
|
||||||
@@ -103,7 +106,7 @@ class ChatPipeline:
|
|||||||
async def process_turn(
|
async def process_turn(
|
||||||
self,
|
self,
|
||||||
audio_16k: np.ndarray,
|
audio_16k: np.ndarray,
|
||||||
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None,
|
on_audio_chunk: Optional[AudioChunkCallback] = None,
|
||||||
on_user_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
on_user_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
||||||
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
||||||
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
||||||
@@ -145,7 +148,7 @@ class ChatPipeline:
|
|||||||
async def process_text_turn(
|
async def process_text_turn(
|
||||||
self,
|
self,
|
||||||
user_text: str,
|
user_text: str,
|
||||||
on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None,
|
on_audio_chunk: Optional[AudioChunkCallback] = None,
|
||||||
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None,
|
||||||
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None,
|
||||||
) -> tuple[dict, np.ndarray, int]:
|
) -> tuple[dict, np.ndarray, int]:
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
@@ -11,8 +11,8 @@ import numpy as np
|
|||||||
from aiortc import RTCPeerConnection, RTCSessionDescription
|
from aiortc import RTCPeerConnection, RTCSessionDescription
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
|
from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
|
||||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ from services.avatar import AvatarService
|
|||||||
from services.llm import LLMService
|
from services.llm import LLMService
|
||||||
from services.tts import TTSService
|
from services.tts import TTSService
|
||||||
from services.vad import VADService
|
from services.vad import VADService
|
||||||
from webrtc.tracks import AudioBus, AvatarAudioTrack, AvatarVideoTrack
|
from webrtc.tracks import AudioBus, AvatarAudioTrack
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
if settings.hf_token:
|
if settings.hf_token:
|
||||||
@@ -51,6 +51,7 @@ pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service)
|
|||||||
audio_bus = AudioBus(sample_rate=48000)
|
audio_bus = AudioBus(sample_rate=48000)
|
||||||
pcs: set[RTCPeerConnection] = set()
|
pcs: set[RTCPeerConnection] = set()
|
||||||
subtitle_clients: set[WebSocket] = set()
|
subtitle_clients: set[WebSocket] = set()
|
||||||
|
animation_clients: set[WebSocket] = set()
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -69,6 +70,8 @@ class RuntimeState:
|
|||||||
last_input_mode: str = "none"
|
last_input_mode: str = "none"
|
||||||
vad_start_count: int = 0
|
vad_start_count: int = 0
|
||||||
vad_end_count: int = 0
|
vad_end_count: int = 0
|
||||||
|
last_animation_mode: str = settings.avatar_driver_mode
|
||||||
|
last_animation_frame_count: int = 0
|
||||||
busy: bool = False
|
busy: bool = False
|
||||||
buffer_16k: list[np.ndarray] = field(default_factory=list)
|
buffer_16k: list[np.ndarray] = field(default_factory=list)
|
||||||
|
|
||||||
@@ -80,6 +83,18 @@ class TextChatRequest(BaseModel):
|
|||||||
text: str
|
text: str
|
||||||
|
|
||||||
|
|
||||||
|
async def _broadcast_json(clients: set[WebSocket], payload: dict) -> None:
|
||||||
|
stale: list[WebSocket] = []
|
||||||
|
message = json.dumps(payload, ensure_ascii=False)
|
||||||
|
for ws in list(clients):
|
||||||
|
try:
|
||||||
|
await ws.send_text(message)
|
||||||
|
except Exception:
|
||||||
|
stale.append(ws)
|
||||||
|
for ws in stale:
|
||||||
|
clients.discard(ws)
|
||||||
|
|
||||||
|
|
||||||
async def _broadcast_subtitle(
|
async def _broadcast_subtitle(
|
||||||
role: str,
|
role: str,
|
||||||
text: str,
|
text: str,
|
||||||
@@ -90,22 +105,54 @@ async def _broadcast_subtitle(
|
|||||||
) -> None:
|
) -> None:
|
||||||
if not text:
|
if not text:
|
||||||
return
|
return
|
||||||
payload = {
|
await _broadcast_json(
|
||||||
"role": role,
|
subtitle_clients,
|
||||||
"text": text,
|
{
|
||||||
"source": source,
|
"role": role,
|
||||||
"partial": partial,
|
"text": text,
|
||||||
"final": final,
|
"source": source,
|
||||||
"ts_ms": int(time.time() * 1000),
|
"partial": partial,
|
||||||
|
"final": final,
|
||||||
|
"ts_ms": int(time.time() * 1000),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _broadcast_animation(payload: dict) -> None:
|
||||||
|
runtime.last_animation_mode = str(payload.get("driver", settings.avatar_driver_mode))
|
||||||
|
runtime.last_animation_frame_count = int(payload.get("frame_count", 0))
|
||||||
|
await _broadcast_json(animation_clients, payload)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_payload() -> dict:
|
||||||
|
return {
|
||||||
|
"state": str(arbitrator.state),
|
||||||
|
"peers": len(pcs),
|
||||||
|
"subtitle_clients": len(subtitle_clients),
|
||||||
|
"animation_clients": len(animation_clients),
|
||||||
|
"pipeline_runs": runtime.pipeline_runs,
|
||||||
|
"last_latency_ms": runtime.last_latency_ms,
|
||||||
|
"last_asr_latency_ms": runtime.last_asr_latency_ms,
|
||||||
|
"last_llm_latency_ms": runtime.last_llm_latency_ms,
|
||||||
|
"last_llm_source": runtime.last_llm_source,
|
||||||
|
"last_tts_latency_ms": runtime.last_tts_latency_ms,
|
||||||
|
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
|
||||||
|
"last_barge_in_ms": runtime.last_barge_in_ms,
|
||||||
|
"barge_in_count": runtime.barge_in_count,
|
||||||
|
"last_input_mode": runtime.last_input_mode,
|
||||||
|
"vad_start_count": runtime.vad_start_count,
|
||||||
|
"vad_end_count": runtime.vad_end_count,
|
||||||
|
"last_user_text": runtime.last_user_text,
|
||||||
|
"last_reply_text": runtime.last_reply_text,
|
||||||
|
"last_animation_mode": runtime.last_animation_mode,
|
||||||
|
"last_animation_frame_count": runtime.last_animation_frame_count,
|
||||||
|
"pipeline_busy": runtime.busy,
|
||||||
|
"llm": llm_service.health,
|
||||||
|
"asr": asr_service.health,
|
||||||
|
"tts": tts_service.health,
|
||||||
|
"vad": vad_service.health,
|
||||||
|
"avatar": avatar_service.health,
|
||||||
}
|
}
|
||||||
stale: list[WebSocket] = []
|
|
||||||
for ws in list(subtitle_clients):
|
|
||||||
try:
|
|
||||||
await ws.send_text(json.dumps(payload, ensure_ascii=False))
|
|
||||||
except Exception:
|
|
||||||
stale.append(ws)
|
|
||||||
for ws in stale:
|
|
||||||
subtitle_clients.discard(ws)
|
|
||||||
|
|
||||||
|
|
||||||
def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray:
|
def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray:
|
||||||
@@ -119,7 +166,6 @@ def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray:
|
|||||||
|
|
||||||
|
|
||||||
def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray:
|
def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray:
|
||||||
# aiortc audio ndarray is typically (channels, samples)
|
|
||||||
if pcm.ndim == 2:
|
if pcm.ndim == 2:
|
||||||
mono = pcm.mean(axis=0)
|
mono = pcm.mean(axis=0)
|
||||||
else:
|
else:
|
||||||
@@ -135,10 +181,28 @@ def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> n
|
|||||||
return np.interp(x_new, x_old, mono).astype(np.float32)
|
return np.interp(x_new, x_old, mono).astype(np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
async def _enqueue_audio_and_animation(
|
||||||
|
seg_audio: np.ndarray,
|
||||||
|
seg_sr: int,
|
||||||
|
seg_text: str,
|
||||||
|
idx: int,
|
||||||
|
total: int,
|
||||||
|
) -> None:
|
||||||
|
audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000)
|
||||||
|
pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16)
|
||||||
|
await audio_bus.enqueue(pcm_seg)
|
||||||
|
animation_payload = await avatar_service.build_controls_from_audio(
|
||||||
|
seg_audio,
|
||||||
|
seg_sr,
|
||||||
|
text=seg_text,
|
||||||
|
chunk_index=idx,
|
||||||
|
total_chunks=total,
|
||||||
|
)
|
||||||
|
await _broadcast_animation(animation_payload)
|
||||||
|
|
||||||
|
|
||||||
async def _run_pipeline_from_buffer() -> None:
|
async def _run_pipeline_from_buffer() -> None:
|
||||||
if runtime.busy:
|
if runtime.busy or not runtime.buffer_16k:
|
||||||
return
|
|
||||||
if not runtime.buffer_16k:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
runtime.busy = True
|
runtime.busy = True
|
||||||
@@ -147,10 +211,6 @@ async def _run_pipeline_from_buffer() -> None:
|
|||||||
runtime.last_input_mode = "voice"
|
runtime.last_input_mode = "voice"
|
||||||
audio_16k = np.concatenate(runtime.buffer_16k, axis=0)
|
audio_16k = np.concatenate(runtime.buffer_16k, axis=0)
|
||||||
runtime.buffer_16k.clear()
|
runtime.buffer_16k.clear()
|
||||||
async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None:
|
|
||||||
audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000)
|
|
||||||
pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16)
|
|
||||||
await audio_bus.enqueue(pcm_seg)
|
|
||||||
|
|
||||||
async def on_user_text_now(user_text: str) -> None:
|
async def on_user_text_now(user_text: str) -> None:
|
||||||
runtime.last_user_text = user_text
|
runtime.last_user_text = user_text
|
||||||
@@ -164,12 +224,11 @@ async def _run_pipeline_from_buffer() -> None:
|
|||||||
|
|
||||||
result, audio, sr = await pipeline.process_turn(
|
result, audio, sr = await pipeline.process_turn(
|
||||||
audio_16k,
|
audio_16k,
|
||||||
on_audio_chunk=on_audio_chunk,
|
on_audio_chunk=_enqueue_audio_and_animation,
|
||||||
on_user_text=on_user_text_now,
|
on_user_text=on_user_text_now,
|
||||||
on_reply_text=on_reply_text_now,
|
on_reply_text=on_reply_text_now,
|
||||||
on_reply_segment=on_reply_segment_now,
|
on_reply_segment=on_reply_segment_now,
|
||||||
)
|
)
|
||||||
asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr))
|
|
||||||
runtime.last_user_text = result["user_text"]
|
runtime.last_user_text = result["user_text"]
|
||||||
runtime.last_reply_text = result["reply_text"]
|
runtime.last_reply_text = result["reply_text"]
|
||||||
runtime.pipeline_runs += 1
|
runtime.pipeline_runs += 1
|
||||||
@@ -179,11 +238,11 @@ async def _run_pipeline_from_buffer() -> None:
|
|||||||
runtime.last_llm_source = str(result.get("llm_source", "unknown"))
|
runtime.last_llm_source = str(result.get("llm_source", "unknown"))
|
||||||
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
|
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
|
||||||
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
|
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
|
||||||
# Fallback: if chunk callback path produced nothing, enqueue full audio.
|
|
||||||
if result.get("audio_samples", 0) <= 1:
|
if result.get("audio_samples", 0) <= 1:
|
||||||
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000)
|
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000)
|
||||||
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16)
|
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16)
|
||||||
await audio_bus.enqueue(pcm_int16)
|
await audio_bus.enqueue(pcm_int16)
|
||||||
|
await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"]))
|
||||||
finally:
|
finally:
|
||||||
runtime.busy = False
|
runtime.busy = False
|
||||||
|
|
||||||
@@ -200,7 +259,6 @@ async def _consume_user_audio(track) -> None:
|
|||||||
runtime.buffer_16k.append(mono_16k)
|
runtime.buffer_16k.append(mono_16k)
|
||||||
vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0)
|
vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0)
|
||||||
|
|
||||||
# Silero VAD requires exact 512 samples at 16k.
|
|
||||||
while vad_buffer.shape[0] >= 512:
|
while vad_buffer.shape[0] >= 512:
|
||||||
chunk = vad_buffer[:512]
|
chunk = vad_buffer[:512]
|
||||||
vad_buffer = vad_buffer[512:]
|
vad_buffer = vad_buffer[512:]
|
||||||
@@ -214,6 +272,7 @@ async def _consume_user_audio(track) -> None:
|
|||||||
was_avatar_speaking = str(arbitrator.state) == "avatar_speaking"
|
was_avatar_speaking = str(arbitrator.state) == "avatar_speaking"
|
||||||
await arbitrator.on_speech_start()
|
await arbitrator.on_speech_start()
|
||||||
await audio_bus.clear()
|
await audio_bus.clear()
|
||||||
|
await _broadcast_animation(await avatar_service.build_reset_payload(reason="speech-start"))
|
||||||
if was_avatar_speaking:
|
if was_avatar_speaking:
|
||||||
runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000)
|
runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000)
|
||||||
runtime.barge_in_count += 1
|
runtime.barge_in_count += 1
|
||||||
@@ -226,30 +285,7 @@ async def _consume_user_audio(track) -> None:
|
|||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
async def health():
|
async def health():
|
||||||
return {
|
return _runtime_payload()
|
||||||
"state": arbitrator.state,
|
|
||||||
"peers": len(pcs),
|
|
||||||
"pipeline_runs": runtime.pipeline_runs,
|
|
||||||
"last_latency_ms": runtime.last_latency_ms,
|
|
||||||
"last_asr_latency_ms": runtime.last_asr_latency_ms,
|
|
||||||
"last_llm_latency_ms": runtime.last_llm_latency_ms,
|
|
||||||
"last_llm_source": runtime.last_llm_source,
|
|
||||||
"last_tts_latency_ms": runtime.last_tts_latency_ms,
|
|
||||||
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
|
|
||||||
"last_barge_in_ms": runtime.last_barge_in_ms,
|
|
||||||
"barge_in_count": runtime.barge_in_count,
|
|
||||||
"last_input_mode": runtime.last_input_mode,
|
|
||||||
"vad_start_count": runtime.vad_start_count,
|
|
||||||
"vad_end_count": runtime.vad_end_count,
|
|
||||||
"last_user_text": runtime.last_user_text,
|
|
||||||
"last_reply_text": runtime.last_reply_text,
|
|
||||||
"pipeline_busy": runtime.busy,
|
|
||||||
"llm": llm_service.health,
|
|
||||||
"asr": asr_service.health,
|
|
||||||
"tts": tts_service.health,
|
|
||||||
"vad": vad_service.health,
|
|
||||||
"avatar": avatar_service.health,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/meta")
|
@app.get("/meta")
|
||||||
@@ -259,34 +295,20 @@ async def meta():
|
|||||||
"https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile),
|
"https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile),
|
||||||
"host": settings.webrtc_host,
|
"host": settings.webrtc_host,
|
||||||
"port": settings.webrtc_port,
|
"port": settings.webrtc_port,
|
||||||
|
"avatar_protocol": settings.avatar_control_protocol,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/avatar/schema")
|
||||||
|
async def avatar_schema():
|
||||||
|
return avatar_service.schema
|
||||||
|
|
||||||
|
|
||||||
@app.get("/events")
|
@app.get("/events")
|
||||||
async def events():
|
async def events():
|
||||||
async def gen():
|
async def gen():
|
||||||
while True:
|
while True:
|
||||||
payload = {
|
yield f"data: {json.dumps(_runtime_payload(), ensure_ascii=False)}\n\n"
|
||||||
"state": str(arbitrator.state),
|
|
||||||
"peers": len(pcs),
|
|
||||||
"pipeline_runs": runtime.pipeline_runs,
|
|
||||||
"last_latency_ms": runtime.last_latency_ms,
|
|
||||||
"last_asr_latency_ms": runtime.last_asr_latency_ms,
|
|
||||||
"last_llm_latency_ms": runtime.last_llm_latency_ms,
|
|
||||||
"last_llm_source": runtime.last_llm_source,
|
|
||||||
"last_tts_latency_ms": runtime.last_tts_latency_ms,
|
|
||||||
"last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms,
|
|
||||||
"last_barge_in_ms": runtime.last_barge_in_ms,
|
|
||||||
"barge_in_count": runtime.barge_in_count,
|
|
||||||
"last_input_mode": runtime.last_input_mode,
|
|
||||||
"vad_start_count": runtime.vad_start_count,
|
|
||||||
"vad_end_count": runtime.vad_end_count,
|
|
||||||
"last_user_text": runtime.last_user_text,
|
|
||||||
"last_reply_text": runtime.last_reply_text,
|
|
||||||
"pipeline_busy": runtime.busy,
|
|
||||||
"avatar": avatar_service.health,
|
|
||||||
}
|
|
||||||
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
|
||||||
await asyncio.sleep(1.0)
|
await asyncio.sleep(1.0)
|
||||||
|
|
||||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||||
@@ -306,9 +328,24 @@ async def ws_subtitles(websocket: WebSocket):
|
|||||||
subtitle_clients.discard(websocket)
|
subtitle_clients.discard(websocket)
|
||||||
|
|
||||||
|
|
||||||
|
@app.websocket("/ws/animation")
|
||||||
|
async def ws_animation(websocket: WebSocket):
|
||||||
|
await websocket.accept()
|
||||||
|
animation_clients.add(websocket)
|
||||||
|
try:
|
||||||
|
await websocket.send_text(json.dumps({"type": "animation_ready", **avatar_service.schema}, ensure_ascii=False))
|
||||||
|
await websocket.send_text(json.dumps(await avatar_service.build_idle_payload(reason="client-connected"), ensure_ascii=False))
|
||||||
|
while True:
|
||||||
|
await websocket.send_text(json.dumps({"type": "ping"}, ensure_ascii=False))
|
||||||
|
await asyncio.sleep(15)
|
||||||
|
except WebSocketDisconnect:
|
||||||
|
animation_clients.discard(websocket)
|
||||||
|
except Exception:
|
||||||
|
animation_clients.discard(websocket)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/demo/run-once")
|
@app.post("/demo/run-once")
|
||||||
async def run_once():
|
async def run_once():
|
||||||
# 3 seconds of silence placeholder; replace with real microphone stream path.
|
|
||||||
fake_audio = np.zeros(16000 * 3, dtype=np.float32)
|
fake_audio = np.zeros(16000 * 3, dtype=np.float32)
|
||||||
await arbitrator.on_speech_start()
|
await arbitrator.on_speech_start()
|
||||||
result = await pipeline.run_once(fake_audio)
|
result = await pipeline.run_once(fake_audio)
|
||||||
@@ -326,11 +363,6 @@ async def chat_text(req: TextChatRequest):
|
|||||||
runtime.busy = True
|
runtime.busy = True
|
||||||
start_t = time.perf_counter()
|
start_t = time.perf_counter()
|
||||||
try:
|
try:
|
||||||
async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None:
|
|
||||||
audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000)
|
|
||||||
pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16)
|
|
||||||
await audio_bus.enqueue(pcm_seg)
|
|
||||||
|
|
||||||
async def on_reply_text_now(reply_text: str) -> None:
|
async def on_reply_text_now(reply_text: str) -> None:
|
||||||
runtime.last_reply_text = reply_text
|
runtime.last_reply_text = reply_text
|
||||||
|
|
||||||
@@ -338,13 +370,13 @@ async def chat_text(req: TextChatRequest):
|
|||||||
await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1))
|
await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1))
|
||||||
|
|
||||||
await _broadcast_subtitle("user", text, "text")
|
await _broadcast_subtitle("user", text, "text")
|
||||||
|
await _broadcast_animation(await avatar_service.build_reset_payload(reason="text-turn-start"))
|
||||||
result, audio, sr = await pipeline.process_text_turn(
|
result, audio, sr = await pipeline.process_text_turn(
|
||||||
text,
|
text,
|
||||||
on_audio_chunk=on_audio_chunk,
|
on_audio_chunk=_enqueue_audio_and_animation,
|
||||||
on_reply_text=on_reply_text_now,
|
on_reply_text=on_reply_text_now,
|
||||||
on_reply_segment=on_reply_segment_now,
|
on_reply_segment=on_reply_segment_now,
|
||||||
)
|
)
|
||||||
asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr))
|
|
||||||
runtime.last_input_mode = "text"
|
runtime.last_input_mode = "text"
|
||||||
runtime.last_user_text = result["user_text"]
|
runtime.last_user_text = result["user_text"]
|
||||||
runtime.last_reply_text = result["reply_text"]
|
runtime.last_reply_text = result["reply_text"]
|
||||||
@@ -355,11 +387,11 @@ async def chat_text(req: TextChatRequest):
|
|||||||
runtime.last_llm_source = str(result.get("llm_source", "unknown"))
|
runtime.last_llm_source = str(result.get("llm_source", "unknown"))
|
||||||
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
|
runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0))
|
||||||
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
|
runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0))
|
||||||
# Fallback: if no chunk was enqueued for some reason, enqueue full audio.
|
|
||||||
if result.get("audio_samples", 0) <= 1:
|
if result.get("audio_samples", 0) <= 1:
|
||||||
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000)
|
audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000)
|
||||||
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16)
|
pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16)
|
||||||
await audio_bus.enqueue(pcm_int16)
|
await audio_bus.enqueue(pcm_int16)
|
||||||
|
await _broadcast_animation(await avatar_service.build_controls_from_audio(audio, sr, text=result["reply_text"]))
|
||||||
return {"ok": True, **result}
|
return {"ok": True, **result}
|
||||||
finally:
|
finally:
|
||||||
runtime.busy = False
|
runtime.busy = False
|
||||||
@@ -382,13 +414,14 @@ async def chat_reset():
|
|||||||
runtime.last_input_mode = "none"
|
runtime.last_input_mode = "none"
|
||||||
runtime.vad_start_count = 0
|
runtime.vad_start_count = 0
|
||||||
runtime.vad_end_count = 0
|
runtime.vad_end_count = 0
|
||||||
|
runtime.last_animation_frame_count = 0
|
||||||
|
await _broadcast_animation(await avatar_service.build_reset_payload(reason="chat-reset"))
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/demo/frame-idle-shape")
|
@app.get("/demo/frame-idle-shape")
|
||||||
async def frame_idle_shape():
|
async def frame_idle_shape():
|
||||||
frame = await avatar_service.idle_frame()
|
return await avatar_service.build_idle_payload(reason="demo")
|
||||||
return {"shape": list(frame.shape)}
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
@@ -401,7 +434,6 @@ async def webrtc_offer(request: Request):
|
|||||||
params = await request.json()
|
params = await request.json()
|
||||||
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
|
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
|
||||||
|
|
||||||
# Single-connection mode: kick existing peers before accepting new one.
|
|
||||||
replaced_previous = len(pcs) > 0
|
replaced_previous = len(pcs) > 0
|
||||||
if replaced_previous:
|
if replaced_previous:
|
||||||
old_peers = list(pcs)
|
old_peers = list(pcs)
|
||||||
@@ -423,7 +455,6 @@ async def webrtc_offer(request: Request):
|
|||||||
if track.kind == "audio":
|
if track.kind == "audio":
|
||||||
asyncio.create_task(_consume_user_audio(track))
|
asyncio.create_task(_consume_user_audio(track))
|
||||||
|
|
||||||
pc.addTrack(AvatarVideoTrack(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
|
|
||||||
pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus))
|
pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus))
|
||||||
|
|
||||||
await pc.setRemoteDescription(offer)
|
await pc.setRemoteDescription(offer)
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
RUN_DIR="$ROOT_DIR/.run"
|
||||||
|
PID_FILE="$RUN_DIR/visual-chat-8018.pid"
|
||||||
|
LOG_FILE="$RUN_DIR/visual-chat-8018.log"
|
||||||
|
ENV_FILE="$ROOT_DIR/.env"
|
||||||
|
|
||||||
|
mkdir -p "$RUN_DIR"
|
||||||
|
|
||||||
|
if [[ -f "$ENV_FILE" ]]; then
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
source "$ENV_FILE"
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
HOST="${WEBRTC_HOST:-0.0.0.0}"
|
||||||
|
PORT="${WEBRTC_PORT:-8018}"
|
||||||
|
SSL_CERTFILE="${SSL_CERTFILE:-$ROOT_DIR/certs/dev-cert.pem}"
|
||||||
|
SSL_KEYFILE="${SSL_KEYFILE:-$ROOT_DIR/certs/dev-key.pem}"
|
||||||
|
|
||||||
|
if [[ -x "$ROOT_DIR/.venv/bin/python" ]]; then
|
||||||
|
PYTHON_BIN="$ROOT_DIR/.venv/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON_BIN="$(command -v python3 || true)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$PYTHON_BIN" ]]; then
|
||||||
|
echo "python3 not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
port_pids() {
|
||||||
|
ss -ltnp 2>/dev/null \
|
||||||
|
| sed -n "s/.*:${PORT} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" \
|
||||||
|
| tr '\n' ' '
|
||||||
|
}
|
||||||
|
|
||||||
|
is_running() {
|
||||||
|
if [[ ! -f "$PID_FILE" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$PID_FILE")"
|
||||||
|
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
start() {
|
||||||
|
if is_running; then
|
||||||
|
echo "already running: pid=$(cat "$PID_FILE")"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local stale
|
||||||
|
stale="$(port_pids)"
|
||||||
|
if [[ -n "${stale// /}" ]]; then
|
||||||
|
echo "killing stale python on port ${PORT}: $stale"
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
kill $stale || true
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local args
|
||||||
|
args=(main:app --host "$HOST" --port "$PORT")
|
||||||
|
if [[ -n "$SSL_CERTFILE" && -n "$SSL_KEYFILE" ]]; then
|
||||||
|
args+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE")
|
||||||
|
fi
|
||||||
|
|
||||||
|
nohup "$PYTHON_BIN" -m uvicorn "${args[@]}" >>"$LOG_FILE" 2>&1 &
|
||||||
|
local pid=$!
|
||||||
|
echo "$pid" >"$PID_FILE"
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
echo "started: pid=$pid"
|
||||||
|
echo "url: https://$HOST:$PORT"
|
||||||
|
echo "log: $LOG_FILE"
|
||||||
|
else
|
||||||
|
echo "failed to start, see log: $LOG_FILE"
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
stop() {
|
||||||
|
if ! is_running; then
|
||||||
|
echo "not running"
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$PID_FILE")"
|
||||||
|
kill "$pid" || true
|
||||||
|
|
||||||
|
for _ in {1..20}; do
|
||||||
|
if ! kill -0 "$pid" 2>/dev/null; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 0.2
|
||||||
|
done
|
||||||
|
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
kill -9 "$pid" || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
echo "stopped"
|
||||||
|
}
|
||||||
|
|
||||||
|
status() {
|
||||||
|
if is_running; then
|
||||||
|
echo "running: pid=$(cat "$PID_FILE")"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "not running"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
logs() {
|
||||||
|
if [[ -f "$LOG_FILE" ]]; then
|
||||||
|
tail -n 120 "$LOG_FILE"
|
||||||
|
else
|
||||||
|
echo "no log file yet"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
restart() {
|
||||||
|
stop || true
|
||||||
|
start
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
start) start ;;
|
||||||
|
stop) stop ;;
|
||||||
|
status) status ;;
|
||||||
|
restart) restart ;;
|
||||||
|
logs) logs ;;
|
||||||
|
*)
|
||||||
|
echo "usage: $0 {start|stop|status|restart|logs}"
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+190
-170
@@ -1,193 +1,213 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import math
|
||||||
import sys
|
import time
|
||||||
import uuid
|
from typing import Any
|
||||||
import wave
|
|
||||||
from collections import deque
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
from config import settings
|
from config import settings
|
||||||
|
|
||||||
|
|
||||||
class AvatarService:
|
class AvatarService:
|
||||||
def __init__(self, width: int = 640, height: int = 360) -> None:
|
def __init__(self) -> None:
|
||||||
self.width = width
|
self.driver_name = "arkit-blendshape-v1"
|
||||||
self.height = height
|
|
||||||
self._frames: deque[np.ndarray] = deque()
|
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
self._job_lock = asyncio.Lock()
|
self._sequence = 0
|
||||||
self._last_error: str | None = None
|
self._last_error: str | None = None
|
||||||
self._base_frame = self._load_base_frame()
|
self._last_frame_count = 0
|
||||||
|
self._last_chunk_duration_ms = 0
|
||||||
|
|
||||||
async def render_frame(self, mouth_open: float = 0.0, speaking: bool = False) -> np.ndarray:
|
async def build_idle_payload(self, reason: str = "idle") -> dict[str, Any]:
|
||||||
frame = self._base_frame.copy()
|
frame = self._build_frame(seq=await self._next_sequence(), time_ms=0, controls=self._neutral_controls())
|
||||||
if speaking:
|
self._last_frame_count = 1
|
||||||
alpha = float(np.clip(mouth_open, 0.0, 1.0))
|
self._last_chunk_duration_ms = 0
|
||||||
# Slightly boost contrast/brightness while speaking to keep a "live" feeling.
|
self._last_error = None
|
||||||
frame = cv2.convertScaleAbs(frame, alpha=1.0 + 0.10 * alpha, beta=2 + int(8 * alpha))
|
return {
|
||||||
# Fallback mouth animation on realistic base frame.
|
"type": "animation_state",
|
||||||
h, w = frame.shape[:2]
|
"driver": self.driver_name,
|
||||||
cx = w // 2
|
"mode": settings.avatar_driver_mode,
|
||||||
cy = int(h * 0.70)
|
"schema": settings.avatar_blendshape_schema,
|
||||||
half_w = max(28, int(w * 0.055))
|
"fps": settings.avatar_fps,
|
||||||
half_h = max(4, int(4 + alpha * h * 0.035))
|
"reason": reason,
|
||||||
|
"frame_count": 1,
|
||||||
|
"frames": [frame],
|
||||||
|
"ts_ms": int(time.time() * 1000),
|
||||||
|
}
|
||||||
|
|
||||||
lip_color = (26, 26, 92)
|
async def build_reset_payload(self, reason: str = "reset") -> dict[str, Any]:
|
||||||
lip_border = (70, 70, 160)
|
payload = await self.build_idle_payload(reason=reason)
|
||||||
cv2.ellipse(frame, (cx, cy), (half_w + 2, half_h + 2), 0, 0, 360, lip_border, -1)
|
payload["type"] = "animation_reset"
|
||||||
cv2.ellipse(frame, (cx, cy), (half_w, half_h), 0, 0, 360, lip_color, -1)
|
return payload
|
||||||
return frame
|
|
||||||
|
|
||||||
async def idle_frame(self) -> np.ndarray:
|
async def build_controls_from_audio(
|
||||||
return await self.render_frame(mouth_open=0.0, speaking=False)
|
self,
|
||||||
|
audio: np.ndarray,
|
||||||
|
sr: int,
|
||||||
|
*,
|
||||||
|
text: str = "",
|
||||||
|
chunk_index: int = 0,
|
||||||
|
total_chunks: int = 1,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if sr <= 0:
|
||||||
|
self._last_error = f"invalid sample rate: {sr}"
|
||||||
|
return await self.build_idle_payload(reason="invalid-sample-rate")
|
||||||
|
|
||||||
async def speaking_frame(self, mouth_open: float = 0.5) -> np.ndarray:
|
mono = np.asarray(audio, dtype=np.float32).reshape(-1)
|
||||||
return await self.render_frame(mouth_open=mouth_open, speaking=True)
|
if mono.size <= 1:
|
||||||
|
return await self.build_idle_payload(reason="empty-audio")
|
||||||
|
|
||||||
async def pop_generated_frame(self) -> np.ndarray | None:
|
mono = np.clip(mono, -1.0, 1.0)
|
||||||
async with self._lock:
|
frame_count = max(1, int(math.ceil((mono.shape[0] / float(sr)) * settings.avatar_fps)))
|
||||||
if self._frames:
|
seqs = await self._reserve_sequences(frame_count)
|
||||||
return self._frames.popleft()
|
frames = self._audio_to_frames(mono, sr, seqs)
|
||||||
return None
|
|
||||||
|
|
||||||
async def enqueue_musetalk_from_audio(self, audio: np.ndarray, sr: int) -> None:
|
self._last_frame_count = len(frames)
|
||||||
if not settings.musetalk_enabled:
|
self._last_chunk_duration_ms = int(round(1000.0 * mono.shape[0] / float(sr)))
|
||||||
return
|
self._last_error = None
|
||||||
if audio.size <= 1 or sr <= 0:
|
return {
|
||||||
return
|
"type": "animation_chunk",
|
||||||
|
"driver": self.driver_name,
|
||||||
async with self._job_lock:
|
"mode": settings.avatar_driver_mode,
|
||||||
try:
|
"schema": settings.avatar_blendshape_schema,
|
||||||
job_id = f"job_{uuid.uuid4().hex[:10]}"
|
"chunk_index": chunk_index,
|
||||||
repo_dir = Path(settings.musetalk_repo_dir)
|
"total_chunks": total_chunks,
|
||||||
work_root = repo_dir / "results" / "visual-chat-jobs" / job_id
|
"sample_rate": sr,
|
||||||
work_root.mkdir(parents=True, exist_ok=True)
|
"fps": settings.avatar_fps,
|
||||||
wav_path = work_root / "input.wav"
|
"text": text,
|
||||||
cfg_path = work_root / "inference.yaml"
|
"frame_count": len(frames),
|
||||||
output_name = f"{job_id}.mp4"
|
"duration_ms": self._last_chunk_duration_ms,
|
||||||
output_path = Path(settings.musetalk_result_dir) / "v15" / output_name
|
"frames": frames,
|
||||||
|
"ts_ms": int(time.time() * 1000),
|
||||||
self._write_wav(wav_path, audio.astype(np.float32, copy=False), sr)
|
}
|
||||||
cfg_path.write_text(
|
|
||||||
(
|
|
||||||
"task_0:\n"
|
|
||||||
f" video_path: \"{settings.musetalk_source_video}\"\n"
|
|
||||||
f" audio_path: \"{wav_path}\"\n"
|
|
||||||
f" result_name: \"{output_name}\"\n"
|
|
||||||
),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
env = os.environ.copy()
|
|
||||||
env.setdefault("HF_HOME", str(repo_dir / "models"))
|
|
||||||
env["PYTHONPATH"] = (
|
|
||||||
f"{repo_dir}:{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(repo_dir)
|
|
||||||
)
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
sys.executable,
|
|
||||||
str(settings.musetalk_infer_script),
|
|
||||||
"--version",
|
|
||||||
"v15",
|
|
||||||
"--inference_config",
|
|
||||||
str(cfg_path),
|
|
||||||
"--result_dir",
|
|
||||||
str(settings.musetalk_result_dir),
|
|
||||||
"--unet_model_path",
|
|
||||||
str(settings.musetalk_unet_model),
|
|
||||||
"--unet_config",
|
|
||||||
str(settings.musetalk_unet_config),
|
|
||||||
"--whisper_dir",
|
|
||||||
str(settings.musetalk_whisper_dir),
|
|
||||||
"--batch_size",
|
|
||||||
"8",
|
|
||||||
"--use_float16",
|
|
||||||
cwd=str(repo_dir),
|
|
||||||
env=env,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE,
|
|
||||||
)
|
|
||||||
_, stderr = await proc.communicate()
|
|
||||||
if proc.returncode != 0:
|
|
||||||
self._last_error = stderr.decode("utf-8", errors="ignore")[-500:]
|
|
||||||
return
|
|
||||||
frames: list[np.ndarray] = []
|
|
||||||
if output_path.exists():
|
|
||||||
frames = self._read_video_frames(str(output_path))
|
|
||||||
if not frames:
|
|
||||||
# Some MuseTalk runs leave image frames instead of final mp4.
|
|
||||||
fallback_dir = Path(settings.musetalk_result_dir) / "v15" / "yongen"
|
|
||||||
if fallback_dir.exists():
|
|
||||||
frames = self._read_image_frames(str(fallback_dir))
|
|
||||||
if not frames:
|
|
||||||
self._last_error = f"MuseTalk produced no readable frames: {output_path}"
|
|
||||||
return
|
|
||||||
|
|
||||||
async with self._lock:
|
|
||||||
self._frames.clear()
|
|
||||||
self._frames.extend(frames)
|
|
||||||
self._last_error = None
|
|
||||||
except Exception as exc: # pragma: no cover
|
|
||||||
self._last_error = str(exc)
|
|
||||||
|
|
||||||
def _read_video_frames(self, path: str) -> list[np.ndarray]:
|
|
||||||
cap = cv2.VideoCapture(path)
|
|
||||||
frames: list[np.ndarray] = []
|
|
||||||
while True:
|
|
||||||
ok, frame = cap.read()
|
|
||||||
if not ok or frame is None:
|
|
||||||
break
|
|
||||||
if frame.shape[1] != self.width or frame.shape[0] != self.height:
|
|
||||||
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR)
|
|
||||||
frames.append(frame)
|
|
||||||
cap.release()
|
|
||||||
return frames
|
|
||||||
|
|
||||||
def _read_image_frames(self, dir_path: str) -> list[np.ndarray]:
|
|
||||||
frames: list[np.ndarray] = []
|
|
||||||
for name in sorted(os.listdir(dir_path)):
|
|
||||||
if not name.lower().endswith(".png"):
|
|
||||||
continue
|
|
||||||
frame = cv2.imread(os.path.join(dir_path, name))
|
|
||||||
if frame is None:
|
|
||||||
continue
|
|
||||||
if frame.shape[1] != self.width or frame.shape[0] != self.height:
|
|
||||||
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR)
|
|
||||||
frames.append(frame)
|
|
||||||
return frames
|
|
||||||
|
|
||||||
def _load_base_frame(self) -> np.ndarray:
|
|
||||||
src = settings.musetalk_source_video
|
|
||||||
if src and os.path.exists(src):
|
|
||||||
cap = cv2.VideoCapture(src)
|
|
||||||
ok, frame = cap.read()
|
|
||||||
cap.release()
|
|
||||||
if ok and frame is not None:
|
|
||||||
if frame.shape[1] != self.width or frame.shape[0] != self.height:
|
|
||||||
frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR)
|
|
||||||
return frame
|
|
||||||
# Last-resort dark background (should rarely happen).
|
|
||||||
fallback = np.zeros((self.height, self.width, 3), dtype=np.uint8)
|
|
||||||
fallback[:, :, :] = (20, 20, 20)
|
|
||||||
return fallback
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _write_wav(path: Path, audio: np.ndarray, sr: int) -> None:
|
|
||||||
pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16)
|
|
||||||
with wave.open(str(path), "wb") as wf:
|
|
||||||
wf.setnchannels(1)
|
|
||||||
wf.setsampwidth(2)
|
|
||||||
wf.setframerate(sr)
|
|
||||||
wf.writeframes(pcm.tobytes())
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def health(self) -> dict:
|
def schema(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"enabled": settings.musetalk_enabled,
|
"driver": self.driver_name,
|
||||||
"queued_frames": len(self._frames),
|
"mode": settings.avatar_driver_mode,
|
||||||
|
"protocol": settings.avatar_control_protocol,
|
||||||
|
"schema": settings.avatar_blendshape_schema,
|
||||||
|
"controls": [
|
||||||
|
"jawOpen",
|
||||||
|
"mouthClose",
|
||||||
|
"mouthFunnel",
|
||||||
|
"mouthPucker",
|
||||||
|
"viseme_aa",
|
||||||
|
"viseme_ee",
|
||||||
|
"viseme_oh",
|
||||||
|
"headYaw",
|
||||||
|
"headPitch",
|
||||||
|
"headRoll",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _audio_to_frames(self, audio: np.ndarray, sr: int, sequences: range) -> list[dict[str, Any]]:
|
||||||
|
samples_per_frame = max(1, int(round(sr / max(1, settings.avatar_fps))))
|
||||||
|
frames: list[dict[str, Any]] = []
|
||||||
|
jaw_prev = 0.0
|
||||||
|
for index, seq in enumerate(sequences):
|
||||||
|
start = index * samples_per_frame
|
||||||
|
end = min(audio.shape[0], start + samples_per_frame)
|
||||||
|
window = audio[start:end]
|
||||||
|
if window.size == 0:
|
||||||
|
window = np.zeros(1, dtype=np.float32)
|
||||||
|
energy, brightness = self._extract_features(window, sr)
|
||||||
|
jaw_open = 0.70 * jaw_prev + 0.30 * energy
|
||||||
|
jaw_prev = jaw_open
|
||||||
|
phase = index / max(1, len(sequences) - 1)
|
||||||
|
frames.append(
|
||||||
|
self._build_frame(
|
||||||
|
seq=seq,
|
||||||
|
time_ms=int(round(index * 1000.0 / settings.avatar_fps)),
|
||||||
|
controls=self._controls_from_features(jaw_open, brightness, phase),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return frames
|
||||||
|
|
||||||
|
def _extract_features(self, window: np.ndarray, sr: int) -> tuple[float, float]:
|
||||||
|
rms = float(np.sqrt(np.mean(window**2))) if window.size else 0.0
|
||||||
|
energy = float(np.clip((rms - 0.01) * 10.0, 0.0, 1.0))
|
||||||
|
if window.size < 16:
|
||||||
|
return energy, 0.5
|
||||||
|
spectrum = np.abs(np.fft.rfft(window * np.hanning(window.size)))
|
||||||
|
if spectrum.size <= 1:
|
||||||
|
return energy, 0.5
|
||||||
|
freqs = np.fft.rfftfreq(window.size, d=1.0 / float(sr))
|
||||||
|
weights = spectrum[1:]
|
||||||
|
freq_values = freqs[1:]
|
||||||
|
denom = float(np.sum(weights)) + 1e-6
|
||||||
|
centroid = float(np.sum(freq_values * weights) / denom)
|
||||||
|
brightness = float(np.clip((centroid - 500.0) / 2500.0, 0.0, 1.0))
|
||||||
|
return energy, brightness
|
||||||
|
|
||||||
|
def _controls_from_features(self, energy: float, brightness: float, phase: float) -> dict[str, float]:
|
||||||
|
jaw_open = float(np.clip(energy, 0.0, 1.0))
|
||||||
|
mouth_close = float(np.clip(1.0 - jaw_open * 0.82, 0.0, 1.0))
|
||||||
|
mouth_funnel = float(np.clip(0.16 + jaw_open * (0.40 - brightness * 0.12), 0.0, 1.0))
|
||||||
|
mouth_pucker = float(np.clip(0.06 + jaw_open * (0.20 + (1.0 - brightness) * 0.30), 0.0, 1.0))
|
||||||
|
viseme_aa = jaw_open
|
||||||
|
viseme_ee = float(np.clip(jaw_open * (0.25 + brightness * 0.95), 0.0, 1.0))
|
||||||
|
viseme_oh = float(np.clip(jaw_open * (0.30 + (1.0 - brightness) * 0.90), 0.0, 1.0))
|
||||||
|
head_yaw = float(np.clip(math.sin(phase * math.pi * 1.7) * 0.02 * max(jaw_open, 0.18), -0.25, 0.25))
|
||||||
|
head_pitch = float(np.clip(math.cos(phase * math.pi * 2.0) * 0.03 * max(jaw_open, 0.15), -0.25, 0.25))
|
||||||
|
head_roll = float(np.clip(math.sin(phase * math.pi) * 0.015, -0.15, 0.15))
|
||||||
|
return {
|
||||||
|
"jawOpen": round(jaw_open, 4),
|
||||||
|
"mouthClose": round(mouth_close, 4),
|
||||||
|
"mouthFunnel": round(mouth_funnel, 4),
|
||||||
|
"mouthPucker": round(mouth_pucker, 4),
|
||||||
|
"viseme_aa": round(viseme_aa, 4),
|
||||||
|
"viseme_ee": round(viseme_ee, 4),
|
||||||
|
"viseme_oh": round(viseme_oh, 4),
|
||||||
|
"headYaw": round(head_yaw, 4),
|
||||||
|
"headPitch": round(head_pitch, 4),
|
||||||
|
"headRoll": round(head_roll, 4),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _build_frame(self, seq: int, time_ms: int, controls: dict[str, float]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"seq": seq,
|
||||||
|
"time_ms": time_ms,
|
||||||
|
"speaking": controls["jawOpen"] > 0.04,
|
||||||
|
"controls": controls,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _neutral_controls(self) -> dict[str, float]:
|
||||||
|
return {
|
||||||
|
"jawOpen": 0.0,
|
||||||
|
"mouthClose": 1.0,
|
||||||
|
"mouthFunnel": 0.0,
|
||||||
|
"mouthPucker": 0.0,
|
||||||
|
"viseme_aa": 0.0,
|
||||||
|
"viseme_ee": 0.0,
|
||||||
|
"viseme_oh": 0.0,
|
||||||
|
"headYaw": 0.0,
|
||||||
|
"headPitch": 0.0,
|
||||||
|
"headRoll": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _next_sequence(self) -> int:
|
||||||
|
async with self._lock:
|
||||||
|
seq = self._sequence
|
||||||
|
self._sequence += 1
|
||||||
|
return seq
|
||||||
|
|
||||||
|
async def _reserve_sequences(self, count: int) -> range:
|
||||||
|
async with self._lock:
|
||||||
|
start = self._sequence
|
||||||
|
self._sequence += count
|
||||||
|
return range(start, start + count)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def health(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"driver": self.driver_name,
|
||||||
|
"mode": settings.avatar_driver_mode,
|
||||||
|
"protocol": settings.avatar_control_protocol,
|
||||||
|
"schema": settings.avatar_blendshape_schema,
|
||||||
|
"last_frame_count": self._last_frame_count,
|
||||||
|
"last_chunk_duration_ms": self._last_chunk_duration_ms,
|
||||||
"last_error": self._last_error,
|
"last_error": self._last_error,
|
||||||
}
|
}
|
||||||
|
|||||||
+19
-3
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from collections import deque
|
from collections import deque
|
||||||
|
|
||||||
@@ -15,23 +16,37 @@ class LLMService:
|
|||||||
self._error: Optional[str] = None
|
self._error: Optional[str] = None
|
||||||
self._client = None
|
self._client = None
|
||||||
self._model = (settings.llm_model or "").strip()
|
self._model = (settings.llm_model or "").strip()
|
||||||
|
self._base_url = (settings.llm_base_url or "").strip()
|
||||||
self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant)
|
self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant)
|
||||||
self._system_prompt = "请用中文口语化简短回复，1-2句。"
|
self._system_prompt = "请用中文口语化简短回复，1-2句。"
|
||||||
|
|
||||||
if not settings.llm_api_key:
|
if not settings.llm_api_key:
|
||||||
|
self._error = "missing LLM api key: set LLM_API_KEY or DEEPSEEK_API_KEY"
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# If DeepSeek key is provided but no explicit base URL override,
|
||||||
|
# route to DeepSeek-compatible OpenAI endpoint by default.
|
||||||
|
if (
|
||||||
|
os.getenv("DEEPSEEK_API_KEY")
|
||||||
|
and not os.getenv("LLM_BASE_URL")
|
||||||
|
and not os.getenv("DEEPSEEK_BASE_URL")
|
||||||
|
and (self._base_url.lower() == "https://api.openai.com/v1" or not self._base_url)
|
||||||
|
):
|
||||||
|
self._base_url = "https://api.deepseek.com/v1"
|
||||||
|
|
||||||
if not self._model:
|
if not self._model:
|
||||||
base = (settings.llm_base_url or "").lower()
|
base = self._base_url.lower()
|
||||||
self._model = "deepseek-chat" if "deepseek.com" in base else "gpt-4o-mini"
|
self._model = "deepseek-chat" if "deepseek" in base else "gpt-4o-mini"
|
||||||
try:
|
try:
|
||||||
from openai import AsyncOpenAI
|
from openai import AsyncOpenAI
|
||||||
|
|
||||||
self._client = AsyncOpenAI(
|
self._client = AsyncOpenAI(
|
||||||
api_key=settings.llm_api_key,
|
api_key=settings.llm_api_key,
|
||||||
base_url=settings.llm_base_url,
|
base_url=self._base_url,
|
||||||
timeout=settings.llm_timeout,
|
timeout=settings.llm_timeout,
|
||||||
)
|
)
|
||||||
self._ready = True
|
self._ready = True
|
||||||
|
self._error = None
|
||||||
except Exception as exc: # pragma: no cover
|
except Exception as exc: # pragma: no cover
|
||||||
self._error = str(exc)
|
self._error = str(exc)
|
||||||
logger.warning("LLM online mode unavailable: %s", exc)
|
logger.warning("LLM online mode unavailable: %s", exc)
|
||||||
@@ -67,6 +82,7 @@ class LLMService:
|
|||||||
return {
|
return {
|
||||||
"ready": self._ready,
|
"ready": self._ready,
|
||||||
"model": self._model,
|
"model": self._model,
|
||||||
|
"base_url": self._base_url,
|
||||||
"history_items": len(self._history),
|
"history_items": len(self._history),
|
||||||
"error": self._error,
|
"error": self._error,
|
||||||
}
|
}
|
||||||
|
|||||||
+681
-34
@@ -1,3 +1,8 @@
|
|||||||
|
import * as THREE from "three";
|
||||||
|
import { OrbitControls } from "three/addons/controls/OrbitControls.js";
|
||||||
|
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
|
||||||
|
import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm";
|
||||||
|
|
||||||
const statusEl = document.getElementById("status");
|
const statusEl = document.getElementById("status");
|
||||||
const outputEl = document.getElementById("output");
|
const outputEl = document.getElementById("output");
|
||||||
const healthBtn = document.getElementById("healthBtn");
|
const healthBtn = document.getElementById("healthBtn");
|
||||||
@@ -5,11 +10,25 @@ const offerBtn = document.getElementById("offerBtn");
|
|||||||
const resetBtn = document.getElementById("resetBtn");
|
const resetBtn = document.getElementById("resetBtn");
|
||||||
const sendBtn = document.getElementById("sendBtn");
|
const sendBtn = document.getElementById("sendBtn");
|
||||||
const chatInput = document.getElementById("chatInput");
|
const chatInput = document.getElementById("chatInput");
|
||||||
const remoteVideo = document.getElementById("remoteVideo");
|
const remoteAudio = document.getElementById("remoteAudio");
|
||||||
remoteVideo.muted = false;
|
const modelFile = document.getElementById("modelFile");
|
||||||
remoteVideo.volume = 1.0;
|
const modelUrl = document.getElementById("modelUrl");
|
||||||
|
const loadModelBtn = document.getElementById("loadModelBtn");
|
||||||
|
const resetViewBtn = document.getElementById("resetViewBtn");
|
||||||
|
const avatarCanvas = document.getElementById("avatarCanvas");
|
||||||
|
const modelStatus = document.getElementById("modelStatus");
|
||||||
|
const modelName = document.getElementById("modelName");
|
||||||
|
const animationBuffer = document.getElementById("animationBuffer");
|
||||||
|
|
||||||
|
remoteAudio.muted = false;
|
||||||
|
remoteAudio.volume = 1.0;
|
||||||
|
|
||||||
const messagesEl = document.getElementById("messages");
|
const messagesEl = document.getElementById("messages");
|
||||||
const connBadge = document.getElementById("connBadge");
|
const connBadge = document.getElementById("connBadge");
|
||||||
|
const animationConn = document.getElementById("animationConn");
|
||||||
|
const animationDriver = document.getElementById("animationDriver");
|
||||||
|
const animationFrames = document.getElementById("animationFrames");
|
||||||
|
const animationPreview = document.getElementById("animationPreview");
|
||||||
const mPeers = document.getElementById("mPeers");
|
const mPeers = document.getElementById("mPeers");
|
||||||
const mBusy = document.getElementById("mBusy");
|
const mBusy = document.getElementById("mBusy");
|
||||||
const mLatency = document.getElementById("mLatency");
|
const mLatency = document.getElementById("mLatency");
|
||||||
@@ -23,6 +42,9 @@ const mTts = document.getElementById("mTts");
|
|||||||
const mBargeIn = document.getElementById("mBargeIn");
|
const mBargeIn = document.getElementById("mBargeIn");
|
||||||
const mInputMode = document.getElementById("mInputMode");
|
const mInputMode = document.getElementById("mInputMode");
|
||||||
const mVadCount = document.getElementById("mVadCount");
|
const mVadCount = document.getElementById("mVadCount");
|
||||||
|
const mAnimClients = document.getElementById("mAnimClients");
|
||||||
|
const mAnimDriver = document.getElementById("mAnimDriver");
|
||||||
|
const mAnimFrames = document.getElementById("mAnimFrames");
|
||||||
const dConn = document.getElementById("dConn");
|
const dConn = document.getElementById("dConn");
|
||||||
const dVad = document.getElementById("dVad");
|
const dVad = document.getElementById("dVad");
|
||||||
const dAsr = document.getElementById("dAsr");
|
const dAsr = document.getElementById("dAsr");
|
||||||
@@ -32,11 +54,15 @@ const dTts = document.getElementById("dTts");
|
|||||||
let pc = null;
|
let pc = null;
|
||||||
let evt = null;
|
let evt = null;
|
||||||
let subtitleWs = null;
|
let subtitleWs = null;
|
||||||
|
let animationWs = null;
|
||||||
let localMicStream = null;
|
let localMicStream = null;
|
||||||
let lastSeenRun = 0;
|
|
||||||
let lastBusy = false;
|
|
||||||
let aiStreamingEl = null;
|
let aiStreamingEl = null;
|
||||||
|
|
||||||
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
|
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
|
||||||
|
const ANIMATION_BUFFER_MS = 120;
|
||||||
|
const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
|
||||||
|
const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
|
||||||
|
const CLOCK = new THREE.Clock();
|
||||||
|
|
||||||
const stateMap = {
|
const stateMap = {
|
||||||
"SessionState.IDLE": "空闲",
|
"SessionState.IDLE": "空闲",
|
||||||
@@ -49,6 +75,56 @@ const stateMap = {
|
|||||||
avatar_speaking: "数字人说话",
|
avatar_speaking: "数字人说话",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true });
|
||||||
|
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||||
|
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
||||||
|
|
||||||
|
const scene = new THREE.Scene();
|
||||||
|
scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5);
|
||||||
|
|
||||||
|
const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100);
|
||||||
|
camera.position.set(0, 1.45, 2.65);
|
||||||
|
|
||||||
|
const controls = new OrbitControls(camera, avatarCanvas);
|
||||||
|
controls.target.set(0, 1.15, 0);
|
||||||
|
controls.enableDamping = true;
|
||||||
|
controls.minDistance = 1.3;
|
||||||
|
controls.maxDistance = 4.5;
|
||||||
|
controls.maxPolarAngle = Math.PI * 0.58;
|
||||||
|
controls.minPolarAngle = Math.PI * 0.22;
|
||||||
|
|
||||||
|
const avatarState = {
|
||||||
|
currentRoot: null,
|
||||||
|
currentVrm: null,
|
||||||
|
morphBindings: new Map(),
|
||||||
|
pendingFrames: [],
|
||||||
|
scheduleCursorMs: performance.now(),
|
||||||
|
lastAppliedSeq: -1,
|
||||||
|
activeControls: {
|
||||||
|
jawOpen: 0,
|
||||||
|
viseme_aa: 0,
|
||||||
|
viseme_ee: 0,
|
||||||
|
viseme_oh: 0,
|
||||||
|
mouthPucker: 0,
|
||||||
|
headYaw: 0,
|
||||||
|
headPitch: 0,
|
||||||
|
headRoll: 0,
|
||||||
|
},
|
||||||
|
headBone: null,
|
||||||
|
neckBone: null,
|
||||||
|
idleRig: null,
|
||||||
|
debugRig: null,
|
||||||
|
debugJaw: null,
|
||||||
|
debugMouth: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const gltfLoader = new GLTFLoader();
|
||||||
|
gltfLoader.register((parser) => new VRMLoaderPlugin(parser));
|
||||||
|
gltfLoader.crossOrigin = "anonymous";
|
||||||
|
|
||||||
|
const TMP_EULER = new THREE.Euler();
|
||||||
|
const TMP_QUATERNION = new THREE.Quaternion();
|
||||||
|
|
||||||
function fmtTime() {
|
function fmtTime() {
|
||||||
return new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
return new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
||||||
}
|
}
|
||||||
@@ -82,6 +158,132 @@ function setDot(el, kind) {
|
|||||||
el.className = `dot ${kind}`;
|
el.className = `dot ${kind}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function initStage() {
|
||||||
|
scene.background = new THREE.Color(0x0b1118);
|
||||||
|
|
||||||
|
const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65);
|
||||||
|
hemiLight.position.set(0, 2.6, 0);
|
||||||
|
scene.add(hemiLight);
|
||||||
|
|
||||||
|
const keyLight = new THREE.DirectionalLight(0xffffff, 1.7);
|
||||||
|
keyLight.position.set(1.4, 2.2, 2.0);
|
||||||
|
scene.add(keyLight);
|
||||||
|
|
||||||
|
const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8);
|
||||||
|
rimLight.position.set(-2.0, 1.4, -1.5);
|
||||||
|
scene.add(rimLight);
|
||||||
|
|
||||||
|
const floor = new THREE.Mesh(
|
||||||
|
new THREE.CircleGeometry(2.8, 48),
|
||||||
|
new THREE.MeshStandardMaterial({
|
||||||
|
color: 0x101b28,
|
||||||
|
roughness: 0.94,
|
||||||
|
metalness: 0.08,
|
||||||
|
transparent: true,
|
||||||
|
opacity: 0.9,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
floor.rotation.x = -Math.PI / 2;
|
||||||
|
floor.position.y = -0.02;
|
||||||
|
scene.add(floor);
|
||||||
|
|
||||||
|
const halo = new THREE.Mesh(
|
||||||
|
new THREE.RingGeometry(1.2, 1.95, 64),
|
||||||
|
new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }),
|
||||||
|
);
|
||||||
|
halo.rotation.x = -Math.PI / 2;
|
||||||
|
halo.position.y = 0.01;
|
||||||
|
scene.add(halo);
|
||||||
|
|
||||||
|
createDebugRig();
|
||||||
|
resizeStage();
|
||||||
|
renderLoop();
|
||||||
|
}
|
||||||
|
|
||||||
|
function createDebugRig() {
|
||||||
|
const group = new THREE.Group();
|
||||||
|
|
||||||
|
const bustMaterial = new THREE.MeshStandardMaterial({
|
||||||
|
color: 0xc7d2e0,
|
||||||
|
roughness: 0.72,
|
||||||
|
metalness: 0.05,
|
||||||
|
});
|
||||||
|
const accentMaterial = new THREE.MeshStandardMaterial({
|
||||||
|
color: 0x0f172a,
|
||||||
|
roughness: 0.35,
|
||||||
|
metalness: 0.18,
|
||||||
|
});
|
||||||
|
const lipMaterial = new THREE.MeshStandardMaterial({
|
||||||
|
color: 0x8b425f,
|
||||||
|
roughness: 0.56,
|
||||||
|
metalness: 0.02,
|
||||||
|
});
|
||||||
|
|
||||||
|
const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial);
|
||||||
|
torso.position.set(0, 0.42, 0);
|
||||||
|
group.add(torso);
|
||||||
|
|
||||||
|
const neck = new THREE.Group();
|
||||||
|
neck.position.set(0, 0.95, 0);
|
||||||
|
group.add(neck);
|
||||||
|
|
||||||
|
const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial);
|
||||||
|
head.scale.set(0.92, 1.03, 0.92);
|
||||||
|
neck.add(head);
|
||||||
|
|
||||||
|
const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial);
|
||||||
|
hair.position.set(0, 0.06, 0);
|
||||||
|
neck.add(hair);
|
||||||
|
|
||||||
|
const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
|
||||||
|
leftEye.position.set(-0.1, 0.03, 0.28);
|
||||||
|
neck.add(leftEye);
|
||||||
|
|
||||||
|
const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
|
||||||
|
rightEye.position.set(0.1, 0.03, 0.28);
|
||||||
|
neck.add(rightEye);
|
||||||
|
|
||||||
|
const jawPivot = new THREE.Group();
|
||||||
|
jawPivot.position.set(0, -0.06, 0.15);
|
||||||
|
neck.add(jawPivot);
|
||||||
|
|
||||||
|
const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial);
|
||||||
|
jaw.position.set(0, -0.08, 0.02);
|
||||||
|
jawPivot.add(jaw);
|
||||||
|
|
||||||
|
const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial);
|
||||||
|
mouth.position.set(0, -0.02, 0.31);
|
||||||
|
neck.add(mouth);
|
||||||
|
|
||||||
|
scene.add(group);
|
||||||
|
avatarState.currentRoot = group;
|
||||||
|
avatarState.debugRig = group;
|
||||||
|
avatarState.debugJaw = jawPivot;
|
||||||
|
avatarState.debugMouth = mouth;
|
||||||
|
avatarState.headBone = neck;
|
||||||
|
avatarState.neckBone = neck;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resizeStage() {
|
||||||
|
const rect = avatarCanvas.getBoundingClientRect();
|
||||||
|
if (!rect.width || !rect.height) return;
|
||||||
|
renderer.setSize(rect.width, rect.height, false);
|
||||||
|
camera.aspect = rect.width / rect.height;
|
||||||
|
camera.updateProjectionMatrix();
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderLoop() {
|
||||||
|
requestAnimationFrame(renderLoop);
|
||||||
|
const delta = CLOCK.getDelta();
|
||||||
|
const elapsed = CLOCK.getElapsedTime();
|
||||||
|
controls.update();
|
||||||
|
flushAnimationQueue();
|
||||||
|
applyProceduralIdlePose(elapsed);
|
||||||
|
applyAnimationControls(avatarState.activeControls);
|
||||||
|
avatarState.currentVrm?.update?.(delta);
|
||||||
|
renderer.render(scene, camera);
|
||||||
|
}
|
||||||
|
|
||||||
function updateMetrics(data) {
|
function updateMetrics(data) {
|
||||||
mPeers.textContent = String(data.peers ?? 0);
|
mPeers.textContent = String(data.peers ?? 0);
|
||||||
mBusy.textContent = data.pipeline_busy ? "是" : "否";
|
mBusy.textContent = data.pipeline_busy ? "是" : "否";
|
||||||
@@ -96,6 +298,9 @@ function updateMetrics(data) {
|
|||||||
mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`;
|
mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`;
|
||||||
mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none";
|
mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none";
|
||||||
mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`;
|
mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`;
|
||||||
|
mAnimClients.textContent = String(data.animation_clients ?? 0);
|
||||||
|
mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--";
|
||||||
|
mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0);
|
||||||
|
|
||||||
setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn");
|
setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn");
|
||||||
setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
|
setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
|
||||||
@@ -104,6 +309,394 @@ function updateMetrics(data) {
|
|||||||
setDot(dTts, data?.tts?.ready ? "ok" : "warn");
|
setDot(dTts, data?.tts?.ready ? "ok" : "warn");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function queueAnimationFrames(payload) {
|
||||||
|
const frames = Array.isArray(payload.frames) ? payload.frames : [];
|
||||||
|
const now = performance.now();
|
||||||
|
const frameWindowMs = (1000 / Math.max(1, payload.fps || 25));
|
||||||
|
|
||||||
|
if (payload.type === "animation_reset" || payload.type === "animation_state") {
|
||||||
|
avatarState.pendingFrames = [];
|
||||||
|
avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS;
|
||||||
|
avatarState.activeControls = {
|
||||||
|
jawOpen: 0,
|
||||||
|
viseme_aa: 0,
|
||||||
|
viseme_ee: 0,
|
||||||
|
viseme_oh: 0,
|
||||||
|
mouthPucker: 0,
|
||||||
|
headYaw: 0,
|
||||||
|
headPitch: 0,
|
||||||
|
headRoll: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs);
|
||||||
|
frames.forEach((frame, index) => {
|
||||||
|
const fallbackTime = index * frameWindowMs;
|
||||||
|
avatarState.pendingFrames.push({
|
||||||
|
dueAt: baseTime + (frame.time_ms ?? fallbackTime),
|
||||||
|
seq: frame.seq ?? index,
|
||||||
|
controls: frame.controls || {},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq);
|
||||||
|
avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs);
|
||||||
|
animationBuffer.textContent = String(avatarState.pendingFrames.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
function flushAnimationQueue() {
|
||||||
|
const now = performance.now();
|
||||||
|
let frameToApply = null;
|
||||||
|
while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) {
|
||||||
|
frameToApply = avatarState.pendingFrames.shift();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) {
|
||||||
|
avatarState.lastAppliedSeq = frameToApply.seq;
|
||||||
|
avatarState.activeControls = {
|
||||||
|
...avatarState.activeControls,
|
||||||
|
...frameToApply.controls,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
animationBuffer.textContent = String(avatarState.pendingFrames.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyAnimationControls(controls) {
|
||||||
|
const safeControls = controls || {};
|
||||||
|
const jawOpen = clampControl(safeControls.jawOpen);
|
||||||
|
const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen);
|
||||||
|
const visemeEe = clampControl(safeControls.viseme_ee);
|
||||||
|
const visemeOh = clampControl(safeControls.viseme_oh);
|
||||||
|
const mouthPucker = clampControl(safeControls.mouthPucker);
|
||||||
|
const headYaw = clampSigned(safeControls.headYaw, 0.35);
|
||||||
|
const headPitch = clampSigned(safeControls.headPitch, 0.3);
|
||||||
|
const headRoll = clampSigned(safeControls.headRoll, 0.24);
|
||||||
|
const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0;
|
||||||
|
const t = performance.now() * 0.001;
|
||||||
|
|
||||||
|
if (avatarState.currentVrm?.expressionManager) {
|
||||||
|
setExpressionValue("aa", visemeAa);
|
||||||
|
setExpressionValue("ee", visemeEe);
|
||||||
|
setExpressionValue("oh", visemeOh);
|
||||||
|
setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (avatarState.morphBindings.size > 0) {
|
||||||
|
setMorphValue("jawOpen", jawOpen);
|
||||||
|
setMorphValue("viseme_aa", visemeAa);
|
||||||
|
setMorphValue("viseme_ee", visemeEe);
|
||||||
|
setMorphValue("viseme_oh", visemeOh);
|
||||||
|
setMorphValue("mouthPucker", mouthPucker);
|
||||||
|
}
|
||||||
|
|
||||||
|
// For plain head meshes (no VRM expressions and no morph targets),
|
||||||
|
// amplify head motion to keep speaking visually obvious.
|
||||||
|
const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09;
|
||||||
|
const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12;
|
||||||
|
const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04;
|
||||||
|
|
||||||
|
applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied);
|
||||||
|
if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) {
|
||||||
|
applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
|
||||||
|
const baseScale = 1.0 + jawOpen * 0.03;
|
||||||
|
avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (avatarState.debugJaw) {
|
||||||
|
avatarState.debugJaw.rotation.x = jawOpen * 0.5;
|
||||||
|
avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015;
|
||||||
|
}
|
||||||
|
if (avatarState.debugMouth) {
|
||||||
|
avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45;
|
||||||
|
avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8;
|
||||||
|
avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setExpressionValue(name, value) {
|
||||||
|
try {
|
||||||
|
avatarState.currentVrm.expressionManager.setValue(name, value);
|
||||||
|
} catch (_) {
|
||||||
|
// no-op: expression name may not exist on the current model
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setMorphValue(controlName, value) {
|
||||||
|
const bindings = avatarState.morphBindings.get(controlName) || [];
|
||||||
|
bindings.forEach((binding) => {
|
||||||
|
binding.mesh.morphTargetInfluences[binding.index] = value;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampControl(value) {
|
||||||
|
return Math.min(1, Math.max(0, Number(value) || 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampSigned(value, limit) {
|
||||||
|
return Math.min(limit, Math.max(-limit, Number(value) || 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetView() {
|
||||||
|
camera.position.set(0, 1.45, 2.65);
|
||||||
|
controls.target.set(0, 1.15, 0);
|
||||||
|
controls.update();
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetAvatarPose() {
|
||||||
|
avatarState.activeControls = {
|
||||||
|
jawOpen: 0,
|
||||||
|
viseme_aa: 0,
|
||||||
|
viseme_ee: 0,
|
||||||
|
viseme_oh: 0,
|
||||||
|
mouthPucker: 0,
|
||||||
|
headYaw: 0,
|
||||||
|
headPitch: 0,
|
||||||
|
headRoll: 0,
|
||||||
|
};
|
||||||
|
applyAnimationControls(avatarState.activeControls);
|
||||||
|
}
|
||||||
|
|
||||||
|
function detachCurrentAvatar() {
|
||||||
|
if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
|
||||||
|
scene.remove(avatarState.currentRoot);
|
||||||
|
}
|
||||||
|
avatarState.pendingFrames = [];
|
||||||
|
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
|
||||||
|
avatarState.lastAppliedSeq = -1;
|
||||||
|
avatarState.idleRig = null;
|
||||||
|
avatarState.currentRoot = avatarState.debugRig;
|
||||||
|
avatarState.currentVrm = null;
|
||||||
|
avatarState.morphBindings.clear();
|
||||||
|
avatarState.headBone = avatarState.debugRig.children[1];
|
||||||
|
avatarState.neckBone = avatarState.debugRig.children[1];
|
||||||
|
avatarState.debugRig.visible = true;
|
||||||
|
modelName.textContent = "debug-avatar";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadModelFromUrl(url) {
|
||||||
|
if (!url) return false;
|
||||||
|
modelStatus.textContent = "模型加载中...";
|
||||||
|
try {
|
||||||
|
const gltf = await gltfLoader.loadAsync(url);
|
||||||
|
attachLoadedAsset(gltf, url.split("/").pop() || url);
|
||||||
|
modelStatus.textContent = "模型已加载";
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
modelStatus.textContent = "模型加载失败，已回退调试头像";
|
||||||
|
addMessage("system", `模型加载失败：${error?.message || error}`);
|
||||||
|
detachCurrentAvatar();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadDefaultAvatarModel() {
|
||||||
|
modelUrl.value = DEFAULT_BODY_MODEL_URL;
|
||||||
|
const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
|
||||||
|
if (!ok) {
|
||||||
|
modelUrl.value = DEFAULT_VRM_FALLBACK_URL;
|
||||||
|
await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadModelFromFile(file) {
|
||||||
|
if (!file) return;
|
||||||
|
const url = URL.createObjectURL(file);
|
||||||
|
modelStatus.textContent = "本地模型加载中...";
|
||||||
|
try {
|
||||||
|
const gltf = await gltfLoader.loadAsync(url);
|
||||||
|
attachLoadedAsset(gltf, file.name);
|
||||||
|
modelStatus.textContent = "本地模型已加载";
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
modelStatus.textContent = "本地模型加载失败，已回退调试头像";
|
||||||
|
addMessage("system", `本地模型加载失败：${error?.message || error}`);
|
||||||
|
detachCurrentAvatar();
|
||||||
|
} finally {
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function attachLoadedAsset(gltf, label) {
|
||||||
|
detachCurrentAvatar();
|
||||||
|
|
||||||
|
const vrm = gltf.userData?.vrm || null;
|
||||||
|
const root = vrm?.scene || gltf.scene;
|
||||||
|
if (!root) {
|
||||||
|
throw new Error("模型中未找到可渲染的 scene");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (vrm) {
|
||||||
|
VRMUtils.removeUnnecessaryVertices(gltf.scene);
|
||||||
|
VRMUtils.combineSkeletons(gltf.scene);
|
||||||
|
VRMUtils.combineMorphs(vrm);
|
||||||
|
VRMUtils.rotateVRM0(vrm);
|
||||||
|
}
|
||||||
|
|
||||||
|
root.traverse((obj) => {
|
||||||
|
obj.frustumCulled = false;
|
||||||
|
});
|
||||||
|
|
||||||
|
root.position.set(0, 0, 0);
|
||||||
|
root.rotation.set(0, 0, 0);
|
||||||
|
root.scale.setScalar(1);
|
||||||
|
scene.add(root);
|
||||||
|
|
||||||
|
avatarState.currentRoot = root;
|
||||||
|
avatarState.currentVrm = vrm;
|
||||||
|
avatarState.debugRig.visible = false;
|
||||||
|
avatarState.morphBindings = collectMorphBindings(root);
|
||||||
|
avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1];
|
||||||
|
avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone;
|
||||||
|
avatarState.idleRig = buildIdleRig(vrm, root);
|
||||||
|
avatarState.pendingFrames = [];
|
||||||
|
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
|
||||||
|
avatarState.lastAppliedSeq = -1;
|
||||||
|
|
||||||
|
fitCameraToObject(root);
|
||||||
|
resetAvatarPose();
|
||||||
|
modelName.textContent = label;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fitCameraToObject(root) {
|
||||||
|
const box = new THREE.Box3().setFromObject(root);
|
||||||
|
const center = new THREE.Vector3();
|
||||||
|
const size = new THREE.Vector3();
|
||||||
|
box.getCenter(center);
|
||||||
|
box.getSize(size);
|
||||||
|
|
||||||
|
const radius = Math.max(size.x, size.y, size.z, 1.2);
|
||||||
|
controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0));
|
||||||
|
camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9);
|
||||||
|
controls.update();
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHeadBone(vrm, root) {
|
||||||
|
return (
|
||||||
|
vrm?.humanoid?.getNormalizedBoneNode?.("head") ||
|
||||||
|
vrm?.humanoid?.getRawBoneNode?.("head") ||
|
||||||
|
root.getObjectByName("Head") ||
|
||||||
|
root.getObjectByName("head") ||
|
||||||
|
root
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveNeckBone(vrm, root) {
|
||||||
|
return (
|
||||||
|
vrm?.humanoid?.getNormalizedBoneNode?.("neck") ||
|
||||||
|
vrm?.humanoid?.getRawBoneNode?.("neck") ||
|
||||||
|
root.getObjectByName("Neck") ||
|
||||||
|
root.getObjectByName("neck") ||
|
||||||
|
resolveHeadBone(vrm, root)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectMorphBindings(root) {
|
||||||
|
const bindings = new Map();
|
||||||
|
const aliases = {
|
||||||
|
jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"],
|
||||||
|
viseme_aa: ["viseme_aa", "aa", "a", "moutha"],
|
||||||
|
viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"],
|
||||||
|
viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"],
|
||||||
|
mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"],
|
||||||
|
};
|
||||||
|
|
||||||
|
root.traverse((node) => {
|
||||||
|
if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return;
|
||||||
|
const entries = Object.entries(node.morphTargetDictionary);
|
||||||
|
Object.entries(aliases).forEach(([controlName, names]) => {
|
||||||
|
entries.forEach(([targetName, index]) => {
|
||||||
|
const normalized = targetName.toLowerCase();
|
||||||
|
if (!names.includes(normalized)) return;
|
||||||
|
if (!bindings.has(controlName)) bindings.set(controlName, []);
|
||||||
|
bindings.get(controlName).push({ mesh: node, index });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return bindings;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildIdleRig(vrm, root) {
|
||||||
|
const lookup = {
|
||||||
|
hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]),
|
||||||
|
spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]),
|
||||||
|
chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]),
|
||||||
|
upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]),
|
||||||
|
neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]),
|
||||||
|
head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]),
|
||||||
|
leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]),
|
||||||
|
rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]),
|
||||||
|
leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]),
|
||||||
|
rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]),
|
||||||
|
leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]),
|
||||||
|
rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]),
|
||||||
|
leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]),
|
||||||
|
rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]),
|
||||||
|
};
|
||||||
|
|
||||||
|
const base = new Map();
|
||||||
|
Object.values(lookup).forEach((bone) => {
|
||||||
|
if (bone && !base.has(bone)) {
|
||||||
|
base.set(bone, bone.quaternion.clone());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return { bones: lookup, base };
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) {
|
||||||
|
return (
|
||||||
|
vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) ||
|
||||||
|
vrm?.humanoid?.getRawBoneNode?.(humanoidName) ||
|
||||||
|
fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) ||
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyProceduralIdlePose(time) {
|
||||||
|
const rig = avatarState.idleRig;
|
||||||
|
if (!rig) return;
|
||||||
|
|
||||||
|
const breathe = Math.sin(time * 1.9) * 0.018;
|
||||||
|
const sway = Math.sin(time * 1.15) * 0.05;
|
||||||
|
const armDrift = Math.sin(time * 1.55) * 0.07;
|
||||||
|
const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06;
|
||||||
|
const handDrift = Math.sin(time * 2.3) * 0.04;
|
||||||
|
|
||||||
|
applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04);
|
||||||
|
applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06);
|
||||||
|
applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08);
|
||||||
|
applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1);
|
||||||
|
|
||||||
|
applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18);
|
||||||
|
applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18);
|
||||||
|
applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0);
|
||||||
|
applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0);
|
||||||
|
applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12);
|
||||||
|
applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12);
|
||||||
|
applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05);
|
||||||
|
applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyBoneRotation(bone, x = 0, y = 0, z = 0) {
|
||||||
|
if (!bone) return;
|
||||||
|
const baseQuaternion = avatarState.idleRig?.base?.get(bone);
|
||||||
|
if (baseQuaternion) {
|
||||||
|
bone.quaternion.copy(baseQuaternion);
|
||||||
|
TMP_EULER.set(x, y, z, "XYZ");
|
||||||
|
TMP_QUATERNION.setFromEuler(TMP_EULER);
|
||||||
|
bone.quaternion.multiply(TMP_QUATERNION);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bone.rotation.x = x;
|
||||||
|
bone.rotation.y = y;
|
||||||
|
bone.rotation.z = z;
|
||||||
|
}
|
||||||
|
|
||||||
async function checkHealth() {
|
async function checkHealth() {
|
||||||
const res = await fetch("/health");
|
const res = await fetch("/health");
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
@@ -123,8 +716,6 @@ function subscribeEvents() {
|
|||||||
outputEl.textContent = JSON.stringify(data, null, 2);
|
outputEl.textContent = JSON.stringify(data, null, 2);
|
||||||
sendBtn.disabled = !!data.pipeline_busy;
|
sendBtn.disabled = !!data.pipeline_busy;
|
||||||
updateMetrics(data);
|
updateMetrics(data);
|
||||||
lastBusy = !!data.pipeline_busy;
|
|
||||||
if (typeof data.pipeline_runs === "number" && data.pipeline_runs > lastSeenRun) lastSeenRun = data.pipeline_runs;
|
|
||||||
} catch (_) {
|
} catch (_) {
|
||||||
// no-op
|
// no-op
|
||||||
}
|
}
|
||||||
@@ -160,6 +751,36 @@ function subscribeSubtitles() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function subscribeAnimation() {
|
||||||
|
if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return;
|
||||||
|
const scheme = location.protocol === "https:" ? "wss" : "ws";
|
||||||
|
animationConn.textContent = "连接中";
|
||||||
|
animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`);
|
||||||
|
animationWs.onopen = () => {
|
||||||
|
animationConn.textContent = "已连接";
|
||||||
|
};
|
||||||
|
animationWs.onmessage = (event) => {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(event.data);
|
||||||
|
if (data.type === "ping") return;
|
||||||
|
if (data.driver) {
|
||||||
|
animationDriver.textContent = data.driver;
|
||||||
|
}
|
||||||
|
const frameCount = data.frame_count ?? data.frames?.length ?? 0;
|
||||||
|
animationFrames.textContent = String(frameCount);
|
||||||
|
const preview = data.frames?.[0]?.controls || null;
|
||||||
|
animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2);
|
||||||
|
queueAnimationFrames(data);
|
||||||
|
} catch (_) {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
};
|
||||||
|
animationWs.onclose = () => {
|
||||||
|
animationConn.textContent = "已断开";
|
||||||
|
setTimeout(subscribeAnimation, 1000);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function connectWebRTC() {
|
async function connectWebRTC() {
|
||||||
if (pc) {
|
if (pc) {
|
||||||
outputEl.textContent = "WebRTC 已连接或正在连接";
|
outputEl.textContent = "WebRTC 已连接或正在连接";
|
||||||
@@ -175,21 +796,22 @@ async function connectWebRTC() {
|
|||||||
});
|
});
|
||||||
const currentPc = pc;
|
const currentPc = pc;
|
||||||
let seenConnected = false;
|
let seenConnected = false;
|
||||||
|
|
||||||
currentPc.onconnectionstatechange = () => {
|
currentPc.onconnectionstatechange = () => {
|
||||||
const st = currentPc.connectionState;
|
const st = currentPc.connectionState;
|
||||||
if (st === "connected") {
|
if (st === "connected") {
|
||||||
seenConnected = true;
|
seenConnected = true;
|
||||||
setConnBadge("已连接", "ok");
|
setConnBadge("已连接", "ok");
|
||||||
localStorage.setItem(AUTO_RECONNECT_KEY, "1");
|
localStorage.setItem(AUTO_RECONNECT_KEY, "1");
|
||||||
}
|
} else if (st === "connecting") {
|
||||||
else if (st === "connecting") setConnBadge("连接中...", "warn");
|
setConnBadge("连接中...", "warn");
|
||||||
else if (st === "failed" || st === "disconnected" || st === "closed") {
|
} else if (st === "failed" || st === "disconnected" || st === "closed") {
|
||||||
setConnBadge(`连接${st}`, "warn");
|
setConnBadge(`连接${st}`, "warn");
|
||||||
if (seenConnected && (st === "disconnected" || st === "failed")) {
|
if (seenConnected && (st === "disconnected" || st === "failed")) {
|
||||||
addMessage("system", "连接已断开，可能被新的客户端连接接管。");
|
addMessage("system", "连接已断开，可能被新的客户端连接接管。");
|
||||||
}
|
}
|
||||||
if (localMicStream) {
|
if (localMicStream) {
|
||||||
localMicStream.getTracks().forEach((t) => t.stop());
|
localMicStream.getTracks().forEach((track) => track.stop());
|
||||||
localMicStream = null;
|
localMicStream = null;
|
||||||
}
|
}
|
||||||
if (pc === currentPc) {
|
if (pc === currentPc) {
|
||||||
@@ -198,13 +820,12 @@ async function connectWebRTC() {
|
|||||||
offerBtn.disabled = false;
|
offerBtn.disabled = false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// Explicitly request remote media m-lines so server can attach tracks.
|
|
||||||
pc.addTransceiver("video", { direction: "recvonly" });
|
|
||||||
pc.ontrack = (event) => {
|
pc.ontrack = (event) => {
|
||||||
if (event.streams && event.streams[0]) {
|
if (event.streams && event.streams[0]) {
|
||||||
remoteVideo.srcObject = event.streams[0];
|
remoteAudio.srcObject = event.streams[0];
|
||||||
remoteVideo.play().catch(() => {
|
remoteAudio.play().catch(() => {
|
||||||
addMessage("system", "远端音视频已到达，如未自动播放请点击视频区域后重试。");
|
addMessage("system", "远端语音已到达，如未自动播放请先与页面交互后重试。");
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -220,12 +841,8 @@ async function connectWebRTC() {
|
|||||||
offerBtn.disabled = false;
|
offerBtn.disabled = false;
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
|
|
||||||
const sendAudioTrack = stream.getAudioTracks()[0];
|
|
||||||
if (sendAudioTrack) {
|
|
||||||
sendAudioTrack.enabled = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
|
||||||
const offer = await pc.createOffer();
|
const offer = await pc.createOffer();
|
||||||
await pc.setLocalDescription(offer);
|
await pc.setLocalDescription(offer);
|
||||||
|
|
||||||
@@ -240,11 +857,7 @@ async function connectWebRTC() {
|
|||||||
const answer = await res.json();
|
const answer = await res.json();
|
||||||
await pc.setRemoteDescription(answer);
|
await pc.setRemoteDescription(answer);
|
||||||
statusEl.textContent = "状态：WebRTC 已连接";
|
statusEl.textContent = "状态：WebRTC 已连接";
|
||||||
outputEl.textContent = JSON.stringify(
|
outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2);
|
||||||
{ connected: true, iceConnectionState: pc.iceConnectionState },
|
|
||||||
null,
|
|
||||||
2
|
|
||||||
);
|
|
||||||
setConnBadge("已连接", "ok");
|
setConnBadge("已连接", "ok");
|
||||||
addMessage("system", "WebRTC 已连接，可开始对话。");
|
addMessage("system", "WebRTC 已连接，可开始对话。");
|
||||||
if (answer.replaced_previous) {
|
if (answer.replaced_previous) {
|
||||||
@@ -256,8 +869,7 @@ async function connectWebRTC() {
|
|||||||
|
|
||||||
async function sendText() {
|
async function sendText() {
|
||||||
const text = (chatInput.value || "").trim();
|
const text = (chatInput.value || "").trim();
|
||||||
if (!text) return;
|
if (!text || sendBtn.disabled) return;
|
||||||
if (sendBtn.disabled) return;
|
|
||||||
sendBtn.disabled = true;
|
sendBtn.disabled = true;
|
||||||
const res = await fetch("/chat/text", {
|
const res = await fetch("/chat/text", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
@@ -277,6 +889,9 @@ async function sendText() {
|
|||||||
async function resetChat() {
|
async function resetChat() {
|
||||||
await fetch("/chat/reset", { method: "POST" });
|
await fetch("/chat/reset", { method: "POST" });
|
||||||
messagesEl.innerHTML = "";
|
messagesEl.innerHTML = "";
|
||||||
|
avatarState.pendingFrames = [];
|
||||||
|
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
|
||||||
|
resetAvatarPose();
|
||||||
addMessage("system", "会话已重置。");
|
addMessage("system", "会话已重置。");
|
||||||
await checkHealth();
|
await checkHealth();
|
||||||
}
|
}
|
||||||
@@ -284,24 +899,56 @@ async function resetChat() {
|
|||||||
function tryAutoReconnect() {
|
function tryAutoReconnect() {
|
||||||
if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
|
if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
if (!pc) connectWebRTC().catch(() => {
|
if (!pc) {
|
||||||
// no-op: user can reconnect manually if permission/policy blocks auto flow
|
connectWebRTC().catch(() => {
|
||||||
});
|
// no-op
|
||||||
|
});
|
||||||
|
}
|
||||||
}, 300);
|
}, 300);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function handleModelFileChange(event) {
|
||||||
|
const [file] = event.target.files || [];
|
||||||
|
if (!file) return;
|
||||||
|
loadModelFromFile(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleModelUrlLoad() {
|
||||||
|
const url = modelUrl.value.trim();
|
||||||
|
if (!url) return;
|
||||||
|
loadModelFromUrl(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
new ResizeObserver(() => resizeStage()).observe(avatarCanvas);
|
||||||
|
window.addEventListener("resize", resizeStage);
|
||||||
|
|
||||||
healthBtn.addEventListener("click", checkHealth);
|
healthBtn.addEventListener("click", checkHealth);
|
||||||
offerBtn.addEventListener("click", connectWebRTC);
|
offerBtn.addEventListener("click", connectWebRTC);
|
||||||
resetBtn.addEventListener("click", resetChat);
|
resetBtn.addEventListener("click", resetChat);
|
||||||
sendBtn.addEventListener("click", sendText);
|
sendBtn.addEventListener("click", sendText);
|
||||||
chatInput.addEventListener("keydown", (e) => {
|
loadModelBtn.addEventListener("click", handleModelUrlLoad);
|
||||||
if (e.key === "Enter") {
|
resetViewBtn.addEventListener("click", resetView);
|
||||||
e.preventDefault();
|
modelFile.addEventListener("change", handleModelFileChange);
|
||||||
|
chatInput.addEventListener("keydown", (event) => {
|
||||||
|
if (event.key === "Enter") {
|
||||||
|
event.preventDefault();
|
||||||
sendText();
|
sendText();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
modelUrl.addEventListener("keydown", (event) => {
|
||||||
|
if (event.key === "Enter") {
|
||||||
|
event.preventDefault();
|
||||||
|
handleModelUrlLoad();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
initStage();
|
||||||
|
resetView();
|
||||||
|
resetAvatarPose();
|
||||||
|
loadDefaultAvatarModel();
|
||||||
checkHealth();
|
checkHealth();
|
||||||
subscribeEvents();
|
subscribeEvents();
|
||||||
subscribeSubtitles();
|
subscribeSubtitles();
|
||||||
|
subscribeAnimation();
|
||||||
setConnBadge("未连接", "neutral");
|
setConnBadge("未连接", "neutral");
|
||||||
tryAutoReconnect();
|
tryAutoReconnect();
|
||||||
|
|||||||
+49
-4
@@ -8,11 +8,44 @@
|
|||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<main class="wrap">
|
<main class="wrap">
|
||||||
<h1>可视化语音聊天系统</h1>
|
<h1>3D 数字人语音聊天系统</h1>
|
||||||
<p id="status">状态：启动中</p>
|
<p id="status">状态：启动中</p>
|
||||||
<div class="video-box">
|
<div class="stage-box">
|
||||||
<video id="remoteVideo" autoplay playsinline></video>
|
<div class="stage-copy">
|
||||||
|
<strong>Three.js + VRM 驱动舞台</strong>
|
||||||
|
<span id="animationConn">未连接</span>
|
||||||
|
</div>
|
||||||
|
<div class="stage-toolbar">
|
||||||
|
<label class="file-pill" for="modelFile">载入本地 VRM/GLB</label>
|
||||||
|
<input id="modelFile" type="file" accept=".vrm,.glb,.gltf" />
|
||||||
|
<input id="modelUrl" type="text" placeholder="输入 VRM/GLB/GLTF 地址，例如官方示例 VRM 或 /web/models/avatar.vrm" />
|
||||||
|
<button id="loadModelBtn">加载模型 URL</button>
|
||||||
|
<button id="resetViewBtn">重置视角</button>
|
||||||
|
</div>
|
||||||
|
<div class="canvas-shell">
|
||||||
|
<canvas id="avatarCanvas"></canvas>
|
||||||
|
<div class="canvas-overlay">
|
||||||
|
<div class="overlay-card">
|
||||||
|
<span>模型状态</span>
|
||||||
|
<strong id="modelStatus">使用调试头像</strong>
|
||||||
|
</div>
|
||||||
|
<div class="overlay-card">
|
||||||
|
<span>驱动器</span>
|
||||||
|
<strong id="animationDriver">--</strong>
|
||||||
|
</div>
|
||||||
|
<div class="overlay-card">
|
||||||
|
<span>最近帧数</span>
|
||||||
|
<strong id="animationFrames">0</strong>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="stage-metrics">
|
||||||
|
<div><span>当前模型</span><strong id="modelName">debug-avatar</strong></div>
|
||||||
|
<div><span>动画缓冲</span><strong id="animationBuffer">0</strong></div>
|
||||||
|
</div>
|
||||||
|
<pre id="animationPreview">等待动画控制数据...</pre>
|
||||||
</div>
|
</div>
|
||||||
|
<audio id="remoteAudio" autoplay></audio>
|
||||||
<div class="actions">
|
<div class="actions">
|
||||||
<button id="healthBtn">检查服务状态</button>
|
<button id="healthBtn">检查服务状态</button>
|
||||||
<button id="offerBtn">连接 WebRTC</button>
|
<button id="offerBtn">连接 WebRTC</button>
|
||||||
@@ -45,12 +78,24 @@
|
|||||||
<div class="metric"><span>打断耗时</span><strong id="mBargeIn">0 ms</strong></div>
|
<div class="metric"><span>打断耗时</span><strong id="mBargeIn">0 ms</strong></div>
|
||||||
<div class="metric"><span>输入模式</span><strong id="mInputMode">none</strong></div>
|
<div class="metric"><span>输入模式</span><strong id="mInputMode">none</strong></div>
|
||||||
<div class="metric"><span>VAD触发</span><strong id="mVadCount">0/0</strong></div>
|
<div class="metric"><span>VAD触发</span><strong id="mVadCount">0/0</strong></div>
|
||||||
|
<div class="metric"><span>动画客户端</span><strong id="mAnimClients">0</strong></div>
|
||||||
|
<div class="metric"><span>动画驱动</span><strong id="mAnimDriver">--</strong></div>
|
||||||
|
<div class="metric"><span>最近动画帧</span><strong id="mAnimFrames">0</strong></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="panel">
|
<div class="panel">
|
||||||
<h3>运行状态</h3>
|
<h3>运行状态</h3>
|
||||||
<pre id="output"></pre>
|
<pre id="output"></pre>
|
||||||
</div>
|
</div>
|
||||||
</main>
|
</main>
|
||||||
<script src="/web/app.js"></script>
|
<script type="importmap">
|
||||||
|
{
|
||||||
|
"imports": {
|
||||||
|
"three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
|
||||||
|
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
|
||||||
|
"@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.5.1/lib/three-vrm.module.min.js"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
<script type="module" src="/web/app.js?v=20260327d"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
+181
-24
@@ -1,32 +1,167 @@
|
|||||||
body {
|
body {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
font-family: sans-serif;
|
font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
|
||||||
background: #0f1115;
|
background:
|
||||||
|
radial-gradient(circle at top, rgba(42, 93, 255, 0.16), transparent 36%),
|
||||||
|
radial-gradient(circle at right, rgba(15, 173, 129, 0.12), transparent 28%),
|
||||||
|
linear-gradient(180deg, #091018 0%, #111827 46%, #0b1018 100%);
|
||||||
color: #f2f2f2;
|
color: #f2f2f2;
|
||||||
}
|
}
|
||||||
|
|
||||||
.wrap {
|
.wrap {
|
||||||
max-width: 900px;
|
max-width: 1120px;
|
||||||
margin: 24px auto;
|
margin: 24px auto;
|
||||||
padding: 0 16px;
|
padding: 0 16px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.video-box {
|
.stage-box {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
aspect-ratio: 16 / 9;
|
border: 1px solid rgba(116, 139, 166, 0.22);
|
||||||
border: 1px solid #333;
|
border-radius: 24px;
|
||||||
border-radius: 10px;
|
background: linear-gradient(180deg, rgba(18, 25, 36, 0.94), rgba(12, 18, 28, 0.96));
|
||||||
background: #1a1f27;
|
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
padding: 18px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 14px;
|
||||||
|
box-shadow: 0 28px 70px rgba(0, 0, 0, 0.32);
|
||||||
}
|
}
|
||||||
|
|
||||||
video {
|
.stage-copy {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-copy strong {
|
||||||
|
font-size: 20px;
|
||||||
|
letter-spacing: 0.01em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-copy span {
|
||||||
|
font-size: 13px;
|
||||||
|
color: #7dd3fc;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-toolbar {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: auto minmax(0, 1fr) auto auto;
|
||||||
|
gap: 12px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-toolbar input[type="text"] {
|
||||||
|
min-width: 0;
|
||||||
|
background: rgba(10, 15, 24, 0.78);
|
||||||
|
color: #e5eef8;
|
||||||
|
border: 1px solid rgba(112, 146, 190, 0.3);
|
||||||
|
border-radius: 12px;
|
||||||
|
padding: 10px 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-toolbar input[type="file"] {
|
||||||
|
position: absolute;
|
||||||
|
width: 1px;
|
||||||
|
height: 1px;
|
||||||
|
opacity: 0;
|
||||||
|
pointer-events: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.file-pill {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
min-height: 40px;
|
||||||
|
border-radius: 999px;
|
||||||
|
border: 1px solid rgba(102, 199, 255, 0.28);
|
||||||
|
background: rgba(9, 60, 86, 0.32);
|
||||||
|
color: #d7f2ff;
|
||||||
|
padding: 0 16px;
|
||||||
|
cursor: pointer;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.canvas-shell {
|
||||||
|
position: relative;
|
||||||
|
aspect-ratio: 16 / 9;
|
||||||
|
border-radius: 18px;
|
||||||
|
overflow: hidden;
|
||||||
|
background:
|
||||||
|
radial-gradient(circle at 50% 28%, rgba(148, 196, 255, 0.18), transparent 24%),
|
||||||
|
linear-gradient(180deg, #152235 0%, #0c131d 52%, #0a1017 100%);
|
||||||
|
border: 1px solid rgba(95, 128, 160, 0.18);
|
||||||
|
}
|
||||||
|
|
||||||
|
#avatarCanvas {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
height: 100%;
|
height: 100%;
|
||||||
object-fit: cover;
|
|
||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.canvas-overlay {
|
||||||
|
position: absolute;
|
||||||
|
left: 14px;
|
||||||
|
right: 14px;
|
||||||
|
bottom: 14px;
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 10px;
|
||||||
|
pointer-events: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.overlay-card {
|
||||||
|
border: 1px solid rgba(144, 170, 197, 0.18);
|
||||||
|
background: rgba(7, 11, 18, 0.68);
|
||||||
|
backdrop-filter: blur(10px);
|
||||||
|
border-radius: 14px;
|
||||||
|
padding: 10px 12px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.overlay-card span {
|
||||||
|
color: #8ea6c3;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.overlay-card strong {
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-metrics {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-metrics div {
|
||||||
|
border: 1px solid rgba(52, 72, 96, 0.7);
|
||||||
|
border-radius: 14px;
|
||||||
|
background: rgba(14, 22, 33, 0.7);
|
||||||
|
padding: 12px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-metrics span {
|
||||||
|
color: #94a3b8;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-metrics strong {
|
||||||
|
font-size: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#animationPreview {
|
||||||
|
flex: 1;
|
||||||
|
min-height: 180px;
|
||||||
|
max-height: 260px;
|
||||||
|
}
|
||||||
|
|
||||||
.actions {
|
.actions {
|
||||||
margin-top: 16px;
|
margin-top: 16px;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -42,10 +177,10 @@ video {
|
|||||||
|
|
||||||
.chat-box input {
|
.chat-box input {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
background: #111827;
|
background: rgba(13, 19, 29, 0.9);
|
||||||
color: #f3f4f6;
|
color: #f3f4f6;
|
||||||
border: 1px solid #374151;
|
border: 1px solid #374151;
|
||||||
border-radius: 8px;
|
border-radius: 12px;
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,9 +190,9 @@ video {
|
|||||||
max-height: 360px;
|
max-height: 360px;
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
border: 1px solid #1f2937;
|
border: 1px solid #1f2937;
|
||||||
border-radius: 10px;
|
border-radius: 16px;
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
background: #0b1220;
|
background: rgba(10, 18, 32, 0.82);
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
gap: 8px;
|
gap: 8px;
|
||||||
@@ -66,8 +201,8 @@ video {
|
|||||||
.diag {
|
.diag {
|
||||||
margin-top: 12px;
|
margin-top: 12px;
|
||||||
border: 1px solid #1f2937;
|
border: 1px solid #1f2937;
|
||||||
border-radius: 10px;
|
border-radius: 16px;
|
||||||
background: #0b0d12;
|
background: rgba(8, 13, 18, 0.78);
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
display: flex;
|
display: flex;
|
||||||
gap: 16px;
|
gap: 16px;
|
||||||
@@ -122,12 +257,12 @@ video {
|
|||||||
|
|
||||||
.msg.user {
|
.msg.user {
|
||||||
align-self: flex-end;
|
align-self: flex-end;
|
||||||
background: #1d4ed8;
|
background: linear-gradient(135deg, #2563eb, #0ea5e9);
|
||||||
}
|
}
|
||||||
|
|
||||||
.msg.ai {
|
.msg.ai {
|
||||||
align-self: flex-start;
|
align-self: flex-start;
|
||||||
background: #374151;
|
background: linear-gradient(135deg, #334155, #475569);
|
||||||
}
|
}
|
||||||
|
|
||||||
.panel {
|
.panel {
|
||||||
@@ -143,8 +278,8 @@ video {
|
|||||||
|
|
||||||
.metric {
|
.metric {
|
||||||
border: 1px solid #1f2937;
|
border: 1px solid #1f2937;
|
||||||
background: #0b0d12;
|
background: rgba(8, 13, 18, 0.78);
|
||||||
border-radius: 8px;
|
border-radius: 14px;
|
||||||
padding: 8px 10px;
|
padding: 8px 10px;
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
@@ -167,10 +302,10 @@ video {
|
|||||||
}
|
}
|
||||||
|
|
||||||
button {
|
button {
|
||||||
background: #2563eb;
|
background: linear-gradient(135deg, #2563eb, #0891b2);
|
||||||
color: white;
|
color: white;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 8px;
|
border-radius: 12px;
|
||||||
padding: 8px 12px;
|
padding: 8px 12px;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
@@ -231,9 +366,31 @@ button:disabled {
|
|||||||
|
|
||||||
pre {
|
pre {
|
||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
background: #0b0d12;
|
background: rgba(8, 13, 18, 0.84);
|
||||||
border: 1px solid #222;
|
border: 1px solid #222;
|
||||||
border-radius: 8px;
|
border-radius: 14px;
|
||||||
padding: 12px;
|
padding: 12px;
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
audio {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 720px) {
|
||||||
|
.metrics {
|
||||||
|
grid-template-columns: repeat(2, minmax(120px, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-toolbar {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stage-metrics {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
|
.canvas-overlay {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
+4
-7
@@ -3,21 +3,18 @@ from __future__ import annotations
|
|||||||
from aiortc import RTCPeerConnection
|
from aiortc import RTCPeerConnection
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
from config import settings
|
|
||||||
from core.state_machine import Arbitrator
|
|
||||||
from services.avatar import AvatarService
|
|
||||||
from webrtc.audio_track import build_audio_track
|
from webrtc.audio_track import build_audio_track
|
||||||
from webrtc.tracks import AudioBus
|
from webrtc.tracks import AudioBus
|
||||||
from webrtc.video_track import build_video_track
|
|
||||||
|
|
||||||
|
|
||||||
def attach_webrtc_tracks(
|
def attach_webrtc_tracks(
|
||||||
pc: RTCPeerConnection,
|
pc: RTCPeerConnection,
|
||||||
avatar_service: AvatarService,
|
avatar_service,
|
||||||
arbitrator: Arbitrator,
|
arbitrator,
|
||||||
audio_bus: AudioBus,
|
audio_bus: AudioBus,
|
||||||
) -> None:
|
) -> None:
|
||||||
pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
|
_ = avatar_service
|
||||||
|
_ = arbitrator
|
||||||
pc.addTrack(build_audio_track(audio_bus))
|
pc.addTrack(build_audio_track(audio_bus))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-39
@@ -6,10 +6,7 @@ from collections import deque
|
|||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from aiortc import MediaStreamTrack
|
from aiortc import MediaStreamTrack
|
||||||
from av import AudioFrame, VideoFrame
|
from av import AudioFrame
|
||||||
|
|
||||||
from core.state_machine import Arbitrator, SessionState
|
|
||||||
from services.avatar import AvatarService
|
|
||||||
|
|
||||||
|
|
||||||
class AudioBus:
|
class AudioBus:
|
||||||
@@ -55,41 +52,6 @@ class AudioBus:
|
|||||||
self._level = 0.0
|
self._level = 0.0
|
||||||
|
|
||||||
|
|
||||||
class AvatarVideoTrack(MediaStreamTrack):
|
|
||||||
kind = "video"
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
avatar_service: AvatarService,
|
|
||||||
arbitrator: Arbitrator,
|
|
||||||
audio_bus: AudioBus,
|
|
||||||
fps: int = 25,
|
|
||||||
) -> None:
|
|
||||||
super().__init__()
|
|
||||||
self.avatar_service = avatar_service
|
|
||||||
self.arbitrator = arbitrator
|
|
||||||
self.audio_bus = audio_bus
|
|
||||||
self.fps = fps
|
|
||||||
self._pts = 0
|
|
||||||
self._time_base = Fraction(1, 90000)
|
|
||||||
|
|
||||||
async def recv(self) -> VideoFrame:
|
|
||||||
await asyncio.sleep(1 / self.fps)
|
|
||||||
generated = await self.avatar_service.pop_generated_frame()
|
|
||||||
if generated is not None:
|
|
||||||
arr = generated
|
|
||||||
else:
|
|
||||||
level = await self.audio_bus.level()
|
|
||||||
speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
|
|
||||||
arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
|
|
||||||
|
|
||||||
frame = VideoFrame.from_ndarray(arr, format="bgr24")
|
|
||||||
frame.pts = self._pts
|
|
||||||
frame.time_base = self._time_base
|
|
||||||
self._pts += int(90000 / self.fps)
|
|
||||||
return frame
|
|
||||||
|
|
||||||
|
|
||||||
class AvatarAudioTrack(MediaStreamTrack):
|
class AvatarAudioTrack(MediaStreamTrack):
|
||||||
kind = "audio"
|
kind = "audio"
|
||||||
|
|
||||||
|
|||||||
+3
-17
@@ -1,21 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from core.state_machine import Arbitrator
|
__all__: list[str] = []
|
||||||
from services.avatar import AvatarService
|
|
||||||
from webrtc.tracks import AudioBus, AvatarVideoTrack
|
|
||||||
|
|
||||||
__all__ = ["AvatarVideoTrack", "build_video_track"]
|
|
||||||
|
|
||||||
|
|
||||||
def build_video_track(
|
def build_video_track(*_args, **_kwargs):
|
||||||
avatar_service: AvatarService,
|
raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")
|
||||||
arbitrator: Arbitrator,
|
|
||||||
audio_bus: AudioBus,
|
|
||||||
fps: int = 25,
|
|
||||||
) -> AvatarVideoTrack:
|
|
||||||
return AvatarVideoTrack(
|
|
||||||
avatar_service=avatar_service,
|
|
||||||
arbitrator=arbitrator,
|
|
||||||
audio_bus=audio_bus,
|
|
||||||
fps=fps,
|
|
||||||
)
|
|
||||||
|
|||||||
Reference in New Issue
Block a user