commit 22a5f318472bd0a8a329dba71896b714432a2785 Author: xsl Date: Fri Mar 27 10:49:34 2026 +0800 init diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md new file mode 100644 index 0000000..486221e --- /dev/null +++ b/DEPLOYMENT_GUIDE.md @@ -0,0 +1,242 @@ +# 可视化语音聊天系统（初版）说明文档 + +本文档覆盖： +- 资源位置（代码、模型、环境） +- 本机启动与部署 +- 局域网/远程访问 +- 常用运维命令 +- 验收与排障 + +--- + +## 1. 项目概览 + +当前项目实现的是单机部署的可视化聊天系统（WebRTC）： +- 用户语音上行（VAD + ASR） +- 文本调用在线 LLM +- 本地 TTS 合成语音 +- 数字人视频下行（MuseTalk 优先，失败回退写实底片） +- 前端字幕与指标面板（SSE + WebSocket） +- 单工仲裁（支持 barge-in 打断） + +代码根目录： +- `/home/xsl/product` + +--- + +## 2. 代码结构（关键目录） + +- `main.py`：后端入口（FastAPI + WebRTC + 全链路编排） +- `config.py`：配置加载（读取 `.env`） +- `core/` + - `state_machine.py`：会话状态机（单工仲裁） + - `pipeline.py`：VAD/ASR/LLM/TTS/字幕/时延逻辑 +- `services/` + - `asr.py`：SenseVoice + - `llm.py`：在线模型 + fallback + - `tts.py`：Kokoro + - `vad.py`：Silero VAD + - `avatar.py`：MuseTalk 推理调度 + 写实回退帧 +- `webrtc/` + - `tracks.py`：实际音视频轨道实现 + - `gateway.py`、`video_track.py`、`audio_track.py`：结构化封装 +- `web/` + - `index.html`、`app.js`、`style.css` +- `scripts/` + - 启停：`service.sh`、`start-public.sh` + - 验收：`run_all_checks.sh`、`qa_check.py`、`smoke_test.py`、`model_probe.py`、`vad_check.py`、`tts_stress.py`、`llm_fallback_check.py` + - 监控：`gpu_monitor.sh` + +--- + +## 3. 环境与模型资源位置 + +### 3.1 Conda 环境 + +当前实际运行环境： +- `/home/xsl/miniconda3/envs/MuseTalk` + +系统启动脚本固定使用该 Python： +- `scripts/start-public.sh` 中 `PY=/home/xsl/miniconda3/envs/MuseTalk/bin/python` + +### 3.2 主要模型路径 + +MuseTalk 模型目录： +- `/home/xsl/work/MuseTalk/models/` + +关键文件： +- UNet：`/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth` +- 配置：`/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json` +- Whisper：`/home/xsl/work/MuseTalk/models/whisper` + +数字人源视频（写实底片 + MuseTalk 驱动源）： +- `/home/xsl/work/MuseTalk/data/video/yongen.mp4` + +MuseTalk 推理脚本： +- `/home/xsl/work/MuseTalk/scripts/inference.py` + +### 3.3 其他环境记录 + +完整环境与模型清单参考： +- `environments.md` + +--- + +## 4. 配置文件 + +主配置文件： +- `/home/xsl/product/.env` + +当前关键项（示例）： +- `LLM_API_KEY` +- `LLM_BASE_URL=https://api.deepseek.com` +- `WEBRTC_HOST=0.0.0.0` +- `WEBRTC_PORT=8080` +- `STUN_URL=stun:stun.l.google.com:19302` +- `SSL_CERTFILE=/home/xsl/product/certs/dev-cert.pem` +- `SSL_KEYFILE=/home/xsl/product/certs/dev-key.pem` + +说明： +- `LLM_MODEL` 为空时会自动按 `base_url` 推断（DeepSeek 默认 `deepseek-chat`）。 + +--- + +## 5. 启动与部署 + +### 5.1 本机启动（推荐） + +```bash +bash /home/xsl/product/scripts/service.sh start +bash /home/xsl/product/scripts/service.sh status +``` + +重启： + +```bash +bash /home/xsl/product/scripts/service.sh restart +``` + +停止： + +```bash +bash /home/xsl/product/scripts/service.sh stop +``` + +日志： + +```bash +bash /home/xsl/product/scripts/service.sh logs +``` + +### 5.2 访问地址 + +- HTTPS（推荐）：`https://<服务器IP>:8080/` +- 本机：`https://127.0.0.1:8080/` + +如果你使用自签证书，浏览器首次需要手动信任。 + +### 5.3 远程/跨机器访问 + +见： +- `REMOTE_ACCESS.md` + +重点： +- 跨机器麦克风通常要求 HTTPS +- WSL2 场景可能需要 Windows 端口转发/防火墙放行 + +--- + +## 6. 一键验收与调试命令 + +### 6.1 一键全检查 + +```bash +bash /home/xsl/product/scripts/run_all_checks.sh +``` + +覆盖： +- 服务状态 +- 文本链路压测 +- LLM fallback +- VAD 检查 +- TTS 20 句稳定性 +- smoke 媒体输出（`wav + mp4`） + +### 6.2 单项脚本 + +```bash +conda run -n MuseTalk python /home/xsl/product/scripts/model_probe.py +conda run -n MuseTalk python /home/xsl/product/scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 20 +conda run -n MuseTalk python /home/xsl/product/scripts/llm_fallback_check.py +conda run -n MuseTalk python /home/xsl/product/scripts/vad_check.py +conda run -n MuseTalk python /home/xsl/product/scripts/tts_stress.py +conda run -n MuseTalk python /home/xsl/product/scripts/smoke_test.py --text "全链路验收" +``` + +### 6.3 长稳与GPU监控 + +1小时长稳： + +```bash +conda run -n MuseTalk python /home/xsl/product/scripts/longrun_test.py --base-url "https://127.0.0.1:8080" --minutes 60 --interval-sec 30 +``` + +GPU采样（CSV）： + +```bash +bash /home/xsl/product/scripts/gpu_monitor.sh /home/xsl/product/outputs/gpu-1h.csv 3600 5 +``` + +--- + +## 7. 前端功能说明 + +网页包含： +- 视频窗口（数字人） +- 文本输入框 +- 字幕消息区（WebSocket） +- 指标面板（SSE） + +当前已支持： +- 页面刷新自动重连 +- 单连接策略（新连接踢掉旧连接） +- 句级字幕流式下发 +- 打断耗时等指标展示 + +--- + +## 8. 当前已知边界 + +- MuseTalk 结果生成可能有一定延迟，期间使用写实底片回退。 +- 部分“主观体验类”验收项（如口型同步主观评分）需要人工实测判定。 +- Qwen3-TTS 对比方案尚未并入主链路（可后续做 A/B 开关）。 + +--- + +## 9. 快速排障 + +1) 页面无声/无画面 +- 先看 `service.sh status` +- 看 `/health` 是否正常 +- 浏览器是否授权麦克风、是否信任 HTTPS 证书 + +2) 连接冲突 +- 系统是单连接模式，新连接会断开旧连接（预期） + +3) 口型不动 +- 查看 `/health` 中 `avatar.queued_frames` 和 `avatar.last_error` +- 若 MuseTalk 未产帧，会回退写实底片 + 口型叠加 + +4) 模型异常 +- 跑 `scripts/model_probe.py` 快速定位 ASR/TTS/VAD + +--- + +## 10. 版本建议 + +如果要做“生产化下一步”，建议优先： +- 增加 TURN（公网复杂 NAT） +- 增加多会话隔离（当前默认单会话） +- 增加 Prometheus/结构化日志 +- 增加模型切换开关（Kokoro/Qwen3-TTS） + diff --git a/REMOTE_ACCESS.md b/REMOTE_ACCESS.md new file mode 100644 index 0000000..9b37775 --- /dev/null +++ b/REMOTE_ACCESS.md @@ -0,0 +1,72 @@ +# 跨机器访问说明 + +## 1) 服务监听 + +已默认监听 `0.0.0.0:8080`，同网段机器可访问： + +- `http://<你的机器IP>:8080/` + +推荐用服务脚本管理（避免旧进程占端口）： + +```bash +bash /home/xsl/product/scripts/service.sh start +bash /home/xsl/product/scripts/service.sh status +bash /home/xsl/product/scripts/service.sh logs +bash /home/xsl/product/scripts/service.sh restart +bash /home/xsl/product/scripts/service.sh stop +``` + +查询本机 IP: + +```bash +ip addr +``` + +## 2) 浏览器麦克风权限（关键） + +跨机器访问时，浏览器通常要求 **HTTPS** 才允许 `getUserMedia`。 + +### 方案 A（推荐）：启用 HTTPS + +```bash +bash /home/xsl/product/scripts/gen-self-signed-cert.sh +``` + +然后编辑 `.env`: + +```env +SSL_CERTFILE=/home/xsl/product/certs/dev-cert.pem +SSL_KEYFILE=/home/xsl/product/certs/dev-key.pem +``` + +启动： + +```bash +bash /home/xsl/product/scripts/service.sh restart +``` + +访问： + +- `https://<你的机器IP>:8080/` + +首次需要在浏览器信任自签证书。 + +### 方案 B：HTTP 测试 + +如果仅做 UI 验证，可先用 HTTP；但麦克风可能被浏览器拒绝。 + +## 3) WSL2 网络注意事项 + +如果服务跑在 WSL2，其他机器无法直接访问时，需要在 Windows 做端口转发与防火墙放行。 + +最简单做法：在 Windows 上运行同样服务，或配置端口代理到 WSL IP。 + +## 4) WebRTC 联通 + +系统已配置 STUN（`.env` 里的 `STUN_URL`），用于获取可达候选地址： + +```env +STUN_URL=stun:stun.l.google.com:19302 +``` + +若跨公网且 NAT 复杂，需增加 TURN（后续可加）。 diff --git a/__pycache__/config.cpython-310.pyc b/__pycache__/config.cpython-310.pyc new file mode 100644 index 0000000..9c2fcf8 Binary files /dev/null and b/__pycache__/config.cpython-310.pyc differ diff --git a/__pycache__/main.cpython-310.pyc b/__pycache__/main.cpython-310.pyc new file mode 100644 index 0000000..a48e754 Binary files /dev/null and b/__pycache__/main.cpython-310.pyc differ diff --git a/certs/dev-cert.pem b/certs/dev-cert.pem new file mode 100644 index 0000000..0ced323 --- /dev/null +++ b/certs/dev-cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqTCCApGgAwIBAgIUbcvkFMlURnbhFIhpH2oqzlhUxTQwDQYJKoZIhvcNAQEL +BQAwZDELMAkGA1UEBhMCQ04xDjAMBgNVBAgMBUxvY2FsMQ4wDAYDVQQHDAVMb2Nh +bDETMBEGA1UECgwKVmlzdWFsQ2hhdDEMMAoGA1UECwwDRGV2MRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMjYwMzI1MTIzMDMzWhcNMjcwMzI1MTIzMDMzWjBkMQswCQYD +VQQGEwJDTjEOMAwGA1UECAwFTG9jYWwxDjAMBgNVBAcMBUxvY2FsMRMwEQYDVQQK +DApWaXN1YWxDaGF0MQwwCgYDVQQLDANEZXYxEjAQBgNVBAMMCWxvY2FsaG9zdDCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKAylFqQNYgJoe54z2gF4sLd +4vohuhvRM8wXuQzYiuEAIllsYghjDuB5l6WjuW/wE2/9l1JAW9i5JRLXUmRQQ2tW +khDjdLHJAemR59C2La0DmUU6RZ5K7Fo6cBXUIsvrc1ZJ9NdabCTLmlyDq5DsVJ2j +DqiUdkuSZr6Sc70mnvhBmdOplvZI45lP62dGfaQwBpGZklP+fMuWI4Ns1WUHcfBf +5LsRYWj/88e9zlwsazyLatME9n/IdxFvYz4Rgtw4+Osu6V7jjVRMKO8qLT80RlO1 +39M3iPRLOnqdQhw0EaMxmOj7Q9vtG+XP47kbkqhwje2wsdIpnY6coZ4zWEXIr9cC +AwEAAaNTMFEwHQYDVR0OBBYEFJ1eHZoc8d1AijuPlFUJbuBf7Uo1MB8GA1UdIwQY +MBaAFJ1eHZoc8d1AijuPlFUJbuBf7Uo1MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBABJ/QEbsRxGrmMZK8RA4CCFMW5U6iNnSfgXYdAhUu2VDbujb +wfUuKCLILbuB+D3hpOI1MwnmlkNnsqym6254sr9FtznDF1TvD0bTnJagzcFn5QQu +VVoV3gX6K4ZcOscwGOVk5fWoIAzFtx10hiSo88xuG2vDTyBwm7AStExkU7NFM+YD +st5/MWzjVMaZ8RiTGFTHY+/JC5JB6xuLlJpDK6HV8LMdRthd3u5Y+ea9omd9wxOL +7qn3LGZJkX2yCy8WjC4DDU+hK+nOr4OQjswIMaJ2a/FItTuvnZkej+xviqMvGhRS +/Ce5vRIs2/SzQ8t0kddEynEM5e2KO3Aa7kew1oo= +-----END CERTIFICATE----- diff --git a/certs/dev-key.pem b/certs/dev-key.pem new file mode 100644 index 0000000..93ad51f --- /dev/null +++ b/certs/dev-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCgMpRakDWICaHu +eM9oBeLC3eL6Ibob0TPMF7kM2IrhACJZbGIIYw7geZelo7lv8BNv/ZdSQFvYuSUS +11JkUENrVpIQ43SxyQHpkefQti2tA5lFOkWeSuxaOnAV1CLL63NWSfTXWmwky5pc +g6uQ7FSdow6olHZLkma+knO9Jp74QZnTqZb2SOOZT+tnRn2kMAaRmZJT/nzLliOD +bNVlB3HwX+S7EWFo//PHvc5cLGs8i2rTBPZ/yHcRb2M+EYLcOPjrLule441UTCjv +Ki0/NEZTtd/TN4j0Szp6nUIcNBGjMZjo+0Pb7Rvlz+O5G5KocI3tsLHSKZ2OnKGe +M1hFyK/XAgMBAAECggEASd2YlzuYW04puuY8qDfSi8y47GHhyRLI4ensWxZ7zux7 +aiU1/K0EPpACUw9voUQfCkvxWq0vTHpuNEZRiMyTkao8tonSlGldNuAedheSbwzS +Pt/0Gt1sJtx8Myx6EPHTYC3Atg3NAJha6B6UXMID8B2v1B6EfysVsfigYk6tDUWi +iF+emJPiwiLxk+blMyWmZCRrVozjIwVbHicloP+BKvsD18g2NG7jy/CkNFSb3AQx +eeP1J4HYQqRjNzAfD3mehyycfwq2OqHl0ys2mVj4nAsdgq7cpG987hiMLYh9czZN ++MMa5ppNR+ueMJZ+mTHVEdSnrFCuqOhcICpwX3Uc9QKBgQDg/ruuwv5DpHIAU9t/ +2U0ljEXK3tMCcjmowPJuMZvg9G0XNahG86S2DoCvx9u3JVuKedSbSyNTOeDFr/XQ +H9bB4tOwoGvDTPFahqEDg5qOO53+NQPC8VuP0HCMJttw71PW/MmXZaOxTGXihbNm +Hslmgx3+tpypXV9NXZ4HDlmZNQKBgQC2RfPKdG0BDLilP+6umW8y0xvpwnRiGYx6 +htGT/CMt2FPy0bfjv1WJqiwnXnEFabaFMnleX9RISX67ewKh/JSE/YdFz2nly7r0 +luRptcaPUNfEfoT4qBbs4wPno7u6oe8SbHqztkaJexC4oqNM1qO1QE926aZRcoJT +dtXF+TOSWwKBgQC68w6R1NYe8GoFWaheH/oZQ+fN/L4KH01HqiMGatQh5BctHNmZ +MuNenshQxtkK4dChTD/bVWChy8q3mFDAwWPZsJbDAVjpa2p0FL9/Qv/ORT0vN2/L +sG6rHcAWxEa8HTBlx5/d24dbT6asxPle1h/0vOfUeHnNxJbmmu4u4L6ULQKBgB5+ +TW7NU38ddbaPn4quOKA0GTNeSMIwrAPDkkRDpk8BJeo1k6ISXGkPgxuC/T1+lf5q +l0tmlMkTIpSS27nl26L0FzFipcC/+KL6q3PT6UgIqlBBKlW/KTawM/MIvVtXw/s9 +EroGAH73i7CX5OHx9qlX/PNT23M8yPjVpKXeLvaJAoGARCFHHW84urPWBhSOd6X/ +fqtZvlciFBaXifMnInCqqDVfRwfNa/ka50zNEQQNhC/6VBzLoo/anzjfhkah9UrB +BmmPZN97iu/fdZ7n6Iqt6nAQCl23/EyTyM/u0rTsm+0a7PjESxGhfyEmiVNRv1V3 +fs5SsXoyZJjIL2if3S+r9lM= +-----END PRIVATE KEY----- diff --git a/config.py b/config.py new file mode 100644 index 0000000..4592c08 --- /dev/null +++ b/config.py @@ -0,0 +1,32 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") + + hf_token: str = "" + llm_api_key: str = "" + llm_base_url: str = "https://api.openai.com/v1" + llm_model: str = "gpt-4o-mini" + llm_timeout: int = 8 + llm_max_tokens: int = 150 + + webrtc_host: str = "0.0.0.0" + webrtc_port: int = 8080 + avatar_fps: int = 25 + stun_url: str = "stun:stun.l.google.com:19302" + cors_origins: str = "*" + ssl_certfile: str = "" + ssl_keyfile: str = "" + + musetalk_enabled: bool = True + musetalk_repo_dir: str = "/home/xsl/work/MuseTalk" + musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py" + musetalk_source_video: str = "/home/xsl/work/MuseTalk/data/video/yongen.mp4" + musetalk_unet_model: str = "/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth" + musetalk_unet_config: str = "/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json" + musetalk_whisper_dir: str = "/home/xsl/work/MuseTalk/models/whisper" + musetalk_result_dir: str = "/home/xsl/work/MuseTalk/results/visual-chat" + + +settings = Settings() diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1 @@ + diff --git a/core/__pycache__/__init__.cpython-310.pyc b/core/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..d4990e2 Binary files /dev/null and b/core/__pycache__/__init__.cpython-310.pyc differ diff --git a/core/__pycache__/pipeline.cpython-310.pyc b/core/__pycache__/pipeline.cpython-310.pyc new file mode 100644 index 0000000..1725f6f Binary files /dev/null and b/core/__pycache__/pipeline.cpython-310.pyc differ diff --git a/core/__pycache__/state_machine.cpython-310.pyc b/core/__pycache__/state_machine.cpython-310.pyc new file mode 100644 index 0000000..19dd5f8 Binary files /dev/null and b/core/__pycache__/state_machine.cpython-310.pyc differ diff --git a/core/pipeline.py b/core/pipeline.py new file mode 100644 index 0000000..299792f --- /dev/null +++ b/core/pipeline.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +import time +from typing import Awaitable, Callable, Optional +import numpy as np + +from core.state_machine import Arbitrator +from services.asr import ASRService +from services.llm import LLMService +from services.tts import TTSService + + +class ChatPipeline: + def __init__( + self, + arbitrator: Arbitrator, + asr: ASRService, + llm: LLMService, + tts: TTSService, + ) -> None: + self.arbitrator = arbitrator + self.asr = asr + self.llm = llm + self.tts = tts + + async def _finish_avatar_after_playback(self, playback_s: float) -> None: + t_end = time.perf_counter() + playback_s + while time.perf_counter() < t_end: + if self.arbitrator.cancel_avatar_event.is_set(): + break + await asyncio.sleep(0.05) + await self.arbitrator.on_avatar_done() + + @staticmethod + def _split_sentences(text: str, max_len: int = 24) -> list[str]: + # Two-stage split: + # 1) hard sentence boundaries + # 2) soft commas / max length chunks for lower first-chunk latency + hard_seps = "。！？!?；;\n" + soft_seps = "，、, " + + stage1 = [] + buf = [] + for ch in text: + buf.append(ch) + if ch in hard_seps: + seg = "".join(buf).strip() + if seg: + stage1.append(seg) + buf = [] + tail = "".join(buf).strip() + if tail: + stage1.append(tail) + if not stage1: + stage1 = [text] + + out = [] + for seg in stage1: + cur = [] + for ch in seg: + cur.append(ch) + cur_len = len(cur) + if ch in soft_seps and cur_len >= 8: + out.append("".join(cur).strip()) + cur = [] + elif cur_len >= max_len: + out.append("".join(cur).strip()) + cur = [] + if cur: + out.append("".join(cur).strip()) + + return [s for s in out if s] + + async def _synthesize_sentence_first( + self, + reply_text: str, + on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_text_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, + should_interrupt: Optional[Callable[[], bool]] = None, + ) -> tuple[np.ndarray, int, int, int]: + segments = self._split_sentences(reply_text) + audio_chunks = [] + sr = 24000 + t0 = time.perf_counter() + first_chunk_ms = 0 + for i, seg in enumerate(segments): + if should_interrupt and should_interrupt(): + break + if on_text_segment is not None: + await on_text_segment(seg, i, len(segments)) + seg_audio, seg_sr = await self.tts.synthesize(seg) + sr = seg_sr + if i == 0: + first_chunk_ms = int((time.perf_counter() - t0) * 1000) + if on_audio_chunk is not None: + await on_audio_chunk(seg_audio, seg_sr) + audio_chunks.append(seg_audio) + tts_total_ms = int((time.perf_counter() - t0) * 1000) + audio = np.concatenate(audio_chunks) if audio_chunks else np.zeros(1, dtype=np.float32) + return audio, sr, first_chunk_ms, tts_total_ms + + async def process_turn( + self, + audio_16k: np.ndarray, + on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_user_text: Optional[Callable[[str], Awaitable[None]]] = None, + on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, + on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, + ) -> tuple[dict, np.ndarray, int]: + await self.arbitrator.on_speech_end() + asr_t0 = time.perf_counter() + user_text = await self.asr.transcribe(audio_16k) + asr_latency_ms = int((time.perf_counter() - asr_t0) * 1000) + if on_user_text is not None: + await on_user_text(user_text) + llm_t0 = time.perf_counter() + reply_text, llm_source = await self.llm.reply_with_meta(user_text) + if on_reply_text is not None: + await on_reply_text(reply_text) + llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000) + audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first( + reply_text, + on_audio_chunk=on_audio_chunk, + on_text_segment=on_reply_segment, + should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(), + ) + await self.arbitrator.on_avatar_start() + playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr))) + # Do not block request completion on playback duration. + asyncio.create_task(self._finish_avatar_after_playback(playback_s)) + payload = { + "user_text": user_text, + "reply_text": reply_text, + "audio_samples": int(audio.shape[0]), + "sample_rate": sr, + "asr_latency_ms": asr_latency_ms, + "llm_latency_ms": llm_latency_ms, + "llm_source": llm_source, + "tts_latency_ms": tts_latency_ms, + "tts_first_chunk_ms": tts_first_chunk_ms, + } + return payload, audio, sr + + async def process_text_turn( + self, + user_text: str, + on_audio_chunk: Optional[Callable[[np.ndarray, int], Awaitable[None]]] = None, + on_reply_text: Optional[Callable[[str], Awaitable[None]]] = None, + on_reply_segment: Optional[Callable[[str, int, int], Awaitable[None]]] = None, + ) -> tuple[dict, np.ndarray, int]: + llm_t0 = time.perf_counter() + reply_text, llm_source = await self.llm.reply_with_meta(user_text) + if on_reply_text is not None: + await on_reply_text(reply_text) + llm_latency_ms = int((time.perf_counter() - llm_t0) * 1000) + audio, sr, tts_first_chunk_ms, tts_latency_ms = await self._synthesize_sentence_first( + reply_text, + on_audio_chunk=on_audio_chunk, + on_text_segment=on_reply_segment, + should_interrupt=lambda: self.arbitrator.cancel_avatar_event.is_set(), + ) + await self.arbitrator.on_avatar_start() + playback_s = max(0.2, min(5.0, float(audio.shape[0]) / float(sr))) + asyncio.create_task(self._finish_avatar_after_playback(playback_s)) + payload = { + "user_text": user_text, + "reply_text": reply_text, + "audio_samples": int(audio.shape[0]), + "sample_rate": sr, + "llm_latency_ms": llm_latency_ms, + "llm_source": llm_source, + "tts_latency_ms": tts_latency_ms, + "tts_first_chunk_ms": tts_first_chunk_ms, + } + return payload, audio, sr + + async def run_once(self, audio_16k: np.ndarray) -> dict: + payload, _, _ = await self.process_turn(audio_16k) + return payload diff --git a/core/state_machine.py b/core/state_machine.py new file mode 100644 index 0000000..adf5cec --- /dev/null +++ b/core/state_machine.py @@ -0,0 +1,35 @@ +import asyncio +from enum import Enum + + +class SessionState(str, Enum): + IDLE = "idle" + USER_SPEAKING = "user_speaking" + THINKING = "thinking" + AVATAR_SPEAKING = "avatar_speaking" + + +class Arbitrator: + def __init__(self) -> None: + self.state = SessionState.IDLE + self._lock = asyncio.Lock() + self.cancel_avatar_event = asyncio.Event() + + async def on_speech_start(self) -> None: + async with self._lock: + if self.state == SessionState.AVATAR_SPEAKING: + self.cancel_avatar_event.set() + self.state = SessionState.USER_SPEAKING + + async def on_speech_end(self) -> None: + async with self._lock: + self.cancel_avatar_event.clear() + self.state = SessionState.THINKING + + async def on_avatar_start(self) -> None: + async with self._lock: + self.state = SessionState.AVATAR_SPEAKING + + async def on_avatar_done(self) -> None: + async with self._lock: + self.state = SessionState.IDLE diff --git a/environments.md b/environments.md new file mode 100644 index 0000000..2ce29d2 --- /dev/null +++ b/environments.md @@ -0,0 +1,198 @@ +# 已配置环境与已下载模型清单 + +> 最后更新：2026-03-25 +> 机器：WSL2, RTX 5090, CUDA 12.9 + +--- + +## 一、Conda 环境总览 + +``` +conda env list +``` + +| 环境名 | 路径 | 用途 | 磁盘占用 | +|--------|------|------|---------| +| `LivePortrait` | `/home/xsl/miniconda3/envs/LivePortrait` | 头部动作驱动 | 8.6GB | +| `MuseTalk` | `/home/xsl/miniconda3/envs/MuseTalk` | 唇形同步（主力） | 11GB | +| `latentsync` | `/home/xsl/miniconda3/envs/latentsync` | 唇形同步（备用） | 9.2GB | +| `condiff-train-hair` | `/home/xsl/miniconda3/envs/condiff-train-hair` | 其他项目（未动） | — | +| `onediff` | `/home/xsl/miniconda3/envs/onediff` | 其他项目（未动） | — | +| `vllm-qwen3` | `/home/xsl/miniconda3/envs/vllm-qwen3` | 其他项目（未动） | — | +| `py310` | `/home/xsl/miniconda3/envs/py310` | 通用 Python 3.10 | — | + +--- + +## 二、LivePortrait 环境 + +**激活**：`conda activate LivePortrait` + +### 关键包版本 + +| 包 | 版本 | +|----|------| +| torch | `2.12.0.dev20260324+cu128`（nightly） | +| CUDA | 12.8 | + +### 已下载权重 + +位置：`/home/xsl/work/LivePortrait/pretrained_weights/`（总计 **1.2GB**） + +``` +pretrained_weights/ +├── insightface/ +│ └── models/buffalo_l/ +│ ├── 2d106det.onnx (4.8MB) 人脸关键点检测 +│ └── det_10g.onnx (17MB) 人脸检测 +├── liveportrait/ +│ ├── base_models/ +│ │ ├── appearance_feature_extractor.pth (3.3MB) +│ │ ├── motion_extractor.pth (108MB) +│ │ ├── spade_generator.pth (212MB) +│ │ └── warping_module.pth (174MB) +│ ├── retargeting_models/ +│ │ └── stitching_retargeting_module.pth (2.3MB) +│ └── landmark.onnx (110MB) +└── liveportrait_animals/ + ├── base_models/ (同上，动物版) + └── base_models_v1.1/ + └── appearance_feature_extractor.pth (3.3MB) +``` + +**来源**：`hf download KlingTeam/LivePortrait --local-dir pretrained_weights` + +--- + +## 三、MuseTalk 环境 + +**激活**：`conda activate MuseTalk` + +### 关键包版本 + +| 包 | 版本 | 备注 | +|----|------|------| +| torch | `2.12.0.dev20260324+cu128` | nightly，sm_120 必须 | +| mmcv | `2.1.0` | 从源码编译，需要 nvcc | +| mmdet | `3.3.0` | ≥3.3.0 才兼容 mmcv 2.1.0 | +| mmpose | `1.3.2` | ≥1.3.0 才兼容 mmcv 2.1.0 | +| transformers | `5.3.0` | ≥4.45.0 才兼容 huggingface_hub 1.x | +| huggingface_hub | `1.7.2` | CLI 命令名为 `hf` | +| diffusers | `0.32.2` | | + +### 已下载权重 + +位置：`/home/xsl/work/MuseTalk/models/`（总计 **5.5GB**） + +``` +models/ +├── musetalkV15/ +│ ├── unet.pth (3.2GB) 主模型 +│ └── musetalk.json 模型配置 +├── sd-vae/ +│ ├── diffusion_pytorch_model.bin (320MB) VAE 解码器 +│ └── config.json +├── dwpose/ +│ └── dw-ll_ucoco_384.pth (389MB) 姿态估计（DWPose） +├── whisper/ +│ ├── pytorch_model.bin (145MB) 音频特征提取 +│ ├── config.json +│ └── preprocessor_config.json +├── syncnet/ +│ └── latentsync_syncnet.pt (1.4GB) 唇形同步评估 +└── face-parse-bisent/ + ├── 79999_iter.pth (51MB) 人脸解析 + └── resnet18-5c106cde.pth (45MB) backbone +``` + +**来源**： +- `musetalkV15` → `hf download TMElyralab/MuseTalk` +- `sd-vae` → `hf download stabilityai/sd-vae-ft-mse` +- `dwpose` → `hf download yzd-v/DWPose` +- `whisper` → `hf download openai/whisper-tiny` +- `syncnet` → `hf download ByteDance/LatentSync` +- `face-parse-bisent` → gdown (Google Drive) + pytorch.org + +--- + +## 四、LatentSync 环境 + +**激活**：`conda activate latentsync` + +### 关键包版本 + +| 包 | 版本 | +|----|------| +| torch | `2.12.0.dev20260324+cu128`（nightly） | +| diffusers | `0.32.2` | + +### 已下载权重 + +位置：`/home/xsl/work/LatentSync/checkpoints/`（总计 **5.4GB**） + +``` +checkpoints/ +├── latentsync_unet.pt (4.8GB) 主扩散模型 +├── whisper/ +│ └── tiny.pt (73MB) Whisper 音频编码器 +└── auxiliary/ + └── models/buffalo_l/ (约330MB) insightface 人脸模型 + ├── 1k3d68.onnx (137MB) + ├── 2d106det.onnx (4.8MB) + ├── det_10g.onnx (17MB) + ├── genderage.onnx (1.3MB) + └── w600k_r50.onnx (167MB) +``` + +**来源**： +- `latentsync_unet.pt` → `hf download ByteDance/LatentSync-1.6` +- `whisper/tiny.pt` → `hf download ByteDance/LatentSync-1.6` +- `buffalo_l` → 运行时自动下载（insightface） + +--- + +## 五、磁盘占用汇总 + +| 类别 | 路径 | 大小 | +|------|------|------| +| LivePortrait 权重 | `work/LivePortrait/pretrained_weights` | 1.2GB | +| LatentSync 权重 | `work/LatentSync/checkpoints` | 5.4GB | +| MuseTalk 权重 | `work/MuseTalk/models` | 5.5GB | +| **模型合计** | | **~12GB** | +| LivePortrait conda 环境 | `envs/LivePortrait` | 8.6GB | +| MuseTalk conda 环境 | `envs/MuseTalk` | 11GB | +| latentsync conda 环境 | `envs/latentsync` | 9.2GB | +| **环境合计** | | **~29GB** | + +--- + +## 六、产品新环境建议 + +如果要为产品重建一套干净环境，**最小必要集合**是： + +```bash +# 一个统一环境（避免两套 conda env 切换） +conda create -n digital-human python=3.10 +conda activate digital-human +conda install -y -c conda-forge ffmpeg + +# PyTorch nightly（RTX 5090 必须） +pip install --pre torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/nightly/cu128 + +# LivePortrait 依赖 +pip install -r /home/xsl/work/LivePortrait/requirements.txt + +# MuseTalk 依赖（含 MMLab 完整链） +pip install -r /home/xsl/work/MuseTalk/requirements.txt +pip install --no-build-isolation chumpy +pip install mmengine +MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation +pip install "mmdet>=3.3.0" "mmpose>=1.3.0" +pip install xtcocotools json-tricks munkres +pip install "transformers>=4.45.0" +``` + +**可复用的模型权重**（无需重新下载）： +- LivePortrait：`/home/xsl/work/LivePortrait/pretrained_weights/` — 直接用 +- MuseTalk：`/home/xsl/work/MuseTalk/models/` — 直接用 +- LatentSync（如需）：`/home/xsl/work/LatentSync/checkpoints/` — 直接用 diff --git a/main.py b/main.py new file mode 100644 index 0000000..611c7c6 --- /dev/null +++ b/main.py @@ -0,0 +1,459 @@ +from __future__ import annotations + +import os +import logging +import asyncio +import json +import time +from dataclasses import dataclass, field + +import numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription +from dotenv import load_dotenv +from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +from config import settings +from core.pipeline import ChatPipeline +from core.state_machine import Arbitrator +from services.asr import ASRService +from services.avatar import AvatarService +from services.llm import LLMService +from services.tts import TTSService +from services.vad import VADService +from webrtc.tracks import AudioBus, AvatarAudioTrack, AvatarVideoTrack + +load_dotenv() +if settings.hf_token: + os.environ["HF_TOKEN"] = settings.hf_token + +logging.basicConfig(level=logging.INFO) +app = FastAPI(title="Visual Voice Chat") +app.mount("/web", StaticFiles(directory="web"), name="web") +app.add_middleware( + CORSMiddleware, + allow_origins=[o.strip() for o in settings.cors_origins.split(",")] if settings.cors_origins else ["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +arbitrator = Arbitrator() +asr_service = ASRService() +llm_service = LLMService() +tts_service = TTSService() +vad_service = VADService() +avatar_service = AvatarService() +pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service) +audio_bus = AudioBus(sample_rate=48000) +pcs: set[RTCPeerConnection] = set() +subtitle_clients: set[WebSocket] = set() + + +@dataclass +class RuntimeState: + last_user_text: str = "" + last_reply_text: str = "" + pipeline_runs: int = 0 + last_latency_ms: int = 0 + last_asr_latency_ms: int = 0 + last_llm_latency_ms: int = 0 + last_llm_source: str = "unknown" + last_tts_latency_ms: int = 0 + last_tts_first_chunk_ms: int = 0 + last_barge_in_ms: int = 0 + barge_in_count: int = 0 + last_input_mode: str = "none" + vad_start_count: int = 0 + vad_end_count: int = 0 + busy: bool = False + buffer_16k: list[np.ndarray] = field(default_factory=list) + + +runtime = RuntimeState() + + +class TextChatRequest(BaseModel): + text: str + + +async def _broadcast_subtitle( + role: str, + text: str, + source: str, + *, + partial: bool = False, + final: bool = True, +) -> None: + if not text: + return + payload = { + "role": role, + "text": text, + "source": source, + "partial": partial, + "final": final, + "ts_ms": int(time.time() * 1000), + } + stale: list[WebSocket] = [] + for ws in list(subtitle_clients): + try: + await ws.send_text(json.dumps(payload, ensure_ascii=False)) + except Exception: + stale.append(ws) + for ws in stale: + subtitle_clients.discard(ws) + + +def _resample_mono(audio: np.ndarray, from_sr: int, to_sr: int) -> np.ndarray: + if from_sr == to_sr: + return audio.astype(np.float32, copy=False) + in_len = audio.shape[0] + out_len = max(1, int(in_len * to_sr / from_sr)) + x_old = np.linspace(0.0, 1.0, in_len, endpoint=False) + x_new = np.linspace(0.0, 1.0, out_len, endpoint=False) + return np.interp(x_new, x_old, audio).astype(np.float32) + + +def _resample_to_16k_mono(pcm: np.ndarray, sample_rate: int, channels: int) -> np.ndarray: + # aiortc audio ndarray is typically (channels, samples) + if pcm.ndim == 2: + mono = pcm.mean(axis=0) + else: + mono = pcm + mono = mono.astype(np.float32) / 32768.0 + if sample_rate == 16000: + return mono + + in_len = mono.shape[0] + out_len = max(1, int(in_len * 16000 / sample_rate)) + x_old = np.linspace(0.0, 1.0, in_len, endpoint=False) + x_new = np.linspace(0.0, 1.0, out_len, endpoint=False) + return np.interp(x_new, x_old, mono).astype(np.float32) + + +async def _run_pipeline_from_buffer() -> None: + if runtime.busy: + return + if not runtime.buffer_16k: + return + + runtime.busy = True + start_t = time.perf_counter() + try: + runtime.last_input_mode = "voice" + audio_16k = np.concatenate(runtime.buffer_16k, axis=0) + runtime.buffer_16k.clear() + async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: + audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) + pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_seg) + + async def on_user_text_now(user_text: str) -> None: + runtime.last_user_text = user_text + await _broadcast_subtitle("user", user_text, "voice") + + async def on_reply_text_now(reply_text: str) -> None: + runtime.last_reply_text = reply_text + + async def on_reply_segment_now(seg_text: str, idx: int, total: int) -> None: + await _broadcast_subtitle("ai", seg_text, "voice", partial=True, final=(idx >= total - 1)) + + result, audio, sr = await pipeline.process_turn( + audio_16k, + on_audio_chunk=on_audio_chunk, + on_user_text=on_user_text_now, + on_reply_text=on_reply_text_now, + on_reply_segment=on_reply_segment_now, + ) + asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) + runtime.last_user_text = result["user_text"] + runtime.last_reply_text = result["reply_text"] + runtime.pipeline_runs += 1 + runtime.last_latency_ms = int((time.perf_counter() - start_t) * 1000) + runtime.last_asr_latency_ms = int(result.get("asr_latency_ms", 0)) + runtime.last_llm_latency_ms = int(result.get("llm_latency_ms", 0)) + runtime.last_llm_source = str(result.get("llm_source", "unknown")) + runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) + runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) + # Fallback: if chunk callback path produced nothing, enqueue full audio. + if result.get("audio_samples", 0) <= 1: + audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) + pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_int16) + finally: + runtime.busy = False + + +async def _consume_user_audio(track) -> None: + vad_buffer = np.zeros(0, dtype=np.float32) + while True: + frame = await track.recv() + pcm = frame.to_ndarray() + sample_rate = getattr(frame, "sample_rate", 48000) or 48000 + layout = getattr(frame, "layout", None) + channels = len(layout.channels) if layout and layout.channels else 1 + mono_16k = _resample_to_16k_mono(pcm, sample_rate, channels) + runtime.buffer_16k.append(mono_16k) + vad_buffer = np.concatenate([vad_buffer, mono_16k], axis=0) + + # Silero VAD requires exact 512 samples at 16k. + while vad_buffer.shape[0] >= 512: + chunk = vad_buffer[:512] + vad_buffer = vad_buffer[512:] + try: + event = vad_service.push_chunk(chunk) + except Exception as exc: + logging.exception("VAD chunk processing failed: %s", exc) + event = None + if event and "start" in event: + barge_t0 = time.perf_counter() + was_avatar_speaking = str(arbitrator.state) == "avatar_speaking" + await arbitrator.on_speech_start() + await audio_bus.clear() + if was_avatar_speaking: + runtime.last_barge_in_ms = int((time.perf_counter() - barge_t0) * 1000) + runtime.barge_in_count += 1 + runtime.vad_start_count += 1 + if event and "end" in event: + await arbitrator.on_speech_end() + runtime.vad_end_count += 1 + asyncio.create_task(_run_pipeline_from_buffer()) + + +@app.get("/health") +async def health(): + return { + "state": arbitrator.state, + "peers": len(pcs), + "pipeline_runs": runtime.pipeline_runs, + "last_latency_ms": runtime.last_latency_ms, + "last_asr_latency_ms": runtime.last_asr_latency_ms, + "last_llm_latency_ms": runtime.last_llm_latency_ms, + "last_llm_source": runtime.last_llm_source, + "last_tts_latency_ms": runtime.last_tts_latency_ms, + "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, + "last_barge_in_ms": runtime.last_barge_in_ms, + "barge_in_count": runtime.barge_in_count, + "last_input_mode": runtime.last_input_mode, + "vad_start_count": runtime.vad_start_count, + "vad_end_count": runtime.vad_end_count, + "last_user_text": runtime.last_user_text, + "last_reply_text": runtime.last_reply_text, + "pipeline_busy": runtime.busy, + "llm": llm_service.health, + "asr": asr_service.health, + "tts": tts_service.health, + "vad": vad_service.health, + "avatar": avatar_service.health, + } + + +@app.get("/meta") +async def meta(): + return { + "stun_url": settings.stun_url, + "https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile), + "host": settings.webrtc_host, + "port": settings.webrtc_port, + } + + +@app.get("/events") +async def events(): + async def gen(): + while True: + payload = { + "state": str(arbitrator.state), + "peers": len(pcs), + "pipeline_runs": runtime.pipeline_runs, + "last_latency_ms": runtime.last_latency_ms, + "last_asr_latency_ms": runtime.last_asr_latency_ms, + "last_llm_latency_ms": runtime.last_llm_latency_ms, + "last_llm_source": runtime.last_llm_source, + "last_tts_latency_ms": runtime.last_tts_latency_ms, + "last_tts_first_chunk_ms": runtime.last_tts_first_chunk_ms, + "last_barge_in_ms": runtime.last_barge_in_ms, + "barge_in_count": runtime.barge_in_count, + "last_input_mode": runtime.last_input_mode, + "vad_start_count": runtime.vad_start_count, + "vad_end_count": runtime.vad_end_count, + "last_user_text": runtime.last_user_text, + "last_reply_text": runtime.last_reply_text, + "pipeline_busy": runtime.busy, + "avatar": avatar_service.health, + } + yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + await asyncio.sleep(1.0) + + return StreamingResponse(gen(), media_type="text/event-stream") + + +@app.websocket("/ws/subtitles") +async def ws_subtitles(websocket: WebSocket): + await websocket.accept() + subtitle_clients.add(websocket) + try: + while True: + await websocket.send_text(json.dumps({"type": "ping"}, ensure_ascii=False)) + await asyncio.sleep(15) + except WebSocketDisconnect: + subtitle_clients.discard(websocket) + except Exception: + subtitle_clients.discard(websocket) + + +@app.post("/demo/run-once") +async def run_once(): + # 3 seconds of silence placeholder; replace with real microphone stream path. + fake_audio = np.zeros(16000 * 3, dtype=np.float32) + await arbitrator.on_speech_start() + result = await pipeline.run_once(fake_audio) + return result + + +@app.post("/chat/text") +async def chat_text(req: TextChatRequest): + text = req.text.strip() + if not text: + return JSONResponse({"ok": False, "message": "text is empty"}, status_code=400) + if runtime.busy: + return JSONResponse({"ok": False, "message": "pipeline busy"}, status_code=429) + + runtime.busy = True + start_t = time.perf_counter() + try: + async def on_audio_chunk(seg_audio: np.ndarray, seg_sr: int) -> None: + audio_48k_seg = _resample_mono(seg_audio.astype(np.float32, copy=False), seg_sr, 48000) + pcm_seg = np.clip(audio_48k_seg * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_seg) + + async def on_reply_text_now(reply_text: str) -> None: + runtime.last_reply_text = reply_text + + async def on_reply_segment_now(seg_text: str, idx: int, total: int) -> None: + await _broadcast_subtitle("ai", seg_text, "text", partial=True, final=(idx >= total - 1)) + + await _broadcast_subtitle("user", text, "text") + result, audio, sr = await pipeline.process_text_turn( + text, + on_audio_chunk=on_audio_chunk, + on_reply_text=on_reply_text_now, + on_reply_segment=on_reply_segment_now, + ) + asyncio.create_task(avatar_service.enqueue_musetalk_from_audio(audio, sr)) + runtime.last_input_mode = "text" + runtime.last_user_text = result["user_text"] + runtime.last_reply_text = result["reply_text"] + runtime.pipeline_runs += 1 + runtime.last_latency_ms = int((time.perf_counter() - start_t) * 1000) + runtime.last_asr_latency_ms = 0 + runtime.last_llm_latency_ms = int(result.get("llm_latency_ms", 0)) + runtime.last_llm_source = str(result.get("llm_source", "unknown")) + runtime.last_tts_latency_ms = int(result.get("tts_latency_ms", 0)) + runtime.last_tts_first_chunk_ms = int(result.get("tts_first_chunk_ms", 0)) + # Fallback: if no chunk was enqueued for some reason, enqueue full audio. + if result.get("audio_samples", 0) <= 1: + audio_48k = _resample_mono(audio.astype(np.float32, copy=False), sr, 48000) + pcm_int16 = np.clip(audio_48k * 32767.0, -32768, 32767).astype(np.int16) + await audio_bus.enqueue(pcm_int16) + return {"ok": True, **result} + finally: + runtime.busy = False + + +@app.post("/chat/reset") +async def chat_reset(): + llm_service.clear_history() + runtime.last_user_text = "" + runtime.last_reply_text = "" + runtime.pipeline_runs = 0 + runtime.last_latency_ms = 0 + runtime.last_asr_latency_ms = 0 + runtime.last_llm_latency_ms = 0 + runtime.last_llm_source = "unknown" + runtime.last_tts_latency_ms = 0 + runtime.last_tts_first_chunk_ms = 0 + runtime.last_barge_in_ms = 0 + runtime.barge_in_count = 0 + runtime.last_input_mode = "none" + runtime.vad_start_count = 0 + runtime.vad_end_count = 0 + return {"ok": True} + + +@app.get("/demo/frame-idle-shape") +async def frame_idle_shape(): + frame = await avatar_service.idle_frame() + return {"shape": list(frame.shape)} + + +@app.get("/") +async def root(): + return FileResponse("web/index.html") + + +@app.post("/webrtc/offer") +async def webrtc_offer(request: Request): + params = await request.json() + offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) + + # Single-connection mode: kick existing peers before accepting new one. + replaced_previous = len(pcs) > 0 + if replaced_previous: + old_peers = list(pcs) + await asyncio.gather(*(peer.close() for peer in old_peers), return_exceptions=True) + for peer in old_peers: + pcs.discard(peer) + + pc = RTCPeerConnection() + pcs.add(pc) + + @pc.on("connectionstatechange") + async def on_connectionstatechange(): + if pc.connectionState in {"failed", "closed", "disconnected"}: + await pc.close() + pcs.discard(pc) + + @pc.on("track") + def on_track(track): + if track.kind == "audio": + asyncio.create_task(_consume_user_audio(track)) + + pc.addTrack(AvatarVideoTrack(avatar_service, arbitrator, audio_bus, settings.avatar_fps)) + pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus)) + + await pc.setRemoteDescription(offer) + answer = await pc.createAnswer() + await pc.setLocalDescription(answer) + + return JSONResponse( + { + "sdp": pc.localDescription.sdp, + "type": pc.localDescription.type, + "replaced_previous": replaced_previous, + } + ) + + +@app.on_event("shutdown") +async def on_shutdown(): + await asyncio.gather(*(pc.close() for pc in list(pcs)), return_exceptions=True) + pcs.clear() + + +if __name__ == "__main__": + import uvicorn + + uvicorn_kwargs = { + "app": app, + "host": settings.webrtc_host, + "port": settings.webrtc_port, + } + if settings.ssl_certfile and settings.ssl_keyfile: + uvicorn_kwargs["ssl_certfile"] = settings.ssl_certfile + uvicorn_kwargs["ssl_keyfile"] = settings.ssl_keyfile + uvicorn.run(**uvicorn_kwargs) diff --git a/outputs/gpu-1h.csv b/outputs/gpu-1h.csv new file mode 100644 index 0000000..8595267 --- /dev/null +++ b/outputs/gpu-1h.csv @@ -0,0 +1,877 @@ +ts,index,name,util_gpu,util_mem,mem_used_mb,mem_total_mb,temp_c,power_w +1774535357,0,NVIDIA GeForce RTX 5090,37,9,9405,32607,50,196.20 +1774535360,0,NVIDIA GeForce RTX 5090,42,11,9419,32607,53,207.35 +1774535362,0,NVIDIA GeForce RTX 5090,2,0,8675,32607,48,98.02 +1774535365,0,NVIDIA GeForce RTX 5090,70,7,8684,32607,56,341.83 +1774535367,0,NVIDIA GeForce RTX 5090,72,7,8704,32607,56,337.02 +1774535370,0,NVIDIA GeForce RTX 5090,0,0,8718,32607,48,71.46 +1774535372,0,NVIDIA GeForce RTX 5090,1,0,8775,32607,47,72.78 +1774535375,0,NVIDIA GeForce RTX 5090,0,0,8669,32607,47,73.92 +1774535377,0,NVIDIA GeForce RTX 5090,0,0,8667,32607,47,71.93 +1774535380,0,NVIDIA GeForce RTX 5090,22,6,9563,32607,48,139.61 +1774535382,0,NVIDIA GeForce RTX 5090,1,0,8666,32607,50,340.08 +1774535385,0,NVIDIA GeForce RTX 5090,7,1,8668,32607,47,70.11 +1774535387,0,NVIDIA GeForce RTX 5090,5,0,8670,32607,47,69.30 +1774535390,0,NVIDIA GeForce RTX 5090,5,1,8670,32607,47,71.17 +1774535392,0,NVIDIA GeForce RTX 5090,9,1,8672,32607,47,83.33 +1774535395,0,NVIDIA GeForce RTX 5090,0,0,8672,32607,47,68.37 +1774535397,0,NVIDIA GeForce RTX 5090,0,0,5341,32607,46,68.13 +1774535400,0,NVIDIA GeForce RTX 5090,0,0,6412,32607,46,71.56 +1774535402,0,NVIDIA GeForce RTX 5090,0,0,6412,32607,46,67.70 +1774535405,0,NVIDIA GeForce RTX 5090,0,0,13128,32607,47,79.09 +1774535407,0,NVIDIA GeForce RTX 5090,0,0,13130,32607,46,67.59 +1774535411,0,NVIDIA GeForce RTX 5090,5,0,8658,32607,46,68.89 +1774535412,0,NVIDIA GeForce RTX 5090,3,0,8658,32607,46,68.81 +1774535416,0,NVIDIA GeForce RTX 5090,0,0,9210,32607,46,68.37 +1774535417,0,NVIDIA GeForce RTX 5090,7,1,8660,32607,46,68.98 +1774535421,0,NVIDIA GeForce RTX 5090,17,7,9290,32607,46,86.12 +1774535422,0,NVIDIA GeForce RTX 5090,10,2,8769,32607,46,88.47 +1774535426,0,NVIDIA GeForce RTX 5090,4,1,8767,32607,46,69.60 +1774535427,0,NVIDIA GeForce RTX 5090,3,0,8767,32607,46,68.94 +1774535431,0,NVIDIA GeForce RTX 5090,35,9,9401,32607,52,198.05 +1774535432,0,NVIDIA GeForce RTX 5090,40,11,9401,32607,49,202.48 +1774535436,0,NVIDIA GeForce RTX 5090,35,9,9400,32607,50,203.84 +1774535437,0,NVIDIA GeForce RTX 5090,37,11,9400,32607,51,195.58 +1774535441,0,NVIDIA GeForce RTX 5090,2,0,8689,32607,49,177.75 +1774535442,0,NVIDIA GeForce RTX 5090,0,0,8687,32607,48,94.34 +1774535446,0,NVIDIA GeForce RTX 5090,67,6,8694,32607,57,338.12 +1774535447,0,NVIDIA GeForce RTX 5090,2,0,8688,32607,50,196.57 +1774535451,0,NVIDIA GeForce RTX 5090,0,0,8689,32607,48,71.60 +1774535452,0,NVIDIA GeForce RTX 5090,1,0,8750,32607,48,75.56 +1774535456,0,NVIDIA GeForce RTX 5090,7,0,8685,32607,48,81.09 +1774535457,0,NVIDIA GeForce RTX 5090,5,0,8699,32607,47,74.28 +1774535461,0,NVIDIA GeForce RTX 5090,26,7,10157,32607,48,166.22 +1774535462,0,NVIDIA GeForce RTX 5090,2,0,8673,32607,49,96.13 +1774535466,0,NVIDIA GeForce RTX 5090,2,0,8774,32607,48,75.22 +1774535467,0,NVIDIA GeForce RTX 5090,1,0,8739,32607,47,76.25 +1774535471,0,NVIDIA GeForce RTX 5090,1,0,8779,32607,47,74.56 +1774535473,0,NVIDIA GeForce RTX 5090,3,0,8812,32607,47,76.80 +1774535476,0,NVIDIA GeForce RTX 5090,1,0,8778,32607,47,70.47 +1774535478,0,NVIDIA GeForce RTX 5090,10,1,8777,32607,47,69.94 +1774535481,0,NVIDIA GeForce RTX 5090,2,0,8779,32607,47,73.33 +1774535483,0,NVIDIA GeForce RTX 5090,0,0,8770,32607,47,72.88 +1774535486,0,NVIDIA GeForce RTX 5090,5,0,8772,32607,47,80.07 +1774535488,0,NVIDIA GeForce RTX 5090,0,0,8762,32607,46,68.46 +1774535491,0,NVIDIA GeForce RTX 5090,0,0,5430,32607,46,67.46 +1774535493,0,NVIDIA GeForce RTX 5090,0,0,6501,32607,46,71.53 +1774535496,0,NVIDIA GeForce RTX 5090,8,0,8970,32607,46,71.22 +1774535498,0,NVIDIA GeForce RTX 5090,4,0,13214,32607,46,81.75 +1774535501,0,NVIDIA GeForce RTX 5090,0,0,8746,32607,46,67.47 +1774535503,0,NVIDIA GeForce RTX 5090,3,0,8744,32607,46,68.11 +1774535506,0,NVIDIA GeForce RTX 5090,0,0,8753,32607,46,68.99 +1774535508,0,NVIDIA GeForce RTX 5090,5,0,8749,32607,46,69.73 +1774535511,0,NVIDIA GeForce RTX 5090,0,0,8749,32607,46,69.46 +1774535513,0,NVIDIA GeForce RTX 5090,6,2,9167,32607,46,74.08 +1774535516,0,NVIDIA GeForce RTX 5090,3,0,8755,32607,46,82.15 +1774535518,0,NVIDIA GeForce RTX 5090,0,0,8861,32607,46,71.22 +1774535521,0,NVIDIA GeForce RTX 5090,1,0,8849,32607,45,68.90 +1774535523,0,NVIDIA GeForce RTX 5090,35,9,9488,32607,51,192.35 +1774535526,0,NVIDIA GeForce RTX 5090,39,11,9488,32607,49,190.74 +1774535528,0,NVIDIA GeForce RTX 5090,34,9,9494,32607,50,202.07 +1774535531,0,NVIDIA GeForce RTX 5090,35,9,9491,32607,51,202.44 +1774535533,0,NVIDIA GeForce RTX 5090,35,9,9489,32607,52,199.64 +1774535537,0,NVIDIA GeForce RTX 5090,1,0,8757,32607,47,73.43 +1774535538,0,NVIDIA GeForce RTX 5090,76,7,8768,32607,55,310.59 +1774535542,0,NVIDIA GeForce RTX 5090,0,0,8758,32607,49,97.44 +1774535543,0,NVIDIA GeForce RTX 5090,0,0,8756,32607,48,72.51 +1774535547,0,NVIDIA GeForce RTX 5090,0,0,8712,32607,48,76.94 +1774535548,0,NVIDIA GeForce RTX 5090,1,0,8682,32607,47,71.24 +1774535552,0,NVIDIA GeForce RTX 5090,31,2,9141,32607,56,116.66 +1774535553,0,NVIDIA GeForce RTX 5090,37,9,9683,32607,48,140.46 +1774535557,0,NVIDIA GeForce RTX 5090,0,0,8627,32607,48,87.96 +1774535558,0,NVIDIA GeForce RTX 5090,1,0,8637,32607,48,71.13 +1774535562,0,NVIDIA GeForce RTX 5090,2,0,8609,32607,47,75.24 +1774535563,0,NVIDIA GeForce RTX 5090,2,0,8635,32607,47,74.92 +1774535567,0,NVIDIA GeForce RTX 5090,5,2,9237,32607,47,106.89 +1774535568,0,NVIDIA GeForce RTX 5090,5,2,8797,32607,47,98.26 +1774535572,0,NVIDIA GeForce RTX 5090,0,0,8625,32607,47,69.55 +1774535573,0,NVIDIA GeForce RTX 5090,0,0,8627,32607,47,69.27 +1774535577,0,NVIDIA GeForce RTX 5090,1,0,8628,32607,47,70.20 +1774535578,0,NVIDIA GeForce RTX 5090,6,1,8629,32607,47,79.64 +1774535582,0,NVIDIA GeForce RTX 5090,1,0,5368,32607,46,69.60 +1774535583,0,NVIDIA GeForce RTX 5090,0,0,5366,32607,46,68.83 +1774535587,0,NVIDIA GeForce RTX 5090,0,0,6462,32607,46,72.54 +1774535588,0,NVIDIA GeForce RTX 5090,0,0,6464,32607,46,67.85 +1774535592,0,NVIDIA GeForce RTX 5090,0,0,13188,32607,46,76.16 +1774535593,0,NVIDIA GeForce RTX 5090,1,0,8694,32607,46,67.39 +1774535597,0,NVIDIA GeForce RTX 5090,0,0,8722,32607,46,70.77 +1774535598,0,NVIDIA GeForce RTX 5090,0,0,8687,32607,46,68.27 +1774535602,0,NVIDIA GeForce RTX 5090,5,0,8631,32607,46,73.17 +1774535604,0,NVIDIA GeForce RTX 5090,11,1,8629,32607,46,72.40 +1774535607,0,NVIDIA GeForce RTX 5090,18,7,9259,32607,46,82.71 +1774535609,0,NVIDIA GeForce RTX 5090,0,0,8726,32607,46,88.91 +1774535612,0,NVIDIA GeForce RTX 5090,0,0,8703,32607,46,69.42 +1774535614,0,NVIDIA GeForce RTX 5090,0,0,8703,32607,45,69.27 +1774535617,0,NVIDIA GeForce RTX 5090,37,11,9342,32607,49,198.53 +1774535619,0,NVIDIA GeForce RTX 5090,34,9,9342,32607,50,193.14 +1774535622,0,NVIDIA GeForce RTX 5090,40,11,9338,32607,50,195.84 +1774535624,0,NVIDIA GeForce RTX 5090,39,11,9337,32607,50,198.38 +1774535627,0,NVIDIA GeForce RTX 5090,38,10,9337,32607,50,198.93 +1774535629,0,NVIDIA GeForce RTX 5090,2,0,8595,32607,48,72.75 +1774535632,0,NVIDIA GeForce RTX 5090,66,6,8603,32607,56,325.51 +1774535634,0,NVIDIA GeForce RTX 5090,68,7,8605,32607,57,331.48 +1774535637,0,NVIDIA GeForce RTX 5090,0,0,8597,32607,48,72.28 +1774535639,0,NVIDIA GeForce RTX 5090,5,1,8674,32607,50,73.24 +1774535642,0,NVIDIA GeForce RTX 5090,0,0,8616,32607,47,75.17 +1774535644,0,NVIDIA GeForce RTX 5090,0,0,8612,32607,47,73.10 +1774535647,0,NVIDIA GeForce RTX 5090,37,9,9613,32607,48,133.67 +1774535649,0,NVIDIA GeForce RTX 5090,76,23,10083,32607,57,305.91 +1774535652,0,NVIDIA GeForce RTX 5090,2,0,8599,32607,48,72.44 +1774535654,0,NVIDIA GeForce RTX 5090,3,0,8684,32607,48,75.02 +1774535658,0,NVIDIA GeForce RTX 5090,3,0,8635,32607,47,75.50 +1774535659,0,NVIDIA GeForce RTX 5090,0,0,8628,32607,47,72.88 +1774535663,0,NVIDIA GeForce RTX 5090,6,3,9140,32607,47,102.14 +1774535664,0,NVIDIA GeForce RTX 5090,0,0,8616,32607,47,70.45 +1774535668,0,NVIDIA GeForce RTX 5090,0,0,8619,32607,47,69.67 +1774535669,0,NVIDIA GeForce RTX 5090,0,0,8619,32607,47,69.82 +1774535673,0,NVIDIA GeForce RTX 5090,6,1,8621,32607,47,80.49 +1774535674,0,NVIDIA GeForce RTX 5090,5,1,8620,32607,47,80.67 +1774535678,0,NVIDIA GeForce RTX 5090,0,0,5282,32607,46,68.25 +1774535679,0,NVIDIA GeForce RTX 5090,0,0,5282,32607,46,68.36 +1774535683,0,NVIDIA GeForce RTX 5090,0,0,6365,32607,46,68.70 +1774535684,0,NVIDIA GeForce RTX 5090,4,0,8062,32607,46,71.99 +1774535688,0,NVIDIA GeForce RTX 5090,2,0,13093,32607,46,69.10 +1774535689,0,NVIDIA GeForce RTX 5090,0,0,8623,32607,46,72.21 +1774535693,0,NVIDIA GeForce RTX 5090,3,0,8615,32607,46,70.25 +1774535694,0,NVIDIA GeForce RTX 5090,1,0,8616,32607,46,70.07 +1774535698,0,NVIDIA GeForce RTX 5090,11,1,8748,32607,46,73.13 +1774535699,0,NVIDIA GeForce RTX 5090,1,0,8737,32607,46,72.89 +1774535703,0,NVIDIA GeForce RTX 5090,1,0,8707,32607,46,87.50 +1774535704,0,NVIDIA GeForce RTX 5090,2,0,8726,32607,46,80.81 +1774535708,0,NVIDIA GeForce RTX 5090,1,0,8759,32607,46,71.95 +1774535709,0,NVIDIA GeForce RTX 5090,1,0,8775,32607,46,86.52 +1774535713,0,NVIDIA GeForce RTX 5090,35,9,9403,32607,51,199.91 +1774535714,0,NVIDIA GeForce RTX 5090,37,10,9383,32607,49,204.86 +1774535718,0,NVIDIA GeForce RTX 5090,37,9,9367,32607,51,204.93 +1774535719,0,NVIDIA GeForce RTX 5090,36,9,9368,32607,50,201.51 +1774535723,0,NVIDIA GeForce RTX 5090,0,0,8678,32607,49,112.27 +1774535724,0,NVIDIA GeForce RTX 5090,0,0,7332,32607,48,82.08 +1774535728,0,NVIDIA GeForce RTX 5090,74,7,7301,32607,56,321.62 +1774535729,0,NVIDIA GeForce RTX 5090,0,0,7303,32607,50,150.62 +1774535733,0,NVIDIA GeForce RTX 5090,0,0,3978,32607,48,69.11 +1774535734,0,NVIDIA GeForce RTX 5090,0,0,3984,32607,48,68.60 +1774535738,0,NVIDIA GeForce RTX 5090,0,0,3986,32607,47,67.93 +1774535740,0,NVIDIA GeForce RTX 5090,0,0,3983,32607,47,67.65 +1774535743,0,NVIDIA GeForce RTX 5090,0,0,4112,32607,47,69.97 +1774535745,0,NVIDIA GeForce RTX 5090,0,0,4085,32607,47,67.44 +1774535748,0,NVIDIA GeForce RTX 5090,1,0,3992,32607,47,69.26 +1774535750,0,NVIDIA GeForce RTX 5090,0,0,4909,32607,47,71.76 +1774535753,0,NVIDIA GeForce RTX 5090,0,0,5337,32607,47,69.51 +1774535755,0,NVIDIA GeForce RTX 5090,0,0,5345,32607,47,67.51 +1774535758,0,NVIDIA GeForce RTX 5090,0,0,6769,32607,47,68.53 +1774535760,0,NVIDIA GeForce RTX 5090,7,0,9535,32607,46,72.89 +1774535763,0,NVIDIA GeForce RTX 5090,0,0,8600,32607,46,69.29 +1774535765,0,NVIDIA GeForce RTX 5090,0,0,8620,32607,46,67.64 +1774535768,0,NVIDIA GeForce RTX 5090,9,1,8630,32607,46,68.71 +1774535770,0,NVIDIA GeForce RTX 5090,3,0,8628,32607,46,69.13 +1774535773,0,NVIDIA GeForce RTX 5090,6,0,8609,32607,46,69.26 +1774535775,0,NVIDIA GeForce RTX 5090,10,1,8609,32607,46,69.93 +1774535779,0,NVIDIA GeForce RTX 5090,0,0,8635,32607,46,84.16 +1774535780,0,NVIDIA GeForce RTX 5090,0,0,8607,32607,46,78.62 +1774535784,0,NVIDIA GeForce RTX 5090,2,0,8720,32607,48,76.83 +1774535785,0,NVIDIA GeForce RTX 5090,0,0,8743,32607,46,68.07 +1774535789,0,NVIDIA GeForce RTX 5090,37,11,9370,32607,49,203.20 +1774535790,0,NVIDIA GeForce RTX 5090,39,10,9391,32607,53,201.49 +1774535794,0,NVIDIA GeForce RTX 5090,39,11,9387,32607,50,198.77 +1774535795,0,NVIDIA GeForce RTX 5090,37,10,9383,32607,51,204.84 +1774535799,0,NVIDIA GeForce RTX 5090,1,0,8641,32607,49,89.78 +1774535800,0,NVIDIA GeForce RTX 5090,1,0,8649,32607,47,70.93 +1774535804,0,NVIDIA GeForce RTX 5090,0,0,8654,32607,54,217.27 +1774535805,0,NVIDIA GeForce RTX 5090,2,0,8636,32607,49,95.39 +1774535809,0,NVIDIA GeForce RTX 5090,0,0,8691,32607,48,75.29 +1774535810,0,NVIDIA GeForce RTX 5090,4,0,8655,32607,48,78.78 +1774535814,0,NVIDIA GeForce RTX 5090,0,0,8675,32607,48,69.48 +1774535815,0,NVIDIA GeForce RTX 5090,25,4,9197,32607,58,134.01 +1774535819,0,NVIDIA GeForce RTX 5090,1,0,8735,32607,51,364.19 +1774535820,0,NVIDIA GeForce RTX 5090,2,0,8727,32607,50,96.93 +1774535824,0,NVIDIA GeForce RTX 5090,4,1,8774,32607,48,74.99 +1774535825,0,NVIDIA GeForce RTX 5090,3,0,8707,32607,48,76.64 +1774535829,0,NVIDIA GeForce RTX 5090,0,0,8674,32607,48,70.19 +1774535830,0,NVIDIA GeForce RTX 5090,17,1,10650,32607,48,102.01 +1774535834,0,NVIDIA GeForce RTX 5090,10,1,8656,32607,47,70.96 +1774535835,0,NVIDIA GeForce RTX 5090,10,1,8673,32607,47,69.86 +1774535839,0,NVIDIA GeForce RTX 5090,0,0,8707,32607,47,70.21 +1774535840,0,NVIDIA GeForce RTX 5090,0,0,8711,32607,47,68.88 +1774535844,0,NVIDIA GeForce RTX 5090,5,1,8706,32607,47,79.20 +1774535845,0,NVIDIA GeForce RTX 5090,5,1,8696,32607,47,79.58 +1774535849,0,NVIDIA GeForce RTX 5090,0,0,8703,32607,47,67.70 +1774535850,0,NVIDIA GeForce RTX 5090,0,0,5363,32607,47,67.98 +1774535854,0,NVIDIA GeForce RTX 5090,0,0,6434,32607,47,71.75 +1774535855,0,NVIDIA GeForce RTX 5090,0,0,6439,32607,47,67.38 +1774535859,0,NVIDIA GeForce RTX 5090,27,1,13135,32607,47,79.64 +1774535860,0,NVIDIA GeForce RTX 5090,0,0,13147,32607,46,67.41 +1774535864,0,NVIDIA GeForce RTX 5090,0,0,8677,32607,46,67.86 +1774535866,0,NVIDIA GeForce RTX 5090,0,0,8677,32607,46,68.71 +1774535869,0,NVIDIA GeForce RTX 5090,9,1,8681,32607,46,68.46 +1774535871,0,NVIDIA GeForce RTX 5090,3,0,8681,32607,46,68.17 +1774535874,0,NVIDIA GeForce RTX 5090,0,0,9851,32607,46,71.61 +1774535876,0,NVIDIA GeForce RTX 5090,5,2,8889,32607,46,87.72 +1774535879,0,NVIDIA GeForce RTX 5090,11,1,8681,32607,46,79.05 +1774535881,0,NVIDIA GeForce RTX 5090,5,0,8796,32607,46,70.33 +1774535884,0,NVIDIA GeForce RTX 5090,40,11,9430,32607,46,114.71 +1774535886,0,NVIDIA GeForce RTX 5090,38,11,9430,32607,51,197.96 +1774535889,0,NVIDIA GeForce RTX 5090,34,9,9431,32607,50,201.02 +1774535891,0,NVIDIA GeForce RTX 5090,37,10,9431,32607,51,201.45 +1774535894,0,NVIDIA GeForce RTX 5090,40,11,9431,32607,51,203.56 +1774535896,0,NVIDIA GeForce RTX 5090,0,0,8687,32607,50,145.93 +1774535899,0,NVIDIA GeForce RTX 5090,60,6,8697,32607,53,187.96 +1774535901,0,NVIDIA GeForce RTX 5090,62,6,8697,32607,56,335.43 +1774535905,0,NVIDIA GeForce RTX 5090,3,0,8752,32607,48,73.61 +1774535906,0,NVIDIA GeForce RTX 5090,2,0,8784,32607,48,71.82 +1774535910,0,NVIDIA GeForce RTX 5090,1,0,8846,32607,48,73.89 +1774535911,0,NVIDIA GeForce RTX 5090,0,0,8845,32607,48,76.39 +1774535915,0,NVIDIA GeForce RTX 5090,7,3,9707,32607,49,131.06 +1774535916,0,NVIDIA GeForce RTX 5090,12,6,9752,32607,49,117.60 +1774535920,0,NVIDIA GeForce RTX 5090,1,0,8725,32607,49,83.15 +1774535921,0,NVIDIA GeForce RTX 5090,0,0,8751,32607,48,71.43 +1774535925,0,NVIDIA GeForce RTX 5090,2,0,8671,32607,48,75.29 +1774535926,0,NVIDIA GeForce RTX 5090,0,0,8669,32607,48,72.52 +1774535930,0,NVIDIA GeForce RTX 5090,0,0,8911,32607,48,111.00 +1774535931,0,NVIDIA GeForce RTX 5090,17,4,9231,32607,48,109.78 +1774535935,0,NVIDIA GeForce RTX 5090,4,0,8730,32607,47,73.78 +1774535936,0,NVIDIA GeForce RTX 5090,0,0,8718,32607,47,70.82 +1774535940,0,NVIDIA GeForce RTX 5090,0,0,8667,32607,47,70.05 +1774535941,0,NVIDIA GeForce RTX 5090,6,0,8715,32607,47,76.69 +1774535945,0,NVIDIA GeForce RTX 5090,6,0,8744,32607,47,79.98 +1774535946,0,NVIDIA GeForce RTX 5090,6,0,8781,32607,47,80.11 +1774535950,0,NVIDIA GeForce RTX 5090,0,0,5445,32607,47,68.64 +1774535951,0,NVIDIA GeForce RTX 5090,0,0,5438,32607,47,68.57 +1774535955,0,NVIDIA GeForce RTX 5090,0,0,6509,32607,46,68.26 +1774535956,0,NVIDIA GeForce RTX 5090,6,0,7566,32607,46,70.45 +1774535960,0,NVIDIA GeForce RTX 5090,0,0,13230,32607,46,67.11 +1774535961,0,NVIDIA GeForce RTX 5090,1,0,8747,32607,46,69.36 +1774535965,0,NVIDIA GeForce RTX 5090,0,0,8769,32607,46,68.93 +1774535966,0,NVIDIA GeForce RTX 5090,1,0,8771,32607,46,68.79 +1774535970,0,NVIDIA GeForce RTX 5090,10,1,8787,32607,46,68.98 +1774535971,0,NVIDIA GeForce RTX 5090,20,2,8788,32607,46,71.14 +1774535975,0,NVIDIA GeForce RTX 5090,6,1,8865,32607,46,84.33 +1774535976,0,NVIDIA GeForce RTX 5090,0,0,8808,32607,46,89.40 +1774535980,0,NVIDIA GeForce RTX 5090,1,0,8999,32607,46,70.62 +1774535981,0,NVIDIA GeForce RTX 5090,1,0,8968,32607,46,67.81 +1774535985,0,NVIDIA GeForce RTX 5090,39,11,9545,32607,50,201.54 +1774535986,0,NVIDIA GeForce RTX 5090,37,10,9537,32607,50,199.26 +1774535990,0,NVIDIA GeForce RTX 5090,35,9,9491,32607,53,200.39 +1774535991,0,NVIDIA GeForce RTX 5090,35,9,9480,32607,51,203.30 +1774535995,0,NVIDIA GeForce RTX 5090,1,0,8733,32607,49,92.03 +1774535996,0,NVIDIA GeForce RTX 5090,1,0,8733,32607,48,71.46 +1774536000,0,NVIDIA GeForce RTX 5090,69,7,8754,32607,57,326.53 +1774536002,0,NVIDIA GeForce RTX 5090,0,0,8737,32607,50,97.22 +1774536005,0,NVIDIA GeForce RTX 5090,0,0,8390,32607,48,74.11 +1774536007,0,NVIDIA GeForce RTX 5090,1,0,8329,32607,48,73.86 +1774536010,0,NVIDIA GeForce RTX 5090,0,0,8325,32607,48,72.57 +1774536012,0,NVIDIA GeForce RTX 5090,14,2,8783,32607,48,93.26 +1774536015,0,NVIDIA GeForce RTX 5090,83,25,9825,32607,59,462.57 +1774536017,0,NVIDIA GeForce RTX 5090,0,0,9825,32607,63,416.02 +1774536020,0,NVIDIA GeForce RTX 5090,1,0,8344,32607,49,71.74 +1774536022,0,NVIDIA GeForce RTX 5090,1,0,8403,32607,48,71.88 +1774536025,0,NVIDIA GeForce RTX 5090,1,0,8336,32607,48,71.27 +1774536027,0,NVIDIA GeForce RTX 5090,0,0,8334,32607,48,70.10 +1774536030,0,NVIDIA GeForce RTX 5090,0,0,8330,32607,48,70.40 +1774536032,0,NVIDIA GeForce RTX 5090,0,0,8331,32607,48,70.64 +1774536035,0,NVIDIA GeForce RTX 5090,1,0,8326,32607,48,70.18 +1774536037,0,NVIDIA GeForce RTX 5090,0,0,8331,32607,47,69.32 +1774536041,0,NVIDIA GeForce RTX 5090,5,0,8331,32607,48,86.83 +1774536042,0,NVIDIA GeForce RTX 5090,6,0,8331,32607,47,79.38 +1774536046,0,NVIDIA GeForce RTX 5090,6,0,8333,32607,47,79.87 +1774536047,0,NVIDIA GeForce RTX 5090,7,0,8332,32607,47,79.74 +1774536051,0,NVIDIA GeForce RTX 5090,0,0,8333,32607,47,67.35 +1774536052,0,NVIDIA GeForce RTX 5090,0,0,5002,32607,47,67.66 +1774536056,0,NVIDIA GeForce RTX 5090,1,0,6073,32607,47,72.20 +1774536057,0,NVIDIA GeForce RTX 5090,0,0,6073,32607,47,67.65 +1774536061,0,NVIDIA GeForce RTX 5090,6,2,12789,32607,47,80.33 +1774536062,0,NVIDIA GeForce RTX 5090,0,0,12788,32607,46,67.38 +1774536066,0,NVIDIA GeForce RTX 5090,3,0,8319,32607,46,69.08 +1774536067,0,NVIDIA GeForce RTX 5090,3,0,8319,32607,46,69.40 +1774536071,0,NVIDIA GeForce RTX 5090,3,0,8322,32607,46,68.60 +1774536072,0,NVIDIA GeForce RTX 5090,2,0,8326,32607,46,78.56 +1774536076,0,NVIDIA GeForce RTX 5090,6,2,8744,32607,46,74.93 +1774536077,0,NVIDIA GeForce RTX 5090,13,3,8645,32607,46,97.88 +1774536081,0,NVIDIA GeForce RTX 5090,2,0,8435,32607,46,70.76 +1774536082,0,NVIDIA GeForce RTX 5090,7,1,8428,32607,46,72.14 +1774536086,0,NVIDIA GeForce RTX 5090,37,11,9062,32607,49,178.89 +1774536087,0,NVIDIA GeForce RTX 5090,30,9,9073,32607,49,186.89 +1774536091,0,NVIDIA GeForce RTX 5090,36,10,9079,32607,52,197.84 +1774536092,0,NVIDIA GeForce RTX 5090,38,10,9073,32607,52,194.06 +1774536096,0,NVIDIA GeForce RTX 5090,33,9,9071,32607,51,194.47 +1774536097,0,NVIDIA GeForce RTX 5090,12,1,8324,32607,51,137.35 +1774536101,0,NVIDIA GeForce RTX 5090,68,6,8336,32607,52,157.97 +1774536102,0,NVIDIA GeForce RTX 5090,70,7,8336,32607,55,307.35 +1774536106,0,NVIDIA GeForce RTX 5090,1,0,8327,32607,49,88.75 +1774536107,0,NVIDIA GeForce RTX 5090,0,0,8329,32607,48,74.20 +1774536111,0,NVIDIA GeForce RTX 5090,1,0,8327,32607,48,78.30 +1774536112,0,NVIDIA GeForce RTX 5090,4,0,8327,32607,48,76.22 +1774536116,0,NVIDIA GeForce RTX 5090,13,2,8849,32607,49,135.38 +1774536117,0,NVIDIA GeForce RTX 5090,37,9,9353,32607,54,135.23 +1774536121,0,NVIDIA GeForce RTX 5090,1,0,8346,32607,50,96.52 +1774536122,0,NVIDIA GeForce RTX 5090,2,0,8363,32607,48,73.25 +1774536126,0,NVIDIA GeForce RTX 5090,3,0,8351,32607,48,75.21 +1774536128,0,NVIDIA GeForce RTX 5090,1,0,8346,32607,48,72.70 +1774536131,0,NVIDIA GeForce RTX 5090,7,1,8631,32607,49,111.73 +1774536133,0,NVIDIA GeForce RTX 5090,2,1,8909,32607,48,109.42 +1774536136,0,NVIDIA GeForce RTX 5090,0,0,8352,32607,48,78.07 +1774536138,0,NVIDIA GeForce RTX 5090,1,0,8337,32607,47,70.38 +1774536141,0,NVIDIA GeForce RTX 5090,0,0,8338,32607,47,69.73 +1774536143,0,NVIDIA GeForce RTX 5090,7,1,8340,32607,47,78.54 +1774536146,0,NVIDIA GeForce RTX 5090,5,0,8340,32607,47,79.57 +1774536148,0,NVIDIA GeForce RTX 5090,5,0,8340,32607,47,79.18 +1774536151,0,NVIDIA GeForce RTX 5090,0,0,5012,32607,47,69.29 +1774536153,0,NVIDIA GeForce RTX 5090,0,0,5012,32607,47,69.22 +1774536156,0,NVIDIA GeForce RTX 5090,0,0,6074,32607,47,69.94 +1774536158,0,NVIDIA GeForce RTX 5090,5,0,5780,32607,47,71.66 +1774536161,0,NVIDIA GeForce RTX 5090,0,0,3641,32607,46,67.49 +1774536163,0,NVIDIA GeForce RTX 5090,0,0,3638,32607,46,67.43 +1774536167,0,NVIDIA GeForce RTX 5090,0,0,3630,32607,46,67.44 +1774536168,0,NVIDIA GeForce RTX 5090,0,0,3630,32607,46,67.09 +1774536172,0,NVIDIA GeForce RTX 5090,0,0,3636,32607,46,66.86 +1774536173,0,NVIDIA GeForce RTX 5090,1,0,3650,32607,46,68.71 +1774536178,0,NVIDIA GeForce RTX 5090,0,0,3698,32607,46,73.85 +1774536183,0,NVIDIA GeForce RTX 5090,2,0,3771,32607,46,70.06 +1774536188,0,NVIDIA GeForce RTX 5090,0,0,3775,32607,46,67.53 +1774536193,0,NVIDIA GeForce RTX 5090,0,0,3775,32607,46,67.67 +1774536198,0,NVIDIA GeForce RTX 5090,0,0,3775,32607,45,67.34 +1774536203,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,68.06 +1774536208,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,68.11 +1774536213,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,67.64 +1774536219,0,NVIDIA GeForce RTX 5090,0,0,3781,32607,45,66.90 +1774536224,0,NVIDIA GeForce RTX 5090,0,0,3781,32607,45,67.10 +1774536229,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,67.28 +1774536234,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,66.64 +1774536239,0,NVIDIA GeForce RTX 5090,1,0,3781,32607,45,66.87 +1774536244,0,NVIDIA GeForce RTX 5090,0,0,3781,32607,45,66.86 +1774536249,0,NVIDIA GeForce RTX 5090,0,0,3781,32607,45,66.09 +1774536254,0,NVIDIA GeForce RTX 5090,0,0,3781,32607,44,66.15 +1774536259,0,NVIDIA GeForce RTX 5090,1,0,3957,32607,45,70.66 +1774536264,0,NVIDIA GeForce RTX 5090,0,0,4057,32607,44,66.17 +1774536269,0,NVIDIA GeForce RTX 5090,0,0,4000,32607,44,64.99 +1774536274,0,NVIDIA GeForce RTX 5090,1,0,4021,32607,44,67.32 +1774536279,0,NVIDIA GeForce RTX 5090,0,0,4026,32607,44,65.69 +1774536284,0,NVIDIA GeForce RTX 5090,0,0,3981,32607,44,65.32 +1774536289,0,NVIDIA GeForce RTX 5090,0,0,4011,32607,44,67.54 +1774536295,0,NVIDIA GeForce RTX 5090,0,0,4007,32607,44,65.88 +1774536300,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,44,65.89 +1774536305,0,NVIDIA GeForce RTX 5090,0,0,4020,32607,44,65.11 +1774536310,0,NVIDIA GeForce RTX 5090,0,0,4062,32607,44,66.44 +1774536315,0,NVIDIA GeForce RTX 5090,1,0,4150,32607,44,66.04 +1774536320,0,NVIDIA GeForce RTX 5090,0,0,4069,32607,44,64.63 +1774536325,0,NVIDIA GeForce RTX 5090,0,0,4040,32607,44,64.83 +1774536330,0,NVIDIA GeForce RTX 5090,0,0,4010,32607,44,64.98 +1774536335,0,NVIDIA GeForce RTX 5090,1,0,4031,32607,44,64.82 +1774536340,0,NVIDIA GeForce RTX 5090,0,0,4045,32607,44,64.97 +1774536345,0,NVIDIA GeForce RTX 5090,0,0,4045,32607,43,64.46 +1774536350,0,NVIDIA GeForce RTX 5090,0,0,4046,32607,43,64.96 +1774536355,0,NVIDIA GeForce RTX 5090,0,0,4116,32607,43,64.34 +1774536360,0,NVIDIA GeForce RTX 5090,0,0,4114,32607,43,64.39 +1774536365,0,NVIDIA GeForce RTX 5090,0,0,4123,32607,43,65.16 +1774536370,0,NVIDIA GeForce RTX 5090,0,0,4078,32607,43,64.18 +1774536376,0,NVIDIA GeForce RTX 5090,2,0,4045,32607,43,67.30 +1774536381,0,NVIDIA GeForce RTX 5090,2,0,4079,32607,43,65.72 +1774536386,0,NVIDIA GeForce RTX 5090,0,0,4122,32607,43,66.13 +1774536391,0,NVIDIA GeForce RTX 5090,0,0,4116,32607,43,65.28 +1774536396,0,NVIDIA GeForce RTX 5090,0,0,4116,32607,43,63.88 +1774536401,0,NVIDIA GeForce RTX 5090,0,0,4097,32607,43,65.04 +1774536406,0,NVIDIA GeForce RTX 5090,1,0,4052,32607,43,65.28 +1774536411,0,NVIDIA GeForce RTX 5090,0,0,4052,32607,43,64.83 +1774536416,0,NVIDIA GeForce RTX 5090,0,0,4052,32607,43,64.92 +1774536421,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,65.00 +1774536426,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,64.87 +1774536431,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,65.25 +1774536436,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,65.14 +1774536441,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,65.24 +1774536446,0,NVIDIA GeForce RTX 5090,0,0,4048,32607,43,65.18 +1774536451,0,NVIDIA GeForce RTX 5090,1,0,4073,32607,43,65.63 +1774536457,0,NVIDIA GeForce RTX 5090,0,0,4045,32607,43,65.48 +1774536462,0,NVIDIA GeForce RTX 5090,0,0,4138,32607,43,66.86 +1774536467,0,NVIDIA GeForce RTX 5090,0,0,4058,32607,43,67.11 +1774536472,0,NVIDIA GeForce RTX 5090,0,0,4078,32607,43,66.79 +1774536477,0,NVIDIA GeForce RTX 5090,0,0,4074,32607,43,66.67 +1774536482,0,NVIDIA GeForce RTX 5090,0,0,4246,32607,43,65.90 +1774536487,0,NVIDIA GeForce RTX 5090,0,0,4047,32607,43,66.73 +1774536492,0,NVIDIA GeForce RTX 5090,0,0,4061,32607,43,66.97 +1774536497,0,NVIDIA GeForce RTX 5090,0,0,4079,32607,43,67.14 +1774536502,0,NVIDIA GeForce RTX 5090,0,0,4051,32607,43,66.53 +1774536507,0,NVIDIA GeForce RTX 5090,0,0,4051,32607,43,66.66 +1774536512,0,NVIDIA GeForce RTX 5090,0,0,4051,32607,43,66.56 +1774536517,0,NVIDIA GeForce RTX 5090,0,0,4119,32607,43,66.93 +1774536522,0,NVIDIA GeForce RTX 5090,0,0,4130,32607,43,67.27 +1774536527,0,NVIDIA GeForce RTX 5090,0,0,4116,32607,43,65.06 +1774536533,0,NVIDIA GeForce RTX 5090,1,0,4064,32607,43,65.25 +1774536538,0,NVIDIA GeForce RTX 5090,0,0,4065,32607,43,65.61 +1774536543,0,NVIDIA GeForce RTX 5090,0,0,4063,32607,43,65.72 +1774536548,0,NVIDIA GeForce RTX 5090,0,0,4086,32607,43,63.40 +1774536553,0,NVIDIA GeForce RTX 5090,1,0,4049,32607,43,68.95 +1774536558,0,NVIDIA GeForce RTX 5090,0,0,4068,32607,43,66.23 +1774536563,0,NVIDIA GeForce RTX 5090,0,0,4108,32607,43,66.00 +1774536568,0,NVIDIA GeForce RTX 5090,0,0,4090,32607,43,65.91 +1774536573,0,NVIDIA GeForce RTX 5090,0,0,4074,32607,43,65.85 +1774536578,0,NVIDIA GeForce RTX 5090,0,0,4089,32607,43,65.71 +1774536583,0,NVIDIA GeForce RTX 5090,0,0,4040,32607,43,64.55 +1774536588,0,NVIDIA GeForce RTX 5090,0,0,4024,32607,43,64.33 +1774536593,0,NVIDIA GeForce RTX 5090,0,0,4030,32607,43,66.56 +1774536598,0,NVIDIA GeForce RTX 5090,0,0,4026,32607,43,66.21 +1774536603,0,NVIDIA GeForce RTX 5090,0,0,4026,32607,43,66.27 +1774536608,0,NVIDIA GeForce RTX 5090,0,0,4030,32607,43,65.04 +1774536614,0,NVIDIA GeForce RTX 5090,0,0,4044,32607,43,66.31 +1774536619,0,NVIDIA GeForce RTX 5090,0,0,4046,32607,43,66.60 +1774536624,0,NVIDIA GeForce RTX 5090,0,0,4030,32607,43,66.73 +1774536629,0,NVIDIA GeForce RTX 5090,0,0,4030,32607,43,65.72 +1774536634,0,NVIDIA GeForce RTX 5090,0,0,4040,32607,43,65.95 +1774536639,0,NVIDIA GeForce RTX 5090,0,0,4045,32607,43,65.18 +1774536644,0,NVIDIA GeForce RTX 5090,0,0,4040,32607,43,64.97 +1774536649,0,NVIDIA GeForce RTX 5090,0,0,4040,32607,43,65.38 +1774536654,0,NVIDIA GeForce RTX 5090,1,0,4029,32607,43,68.26 +1774536659,0,NVIDIA GeForce RTX 5090,0,0,4066,32607,43,64.91 +1774536664,0,NVIDIA GeForce RTX 5090,0,0,4034,32607,43,64.85 +1774536669,0,NVIDIA GeForce RTX 5090,0,0,4024,32607,43,64.21 +1774536674,0,NVIDIA GeForce RTX 5090,0,0,4010,32607,43,64.83 +1774536679,0,NVIDIA GeForce RTX 5090,0,0,4038,32607,43,64.18 +1774536684,0,NVIDIA GeForce RTX 5090,2,0,4530,32607,43,67.12 +1774536689,0,NVIDIA GeForce RTX 5090,0,0,4550,32607,43,64.59 +1774536694,0,NVIDIA GeForce RTX 5090,0,0,4534,32607,43,63.98 +1774536700,0,NVIDIA GeForce RTX 5090,0,0,4518,32607,43,64.05 +1774536705,0,NVIDIA GeForce RTX 5090,0,0,4521,32607,43,65.75 +1774536710,0,NVIDIA GeForce RTX 5090,0,0,4512,32607,43,63.89 +1774536715,0,NVIDIA GeForce RTX 5090,0,0,4461,32607,43,64.45 +1774536720,0,NVIDIA GeForce RTX 5090,0,0,4486,32607,42,63.90 +1774536725,0,NVIDIA GeForce RTX 5090,1,0,4418,32607,43,65.90 +1774536730,0,NVIDIA GeForce RTX 5090,2,0,4417,32607,43,66.95 +1774536735,0,NVIDIA GeForce RTX 5090,0,0,4498,32607,43,65.27 +1774536740,0,NVIDIA GeForce RTX 5090,0,0,4496,32607,43,65.28 +1774536745,0,NVIDIA GeForce RTX 5090,0,0,4415,32607,42,64.83 +1774536750,0,NVIDIA GeForce RTX 5090,0,0,4418,32607,43,65.12 +1774536755,0,NVIDIA GeForce RTX 5090,0,0,4399,32607,43,65.54 +1774536760,0,NVIDIA GeForce RTX 5090,0,0,4398,32607,42,65.57 +1774536765,0,NVIDIA GeForce RTX 5090,2,0,4403,32607,43,66.69 +1774536770,0,NVIDIA GeForce RTX 5090,0,0,4472,32607,42,64.96 +1774536775,0,NVIDIA GeForce RTX 5090,0,0,4477,32607,42,63.24 +1774536780,0,NVIDIA GeForce RTX 5090,0,0,4477,32607,42,63.21 +1774536785,0,NVIDIA GeForce RTX 5090,0,0,4477,32607,42,63.40 +1774536790,0,NVIDIA GeForce RTX 5090,0,0,4473,32607,42,64.31 +1774536795,0,NVIDIA GeForce RTX 5090,0,0,4532,32607,42,63.42 +1774536800,0,NVIDIA GeForce RTX 5090,0,0,4538,32607,42,64.19 +1774536805,0,NVIDIA GeForce RTX 5090,0,0,4543,32607,42,63.49 +1774536810,0,NVIDIA GeForce RTX 5090,0,0,4549,32607,42,64.47 +1774536815,0,NVIDIA GeForce RTX 5090,2,0,4621,32607,42,64.93 +1774536820,0,NVIDIA GeForce RTX 5090,1,0,4635,32607,42,66.09 +1774536825,0,NVIDIA GeForce RTX 5090,2,0,4626,32607,42,66.61 +1774536831,0,NVIDIA GeForce RTX 5090,0,0,4610,32607,42,64.67 +1774536836,0,NVIDIA GeForce RTX 5090,0,0,4596,32607,42,65.50 +1774536841,0,NVIDIA GeForce RTX 5090,0,0,4575,32607,42,63.87 +1774536846,0,NVIDIA GeForce RTX 5090,0,0,4499,32607,42,65.36 +1774536851,0,NVIDIA GeForce RTX 5090,0,0,4444,32607,42,65.88 +1774536856,0,NVIDIA GeForce RTX 5090,0,0,4441,32607,42,65.62 +1774536861,0,NVIDIA GeForce RTX 5090,1,0,4444,32607,42,65.95 +1774536866,0,NVIDIA GeForce RTX 5090,0,0,4458,32607,42,64.76 +1774536871,0,NVIDIA GeForce RTX 5090,1,0,4503,32607,42,66.25 +1774536876,0,NVIDIA GeForce RTX 5090,1,0,4467,32607,42,66.62 +1774536881,0,NVIDIA GeForce RTX 5090,0,0,4464,32607,42,64.83 +1774536886,0,NVIDIA GeForce RTX 5090,0,0,4430,32607,42,64.49 +1774536891,0,NVIDIA GeForce RTX 5090,0,0,4448,32607,42,64.24 +1774536896,0,NVIDIA GeForce RTX 5090,0,0,4440,32607,42,64.45 +1774536901,0,NVIDIA GeForce RTX 5090,0,0,4485,32607,42,64.98 +1774536906,0,NVIDIA GeForce RTX 5090,2,0,4505,32607,42,68.89 +1774536911,0,NVIDIA GeForce RTX 5090,0,0,4489,32607,42,64.12 +1774536916,0,NVIDIA GeForce RTX 5090,0,0,4470,32607,42,65.12 +1774536921,0,NVIDIA GeForce RTX 5090,3,0,4448,32607,42,66.38 +1774536926,0,NVIDIA GeForce RTX 5090,2,0,4466,32607,42,66.16 +1774536931,0,NVIDIA GeForce RTX 5090,0,0,4501,32607,42,65.57 +1774536936,0,NVIDIA GeForce RTX 5090,0,0,4462,32607,42,65.03 +1774536941,0,NVIDIA GeForce RTX 5090,0,0,4481,32607,42,64.62 +1774536946,0,NVIDIA GeForce RTX 5090,0,0,4501,32607,42,64.31 +1774536951,0,NVIDIA GeForce RTX 5090,2,0,4499,32607,42,65.21 +1774536957,0,NVIDIA GeForce RTX 5090,0,0,4559,32607,42,65.51 +1774536962,0,NVIDIA GeForce RTX 5090,1,0,4547,32607,42,64.94 +1774536967,0,NVIDIA GeForce RTX 5090,1,0,4715,32607,42,65.75 +1774536972,0,NVIDIA GeForce RTX 5090,0,0,4743,32607,42,64.52 +1774536977,0,NVIDIA GeForce RTX 5090,0,0,4712,32607,42,63.86 +1774536982,0,NVIDIA GeForce RTX 5090,0,0,4582,32607,42,63.45 +1774536987,0,NVIDIA GeForce RTX 5090,0,0,4557,32607,42,63.44 +1774536992,0,NVIDIA GeForce RTX 5090,0,0,4677,32607,42,67.30 +1774536997,0,NVIDIA GeForce RTX 5090,0,0,4704,32607,42,65.35 +1774537002,0,NVIDIA GeForce RTX 5090,0,0,4706,32607,42,63.92 +1774537007,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,62.92 +1774537012,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,62.77 +1774537017,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,62.90 +1774537022,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,62.93 +1774537027,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,63.22 +1774537032,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,62.82 +1774537037,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,63.12 +1774537042,0,NVIDIA GeForce RTX 5090,0,0,4554,32607,42,63.40 +1774537047,0,NVIDIA GeForce RTX 5090,0,0,4656,32607,42,64.56 +1774537052,0,NVIDIA GeForce RTX 5090,0,0,4661,32607,42,62.95 +1774537057,0,NVIDIA GeForce RTX 5090,0,0,4656,32607,42,64.69 +1774537062,0,NVIDIA GeForce RTX 5090,0,0,4656,32607,42,64.58 +1774537067,0,NVIDIA GeForce RTX 5090,0,0,4656,32607,42,64.37 +1774537072,0,NVIDIA GeForce RTX 5090,0,0,4656,32607,42,64.39 +1774537077,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.52 +1774537082,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.15 +1774537088,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.09 +1774537093,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.34 +1774537098,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.12 +1774537103,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.13 +1774537108,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.38 +1774537113,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.57 +1774537118,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.74 +1774537123,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.21 +1774537128,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.23 +1774537133,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.17 +1774537138,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.69 +1774537143,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.43 +1774537148,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.53 +1774537153,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.59 +1774537158,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.43 +1774537163,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.57 +1774537168,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.78 +1774537173,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.22 +1774537178,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.04 +1774537183,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,62.94 +1774537188,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.27 +1774537193,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,62.99 +1774537198,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,62.93 +1774537203,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.39 +1774537209,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,65.22 +1774537214,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.22 +1774537219,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.32 +1774537224,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.30 +1774537229,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,65.06 +1774537234,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,65.04 +1774537239,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.51 +1774537244,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.43 +1774537249,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.58 +1774537254,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.63 +1774537259,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.89 +1774537264,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.35 +1774537269,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.08 +1774537274,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.46 +1774537279,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.63 +1774537284,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.58 +1774537289,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.04 +1774537294,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.92 +1774537299,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.63 +1774537304,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.47 +1774537309,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.41 +1774537314,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.39 +1774537319,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.94 +1774537325,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.73 +1774537330,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.23 +1774537335,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.85 +1774537340,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,63.79 +1774537345,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.12 +1774537350,0,NVIDIA GeForce RTX 5090,0,0,4651,32607,42,64.47 +1774537355,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.60 +1774537360,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.39 +1774537365,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.89 +1774537370,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.25 +1774537375,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.74 +1774537380,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.31 +1774537385,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.74 +1774537390,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.69 +1774537395,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.95 +1774537400,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.22 +1774537405,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.68 +1774537410,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.21 +1774537415,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.45 +1774537420,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.00 +1774537425,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.51 +1774537430,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.07 +1774537435,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.27 +1774537440,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.47 +1774537445,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.52 +1774537450,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.05 +1774537456,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.73 +1774537461,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.00 +1774537466,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.36 +1774537471,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.67 +1774537476,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.09 +1774537481,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.22 +1774537486,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.43 +1774537491,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.85 +1774537496,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.34 +1774537501,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.57 +1774537506,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.45 +1774537511,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.62 +1774537516,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.62 +1774537521,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.28 +1774537526,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.25 +1774537531,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.54 +1774537536,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.32 +1774537541,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.71 +1774537546,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.57 +1774537551,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.53 +1774537556,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.71 +1774537561,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.56 +1774537566,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.91 +1774537572,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.67 +1774537577,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.56 +1774537582,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.07 +1774537587,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.15 +1774537592,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.03 +1774537597,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.07 +1774537602,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.21 +1774537607,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.94 +1774537612,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.95 +1774537617,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.63 +1774537622,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.96 +1774537627,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.01 +1774537632,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.53 +1774537637,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.58 +1774537642,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.92 +1774537647,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.94 +1774537652,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,63.93 +1774537657,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.06 +1774537662,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.29 +1774537667,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.00 +1774537672,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.11 +1774537677,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.15 +1774537682,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.34 +1774537687,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.63 +1774537692,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.53 +1774537698,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.33 +1774537703,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.14 +1774537708,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.42 +1774537713,0,NVIDIA GeForce RTX 5090,0,0,4647,32607,42,64.69 +1774537718,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,65.03 +1774537723,0,NVIDIA GeForce RTX 5090,1,0,4648,32607,42,65.71 +1774537728,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.92 +1774537733,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.89 +1774537738,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.38 +1774537743,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.67 +1774537748,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.61 +1774537753,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.75 +1774537758,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,65.10 +1774537763,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.69 +1774537768,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.83 +1774537773,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,65.24 +1774537778,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.86 +1774537783,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.89 +1774537788,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.57 +1774537793,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,65.12 +1774537798,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.61 +1774537803,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.85 +1774537808,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.32 +1774537814,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.91 +1774537819,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.96 +1774537824,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.07 +1774537829,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.20 +1774537834,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.85 +1774537839,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.89 +1774537844,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.67 +1774537849,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.81 +1774537854,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.96 +1774537859,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.17 +1774537864,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.24 +1774537869,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.18 +1774537874,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.14 +1774537879,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.21 +1774537884,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.27 +1774537889,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.55 +1774537894,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.39 +1774537899,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.44 +1774537904,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.08 +1774537909,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.64 +1774537914,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.08 +1774537919,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.27 +1774537924,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.21 +1774537929,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.49 +1774537934,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.43 +1774537940,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.25 +1774537945,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.27 +1774537950,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.89 +1774537955,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.52 +1774537960,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.27 +1774537965,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.09 +1774537970,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.48 +1774537975,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.90 +1774537980,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,65.10 +1774537985,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.53 +1774537990,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.89 +1774537995,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.41 +1774538000,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.36 +1774538005,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,63.82 +1774538010,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.43 +1774538015,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.08 +1774538020,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.51 +1774538025,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.77 +1774538030,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.83 +1774538035,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.91 +1774538040,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.07 +1774538045,0,NVIDIA GeForce RTX 5090,0,0,4648,32607,42,64.31 +1774538050,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.39 +1774538055,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.35 +1774538060,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.78 +1774538066,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.66 +1774538071,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.33 +1774538076,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.72 +1774538081,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.44 +1774538086,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.93 +1774538091,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.41 +1774538096,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.36 +1774538101,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.42 +1774538106,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.26 +1774538111,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.74 +1774538116,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.36 +1774538121,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.17 +1774538126,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.94 +1774538131,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.40 +1774538136,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.29 +1774538141,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.57 +1774538146,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.18 +1774538151,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.31 +1774538156,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.15 +1774538161,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.79 +1774538166,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.39 +1774538171,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.62 +1774538176,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.74 +1774538181,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.18 +1774538186,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.95 +1774538192,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.87 +1774538197,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.29 +1774538202,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.41 +1774538207,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.34 +1774538212,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.53 +1774538217,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.78 +1774538222,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.08 +1774538227,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.24 +1774538232,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.35 +1774538237,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.77 +1774538242,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.01 +1774538247,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.35 +1774538252,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.52 +1774538257,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.55 +1774538262,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.38 +1774538267,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.35 +1774538272,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.11 +1774538277,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.89 +1774538282,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.29 +1774538287,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.97 +1774538292,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.97 +1774538297,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.40 +1774538302,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.39 +1774538307,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.39 +1774538312,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.92 +1774538318,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.18 +1774538323,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.08 +1774538328,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.82 +1774538333,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.83 +1774538338,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.99 +1774538343,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.39 +1774538348,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.23 +1774538353,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.18 +1774538358,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.31 +1774538363,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.72 +1774538368,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.25 +1774538373,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.94 +1774538378,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.06 +1774538383,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.16 +1774538388,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.05 +1774538393,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.02 +1774538398,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.02 +1774538403,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.07 +1774538408,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.20 +1774538413,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.06 +1774538418,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.18 +1774538423,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.44 +1774538428,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.16 +1774538434,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.32 +1774538439,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.06 +1774538444,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.16 +1774538449,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.07 +1774538454,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.27 +1774538459,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.29 +1774538464,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.41 +1774538469,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.29 +1774538474,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.44 +1774538479,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.89 +1774538484,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.31 +1774538489,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.46 +1774538494,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.43 +1774538499,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.56 +1774538504,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.34 +1774538509,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.56 +1774538514,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.81 +1774538519,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.32 +1774538524,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,65.20 +1774538529,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.90 +1774538534,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.57 +1774538539,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.47 +1774538544,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.01 +1774538549,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.93 +1774538555,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.68 +1774538560,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,65.24 +1774538565,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.96 +1774538570,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.68 +1774538575,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.98 +1774538580,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.84 +1774538585,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.04 +1774538590,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.61 +1774538595,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,63.97 +1774538600,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.42 +1774538605,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.93 +1774538610,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.69 +1774538615,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.64 +1774538620,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.29 +1774538625,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.75 +1774538630,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.78 +1774538635,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.40 +1774538640,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.84 +1774538645,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.02 +1774538650,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.74 +1774538655,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.00 +1774538660,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.34 +1774538665,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.04 +1774538670,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.69 +1774538676,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.70 +1774538681,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.48 +1774538686,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,65.29 +1774538691,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,65.13 +1774538696,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,65.04 +1774538701,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.92 +1774538706,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.32 +1774538711,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.54 +1774538716,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.58 +1774538721,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.53 +1774538726,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.70 +1774538731,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.37 +1774538736,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.95 +1774538741,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.91 +1774538746,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.46 +1774538751,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,42,64.55 +1774538756,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.01 +1774538761,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.48 +1774538766,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.53 +1774538771,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.38 +1774538776,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.89 +1774538781,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.35 +1774538786,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.00 +1774538791,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.50 +1774538797,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.08 +1774538802,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.09 +1774538807,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.66 +1774538812,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.37 +1774538817,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.80 +1774538822,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.17 +1774538827,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.52 +1774538832,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.76 +1774538837,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.47 +1774538842,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.53 +1774538847,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.90 +1774538852,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.19 +1774538857,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.31 +1774538862,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.98 +1774538867,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.28 +1774538872,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.10 +1774538877,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.82 +1774538882,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.91 +1774538887,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.88 +1774538892,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.19 +1774538897,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.89 +1774538902,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.78 +1774538907,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.74 +1774538913,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.82 +1774538918,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.83 +1774538923,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.61 +1774538928,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.76 +1774538933,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.77 +1774538938,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,64.17 +1774538943,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.77 +1774538948,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.90 +1774538953,0,NVIDIA GeForce RTX 5090,0,0,4696,32607,41,63.59 diff --git a/outputs/smoke_avatar.mp4 b/outputs/smoke_avatar.mp4 new file mode 100644 index 0000000..3c6ae06 Binary files /dev/null and b/outputs/smoke_avatar.mp4 differ diff --git a/outputs/smoke_reply.wav b/outputs/smoke_reply.wav new file mode 100644 index 0000000..71c840b Binary files /dev/null and b/outputs/smoke_reply.wav differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f4aa64c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi +uvicorn[standard] +aiortc +numpy +soundfile +python-dotenv +pydantic +pydantic-settings +tenacity +silero-vad +funasr +kokoro +opencv-python-headless<4.13 +openai diff --git a/research-notes.md b/research-notes.md new file mode 100644 index 0000000..d4ff3ac --- /dev/null +++ b/research-notes.md @@ -0,0 +1,517 @@ +# 数字人视频生成 — 研究与验证记录 + +> 本文档记录了从零开始探索「单图 + 音频 → 说话视频」的完整过程， +> 包括工具选型、环境踩坑、最终跑通的完整流程，以及对产品化的建议。 +> +> 日期：2026-03-25 +> 硬件：RTX 5090 (Blackwell, sm_120), CUDA 12.9, WSL2 + +--- + +## 一、目标 + +给定一张人物图片和一段音频，生成该人物"说话"的视频： +- 嘴型与音频内容同步 +- 头部有自然动作（非僵硬的照片） +- 可扩展至实时/直播场景 + +--- + +## 二、工具选型过程 + +### 2.1 最初方向：LivePortrait + +- **项目**：https://github.com/KlingTeam/LivePortrait（快手） +- **本质**：视频驱动的人脸动画，需要一个"驱动视频"来控制人物表情和头部运动 +- **误区澄清**：网上有描述称 LivePortrait 支持"音频驱动"，这是**不准确的**。它只能从驱动视频中提取音频附加到输出，无法用音频生成嘴型 +- **结论**：LivePortrait 适合生成头部动作底片，不能单独完成任务 + +### 2.2 调研其他工具 + +| 工具 | 厂商 | 核心能力 | 实时性 | 结论 | +|------|------|---------|--------|------| +| LatentSync 1.6 | 字节跳动 | 扩散模型唇形同步，512x512 | ❌ 约4fps | 质量好，但太慢 | +| MuseTalk 1.5 | 腾讯 | 单步推理唇形同步 | ⚠️ 约7fps | 速度可接受，质量好 | +| Live Avatar | 阿里 | 140B 扩散模型 | ❌ 需集群 | 单卡不可行 | +| SadTalker | 开源 | 音频驱动全脸动画 | ❌ | 未测试 | +| Sonic | 腾讯 | 音频驱动 | 未知 | 项目在 `/home/xsl/work/Sonic`，未测试 | + +### 2.3 最终方案 + +**两阶段流水线：** + +``` +hairstyle-result.jpg + ↓ + [LivePortrait] ← 驱动视频 (d3.mp4, 自然头部动作) + ↓ + 头动底片视频 (hairstyle-result--d3.mp4, 11.8s) + ↓ + [MuseTalk] ← 音频 (test_speech_10s.wav) + ↓ + 最终视频 (嘴型同步 + 头部运动) +``` + +--- + +## 三、环境配置（重要踩坑记录） + +### 3.1 RTX 5090 的核心问题 + +RTX 5090 是 Blackwell 架构（sm_120），**PyTorch 稳定版（2.6 及以下）不支持**。 +所有项目都必须使用 PyTorch nightly cu128： + +```bash +pip install --pre torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/nightly/cu128 +# 已验证版本：torch-2.12.0.dev20260324+cu128 +``` + +**这是最先要做的事，否则所有 CUDA 推理都会报错：** +``` +CUDA error: no kernel image is available for execution on the device +``` + +### 3.2 HuggingFace CLI 命令名 + +在某些 conda 环境中，命令名是 `hf` 而不是 `huggingface-cli`： + +```bash +which hf # /home/xsl/miniconda3/envs/MuseTalk/bin/hf +hf --version # 1.7.2 +``` + +下载需要 Token： +```bash +export HF_TOKEN= +hf download --local-dir +``` + +--- + +## 四、LivePortrait 部署 + +### 4.1 环境 + +```bash +conda create -n LivePortrait python=3.10 -y +conda activate LivePortrait +pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128 +cd /home/xsl/work/LivePortrait +pip install -r requirements.txt +``` + +### 4.2 权重 + +```bash +# 约 1.1GB +huggingface-cli download KlingTeam/LivePortrait \ + --local-dir pretrained_weights \ + --exclude "*.git*" "README.md" "docs" +``` + +目录结构： +``` +pretrained_weights/ +├── insightface/ (~21MB) +├── liveportrait/ (~608MB) +└── liveportrait_animals/ (~500MB) +``` + +### 4.3 生成头动底片 + +```bash +conda activate LivePortrait +cd /home/xsl/work/LivePortrait + +python inference.py \ + -s /path/to/portrait.jpg \ + -d assets/examples/driving/d3.mp4 \ + --flag_crop_driving_video \ + -o /home/xsl/work/head_motion_base +``` + +**驱动视频参考**（位于 `assets/examples/driving/`）： + +| 文件 | 时长 | 特点 | +|------|------|------| +| d3.mp4 | 11.8s | 自然说话动作，与10s音频匹配 | +| d6.mp4 | 33.6s | 长片段，适合循环 | +| d10.mp4 | 15.0s | 较长 | + +**技巧**：选择时长接近音频时长的驱动视频；`--flag_crop_driving_video` 会自动裁剪驱动视频。 + +### 4.4 已知警告（可忽略） + +``` +[E:onnxruntime] Failed to load library libonnxruntime_providers_cuda.so +``` +onnxruntime-gpu 依赖 CUDA 11.x，与系统 CUDA 12.9 不兼容，人脸检测自动回退 CPU，不影响结果。 + +--- + +## 五、LatentSync 部署（已验证，非推荐方案） + +> **结论**：质量好，但速度太慢（4fps），不适合产品化。作为备选方案保留。 + +### 5.1 环境 + +```bash +conda create -y -n latentsync python=3.10.13 +conda activate latentsync +conda install -y -c conda-forge ffmpeg + +pip install -r /home/xsl/work/LatentSync/requirements.txt + +# RTX 5090 必须卸载稳定版再装 nightly +pip uninstall torch torchvision torchaudio -y +pip install --pre torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/nightly/cu128 +``` + +### 5.2 权重（约 4.9GB） + +```bash +cd /home/xsl/work/LatentSync +huggingface-cli download ByteDance/LatentSync-1.6 whisper/tiny.pt --local-dir checkpoints +huggingface-cli download ByteDance/LatentSync-1.6 latentsync_unet.pt --local-dir checkpoints +``` + +### 5.3 输入准备 + +LatentSync 需要**视频**（非图片）+ **WAV 音频**： + +```bash +# 图片 → 视频 +ffmpeg -loop 1 -i portrait.jpg \ + -t \ + -vf "scale=512:512:force_original_aspect_ratio=decrease,pad=512:512:(ow-iw)/2:(oh-ih)/2" \ + -r 25 -c:v libx264 -pix_fmt yuv420p input_video.mp4 -y + +# mp3 → wav +ffmpeg -i audio.mp3 -ar 16000 -ac 1 input_audio.wav -y +``` + +### 5.4 推理 + +```bash +conda activate latentsync +cd /home/xsl/work/LatentSync +python -m scripts.inference \ + --unet_config_path "configs/unet/stage2_512.yaml" \ + --inference_ckpt_path "checkpoints/latentsync_unet.pt" \ + --inference_steps 20 \ + --guidance_scale 1.5 \ + --enable_deepcache \ + --video_path "assets/input_video.mp4" \ + --audio_path "assets/input_audio.wav" \ + --video_out_path "video_out.mp4" +``` + +--- + +## 六、MuseTalk 部署（推荐方案） + +### 6.1 项目路径 + +``` +/home/xsl/work/MuseTalk +``` + +### 6.2 环境搭建（完整步骤，含所有踩坑修复） + +```bash +conda create -y -n MuseTalk python=3.10 +conda activate MuseTalk +conda install -y -c conda-forge ffmpeg + +# 1. PyTorch nightly（RTX 5090 必须） +pip install --pre torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/nightly/cu128 + +# 2. 项目依赖 +cd /home/xsl/work/MuseTalk +pip install -r requirements.txt + +# 3. 修复 chumpy 构建问题 +pip install --no-build-isolation chumpy + +# 4. MMLab（版本必须严格匹配，不可随意升降） +pip install mmengine +MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation # 需要系统 nvcc +pip install "mmdet>=3.3.0" # ≥3.3.0 才兼容 mmcv 2.1.0 +pip install "mmpose>=1.3.0" # ≥1.3.0 才兼容 mmcv 2.1.0 +pip install xtcocotools json-tricks munkres + +# 5. 更新 transformers（旧版与 huggingface-hub 1.x 冲突） +pip install "transformers>=4.45.0" +``` + +### 6.3 MMLab 版本兼容矩阵（关键） + +``` +mmcv 2.1.0 ←→ mmdet ≥3.3.0 ←→ mmpose ≥1.3.0 +``` + +| 踩坑 | 原因 | 修复 | +|------|------|------| +| mmcv 构建失败 | nightly PyTorch 没有预编译轮子 | `MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation`（需系统 nvcc） | +| `MMCV==2.1.0 is used but incompatible` | mmdet 3.1.0 要求 mmcv < 2.1.0 | 升级到 mmdet ≥3.3.0 | +| `mmpose 1.1.0 incompatible` | 同上 | 升级到 mmpose ≥1.3.0 | +| `transformers ImportError` | huggingface-hub 1.x 与旧 transformers 冲突 | 升级 transformers ≥4.45.0 | + +### 6.4 权重下载 + +```bash +cd /home/xsl/work/MuseTalk +export HF_TOKEN= + +# 注意：此环境中命令是 hf，不是 huggingface-cli +hf download TMElyralab/MuseTalk \ + "musetalkV15/musetalk.json" "musetalkV15/unet.pth" \ + --local-dir models + +hf download stabilityai/sd-vae-ft-mse \ + config.json diffusion_pytorch_model.bin \ + --local-dir models/sd-vae + +hf download yzd-v/DWPose \ + dw-ll_ucoco_384.pth \ + --local-dir models/dwpose + +hf download openai/whisper-tiny \ + config.json pytorch_model.bin preprocessor_config.json \ + --local-dir models/whisper + +hf download ByteDance/LatentSync \ + latentsync_syncnet.pt \ + --local-dir models/syncnet + +# face-parse 权重（不在 HuggingFace） +pip install gdown +gdown --id 154JgKpzCPW82qINcVieuPH3fZ2e0P812 -O models/face-parse-bisent/79999_iter.pth +curl -L https://download.pytorch.org/models/resnet18-5c106cde.pth \ + -o models/face-parse-bisent/resnet18-5c106cde.pth +``` + +权重总大小约 **5GB**： +``` +models/ +├── musetalkV15/ +│ ├── unet.pth (3.2GB) +│ └── musetalk.json +├── sd-vae/ +│ ├── diffusion_pytorch_model.bin (320MB) +│ └── config.json +├── dwpose/ +│ └── dw-ll_ucoco_384.pth (389MB) +├── whisper/ +│ └── pytorch_model.bin (145MB) +├── syncnet/ +│ └── latentsync_syncnet.pt (1.4GB) +└── face-parse-bisent/ + ├── 79999_iter.pth (51MB) + └── resnet18-5c106cde.pth (45MB) +``` + +### 6.5 PyTorch 2.6+ 兼容性补丁（必须） + +PyTorch 2.6+ 将 `torch.load` 的 `weights_only` 默认值改为 `True`，导致 MuseTalk 加载旧格式权重失败。 +已在 `scripts/inference.py` 中加入 monkey-patch： + +```python +# 在 inference.py 开头的 import torch 后面加入： +_orig_torch_load = torch.load +def _patched_torch_load(f, *args, **kwargs): + kwargs.setdefault('weights_only', False) + return _orig_torch_load(f, *args, **kwargs) +torch.load = _patched_torch_load +``` + +同样的补丁也需要加到 `scripts/realtime_inference.py`（如果使用实时模式）。 + +### 6.6 推理配置文件 + +创建 YAML 配置文件（`configs/inference/my_task.yaml`）： + +```yaml +task_0: + video_path: "/path/to/head_motion_video.mp4" + audio_path: "/path/to/audio.wav" + bbox_shift: 0 # 可选：调整嘴部区域位置，负值下移，正值上移 +``` + +### 6.7 运行推理 + +```bash +conda activate MuseTalk +cd /home/xsl/work/MuseTalk + +python -m scripts.inference \ + --inference_config configs/inference/my_task.yaml \ + --result_dir results/output \ + --unet_model_path models/musetalkV15/unet.pth \ + --unet_config models/musetalkV15/musetalk.json \ + --version v15 +``` + +输出路径：`results/output/v15/.mp4` + +--- + +## 七、完整流水线（已验证示例） + +### 输入 + +- 图片：`/home/xsl/work/LivePortrait/hairstyle-result.jpg`（1080x1920，正面人像） +- 音频：`/home/xsl/work/LivePortrait/test_speech_10s.mp3`（11.8秒，中文语音） + +### Step 1：生成头动底片 + +```bash +conda activate LivePortrait +cd /home/xsl/work/LivePortrait + +python inference.py \ + -s /home/xsl/work/LivePortrait/hairstyle-result.jpg \ + -d assets/examples/driving/d3.mp4 \ + --flag_crop_driving_video \ + -o /home/xsl/work/head_motion_base +``` + +输出：`/home/xsl/work/head_motion_base/hairstyle-result--d3.mp4`（11.8s, 2.1MB） + +### Step 2：音频转 WAV + +```bash +ffmpeg -i /home/xsl/work/LivePortrait/test_speech_10s.mp3 \ + -ar 16000 -ac 1 /tmp/input_audio.wav -y +``` + +### Step 3：MuseTalk 嘴型同步 + +```bash +conda activate MuseTalk +cd /home/xsl/work/MuseTalk + +cat > configs/inference/hairstyle.yaml << 'EOF' +task_0: + video_path: "/home/xsl/work/head_motion_base/hairstyle-result--d3.mp4" + audio_path: "/tmp/input_audio.wav" +EOF + +python -m scripts.inference \ + --inference_config configs/inference/hairstyle.yaml \ + --result_dir results/hairstyle \ + --unet_model_path models/musetalkV15/unet.pth \ + --unet_config models/musetalkV15/musetalk.json \ + --version v15 +``` + +输出：`/home/xsl/work/MuseTalk/results/hairstyle/v15/hairstyle-result--d3_input_audio.mp4` + +### 性能数据 + +| 阶段 | 耗时 | 帧数 | 等效速度 | +|------|------|------|---------| +| LivePortrait 头动生成 | ~30s | 295帧 | ~10fps | +| MuseTalk 嘴型合成 | ~50s | 354帧 | ~7fps | +| 总计 | ~80s | — | 生成12s视频 | + +--- + +## 八、产品化建议 + +### 8.1 当前方案的局限 + +1. **非实时**：总体约 7fps 生成速度，12秒视频需要 80 秒处理 +2. **两套环境**：LivePortrait 和 MuseTalk 分别用不同 conda 环境，pipeline 不够整洁 +3. **头动单调**：驱动视频固定，长时间使用会重复 +4. **图片分辨率**：MuseTalk 处理区域限定在 256x256 嘴部区域 + +### 8.2 提速方向 + +- **MuseTalk 实时模式**：`scripts/realtime_inference.py` 支持流式处理，延迟更低 +- **头动循环库**：预先用 LivePortrait 生成多种头动视频（点头、摇头、思考等），运行时随机选取 +- **TensorRT 量化**：对 MuseTalk UNet 做 TRT 优化，预计可提速 2-3x + +### 8.3 架构建议（产品级） + +``` +音频输入 + ↓ +[ASR/VAD] → 静音检测、分段 + ↓ +[MuseTalk 实时流] ← 头动视频循环 + ↓ +[后处理：美颜/超分]（可选） + ↓ +视频输出流 +``` + +### 8.4 替代方案调研建议 + +在正式产品开发前，建议调研以下工具： +- **Sonic**（`/home/xsl/work/Sonic`）：已有项目，未测试，可能支持更好的实时性 +- **EchoMimic**：阿里的音频驱动方案，支持半身动作 +- **AniPortrait**：同时驱动头部和嘴型 + +--- + +## 九、目录结构总览 + +``` +/home/xsl/work/ +├── LivePortrait/ # 头部动作驱动 +│ ├── pretrained_weights/ (~1.1GB) +│ ├── hairstyle-result.jpg # 测试图片 +│ ├── test_speech_10s.mp3 # 测试音频 +│ └── DEPLOYMENT.md # 详细部署笔记 +│ +├── LatentSync/ # 扩散模型唇形同步（备用） +│ ├── checkpoints/ (~4.9GB) +│ └── video_out.mp4 # 测试输出 +│ +├── MuseTalk/ # 单步推理唇形同步（推荐） +│ ├── models/ (~5GB) +│ └── results/hairstyle/v15/hairstyle-result--d3_input_audio.mp4 +│ +├── head_motion_base/ # LivePortrait 生成的头动底片 +│ └── hairstyle-result--d3.mp4 +│ +└── Sonic/ # 待调研 +``` + +### Conda 环境 + +```bash +conda env list +# LivePortrait /home/xsl/miniconda3/envs/LivePortrait +# latentsync /home/xsl/miniconda3/envs/latentsync +# MuseTalk /home/xsl/miniconda3/envs/MuseTalk +``` + +--- + +## 十、快速复现命令（Cheat Sheet） + +```bash +# === 生成一个说话视频（完整流程）=== + +# 1. 生成头动底片 +conda activate LivePortrait && cd /home/xsl/work/LivePortrait +python inference.py -s portrait.jpg -d assets/examples/driving/d3.mp4 \ + --flag_crop_driving_video -o /tmp/head_motion + +# 2. 音频转 WAV +ffmpeg -i input.mp3 -ar 16000 -ac 1 /tmp/audio.wav -y + +# 3. 嘴型同步 +conda activate MuseTalk && cd /home/xsl/work/MuseTalk +python -m scripts.inference \ + --inference_config <(echo "task_0:\n video_path: /tmp/head_motion/*.mp4\n audio_path: /tmp/audio.wav") \ + --result_dir /tmp/output \ + --unet_model_path models/musetalkV15/unet.pth \ + --unet_config models/musetalkV15/musetalk.json \ + --version v15 +``` diff --git a/scripts/gen-self-signed-cert.sh b/scripts/gen-self-signed-cert.sh new file mode 100755 index 0000000..65f9ae4 --- /dev/null +++ b/scripts/gen-self-signed-cert.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /home/xsl/product +mkdir -p certs + +openssl req -x509 -nodes -newkey rsa:2048 -days 365 \ + -keyout certs/dev-key.pem \ + -out certs/dev-cert.pem \ + -subj "/C=CN/ST=Local/L=Local/O=VisualChat/OU=Dev/CN=localhost" + +echo "Generated:" +echo " /home/xsl/product/certs/dev-cert.pem" +echo " /home/xsl/product/certs/dev-key.pem" +echo "Set SSL_CERTFILE and SSL_KEYFILE in .env to enable HTTPS." diff --git a/scripts/gpu_monitor.sh b/scripts/gpu_monitor.sh new file mode 100755 index 0000000..08f034a --- /dev/null +++ b/scripts/gpu_monitor.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +OUT="${1:-/home/xsl/product/outputs/gpu-metrics.csv}" +SECONDS_TOTAL="${2:-3600}" +INTERVAL="${3:-5}" + +mkdir -p "$(dirname "$OUT")" +echo "ts,index,name,util_gpu,util_mem,mem_used_mb,mem_total_mb,temp_c,power_w" > "$OUT" + +end_ts=$(( $(date +%s) + SECONDS_TOTAL )) +while [[ "$(date +%s)" -lt "$end_ts" ]]; do + now="$(date +%s)" + nvidia-smi --query-gpu=index,name,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu,power.draw \ + --format=csv,noheader,nounits \ + | awk -v ts="$now" -F', ' '{print ts","$1","$2","$3","$4","$5","$6","$7","$8}' >> "$OUT" + sleep "$INTERVAL" +done + +echo "saved: $OUT" diff --git a/scripts/llm_fallback_check.py b/scripts/llm_fallback_check.py new file mode 100644 index 0000000..4f270c9 --- /dev/null +++ b/scripts/llm_fallback_check.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import os +import subprocess +import sys + + +def main() -> None: + code = r""" +import asyncio +from services.llm import LLMService + +async def run(): + svc = LLMService() + text, source = await svc.reply_with_meta("测试降级") + print("source:", source) + print("reply:", text) + +asyncio.run(run()) +""" + env = os.environ.copy() + env["LLM_BASE_URL"] = "http://127.0.0.1:9" + env["LLM_TIMEOUT"] = "1" + proc = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True) + print(proc.stdout.strip()) + if proc.returncode != 0: + print(proc.stderr.strip()) + raise SystemExit(proc.returncode) + if "source: fallback" not in proc.stdout: + raise SystemExit("fallback check failed: source is not fallback") + print("fallback check: ok") + + +if __name__ == "__main__": + main() diff --git a/scripts/longrun_test.py b/scripts/longrun_test.py new file mode 100644 index 0000000..70e3953 --- /dev/null +++ b/scripts/longrun_test.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import argparse +import json +import ssl +import time +import urllib.request + + +def _json_request(url: str, method: str = "GET", payload: dict | None = None, timeout: int = 30) -> dict: + body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(url=url, method=method, data=body) + if payload is not None: + req.add_header("Content-Type", "application/json") + ctx = ssl._create_unverified_context() if url.startswith("https://") else None + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Long-run stability test") + parser.add_argument("--base-url", default="https://127.0.0.1:8080") + parser.add_argument("--minutes", type=int, default=60) + parser.add_argument("--interval-sec", type=int, default=30) + args = parser.parse_args() + + end_ts = time.time() + args.minutes * 60 + rounds = 0 + while time.time() < end_ts: + rounds += 1 + data = _json_request( + f"{args.base_url}/chat/text", + method="POST", + payload={"text": f"长稳测试第{rounds}轮，请简短回复。"}, + timeout=60, + ) + health = _json_request(f"{args.base_url}/health") + print( + f"round={rounds} ok={data.get('ok')} " + f"last_latency={health.get('last_latency_ms')} " + f"llm={health.get('last_llm_latency_ms')} " + f"tts={health.get('last_tts_latency_ms')}" + ) + time.sleep(args.interval_sec) + + print("longrun done, rounds:", rounds) + + +if __name__ == "__main__": + main() diff --git a/scripts/model_probe.py b/scripts/model_probe.py new file mode 100644 index 0000000..9a5ce2b --- /dev/null +++ b/scripts/model_probe.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import asyncio +import os +import sys +import numpy as np + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from services.asr import ASRService +from services.tts import TTSService +from services.vad import VADService + + +async def main() -> None: + asr = ASRService() + tts = TTSService() + vad = VADService() + + dummy_16k = np.zeros(16000, dtype=np.float32) + asr_text = await asr.transcribe(dummy_16k) + tts_audio, tts_sr = await tts.synthesize("你好，这是模型探测。") + vad_event = vad.push_chunk(np.zeros(512, dtype=np.float32)) + + print("asr.health:", asr.health) + print("asr.text:", asr_text[:80]) + print("tts.health:", tts.health) + print("tts.samples:", int(tts_audio.shape[0]), "sr:", tts_sr) + print("vad.health:", vad.health, "vad.event:", vad_event) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/qa_check.py b/scripts/qa_check.py new file mode 100644 index 0000000..e707bcf --- /dev/null +++ b/scripts/qa_check.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import argparse +import json +import ssl +import time +import urllib.error +import urllib.request + + +def _json_request(url: str, method: str = "GET", payload: dict | None = None, timeout: int = 20) -> dict: + body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(url=url, method=method, data=body) + if payload is not None: + req.add_header("Content-Type", "application/json") + ctx = ssl._create_unverified_context() if url.startswith("https://") else None + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Quick QA for text pipeline and metrics") + parser.add_argument("--base-url", default="https://127.0.0.1:8080") + parser.add_argument("--rounds", type=int, default=5) + args = parser.parse_args() + + health = _json_request(f"{args.base_url}/health") + print("health.ok:", bool(health.get("llm"))) + latencies = [] + for i in range(args.rounds): + t0 = time.perf_counter() + data = _json_request( + f"{args.base_url}/chat/text", + method="POST", + payload={"text": f"第{i+1}轮压测，请简短回复。"}, + timeout=60, + ) + dt = int((time.perf_counter() - t0) * 1000) + latencies.append(dt) + print( + f"round={i+1} ok={data.get('ok')} llm={data.get('llm_latency_ms')} " + f"tts={data.get('tts_latency_ms')} total={dt}" + ) + + p50 = sorted(latencies)[len(latencies) // 2] if latencies else 0 + print("rounds:", args.rounds, "p50_ms:", p50, "max_ms:", max(latencies) if latencies else 0) + + health2 = _json_request(f"{args.base_url}/health") + print( + "final.health:", + { + "pipeline_runs": health2.get("pipeline_runs"), + "last_latency_ms": health2.get("last_latency_ms"), + "last_llm_latency_ms": health2.get("last_llm_latency_ms"), + "last_tts_latency_ms": health2.get("last_tts_latency_ms"), + }, + ) + + +if __name__ == "__main__": + try: + main() + except urllib.error.URLError as exc: + raise SystemExit(f"request failed: {exc}") diff --git a/scripts/run_all_checks.sh b/scripts/run_all_checks.sh new file mode 100755 index 0000000..218c96c --- /dev/null +++ b/scripts/run_all_checks.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /home/xsl/product +PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python" + +echo "[1/6] service status" +bash scripts/service.sh status || true + +echo "[2/6] quick pipeline qa" +"$PY" scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 5 + +echo "[3/6] llm fallback" +"$PY" scripts/llm_fallback_check.py + +echo "[4/6] vad check" +"$PY" scripts/vad_check.py + +echo "[5/6] tts stress" +"$PY" scripts/tts_stress.py + +echo "[6/6] smoke media output" +"$PY" scripts/smoke_test.py --text "全链路验收测试，请简短回复" + +echo "all checks finished." diff --git a/scripts/service.sh b/scripts/service.sh new file mode 100755 index 0000000..66e1519 --- /dev/null +++ b/scripts/service.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="/home/xsl/product" +PID_FILE="$ROOT/.run/visual-chat.pid" +LOG_FILE="$ROOT/.run/visual-chat.log" +START_CMD="$ROOT/scripts/start-public.sh" + +mkdir -p "$ROOT/.run" + +is_running() { + if [[ ! -f "$PID_FILE" ]]; then + return 1 + fi + local pid + pid="$(cat "$PID_FILE")" + if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then + return 0 + fi + return 1 +} + +start() { + if is_running; then + echo "already running: pid=$(cat "$PID_FILE")" + return 0 + fi + + # If stale process still binds configured port, stop it first. + local port + port="$(awk -F= '/^WEBRTC_PORT=/{print $2}' "$ROOT/.env" 2>/dev/null || true)" + port="${port:-8080}" + local pids + pids="$(ss -ltnp 2>/dev/null | sed -n "s/.*:${port} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" | tr '\n' ' ')" + if [[ -n "${pids// /}" ]]; then + echo "killing stale python on port $port: $pids" + # shellcheck disable=SC2086 + kill $pids || true + sleep 1 + fi + + nohup bash "$START_CMD" >>"$LOG_FILE" 2>&1 & + local pid=$! + echo "$pid" >"$PID_FILE" + sleep 1 + if kill -0 "$pid" 2>/dev/null; then + echo "started: pid=$pid" + echo "log: $LOG_FILE" + else + echo "failed to start, check log: $LOG_FILE" + rm -f "$PID_FILE" + return 1 + fi +} + +stop() { + if ! is_running; then + echo "not running" + rm -f "$PID_FILE" + return 0 + fi + local pid + pid="$(cat "$PID_FILE")" + kill "$pid" || true + for _ in {1..20}; do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.2 + done + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" || true + fi + rm -f "$PID_FILE" + echo "stopped" +} + +status() { + if is_running; then + echo "running: pid=$(cat "$PID_FILE")" + return 0 + fi + echo "not running" + return 1 +} + +restart() { + stop || true + start +} + +logs() { + if [[ -f "$LOG_FILE" ]]; then + tail -n 80 "$LOG_FILE" + else + echo "no log file yet" + fi +} + +case "${1:-}" in + start) start ;; + stop) stop ;; + restart) restart ;; + status) status ;; + logs) logs ;; + *) + echo "usage: $0 {start|stop|restart|status|logs}" + exit 2 + ;; +esac diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py new file mode 100644 index 0000000..ce3af81 --- /dev/null +++ b/scripts/smoke_test.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import argparse +import asyncio +import math +import os +import sys +import wave + +import cv2 +import numpy as np + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from core.pipeline import ChatPipeline +from core.state_machine import Arbitrator +from services.asr import ASRService +from services.avatar import AvatarService +from services.llm import LLMService +from services.tts import TTSService + + +def _write_wav(path: str, audio: np.ndarray, sr: int) -> None: + pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16) + with wave.open(path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sr) + wf.writeframes(pcm.tobytes()) + + +async def _write_avatar_video( + avatar: AvatarService, audio: np.ndarray, sr: int, out_path: str, fps: int +) -> tuple[int, float]: + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + writer = cv2.VideoWriter(out_path, fourcc, float(fps), (avatar.width, avatar.height)) + if not writer.isOpened(): + raise RuntimeError(f"failed to open video writer: {out_path}") + + duration_s = float(audio.shape[0]) / float(sr) if sr > 0 else 0.0 + total_frames = max(1, int(math.ceil(duration_s * fps))) + samples_per_frame = max(1, int(sr / fps)) + + for i in range(total_frames): + s = i * samples_per_frame + e = min(audio.shape[0], (i + 1) * samples_per_frame) + seg = audio[s:e] + if seg.size == 0: + level = 0.0 + else: + rms = float(np.sqrt(np.mean(seg.astype(np.float32) ** 2))) + level = float(np.clip(rms * 10.0, 0.0, 1.0)) + frame = await avatar.render_frame(mouth_open=level, speaking=True) + writer.write(frame) + + # add 0.4s idle tail for easy visual check + tail_frames = max(1, int(0.4 * fps)) + for _ in range(tail_frames): + frame = await avatar.render_frame(mouth_open=0.0, speaking=False) + writer.write(frame) + + writer.release() + return total_frames + tail_frames, duration_s + 0.4 + + +async def main() -> None: + parser = argparse.ArgumentParser(description="Offline full-pipeline smoke test") + parser.add_argument("--text", default="你好，请做一个十秒内的简短自我介绍。") + parser.add_argument("--out-dir", default="outputs") + parser.add_argument("--fps", type=int, default=25) + args = parser.parse_args() + + os.makedirs(args.out_dir, exist_ok=True) + wav_path = os.path.join(args.out_dir, "smoke_reply.wav") + mp4_path = os.path.join(args.out_dir, "smoke_avatar.mp4") + + arbitrator = Arbitrator() + pipeline = ChatPipeline( + arbitrator=arbitrator, + asr=ASRService(), + llm=LLMService(), + tts=TTSService(), + ) + avatar = AvatarService() + + result, audio, sr = await pipeline.process_text_turn(args.text) + _write_wav(wav_path, audio, sr) + frame_count, video_s = await _write_avatar_video(avatar, audio, sr, mp4_path, args.fps) + + print("=== smoke test done ===") + print(f"user_text: {result.get('user_text', '')}") + print(f"reply_text: {result.get('reply_text', '')}") + print(f"llm_source: {result.get('llm_source', 'unknown')}") + print(f"audio_samples: {result.get('audio_samples', 0)}, sample_rate: {sr}") + print(f"video_frames: {frame_count}, video_seconds: {video_s:.2f}") + print(f"wav: {wav_path}") + print(f"mp4: {mp4_path}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/start-public.sh b/scripts/start-public.sh new file mode 100755 index 0000000..e52e15b --- /dev/null +++ b/scripts/start-public.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /home/xsl/product + +PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python" + +if [[ -f ".env" ]]; then + # shellcheck disable=SC2046 + export $(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' .env | xargs) +fi + +ARGS=(main:app --host "${WEBRTC_HOST:-0.0.0.0}" --port "${WEBRTC_PORT:-8080}") + +if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then + ARGS+=(--ssl-certfile "$SSL_CERTFILE" --ssl-keyfile "$SSL_KEYFILE") +fi + +exec "$PY" -m uvicorn "${ARGS[@]}" diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000..3e29c67 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /home/xsl/product +python -m uvicorn main:app --host 0.0.0.0 --port 8080 diff --git a/scripts/tts_stress.py b/scripts/tts_stress.py new file mode 100644 index 0000000..31de5c8 --- /dev/null +++ b/scripts/tts_stress.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import asyncio +import os +import sys +import time + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from services.tts import TTSService + + +async def main() -> None: + svc = TTSService() + texts = [f"第{i+1}句，语音稳定性压力测试。" for i in range(20)] + costs = [] + for i, text in enumerate(texts): + t0 = time.perf_counter() + audio, sr = await svc.synthesize(text) + dt = int((time.perf_counter() - t0) * 1000) + costs.append(dt) + print(f"round={i+1} ms={dt} samples={audio.shape[0]} sr={sr}") + p50 = sorted(costs)[len(costs) // 2] + print("tts.health:", svc.health) + print("tts stress p50_ms:", p50, "max_ms:", max(costs)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/vad_check.py b/scripts/vad_check.py new file mode 100644 index 0000000..4c7232e --- /dev/null +++ b/scripts/vad_check.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import math +import os +import sys +import time + +import numpy as np + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from services.vad import VADService + + +def main() -> None: + vad = VADService() + sr = 16000 + chunk = 512 + + # 1s silence -> 1s speech-like sine -> 1s silence + t = np.arange(sr, dtype=np.float32) / sr + speech = 0.20 * np.sin(2 * math.pi * 220 * t).astype(np.float32) + seq = [np.zeros(sr, dtype=np.float32), speech, np.zeros(sr, dtype=np.float32)] + stream = np.concatenate(seq) + + starts = 0 + ends = 0 + first_start_ms = -1 + t0 = time.perf_counter() + for i in range(0, stream.shape[0] - chunk + 1, chunk): + evt = vad.push_chunk(stream[i : i + chunk]) + if evt and "start" in evt: + starts += 1 + if first_start_ms < 0: + first_start_ms = int((time.perf_counter() - t0) * 1000) + if evt and "end" in evt: + ends += 1 + + print("vad.health:", vad.health) + print("vad.starts:", starts, "vad.ends:", ends, "first_start_ms:", first_start_ms) + if starts < 1 or ends < 1: + raise SystemExit("vad check failed: start/end not detected") + print("vad check: ok") + + +if __name__ == "__main__": + main() diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/services/__init__.py @@ -0,0 +1 @@ + diff --git a/services/__pycache__/__init__.cpython-310.pyc b/services/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..9e8f688 Binary files /dev/null and b/services/__pycache__/__init__.cpython-310.pyc differ diff --git a/services/__pycache__/asr.cpython-310.pyc b/services/__pycache__/asr.cpython-310.pyc new file mode 100644 index 0000000..efdd255 Binary files /dev/null and b/services/__pycache__/asr.cpython-310.pyc differ diff --git a/services/__pycache__/avatar.cpython-310.pyc b/services/__pycache__/avatar.cpython-310.pyc new file mode 100644 index 0000000..a66c1fd Binary files /dev/null and b/services/__pycache__/avatar.cpython-310.pyc differ diff --git a/services/__pycache__/llm.cpython-310.pyc b/services/__pycache__/llm.cpython-310.pyc new file mode 100644 index 0000000..6f3e500 Binary files /dev/null and b/services/__pycache__/llm.cpython-310.pyc differ diff --git a/services/__pycache__/tts.cpython-310.pyc b/services/__pycache__/tts.cpython-310.pyc new file mode 100644 index 0000000..b706717 Binary files /dev/null and b/services/__pycache__/tts.cpython-310.pyc differ diff --git a/services/__pycache__/vad.cpython-310.pyc b/services/__pycache__/vad.cpython-310.pyc new file mode 100644 index 0000000..57e7aca Binary files /dev/null and b/services/__pycache__/vad.cpython-310.pyc differ diff --git a/services/asr.py b/services/asr.py new file mode 100644 index 0000000..a7045d7 --- /dev/null +++ b/services/asr.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import logging +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +class ASRService: + def __init__(self) -> None: + self._model = None + self._post = None + self._ready = False + self._attempted = False + self._init_error: Optional[str] = None + + def _ensure_loaded(self) -> None: + if self._ready or self._attempted: + return + self._attempted = True + try: + from funasr import AutoModel + from funasr.utils.postprocess_utils import rich_transcription_postprocess + + self._model = AutoModel( + model="FunAudioLLM/SenseVoiceSmall", + device="cuda:0", + hub="hf", + ) + self._post = rich_transcription_postprocess + self._ready = True + except Exception as exc: # pragma: no cover + self._init_error = str(exc) + logger.warning("ASR fallback mode: %s", exc) + + async def transcribe(self, audio_16k: np.ndarray) -> str: + self._ensure_loaded() + if self._ready and self._model is not None and self._post is not None: + res = self._model.generate( + input=audio_16k, + cache={}, + language="zh", + use_itn=True, + ) + return self._post(res[0]["text"]) + + # Fallback for initial bring-up + return "（ASR降级）语音已接收" + + @property + def health(self) -> dict: + return {"ready": self._ready, "attempted": self._attempted, "error": self._init_error} diff --git a/services/avatar.py b/services/avatar.py new file mode 100644 index 0000000..b19a35d --- /dev/null +++ b/services/avatar.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import asyncio +import os +import sys +import uuid +import wave +from collections import deque +from pathlib import Path + +import cv2 +import numpy as np + +from config import settings + + +class AvatarService: + def __init__(self, width: int = 640, height: int = 360) -> None: + self.width = width + self.height = height + self._frames: deque[np.ndarray] = deque() + self._lock = asyncio.Lock() + self._job_lock = asyncio.Lock() + self._last_error: str | None = None + self._base_frame = self._load_base_frame() + + async def render_frame(self, mouth_open: float = 0.0, speaking: bool = False) -> np.ndarray: + frame = self._base_frame.copy() + if speaking: + alpha = float(np.clip(mouth_open, 0.0, 1.0)) + # Slightly boost contrast/brightness while speaking to keep a "live" feeling. + frame = cv2.convertScaleAbs(frame, alpha=1.0 + 0.10 * alpha, beta=2 + int(8 * alpha)) + # Fallback mouth animation on realistic base frame. + h, w = frame.shape[:2] + cx = w // 2 + cy = int(h * 0.70) + half_w = max(28, int(w * 0.055)) + half_h = max(4, int(4 + alpha * h * 0.035)) + + lip_color = (26, 26, 92) + lip_border = (70, 70, 160) + cv2.ellipse(frame, (cx, cy), (half_w + 2, half_h + 2), 0, 0, 360, lip_border, -1) + cv2.ellipse(frame, (cx, cy), (half_w, half_h), 0, 0, 360, lip_color, -1) + return frame + + async def idle_frame(self) -> np.ndarray: + return await self.render_frame(mouth_open=0.0, speaking=False) + + async def speaking_frame(self, mouth_open: float = 0.5) -> np.ndarray: + return await self.render_frame(mouth_open=mouth_open, speaking=True) + + async def pop_generated_frame(self) -> np.ndarray | None: + async with self._lock: + if self._frames: + return self._frames.popleft() + return None + + async def enqueue_musetalk_from_audio(self, audio: np.ndarray, sr: int) -> None: + if not settings.musetalk_enabled: + return + if audio.size <= 1 or sr <= 0: + return + + async with self._job_lock: + try: + job_id = f"job_{uuid.uuid4().hex[:10]}" + repo_dir = Path(settings.musetalk_repo_dir) + work_root = repo_dir / "results" / "visual-chat-jobs" / job_id + work_root.mkdir(parents=True, exist_ok=True) + wav_path = work_root / "input.wav" + cfg_path = work_root / "inference.yaml" + output_name = f"{job_id}.mp4" + output_path = Path(settings.musetalk_result_dir) / "v15" / output_name + + self._write_wav(wav_path, audio.astype(np.float32, copy=False), sr) + cfg_path.write_text( + ( + "task_0:\n" + f" video_path: \"{settings.musetalk_source_video}\"\n" + f" audio_path: \"{wav_path}\"\n" + f" result_name: \"{output_name}\"\n" + ), + encoding="utf-8", + ) + + env = os.environ.copy() + env.setdefault("HF_HOME", str(repo_dir / "models")) + env["PYTHONPATH"] = ( + f"{repo_dir}:{env['PYTHONPATH']}" if env.get("PYTHONPATH") else str(repo_dir) + ) + proc = await asyncio.create_subprocess_exec( + sys.executable, + str(settings.musetalk_infer_script), + "--version", + "v15", + "--inference_config", + str(cfg_path), + "--result_dir", + str(settings.musetalk_result_dir), + "--unet_model_path", + str(settings.musetalk_unet_model), + "--unet_config", + str(settings.musetalk_unet_config), + "--whisper_dir", + str(settings.musetalk_whisper_dir), + "--batch_size", + "8", + "--use_float16", + cwd=str(repo_dir), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + self._last_error = stderr.decode("utf-8", errors="ignore")[-500:] + return + frames: list[np.ndarray] = [] + if output_path.exists(): + frames = self._read_video_frames(str(output_path)) + if not frames: + # Some MuseTalk runs leave image frames instead of final mp4. + fallback_dir = Path(settings.musetalk_result_dir) / "v15" / "yongen" + if fallback_dir.exists(): + frames = self._read_image_frames(str(fallback_dir)) + if not frames: + self._last_error = f"MuseTalk produced no readable frames: {output_path}" + return + + async with self._lock: + self._frames.clear() + self._frames.extend(frames) + self._last_error = None + except Exception as exc: # pragma: no cover + self._last_error = str(exc) + + def _read_video_frames(self, path: str) -> list[np.ndarray]: + cap = cv2.VideoCapture(path) + frames: list[np.ndarray] = [] + while True: + ok, frame = cap.read() + if not ok or frame is None: + break + if frame.shape[1] != self.width or frame.shape[0] != self.height: + frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) + frames.append(frame) + cap.release() + return frames + + def _read_image_frames(self, dir_path: str) -> list[np.ndarray]: + frames: list[np.ndarray] = [] + for name in sorted(os.listdir(dir_path)): + if not name.lower().endswith(".png"): + continue + frame = cv2.imread(os.path.join(dir_path, name)) + if frame is None: + continue + if frame.shape[1] != self.width or frame.shape[0] != self.height: + frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) + frames.append(frame) + return frames + + def _load_base_frame(self) -> np.ndarray: + src = settings.musetalk_source_video + if src and os.path.exists(src): + cap = cv2.VideoCapture(src) + ok, frame = cap.read() + cap.release() + if ok and frame is not None: + if frame.shape[1] != self.width or frame.shape[0] != self.height: + frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_LINEAR) + return frame + # Last-resort dark background (should rarely happen). + fallback = np.zeros((self.height, self.width, 3), dtype=np.uint8) + fallback[:, :, :] = (20, 20, 20) + return fallback + + @staticmethod + def _write_wav(path: Path, audio: np.ndarray, sr: int) -> None: + pcm = np.clip(audio * 32767.0, -32768, 32767).astype(np.int16) + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sr) + wf.writeframes(pcm.tobytes()) + + @property + def health(self) -> dict: + return { + "enabled": settings.musetalk_enabled, + "queued_frames": len(self._frames), + "last_error": self._last_error, + } diff --git a/services/llm.py b/services/llm.py new file mode 100644 index 0000000..cc1f9b3 --- /dev/null +++ b/services/llm.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import logging +from typing import Optional +from collections import deque + +from config import settings + +logger = logging.getLogger(__name__) + + +class LLMService: + def __init__(self) -> None: + self._ready = False + self._error: Optional[str] = None + self._client = None + self._model = (settings.llm_model or "").strip() + self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant) + self._system_prompt = "请用中文口语化简短回复，1-2句。" + + if not settings.llm_api_key: + return + if not self._model: + base = (settings.llm_base_url or "").lower() + self._model = "deepseek-chat" if "deepseek.com" in base else "gpt-4o-mini" + try: + from openai import AsyncOpenAI + + self._client = AsyncOpenAI( + api_key=settings.llm_api_key, + base_url=settings.llm_base_url, + timeout=settings.llm_timeout, + ) + self._ready = True + except Exception as exc: # pragma: no cover + self._error = str(exc) + logger.warning("LLM online mode unavailable: %s", exc) + + async def reply_with_meta(self, text: str) -> tuple[str, str]: + if not self._ready or self._client is None: + return f"收到：{text}。这是本地兜底回复。", "fallback" + + try: + resp = await self._client.chat.completions.create( + model=self._model, + messages=[{"role": "system", "content": self._system_prompt}, *list(self._history), {"role": "user", "content": text}], + max_tokens=settings.llm_max_tokens, + temperature=0.7, + ) + content = (resp.choices[0].message.content or "").strip() + if content: + self._history.append({"role": "user", "content": text}) + self._history.append({"role": "assistant", "content": content}) + return content, "online" + except Exception as exc: # pragma: no cover + self._error = str(exc) + logger.warning("LLM request failed, fallback used: %s", exc) + + return f"收到：{text}。这是本地兜底回复。", "fallback" + + async def reply(self, text: str) -> str: + content, _ = await self.reply_with_meta(text) + return content + + @property + def health(self) -> dict: + return { + "ready": self._ready, + "model": self._model, + "history_items": len(self._history), + "error": self._error, + } + + def clear_history(self) -> None: + self._history.clear() diff --git a/services/tts.py b/services/tts.py new file mode 100644 index 0000000..1615027 --- /dev/null +++ b/services/tts.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import logging +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +class TTSService: + def __init__(self) -> None: + self._pipeline = None + self._ready = False + self._attempted = False + self._init_error: Optional[str] = None + + def _ensure_loaded(self) -> None: + if self._ready or self._attempted: + return + self._attempted = True + try: + import kokoro + + self._pipeline = kokoro.KPipeline(lang_code="z") + self._ready = True + except Exception as exc: # pragma: no cover + self._init_error = str(exc) + logger.warning("TTS fallback mode: %s", exc) + + async def synthesize(self, text: str) -> tuple[np.ndarray, int]: + self._ensure_loaded() + if self._ready and self._pipeline is not None: + chunks = [] + for _, _, audio in self._pipeline(text, voice="zf_xiaoxiao"): + chunks.append(audio) + if chunks: + return np.concatenate(chunks), 24000 + + # 0.5s silence fallback for flow verification + return np.zeros(12000, dtype=np.float32), 24000 + + @property + def health(self) -> dict: + return {"ready": self._ready, "attempted": self._attempted, "error": self._init_error} diff --git a/services/vad.py b/services/vad.py new file mode 100644 index 0000000..e1763ea --- /dev/null +++ b/services/vad.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Optional + + +class VADService: + def __init__(self) -> None: + self._model = None + self._iterator = None + self._ready = False + self._attempted = False + self._error: Optional[str] = None + + def _ensure_loaded(self) -> None: + if self._ready or self._attempted: + return + self._attempted = True + try: + from silero_vad import VADIterator, load_silero_vad + + self._model = load_silero_vad() + self._iterator = VADIterator( + self._model, + threshold=0.5, + sampling_rate=16000, + min_silence_duration_ms=300, + speech_pad_ms=30, + ) + self._ready = True + except Exception as exc: # pragma: no cover + self._error = str(exc) + + def push_chunk(self, pcm_chunk): + self._ensure_loaded() + if self._ready and self._iterator is not None: + return self._iterator(pcm_chunk) + return None + + @property + def health(self) -> dict: + return {"ready": self._ready, "attempted": self._attempted, "error": self._error} diff --git a/visual-chat-task-list.md b/visual-chat-task-list.md new file mode 100644 index 0000000..ba77156 --- /dev/null +++ b/visual-chat-task-list.md @@ -0,0 +1,724 @@ +# 智能可视化语音聊天系统 — 详细任务清单（本机实时版） + +> **目标**：在本机（RTX 5090 / WSL2）实现"浏览器访问 → WebRTC 传输 → 单工语音对话 → 数字人口型同步"的**实时**可用系统。 +> **基础**：`environments.md` 与 `research-notes.md` 中已验证的环境、模型和踩坑修复全部复用。 +> **最后更新**：2026-03-25 — 经过逐组件可行性硬核验证后修订（第三版）。 + +--- + +## 0. 逐组件可行性验证报告（执行前必读） + +### 0.1 组件全景审计表 + +| 组件 | 选型 | 许可证 | 纯本地 | RTX 5090 验证 | 实时性证据 | 风险等级 | +|------|------|--------|--------|---------------|-----------|---------| +| **VAD** | Silero VAD 6.x | MIT | 是(CPU) | N/A(纯CPU) | 流式 VADIterator，32ms 帧级延迟 | 无 | +| **ASR** | SenseVoice-Small (FunASR) | Apache 2.0 | 是(GPU) | 已有用户确认 cu128 可跑 | 中文 CER 4.2%，CPU 14x实时，GPU 更快 | 低 | +| **LLM** | 在线 API (OpenAI 兼容) | N/A | 否(设计如此) | N/A | 流式，首 token ~0.5-1.5s | 中(网络) | +| **TTS** | Kokoro-82M-v1.1-zh | Apache 2.0 | 是(GPU) | 确认支持 CUDA 12.8 RTX 50 系 | 210x实时(4090)，<2GB VRAM | 低 | +| **TTS 升级** | Qwen3-TTS 0.6B | Apache 2.0 | 是(GPU) | 确认可跑，需 torch.compile 优化 | 97ms 首音频，RTF<0.25(优化后) | 中 | +| **口型** | MuseTalk v1.5 | MIT | 是(GPU) | 已在你机器验证通过 | 30fps+(V100)，RTX 5090 更快 | 中(需改流式) | +| **头动底片** | LivePortrait | MIT | 是(离线预处理) | 已在你机器验证通过 | 离线，无实时要求 | 无 | +| **WebRTC** | aiortc | BSD-3-Clause | 是 | N/A(纯Python) | asyncio 原生，H.264 已优化 | 低 | + +**结论：全部组件均为开源许可（MIT / Apache 2.0 / BSD-3），全部可本地运行，全部有 RTX 5090 兼容路径。** + +### 0.2 两个致命问题及修正 + +#### 问题一：faster-whisper 中文识别率灾难 + +| | faster-whisper large-v3-turbo | SenseVoice-Small | +|---|---|---| +| 中文 WER/CER | **WER 77%**（几乎不可用） | **CER 4.2%**（优秀） | +| 中文方言 | 不支持 | 7 大方言 + 26 种口音 | +| 推理速度(GPU) | ~50x 实时 | RTF 0.0076（130x 实时） | +| 模型大小 | ~1.5GB | ~234MB | +| 许可证 | MIT | Apache 2.0 | + +**决定：ASR 从 faster-whisper 改为 SenseVoice-Small。** faster-whisper 的 turbo 模型砍掉了 28/32 层 decoder，导致中文（需要更复杂解码的语言）准确率暴跌，完全不可接受。 + +#### 问题二：CosyVoice2 实际延迟远超宣传 + +| | 宣传值 | 开源版实测值(社区) | +|---|---|---| +| 流式首块延迟 | 150ms | **1 - 4.5 秒** | +| 原因 | 阿里云预训练音色，无特征提取 | 开源 zero-shot 需提取 token/embedding | +| 流式实现 | 优化的 chunk 流 | 累积式 chunk（1A, 2A, 3A...），无 KV cache | + +**决定：TTS 从 CosyVoice2 改为 Kokoro-82M（快速启动） + Qwen3-TTS 0.6B（质量升级路径）。** + +### 0.3 修正后的端到端延迟预算 + +| 阶段 | 选型 | 预估延迟 | 数据来源 | +|------|------|----------|---------| +| VAD 尾部检测 | Silero VAD | ~300ms | 默认 silence_duration_ms=300 | +| ASR 转写(3s) | SenseVoice-Small GPU | ~50-150ms | RTF 0.0076，3s × 0.0076 ≈ 23ms + 开销 | +| LLM 首 token | 在线流式 | ~500-1500ms | 取决于模型/网络 | +| TTS 首音频块 | Kokoro-82M | ~20-50ms | 210x 实时 on 4090 | +| MuseTalk 首帧批 | MuseTalk batch=4 | ~100-200ms | 30fps+ → 4帧 ≈ 130ms | +| WebRTC 编码传输 | aiortc H.264 | ~50-100ms | 软编码 | +| **合计** | | **~1.0 - 2.3s** | | + +> 比上版预算更优，因为 SenseVoice 比 faster-whisper 更快，Kokoro 比 CosyVoice2 快一个数量级。 + +### 0.4 GPU 显存预算（RTX 5090 = 32GB） + +| 模型 | 精度 | VRAM | 常驻 | +|------|------|------|------| +| MuseTalk (UNet + VAE + Whisper) | fp16 | ~3-4GB | 是 | +| SenseVoice-Small | fp16 | ~0.8GB | 是 | +| Kokoro-82M | fp16 | ~1.5GB | 是 | +| PyTorch CUDA 上下文 | — | ~1GB | 固定 | +| **合计** | | **~6-7GB** | | + +> 32GB VRAM 用不到 1/4，若后续升级 Qwen3-TTS 0.6B（+3GB）仍有充裕余量。 + +### 0.5 "开源可控"确认清单 + +| 组件 | 许可证 | 可商用 | 代码仓库 | 离线可用 | +|------|--------|--------|----------|---------| +| Silero VAD | MIT | 是 | github.com/snakers4/silero-vad | 是 | +| SenseVoice-Small | Apache 2.0 | 是 | github.com/FunAudioLLM/SenseVoice | 是 | +| Kokoro-82M | Apache 2.0 | 是 | github.com/hexgrad/kokoro | 是 | +| Qwen3-TTS | Apache 2.0 | 是 | github.com/QwenLM/Qwen3-TTS | 是 | +| MuseTalk v1.5 | MIT | 是 | github.com/TMElyralab/MuseTalk | 是 | +| LivePortrait | MIT | 是 | github.com/KlingTeam/LivePortrait | 是 | +| aiortc | BSD-3 | 是 | github.com/aiortc/aiortc | 是 | + +**零云端依赖（LLM 除外，且 LLM 有本地兜底），零非开源组件。** + +### 0.6 "稳健落地"保障策略 + +| 风险 | 发生时 | 保底方案 | 切换成本 | +|------|--------|----------|---------| +| SenseVoice 在 nightly PyTorch 编译失败 | M0 | 用 MuseTalk 自带 whisper-tiny 做 ASR（精度低但能跑） | 改一行 import | +| Kokoro 中文质量不达标 | M1 | 切 Qwen3-TTS 0.6B（已验证 RTX 5090 可跑） | 换一个 TTS 服务封装 | +| Qwen3-TTS RTX 5090 推理慢 | M4 | torch.compile + CUDA graphs 优化（社区已有方案，RTF<0.25） | 加 2 行代码 | +| MuseTalk 流式改造困难 | M1 | 先用"分段文件处理"：TTS 输出完整 WAV → MuseTalk 批量合成 → 整段播放 | 延迟增加 2-3s 但可用 | +| 在线 LLM 不可用 | 运行时 | 本地预设话术兜底，8s 超时自动触发 | 已内建 | +| aiortc 软编码瓶颈 | M2 | 改用 ffmpeg + NVENC 硬编码管道 | 约 1 天工作量 | + +--- + +## 1. 总体架构 + +### 1.1 系统拓扑 + +``` +浏览器 (Chrome) + │ + │ WebRTC (ICE/DTLS/SRTP) + │ + ▼ +┌─────────────────────────────────────────────────────┐ +│ webrtc-gateway (aiortc + FastAPI) │ +│ ┌──────────┐ ┌──────────────────────┐ │ +│ │ 上行音频 │ │ 下行视频 + 音频轨 │ │ +│ └────┬─────┘ └──────────▲───────────┘ │ +└───────┼───────────────────────────────┼─────────────┘ + │ │ + ▼ │ +┌──────────────┐ ┌──────┴──────┐ +│ Silero VAD │ │ mixer │ +│ (CPU,MIT) │ │ (音视频合流) │ +└──────┬───────┘ └──▲───────▲──┘ + │ speech.end │ │ + ▼ │ │ +┌──────────────┐ ┌──────┴──┐ ┌──┴──────────┐ +│ SenseVoice │ │MuseTalk │ │ TTS 音频 │ +│ ASR (GPU) │ │ 口型渲染│ │ (Kokoro) │ +│ Apache 2.0 │ │ (GPU) │ │ Apache 2.0 │ +└──────┬───────┘ │ MIT │ └──────▲──────┘ + │ text └────▲────┘ │ + ▼ │ wav │ text +┌──────────────┐ ┌─────┴──────┐ │ +│ LLM Client │──text──▶│ Kokoro TTS ├───────┘ +│ (在线 API) │ │ (GPU) │ +└──────────────┘ └────────────┘ +``` + +### 1.2 单工状态机 + +``` + speech.start speech.end + ┌────┐ ──────────▶ ┌──────────────┐ ────────▶ ┌──────────┐ + │IDLE│ │USER_SPEAKING │ │THINKING │ + └──▲─┘ ◀────────── └──────────────┘ └────┬─────┘ + │ barge-in │ tts.ready + │ (用户抢话) ▼ + │ ┌─────────────────┐ ┌────────────────┐ + └───────────────│ │◀────────│AVATAR_SPEAKING │ + playback.done │ (回到 IDLE) │ └────────────────┘ + └─────────────────┘ +``` + +**状态规则**： +- `IDLE`：数字人播放闭嘴循环视频，下行静音 +- `USER_SPEAKING`：数字人立即切闭嘴循环（如正在说话则中断），录音开始 +- `THINKING`：录音结束，ASR → LLM → TTS 管线启动，数字人继续闭嘴循环 +- `AVATAR_SPEAKING`：MuseTalk 输出帧 + TTS 音频同步下发 +- 任何状态下检测到 `speech.start` → 立即跳转 `USER_SPEAKING`（barge-in） + +### 1.3 进程内通信（不用消息中间件） + +单机单进程，所有模块通过 `asyncio.Queue` 和 `asyncio.Event` 通信，延迟最低： + +```python +audio_chunk_queue: asyncio.Queue[bytes] # VAD → ASR +text_queue: asyncio.Queue[str] # ASR → LLM +tts_chunk_queue: asyncio.Queue[bytes] # LLM→TTS → mixer +avatar_frame_queue: asyncio.Queue[np.ndarray] # TTS→MuseTalk → mixer +state_event: asyncio.Event # 状态变更通知 +``` + +--- + +## 2. 环境与依赖 + +### 2.1 创建统一 conda 环境 + +```bash +conda create -n digital-human python=3.10 -y +conda activate digital-human +conda install -y -c conda-forge ffmpeg + +# PyTorch nightly（RTX 5090 必须，已验证版本） +pip install --pre torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/nightly/cu128 +``` + +### 2.2 安装各模块依赖 + +```bash +# MuseTalk 依赖链（按 research-notes.md 已验证） +cd /home/xsl/work/MuseTalk +pip install -r requirements.txt +pip install --no-build-isolation chumpy +pip install mmengine +MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation +pip install "mmdet>=3.3.0" "mmpose>=1.3.0" +pip install xtcocotools json-tricks munkres +pip install "transformers>=4.45.0" + +# ASR - SenseVoice +pip install funasr + +# TTS - Kokoro +pip install kokoro soundfile + +# TTS 升级备选 - Qwen3-TTS（M4 阶段再装） +# pip install qwen3-tts + +# VAD +pip install silero-vad + +# WebRTC + Web 服务 +pip install aiortc aiohttp uvicorn fastapi + +# 工具库 +pip install pydantic python-dotenv tenacity numpy opencv-python-headless +``` + +### 2.3 模型权重 + +| 模型 | 路径 | 状态 | 大小 | +|------|------|------|------| +| MuseTalk v1.5 全套 | `/home/xsl/work/MuseTalk/models/` | 已有 | 5.5GB | +| LivePortrait 权重 | `/home/xsl/work/LivePortrait/pretrained_weights/` | 已有 | 1.2GB | +| SenseVoice-Small | 首次调用自动下载 | **新增** | ~234MB | +| Kokoro-82M-v1.1-zh | 首次调用自动下载 | **新增** | ~180MB | +| Silero VAD | 首次调用自动下载 | **自动** | ~2MB | + +```bash +# SenseVoice 模型会在首次 AutoModel() 调用时自动下载 +# Kokoro 模型会在首次 import kokoro 时自动下载 +# 也可以手动预下载： +python -c "from funasr import AutoModel; AutoModel(model='FunAudioLLM/SenseVoiceSmall', device='cuda:0')" +python -c "import kokoro; kokoro.KPipeline(lang_code='z')" +``` + +### 2.4 MuseTalk 补丁（必须，已验证） + +`torch.load(weights_only=False)` monkey patch — 保持 research-notes.md 中已验证的方案。 + +### 2.5 验收清单 + +- [x] `python -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name())"` → True, NVIDIA GeForce RTX 5090 +- [x] SenseVoice 加载模型 + 转写一段中文音频成功 +- [x] Kokoro 合成一句中文语音成功 +- [x] MuseTalk 单样例推理输出视频成功 + +--- + +## 3. VAD 语音活动检测（Silero VAD, MIT） + +### 3.1 实现 + +```python +from silero_vad import load_silero_vad, VADIterator + +model = load_silero_vad() +vad_iterator = VADIterator( + model, + threshold=0.5, + sampling_rate=16000, + min_silence_duration_ms=300, # 尾部静音判定 + speech_pad_ms=30, +) + +# 流式处理 WebRTC 上行音频帧 +for audio_chunk in webrtc_audio_stream: + speech_dict = vad_iterator(audio_chunk) + if speech_dict: + if 'start' in speech_dict: + arbitrator.on_speech_start() + if 'end' in speech_dict: + arbitrator.on_speech_end(recorded_audio) +``` + +### 3.2 验收 + +- [x] 静音环境不误触发 +- [x] 正常说话稳定触发 +- [x] 说话结束后 ~300ms 内检测到 speech.end + +--- + +## 4. ASR 语音识别（SenseVoice-Small, Apache 2.0, 本地 GPU） + +### 4.1 模型加载 + +```python +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + +asr_model = AutoModel( + model="FunAudioLLM/SenseVoiceSmall", + device="cuda:0", + hub="hf", +) +``` + +### 4.2 推理接口 + +```python +async def transcribe(audio_array: np.ndarray) -> str: + """16kHz mono float32 numpy array → 文本""" + res = asr_model.generate( + input=audio_array, + cache={}, + language="zh", + use_itn=True, + ) + return rich_transcription_postprocess(res[0]["text"]) +``` + +### 4.3 性能预期 + +- AISHELL-1 中文 CER: 4.2%（比 whisper-turbo 的 77% WER 好一个量级） +- 3 秒语音推理延迟: ~50-150ms (GPU) +- VRAM: ~0.8GB + +### 4.4 验收 + +- [ ] 10 条中文测试语音，转写准确率主观 > 90% +- [ ] 单段 3 秒语音延迟 < 300ms + +--- + +## 5. LLM 对话（在线 API，流式输出） + +### 5.1 客户端实现 + +```python +import openai + +client = openai.AsyncOpenAI( + api_key=os.getenv("LLM_API_KEY"), + base_url=os.getenv("LLM_BASE_URL"), +) + +async def chat_stream(messages: list[dict]) -> AsyncIterator[str]: + response = await client.chat.completions.create( + model=os.getenv("LLM_MODEL", "gpt-4o-mini"), + messages=messages, + stream=True, + max_tokens=150, + temperature=0.7, + ) + async for chunk in response: + if chunk.choices[0].delta.content: + yield chunk.choices[0].delta.content +``` + +### 5.2 流式分句送 TTS + +- LLM 流式输出 token → 累积到句子边界（。！？，\n）→ 立即送 TTS +- 系统提示词要求"用简短口语回复，每句不超过 30 字" + +### 5.3 降级策略 + +- 超时 8s 或网络错误 → 返回预设兜底话术 +- 用 `tenacity` 做 1 次快速重试（2s 超时） + +### 5.4 验收 + +- [x] 正常首 token < 2 秒 +- [x] 断网 3 秒内返回兜底话术 +- [x] 20 轮对话上下文连贯 + +--- + +## 6. TTS 语音合成（Kokoro-82M, Apache 2.0, 本地 GPU） + +### 6.1 模型加载 + +```python +import kokoro + +pipeline = kokoro.KPipeline(lang_code='z') # z = 中文 +``` + +### 6.2 合成接口 + +```python +async def synthesize(text: str) -> tuple[np.ndarray, int]: + """文本 → (audio_array, sample_rate)""" + generator = pipeline(text, voice='zf_xiaoxiao') # 中文女声 + audio_chunks = [] + for _, _, chunk in generator: + audio_chunks.append(chunk) + audio = np.concatenate(audio_chunks) + return audio, 24000 # Kokoro 输出 24kHz +``` + +### 6.3 音频格式对齐 + +- Kokoro 输出 24kHz → 需重采样到 16kHz 供 MuseTalk whisper +- 下行音频可保持 24kHz（WebRTC 支持） + +### 6.4 升级路径（Qwen3-TTS） + +M4 阶段如果 Kokoro 中文质量不满意，切换到 Qwen3-TTS 0.6B： +- 97ms 首音频延迟 +- 更好的中文自然度 +- 需额外 ~3GB VRAM（总计仍 <10GB） +- RTX 5090 需 `torch.compile` + CUDA graphs 优化（社区已验证方案） + +### 6.5 验收 + +- [ ] 40 字中文文本合成延迟 < 500ms +- [ ] 音质清晰，无明显机器感 +- [x] 连续合成 20 句无崩溃 + +--- + +## 7. 数字人渲染（MuseTalk v1.5, MIT, 本地 GPU） + +### 7.1 离线预处理：底片库（一次性） + +**Step 1**: 用 LivePortrait 生成闭嘴自然动作循环视频 + +```bash +conda activate digital-human +cd /home/xsl/work/LivePortrait + +for drv in d3.mp4 d6.mp4 d10.mp4; do + python inference.py \ + -s /path/to/avatar_photo.jpg \ + -d assets/examples/driving/$drv \ + --flag_crop_driving_video \ + -o /home/xsl/product/assets/avatar_loops/ +done +``` + +**Step 2**: 用 MuseTalk 预处理（提取 latents/coords/masks → 缓存） + +```bash +cd /home/xsl/work/MuseTalk +python -m scripts.realtime_inference \ + --inference_config configs/inference/realtime.yaml \ + --version v15 \ + --unet_model_path models/musetalkV15/unet.pth \ + --unet_config models/musetalkV15/musetalk.json +``` + +预处理产物缓存后，运行时直接加载，不再需要人脸检测和解析。 + +### 7.2 实时口型合成：改造 MuseTalk + +**核心**：将 `Avatar.inference()` 从完整音频文件改为流式音频 chunk。 + +```python +class RealtimeAvatar: + def __init__(self, prepared_data_path): + # 加载预处理缓存 + self.latents = torch.load(f"{prepared_data_path}/latents.pt") + self.coords = pickle.load(open(f"{prepared_data_path}/coords.pkl", 'rb')) + self.frames = [...] # 预加载原始帧 + self.masks = [...] # 预加载 mask + self.frame_idx = 0 + + async def process_audio_chunk(self, pcm_16k: np.ndarray) -> list[np.ndarray]: + """接收 TTS PCM 块 → 返回口型帧列表""" + whisper_features = audio_processor.get_audio_feature(pcm_16k) + whisper_chunks = audio_processor.get_whisper_chunk(whisper_features, ...) + + output_frames = [] + for batch in datagen(whisper_chunks, self.latents, batch_size=4): + pred = unet.model(batch_latent, timesteps, encoder_hidden_states=audio_feat) + recon = vae.decode_latents(pred) + for frame in recon: + blended = blend_frame(frame, self.frame_idx) + output_frames.append(blended) + self.frame_idx = (self.frame_idx + 1) % len(self.frames) + return output_frames + + def get_idle_frame(self) -> np.ndarray: + """闭嘴循环帧（不过 UNet）""" + frame = self.frames[self.frame_idx % len(self.frames)] + self.frame_idx = (self.frame_idx + 1) % len(self.frames) + return frame +``` + +### 7.3 帧率与同步 + +- 25fps（与底片一致），每帧 40ms +- TTS 音频时长 → 计算帧数 → MuseTalk 生成对应数量帧 +- asyncio 定时器控制帧发送节奏 + +### 7.4 验收 + +- [ ] 闭嘴循环 60 秒无卡顿 +- [ ] 5 秒 TTS 音频口型帧 200ms 内开始输出 +- [ ] 口型同步主观无 >200ms 错位 + +--- + +## 8. WebRTC 网关与前端 + +### 8.1 后端（aiortc + FastAPI, BSD-3） + +```python +@app.post("/webrtc/offer") +async def webrtc_offer(request: Request): + params = await request.json() + offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) + pc = RTCPeerConnection() + + video_track = AvatarVideoTrack() # 从 MuseTalk 帧队列读取 + audio_track = AvatarAudioTrack() # 从 TTS 音频队列读取 + pc.addTrack(video_track) + pc.addTrack(audio_track) + + @pc.on("track") + def on_track(track): + if track.kind == "audio": + asyncio.ensure_future(process_incoming_audio(track)) + + await pc.setRemoteDescription(offer) + answer = await pc.createAnswer() + await pc.setLocalDescription(answer) + return {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} +``` + +### 8.2 前端页面（纯 HTML/JS，无框架依赖） + +``` +┌──────────────────────────────┐ +│ 数字人视频 │ +│ (WebRTC video 标签) │ +│ │ +├──────────────────────────────┤ +│ 🎤 正在倾听... │ ← 状态指示 +├──────────────────────────────┤ +│ 你: "今天天气怎么样？" │ ← ASR 字幕 +│ AI: "今天天气不错呢..." │ ← LLM 回复字幕 +└──────────────────────────────┘ +``` + +- 页面加载 → 自动建 WebRTC 连接 → 请求麦克风权限 +- 下行视频+音频 → `