diff --git a/.env b/.env index 08af570..2a06d12 100644 --- a/.env +++ b/.env @@ -14,15 +14,14 @@ TTS_SENTENCE_MAX_LEN=64 TTS_SEGMENT_PAUSE_MS=90 TTS_FADE_MS=12 -# Web service -WEBRTC_HOST=0.0.0.0 -WEBRTC_PORT=8018 -STUN_URL=stun:stun.l.google.com:19302 +# Web service（与当前代码一致，仅 HTTP_HOST / HTTP_PORT） +HTTP_HOST=0.0.0.0 +HTTP_PORT=8018 CORS_ORIGINS=* -# HTTPS (for cross-device mic/WebRTC) -SSL_CERTFILE=/home/xsl/code/product/certs/dev-cert.pem -SSL_KEYFILE=/home/xsl/code/product/certs/dev-key.pem +# HTTPS（跨设备访问麦克风/页面时常需 TLS；路径相对项目根目录） +SSL_CERTFILE=certs/dev-cert.pem +SSL_KEYFILE=certs/dev-key.pem # 3D avatar driver mode AVATAR_DRIVER_MODE=blendshape_stream diff --git a/.env.example b/.env.example deleted file mode 100644 index 91b52db..0000000 --- a/.env.example +++ /dev/null @@ -1,28 +0,0 @@ -LLM_API_KEY= -LLM_BASE_URL=https://api.deepseek.com/v1 -LLM_MODEL=deepseek-chat -LLM_TIMEOUT=8 -LLM_MAX_TOKENS=150 -LLM_TEMPERATURE=0.55 -LLM_SYSTEM_PROMPT=你是一个自然、友好的中文语音助手。请像真人当面聊天一样回答，口语化、简洁、自然，通常1到2句。不要使用书面公告腔，不要分点，不要长段解释，不要夸张语气词。不要虚构身份关系，不要自称家人、恋人或其他私人身份。避免客服套话，比如“随时为你服务”、“很高兴为你服务”、“请问有什么可以帮您”。优先使用短句和自然停顿，适合直接语音播报。 - -TTS_VOICE=zf_xiaoxiao -TTS_SPEED=0.90 -TTS_SENTENCE_MAX_LEN=64 -TTS_SEGMENT_PAUSE_MS=90 -TTS_FADE_MS=12 - -WEBRTC_HOST=0.0.0.0 -WEBRTC_PORT=8080 -STUN_URL=stun:stun.l.google.com:19302 -CORS_ORIGINS=* - -AVATAR_FPS=25 -AVATAR_DRIVER_MODE=blendshape_stream -AVATAR_CONTROL_PROTOCOL=ws -AVATAR_BLENDSHAPE_SCHEMA=arkit - -SSL_CERTFILE= -SSL_KEYFILE= - -HF_TOKEN= \ No newline at end of file diff --git a/.run/visual-chat-8018.log b/.run/visual-chat-8018.log index fec0d2c..60d6047 100644 --- a/.run/visual-chat-8018.log +++ b/.run/visual-chat-8018.log @@ -32,3 +32,8 @@ INFO: connection open INFO: connection closed INFO: connection closed INFO: connection closed +INFO: Shutting down +INFO: Waiting for connections to close. (CTRL+C to force quit) +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [3446] diff --git a/.run/visual-chat-8018.pid b/.run/visual-chat-8018.pid deleted file mode 100644 index 6cf44d4..0000000 --- a/.run/visual-chat-8018.pid +++ /dev/null @@ -1 +0,0 @@ -3446 diff --git a/README.md b/README.md index 74c2d77..0933cde 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,11 @@ export LLM_BASE_URL="https://api.deepseek.com/v1" export LLM_MODEL="deepseek-chat" ``` -也可以直接复制 [.env.example](.env.example) 作为本地配置模板。 +复制 [.env.example](.env.example) 为 `.env`，填入有效的 `LLM_API_KEY`。 + +**本地权重（除 LLM 外全部在仓库内 `models/`）**：首次可用联网环境执行 `python scripts/vendor_hf_models.py`；其中 VAD 的 `silero_vad.jit` 会从已安装的 `silero-vad` 包复制到 `models/vad/`。ASR 使用 `models/asr/SenseVoiceSmall/`，TTS 使用 `models/tts/Kokoro-82M/`。 + +完整部署步骤（新机器、模型打包、HTTPS、防火墙、排错）见 **[docs/deployment.md](docs/deployment.md)**。 语音如果偏快、偏硬，可以优先调这几个参数： @@ -88,6 +92,8 @@ export LLM_MODEL="deepseek-chat" - `TTS_SEGMENT_PAUSE_MS`：默认 `90`，给分段之间留一点停顿，但不要太拖。 - `TTS_FADE_MS`：默认 `12`，减轻段间拼接的突兀感。 +部署到任意机器时：在项目根目录运行（保证 `models/`、`web/`、`certs/` 等相对路径有效）。证书可在 `.env` 里写相对路径（例如 `certs/dev-cert.pem`），会按项目根目录解析。也可用环境变量 `VISUAL_CHAT_PYTHON` 指定解释器。 + 本地启动： ```bash @@ -119,7 +125,7 @@ python scripts/smoke_test.py --text "你好，请做一个简短自我介绍。" - `http://127.0.0.1:8080/` - `http://127.0.0.1:8080/health` -如果 `.env` 里已经配置了 `WEBRTC_PORT=8018` 与 `SSL_CERTFILE` / `SSL_KEYFILE`，实际访问地址通常会变成： +如果 `.env` 里已经配置了 `HTTP_PORT=8018` 与 `SSL_CERTFILE` / `SSL_KEYFILE`，实际访问地址通常会变成： - `https://:8018/` - `https://:8018/health` @@ -135,7 +141,7 @@ bash scripts/gen-self-signed-cert.sh 监听规则： - 服务监听地址固定按 `0.0.0.0` 规则执行。 -- 即使把 `WEBRTC_HOST` 写成 `127.0.0.1`、`localhost` 或 `::1`，启动时也会自动归一化成 `0.0.0.0`，避免局域网访问被误伤。 +- 即使把 `HTTP_HOST` 写成 `127.0.0.1`、`localhost` 或 `::1`，启动时也会自动归一化成 `0.0.0.0`，避免局域网访问被误伤。 ## 核心接口 @@ -143,7 +149,7 @@ bash scripts/gen-self-signed-cert.sh - `POST /chat/text`: 文本输入链路，跳过 ASR/VAD。 - `POST /chat/reset`: 清空上下文和运行时状态。 - `GET /health`: 返回后端运行指标和组件健康状态。 -- `GET /meta`: 返回 STUN、HTTPS、端口和动画协议配置。 +- `GET /meta`: 返回 HTTPS、监听端口和动画协议配置。 - `GET /events`: Server-Sent Events 运行状态流。 - `WS /ws/audio`: 双向语音通道，上传麦克风 PCM 并接收回复语音。 - `WS /ws/subtitles`: 字幕流。 diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc index 6fd8e0f..a9a833b 100644 Binary files a/__pycache__/config.cpython-312.pyc and b/__pycache__/config.cpython-312.pyc differ diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc index 9928b06..022d505 100644 Binary files a/__pycache__/main.cpython-312.pyc and b/__pycache__/main.cpython-312.pyc differ diff --git a/config.py b/config.py index 3396a8a..ff8c68b 100644 --- a/config.py +++ b/config.py @@ -1,11 +1,35 @@ -from pydantic import AliasChoices, Field, field_validator +from pathlib import Path + +from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict +PROJECT_ROOT = Path(__file__).resolve().parent + + +def asr_bundle_dir() -> Path: + return PROJECT_ROOT / "models" / "asr" / "SenseVoiceSmall" + + +def kokoro_bundle_dir() -> Path: + return PROJECT_ROOT / "models" / "tts" / "Kokoro-82M" + + +def vad_bundle_path() -> Path: + return PROJECT_ROOT / "models" / "vad" / "silero_vad.jit" + class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") - @field_validator("webrtc_host", mode="before") + @field_validator("llm_api_key", "llm_base_url", "llm_model", mode="before") + @classmethod + def llm_required_nonempty(cls, value: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("LLM_API_KEY, LLM_BASE_URL and LLM_MODEL must be set (non-empty).") + return text + + @field_validator("http_host", mode="before") @classmethod def normalize_listen_host(cls, value: str) -> str: host = str(value or "0.0.0.0").strip().lower() @@ -13,19 +37,20 @@ class Settings(BaseSettings): return "0.0.0.0" return host or "0.0.0.0" - hf_token: str = "" - llm_api_key: str = Field( - default="", - validation_alias=AliasChoices("LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY"), - ) - llm_base_url: str = Field( - default="https://api.deepseek.com/v1", - validation_alias=AliasChoices("LLM_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"), - ) - llm_model: str = Field( - default="deepseek-chat", - validation_alias=AliasChoices("LLM_MODEL", "DEEPSEEK_MODEL", "OPENAI_MODEL"), - ) + @field_validator("ssl_certfile", "ssl_keyfile", mode="after") + @classmethod + def resolve_ssl_path(cls, value: str) -> str: + text = (value or "").strip() + if not text: + return "" + p = Path(text) + if not p.is_absolute(): + p = PROJECT_ROOT / p + return str(p.resolve()) + + llm_api_key: str = Field(..., validation_alias="LLM_API_KEY") + llm_base_url: str = Field(..., validation_alias="LLM_BASE_URL") + llm_model: str = Field(..., validation_alias="LLM_MODEL") llm_timeout: int = 8 llm_max_tokens: int = 150 llm_temperature: float = 0.55 @@ -44,26 +69,15 @@ class Settings(BaseSettings): tts_segment_pause_ms: int = 90 tts_fade_ms: int = 12 - webrtc_host: str = "0.0.0.0" - webrtc_port: int = 8080 + http_host: str = Field(default="0.0.0.0", validation_alias="HTTP_HOST") + http_port: int = Field(default=8080, validation_alias="HTTP_PORT") avatar_fps: int = 25 avatar_driver_mode: str = "blendshape_stream" avatar_control_protocol: str = "ws" avatar_blendshape_schema: str = "arkit" - stun_url: str = "stun:stun.l.google.com:19302" cors_origins: str = "*" ssl_certfile: str = "" ssl_keyfile: str = "" - # Legacy MuseTalk settings retained for migration reference only. - musetalk_enabled: bool = True - musetalk_repo_dir: str = "/home/xsl/work/MuseTalk" - musetalk_infer_script: str = "/home/xsl/work/MuseTalk/scripts/inference.py" - musetalk_source_video: str = "/home/xsl/work/MuseTalk/data/video/yongen.mp4" - musetalk_unet_model: str = "/home/xsl/work/MuseTalk/models/musetalkV15/unet.pth" - musetalk_unet_config: str = "/home/xsl/work/MuseTalk/models/musetalkV15/musetalk.json" - musetalk_whisper_dir: str = "/home/xsl/work/MuseTalk/models/whisper" - musetalk_result_dir: str = "/home/xsl/work/MuseTalk/results/visual-chat" - settings = Settings() diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..56b4444 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,146 @@ +# 部署说明 + +本文说明如何把「3D 数字人语音聊天」服务端部署到一台新机器（Linux / WSL / macOS 等类 Unix 环境）。**唯一需要能访问公网的依赖是 LLM API**；语音相关权重应放在项目内 `models/` 目录，不在运行时从公网拉取。 + +## 1. 环境与依赖 + +- **Python**：建议 3.12（与当前 `requirements.txt` 一致）。 +- **系统工具**：`openssl`（生成自签证书）；可选 `ffmpeg`（若你后续扩展音视频流程）。 +- **硬件**：CPU 可运行；有 NVIDIA GPU 时 ASR 等会优先用 CUDA（无卡则回退 CPU，可能较慢）。 +- **磁盘**：`models/` 中 ASR + TTS 权重合计约 **1.2GB+**，请预留空间。 + +## 2. 获取代码与虚拟环境 + +```bash +git clone <你的仓库地址> product +cd product + +python3 -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate + +pip install -U pip +pip install -r requirements.txt +``` + +说明：依赖里已包含 `huggingface_hub`（由其它包装带动）。若单独只为下载权重，也可 `pip install huggingface_hub`。 + +## 3. 准备本地模型（`models/`） + +首次在一台**能访问 Hugging Face**的机器上执行（仅需一次，或把整份 `models/` 目录打包拷贝到离线机）： + +```bash +source .venv/bin/activate +python scripts/vendor_hf_models.py +``` + +该脚本会： + +- 从已安装的 `silero-vad` 包复制 **`models/vad/silero_vad.jit`** +- 拉取 **ASR**：`models/asr/SenseVoiceSmall/` +- 拉取 **TTS（Kokoro）**：`models/tts/Kokoro-82M/`（含 `voices/` 下各 `.pt` 音色） + +若目标机无外网，可在有网机器上跑完上述命令后，**整体复制 `models/`** 到部署机同一路径（相对于项目根）。 + +仅补全 VAD、不拉 HF（需已 `pip install silero-vad`）： + +```bash +python scripts/vendor_hf_models.py --skip-hf +``` + +## 4. 配置环境变量（`.env`） + +```bash +cp .env.example .env +``` + +**必须非空**（见 `config.py` 校验）： + +| 变量 | 含义 | +|------|------| +| `LLM_API_KEY` | OpenAI 兼容接口的 API Key | +| `LLM_BASE_URL` | 例如 `https://api.deepseek.com/v1` | +| `LLM_MODEL` | 例如 `deepseek-chat` | + +**HTTP 服务**（不要使用已废弃的 `WEBRTC_*`）： + +| 变量 | 含义 | +|------|------| +| `HTTP_HOST` | 监听地址，一般 `0.0.0.0` | +| `HTTP_PORT` | 端口，例如 `8080` 或 `8018` | + +**HTTPS（推荐用于跨设备访问页面与麦克风）**： + +- 证书路径建议写**相对项目根**，便于迁移，例如： + - `SSL_CERTFILE=certs/dev-cert.pem` + - `SSL_KEYFILE=certs/dev-key.pem` +- 程序会把相对路径解析为项目根下的绝对路径。 + +**TTS 音色**：`TTS_VOICE` 必须与 `models/tts/Kokoro-82M/voices/<名称>.pt` 一致（默认示例为 `zf_xiaoxiao`）。 + +完整字段说明见仓库根目录 [`.env.example`](../.env.example)。 + +## 5. 生成自签证书（可选但常需要） + +在**项目根目录**执行： + +```bash +bash scripts/gen-self-signed-cert.sh +``` + +生成 `certs/dev-cert.pem` 与 `certs/dev-key.pem`，并在 `.env` 中配置上一节的 `SSL_CERTFILE`、`SSL_KEYFILE`。 + +默认证书主题为 `CN=localhost`。用局域网 IP 访问时浏览器可能提示证书与地址不符，属自签名常见情况，可在浏览器中选择继续访问；生产环境请改用正规 CA 或内网 PKI 签发的证书。 + +## 6. 启动服务 + +**开发 / 前台运行：** + +```bash +source .venv/bin/activate +python main.py +``` + +或使用脚本（会读 `.env` 中的 `HTTP_HOST`、`HTTP_PORT`）： + +```bash +bash scripts/start-public.sh +``` + +**HTTPS**：当 `.env` 中 `SSL_CERTFILE`、`SSL_KEYFILE` 均有效且文件存在时，`main.py` 会通过 uvicorn 加载证书（与 `config` 中解析后的路径一致）。 + +启动后在本机浏览器访问： + +- `http(s)://127.0.0.1:/` +- `http(s)://<服务器局域网IP>:/`（需防火墙放行该端口） + +## 7. 防火墙与端口 + +确保部署机对客户端开放 **`HTTP_PORT`（TCP）**。若前面有云厂商安全组，需同步放行。 + +## 8. 目录与可移植性约定 + +- **项目根**：所有相对路径（SSL、`models/` 布局、`web/` 静态资源）均以**含有 `main.py` 与 `config.py` 的目录**为基准；启动时工作目录应为此目录。 +- **可选**：环境变量 `VISUAL_CHAT_PYTHON` 可指向指定 Python 解释器（部分 `scripts/*.sh` 会优先使用）。 + +## 9. 验证 + +```bash +curl -s "http://127.0.0.1:${HTTP_PORT:-8080}/health" | head +``` + +或用仓库内脚本（需根据实际 URL 调整）： + +```bash +python scripts/qa_check.py --base-url "https://127.0.0.1:8080" --rounds 3 +``` + +（若仅 HTTP，把 `base-url` 改成 `http://...`。） + +## 10. 常见问题 + +- **`.env` 里 `LLM_*` 为空**：应用在加载 `config` 时会校验失败，请务必填写有效值。 +- **ASR/TTS 不工作**：检查 `models/asr/SenseVoiceSmall/` 是否含 `configuration.json`；`models/tts/Kokoro-82M/` 是否含 `config.json`、权重 `.pth` 与 `voices/.pt`。 +- **麦克风在别的设备上不可用**：多数浏览器要求 **HTTPS** 与用户手势；请启用 HTTPS 并在页面内点击「连接语音通道」等操作。 +- **旧变量名**：`WEBRTC_HOST`、`WEBRTC_PORT`、`STUN_URL` 已废弃，请只使用 `HTTP_HOST`、`HTTP_PORT`。 + +更细的语音/文本流水线时序见 [development-guide.md](development-guide.md)。 diff --git a/environments.md b/environments.md index 2ce29d2..c7552dc 100644 --- a/environments.md +++ b/environments.md @@ -13,13 +13,13 @@ conda env list | 环境名 | 路径 | 用途 | 磁盘占用 | |--------|------|------|---------| -| `LivePortrait` | `/home/xsl/miniconda3/envs/LivePortrait` | 头部动作驱动 | 8.6GB | -| `MuseTalk` | `/home/xsl/miniconda3/envs/MuseTalk` | 唇形同步（主力） | 11GB | -| `latentsync` | `/home/xsl/miniconda3/envs/latentsync` | 唇形同步（备用） | 9.2GB | -| `condiff-train-hair` | `/home/xsl/miniconda3/envs/condiff-train-hair` | 其他项目（未动） | — | -| `onediff` | `/home/xsl/miniconda3/envs/onediff` | 其他项目（未动） | — | -| `vllm-qwen3` | `/home/xsl/miniconda3/envs/vllm-qwen3` | 其他项目（未动） | — | -| `py310` | `/home/xsl/miniconda3/envs/py310` | 通用 Python 3.10 | — | +| `LivePortrait` | `~/miniconda3/envs/LivePortrait` | 头部动作驱动 | 8.6GB | +| `MuseTalk` | `~/miniconda3/envs/MuseTalk` | 唇形同步（主力） | 11GB | +| `latentsync` | `~/miniconda3/envs/latentsync` | 唇形同步（备用） | 9.2GB | +| `condiff-train-hair` | `~/miniconda3/envs/condiff-train-hair` | 其他项目（未动） | — | +| `onediff` | `~/miniconda3/envs/onediff` | 其他项目（未动） | — | +| `vllm-qwen3` | `~/miniconda3/envs/vllm-qwen3` | 其他项目（未动） | — | +| `py310` | `~/miniconda3/envs/py310` | 通用 Python 3.10 | — | --- @@ -36,7 +36,7 @@ conda env list ### 已下载权重 -位置：`/home/xsl/work/LivePortrait/pretrained_weights/`（总计 **1.2GB**） +位置：`~/work/LivePortrait/pretrained_weights/`（总计 **1.2GB**） ``` pretrained_weights/ @@ -81,7 +81,7 @@ pretrained_weights/ ### 已下载权重 -位置：`/home/xsl/work/MuseTalk/models/`（总计 **5.5GB**） +位置：`~/work/MuseTalk/models/`（总计 **5.5GB**） ``` models/ @@ -127,7 +127,7 @@ models/ ### 已下载权重 -位置：`/home/xsl/work/LatentSync/checkpoints/`（总计 **5.4GB**） +位置：`~/work/LatentSync/checkpoints/`（总计 **5.4GB**） ``` checkpoints/ @@ -180,10 +180,10 @@ pip install --pre torch torchvision torchaudio \ --index-url https://download.pytorch.org/whl/nightly/cu128 # LivePortrait 依赖 -pip install -r /home/xsl/work/LivePortrait/requirements.txt +pip install -r ~/work/LivePortrait/requirements.txt # MuseTalk 依赖（含 MMLab 完整链） -pip install -r /home/xsl/work/MuseTalk/requirements.txt +pip install -r ~/work/MuseTalk/requirements.txt pip install --no-build-isolation chumpy pip install mmengine MMCV_WITH_OPS=1 pip install mmcv==2.1.0 --no-build-isolation @@ -193,6 +193,6 @@ pip install "transformers>=4.45.0" ``` **可复用的模型权重**（无需重新下载）： -- LivePortrait：`/home/xsl/work/LivePortrait/pretrained_weights/` — 直接用 -- MuseTalk：`/home/xsl/work/MuseTalk/models/` — 直接用 -- LatentSync（如需）：`/home/xsl/work/LatentSync/checkpoints/` — 直接用 +- LivePortrait：`~/work/LivePortrait/pretrained_weights/` — 直接用 +- MuseTalk：`~/work/MuseTalk/models/` — 直接用 +- LatentSync（如需）：`~/work/LatentSync/checkpoints/` — 直接用 diff --git a/main.py b/main.py index 99e121b..b90aa20 100644 --- a/main.py +++ b/main.py @@ -4,15 +4,13 @@ import asyncio import io import json import logging -import os import time import wave from dataclasses import dataclass, field import numpy as np -from aiortc import RTCPeerConnection, RTCSessionDescription from dotenv import load_dotenv -from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect +from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles @@ -26,11 +24,8 @@ from services.avatar import AvatarService from services.llm import LLMService from services.tts import TTSService from services.vad import VADService -from webrtc.tracks import AudioBus, AvatarAudioTrack load_dotenv() -if settings.hf_token: - os.environ["HF_TOKEN"] = settings.hf_token logging.basicConfig(level=logging.INFO) app = FastAPI(title="Visual Voice Chat") @@ -50,8 +45,6 @@ tts_service = TTSService() vad_service = VADService() avatar_service = AvatarService() pipeline = ChatPipeline(arbitrator, asr_service, llm_service, tts_service) -audio_bus = AudioBus(sample_rate=48000) -pcs: set[RTCPeerConnection] = set() subtitle_clients: set[WebSocket] = set() animation_clients: set[WebSocket] = set() audio_clients: set[WebSocket] = set() @@ -146,7 +139,6 @@ async def _broadcast_animation(payload: dict) -> None: def _runtime_payload() -> dict: return { "state": str(arbitrator.state), - "peers": len(pcs), "audio_clients": len(audio_clients), "subtitle_clients": len(subtitle_clients), "animation_clients": len(animation_clients), @@ -341,7 +333,6 @@ async def _process_user_audio_chunk(mono_16k: np.ndarray, vad_buffer: np.ndarray barge_t0 = time.perf_counter() was_avatar_speaking = arbitrator.state == SessionState.AVATAR_SPEAKING await arbitrator.on_speech_start() - await audio_bus.clear() await _broadcast_audio_reset("speech-start") await _broadcast_animation(await avatar_service.build_reset_payload(reason="speech-start")) if was_avatar_speaking: @@ -357,18 +348,6 @@ async def _process_user_audio_chunk(mono_16k: np.ndarray, vad_buffer: np.ndarray return vad_buffer -async def _consume_user_audio(track) -> None: - vad_buffer = np.zeros(0, dtype=np.float32) - while True: - frame = await track.recv() - pcm = frame.to_ndarray() - sample_rate = getattr(frame, "sample_rate", 48000) or 48000 - layout = getattr(frame, "layout", None) - channels = len(layout.channels) if layout and layout.channels else 1 - mono_16k = _resample_to_16k_mono(pcm, sample_rate, channels) - vad_buffer = await _process_user_audio_chunk(mono_16k, vad_buffer) - - @app.get("/health") async def health(): return _runtime_payload() @@ -377,10 +356,9 @@ async def health(): @app.get("/meta") async def meta(): return { - "stun_url": settings.stun_url, "https_enabled": bool(settings.ssl_certfile and settings.ssl_keyfile), - "host": settings.webrtc_host, - "port": settings.webrtc_port, + "host": settings.http_host, + "port": settings.http_port, "avatar_protocol": settings.avatar_control_protocol, } @@ -474,7 +452,6 @@ async def ws_audio(websocket: WebSocket): elif event_type == "audio_reset": _clear_buffered_audio() runtime.pending_voice_turn = False - await audio_bus.clear() await _broadcast_audio_reset("client-reset") elif event_type == "ping": await websocket.send_text(json.dumps({"type": "pong"}, ensure_ascii=False)) @@ -567,7 +544,6 @@ async def chat_reset(): runtime.vad_start_count = 0 runtime.vad_end_count = 0 runtime.last_animation_frame_count = 0 - await audio_bus.clear() await _broadcast_audio_reset("chat-reset") await _broadcast_animation(await avatar_service.build_reset_payload(reason="chat-reset")) return {"ok": True} @@ -583,60 +559,13 @@ async def root(): return FileResponse("web/index.html") -@app.post("/webrtc/offer") -async def webrtc_offer(request: Request): - params = await request.json() - offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"]) - - replaced_previous = len(pcs) > 0 - if replaced_previous: - old_peers = list(pcs) - await asyncio.gather(*(peer.close() for peer in old_peers), return_exceptions=True) - for peer in old_peers: - pcs.discard(peer) - - pc = RTCPeerConnection() - pcs.add(pc) - - @pc.on("connectionstatechange") - async def on_connectionstatechange(): - if pc.connectionState in {"failed", "closed", "disconnected"}: - await pc.close() - pcs.discard(pc) - - @pc.on("track") - def on_track(track): - if track.kind == "audio": - asyncio.create_task(_consume_user_audio(track)) - - pc.addTrack(AvatarAudioTrack(audio_bus=audio_bus)) - - await pc.setRemoteDescription(offer) - answer = await pc.createAnswer() - await pc.setLocalDescription(answer) - - return JSONResponse( - { - "sdp": pc.localDescription.sdp, - "type": pc.localDescription.type, - "replaced_previous": replaced_previous, - } - ) - - -@app.on_event("shutdown") -async def on_shutdown(): - await asyncio.gather(*(pc.close() for pc in list(pcs)), return_exceptions=True) - pcs.clear() - - if __name__ == "__main__": import uvicorn uvicorn_kwargs = { "app": app, - "host": settings.webrtc_host, - "port": settings.webrtc_port, + "host": settings.http_host, + "port": settings.http_port, } if settings.ssl_certfile and settings.ssl_keyfile: uvicorn_kwargs["ssl_certfile"] = settings.ssl_certfile diff --git a/requirements.txt b/requirements.txt index f4aa64c..82d8737 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ fastapi uvicorn[standard] -aiortc numpy soundfile python-dotenv diff --git a/scripts/run_all_checks.sh b/scripts/run_all_checks.sh index 56c1ffe..186c0dc 100755 --- a/scripts/run_all_checks.sh +++ b/scripts/run_all_checks.sh @@ -4,7 +4,28 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -PY="/home/xsl/miniconda3/envs/MuseTalk/bin/python" + +pick_python() { + if [[ -n "${VISUAL_CHAT_PYTHON:-}" && -x "${VISUAL_CHAT_PYTHON}" ]]; then + echo "${VISUAL_CHAT_PYTHON}" + return 0 + fi + if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then + echo "${VIRTUAL_ENV}/bin/python" + return 0 + fi + if [[ -x "$ROOT/.venv/bin/python" ]]; then + echo "$ROOT/.venv/bin/python" + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + command -v python3 + return 0 + fi + command -v python +} + +PY="$(pick_python)" if [[ -f ".env" ]]; then set -a @@ -17,7 +38,7 @@ SCHEME="http" if [[ -n "${SSL_CERTFILE:-}" && -n "${SSL_KEYFILE:-}" ]]; then SCHEME="https" fi -BASE_URL="${BASE_URL:-${SCHEME}://127.0.0.1:${WEBRTC_PORT:-8080}}" +BASE_URL="${BASE_URL:-${SCHEME}://127.0.0.1:${HTTP_PORT:-8080}}" echo "[1/6] service status" bash scripts/service.sh status || true diff --git a/scripts/service-8018.sh b/scripts/service-8018.sh index 67437d7..74d91d6 100644 --- a/scripts/service-8018.sh +++ b/scripts/service-8018.sh @@ -28,8 +28,8 @@ normalize_host() { esac } -HOST="$(normalize_host "${WEBRTC_HOST:-0.0.0.0}")" -PORT="${WEBRTC_PORT:-8018}" +HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")" +PORT="${HTTP_PORT:-8018}" SSL_CERTFILE="${SSL_CERTFILE:-$ROOT_DIR/certs/dev-cert.pem}" SSL_KEYFILE="${SSL_KEYFILE:-$ROOT_DIR/certs/dev-key.pem}" diff --git a/scripts/service.sh b/scripts/service.sh index 79f8216..e1efb95 100755 --- a/scripts/service.sh +++ b/scripts/service.sh @@ -28,7 +28,7 @@ start() { # If stale process still binds configured port, stop it first. local port - port="$(awk -F= '/^WEBRTC_PORT=/{print $2}' "$ROOT/.env" 2>/dev/null || true)" + port="$(awk -F= '/^HTTP_PORT=/{print $2}' "$ROOT/.env" 2>/dev/null || true)" port="${port:-8080}" local pids pids="$(ss -ltnp 2>/dev/null | sed -n "s/.*:${port} .*users:((\"python\",pid=\([0-9]\+\),.*/\1/p" | tr '\n' ' ')" diff --git a/scripts/start-public.sh b/scripts/start-public.sh index 998e331..84d73b0 100755 --- a/scripts/start-public.sh +++ b/scripts/start-public.sh @@ -21,8 +21,8 @@ pick_python() { echo "${VIRTUAL_ENV}/bin/python" return 0 fi - if [[ -x "/home/xsl/miniconda3/envs/MuseTalk/bin/python" ]]; then - echo "/home/xsl/miniconda3/envs/MuseTalk/bin/python" + if [[ -x "$ROOT/.venv/bin/python" ]]; then + echo "$ROOT/.venv/bin/python" return 0 fi if command -v python3 >/dev/null 2>&1; then @@ -44,8 +44,8 @@ normalize_host() { esac } -HOST="$(normalize_host "${WEBRTC_HOST:-0.0.0.0}")" -PORT="${WEBRTC_PORT:-8080}" +HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")" +PORT="${HTTP_PORT:-8080}" PY="$(pick_python)" SCHEME="http" ARGS=(main:app --host "$HOST" --port "$PORT") diff --git a/scripts/start-test.sh b/scripts/start-test.sh index 7b43953..cd9e61c 100644 --- a/scripts/start-test.sh +++ b/scripts/start-test.sh @@ -34,8 +34,8 @@ pick_python() { echo "${VIRTUAL_ENV}/bin/python" return 0 fi - if [[ -x "/home/xsl/miniconda3/envs/MuseTalk/bin/python" ]]; then - echo "/home/xsl/miniconda3/envs/MuseTalk/bin/python" + if [[ -x "$ROOT/.venv/bin/python" ]]; then + echo "$ROOT/.venv/bin/python" return 0 fi if command -v python3 >/dev/null 2>&1; then @@ -46,8 +46,8 @@ pick_python() { } PY="$(pick_python)" -HOST="$(normalize_host "${WEBRTC_HOST:-0.0.0.0}")" -PORT="${WEBRTC_PORT:-8080}" +HOST="$(normalize_host "${HTTP_HOST:-0.0.0.0}")" +PORT="${HTTP_PORT:-8080}" SCHEME="http" ARGS=(main:app --host "$HOST" --port "$PORT") diff --git a/scripts/vendor_hf_models.py b/scripts/vendor_hf_models.py new file mode 100644 index 0000000..a02c1a4 --- /dev/null +++ b/scripts/vendor_hf_models.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Populate ./models from local silero wheel + Hugging Face snapshots (ASR/TTS only; needs network for HF).""" +from __future__ import annotations + +import argparse +import importlib.resources as ir +import sys +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +_DEFAULT_ASR = _ROOT / "models" / "asr" / "SenseVoiceSmall" +_DEFAULT_KOKORO = _ROOT / "models" / "tts" / "Kokoro-82M" +_VAD_JIT = _ROOT / "models" / "vad" / "silero_vad.jit" + + +def _copy_silero_vad() -> None: + _VAD_JIT.parent.mkdir(parents=True, exist_ok=True) + src = ir.files("silero_vad.data").joinpath("silero_vad.jit") + _VAD_JIT.write_bytes(src.read_bytes()) + print(f"VAD -> {_VAD_JIT}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--asr-repo", default="FunAudioLLM/SenseVoiceSmall") + parser.add_argument("--kokoro-repo", default="hexgrad/Kokoro-82M") + parser.add_argument("--asr-dir", type=Path, default=_DEFAULT_ASR) + parser.add_argument("--kokoro-dir", type=Path, default=_DEFAULT_KOKORO) + parser.add_argument("--asr-only", action="store_true") + parser.add_argument("--kokoro-only", action="store_true") + parser.add_argument("--skip-hf", action="store_true", help="Only copy Silero VAD from installed wheel") + args = parser.parse_args() + + try: + _copy_silero_vad() + except Exception as exc: + print(f"Silero VAD copy failed (pip install silero-vad): {exc}", file=sys.stderr) + return 1 + + if args.skip_hf: + print("Done (HF skipped).") + return 0 + + try: + from huggingface_hub import snapshot_download + except ImportError: + print("Install huggingface_hub for ASR/TTS downloads.", file=sys.stderr) + return 1 + + if not args.kokoro_only: + args.asr_dir.mkdir(parents=True, exist_ok=True) + print(f"ASR -> {args.asr_dir}") + snapshot_download(repo_id=args.asr_repo, local_dir=str(args.asr_dir)) + if not args.asr_only: + args.kokoro_dir.mkdir(parents=True, exist_ok=True) + print(f"Kokoro -> {args.kokoro_dir}") + snapshot_download(repo_id=args.kokoro_repo, local_dir=str(args.kokoro_dir)) + + print("Done.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/__pycache__/asr.cpython-312.pyc b/services/__pycache__/asr.cpython-312.pyc index dbffe4b..805d1a8 100644 Binary files a/services/__pycache__/asr.cpython-312.pyc and b/services/__pycache__/asr.cpython-312.pyc differ diff --git a/services/__pycache__/llm.cpython-312.pyc b/services/__pycache__/llm.cpython-312.pyc index dff60e5..8303918 100644 Binary files a/services/__pycache__/llm.cpython-312.pyc and b/services/__pycache__/llm.cpython-312.pyc differ diff --git a/services/__pycache__/tts.cpython-312.pyc b/services/__pycache__/tts.cpython-312.pyc index b3d6353..1aea2c6 100644 Binary files a/services/__pycache__/tts.cpython-312.pyc and b/services/__pycache__/tts.cpython-312.pyc differ diff --git a/services/__pycache__/vad.cpython-312.pyc b/services/__pycache__/vad.cpython-312.pyc index bc761b2..75a642c 100644 Binary files a/services/__pycache__/vad.cpython-312.pyc and b/services/__pycache__/vad.cpython-312.pyc differ diff --git a/services/asr.py b/services/asr.py index c2cba6b..4602f38 100644 --- a/services/asr.py +++ b/services/asr.py @@ -1,13 +1,25 @@ from __future__ import annotations import logging +from pathlib import Path from typing import Optional import numpy as np +from config import asr_bundle_dir + logger = logging.getLogger(__name__) +def _asr_model_path() -> Path: + p = asr_bundle_dir().resolve() + if not p.is_dir(): + raise FileNotFoundError(f"ASR model directory missing: {p}") + if not (p / "configuration.json").is_file(): + raise FileNotFoundError(f"ASR bundle incomplete (no configuration.json): {p}") + return p + + class ASRService: def __init__(self) -> None: self._model = None @@ -21,16 +33,26 @@ class ASRService: if self._ready or self._attempted: return self._attempted = True + try: + model_dir = _asr_model_path() + except FileNotFoundError as exc: + self._init_error = str(exc) + self._device = "unavailable" + logger.error("%s", exc) + return + from funasr import AutoModel from funasr.utils.postprocess_utils import rich_transcription_postprocess + model_ref = str(model_dir) errors: list[str] = [] for device in ("cuda:0", "cpu"): try: self._model = AutoModel( - model="FunAudioLLM/SenseVoiceSmall", + model=model_ref, device=device, hub="hf", + disable_update=True, ) self._post = rich_transcription_postprocess self._ready = True @@ -56,8 +78,6 @@ class ASRService: use_itn=True, ) return self._post(res[0]["text"]) - - # When ASR is unavailable, avoid emitting fake user text that would trigger a bogus LLM reply. return "" @property diff --git a/services/llm.py b/services/llm.py index c00b6a0..26b616f 100644 --- a/services/llm.py +++ b/services/llm.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import os import re from typing import Optional from collections import deque @@ -21,23 +20,6 @@ class LLMService: self._history: deque[dict] = deque(maxlen=12) # 6 turns (user+assistant) self._system_prompt = (settings.llm_system_prompt or "").strip() or "请用中文口语化简短回复，1-2句。" - if not settings.llm_api_key: - self._error = "missing LLM api key: set LLM_API_KEY or DEEPSEEK_API_KEY" - return - - # If DeepSeek key is provided but no explicit base URL override, - # route to DeepSeek-compatible OpenAI endpoint by default. - if ( - os.getenv("DEEPSEEK_API_KEY") - and not os.getenv("LLM_BASE_URL") - and not os.getenv("DEEPSEEK_BASE_URL") - and (self._base_url.lower() == "https://api.openai.com/v1" or not self._base_url) - ): - self._base_url = "https://api.deepseek.com/v1" - - if not self._model: - base = self._base_url.lower() - self._model = "deepseek-chat" if "deepseek" in base else "gpt-4o-mini" try: from openai import AsyncOpenAI @@ -50,7 +32,7 @@ class LLMService: self._error = None except Exception as exc: # pragma: no cover self._error = str(exc) - logger.warning("LLM online mode unavailable: %s", exc) + logger.warning("LLM client init failed: %s", exc) @staticmethod def _normalize_reply_text(text: str) -> str: @@ -70,7 +52,7 @@ class LLMService: async def reply_with_meta(self, text: str) -> tuple[str, str]: if not self._ready or self._client is None: - return f"收到：{text}。这是本地兜底回复。", "fallback" + return "", "unavailable" try: resp = await self._client.chat.completions.create( @@ -86,9 +68,9 @@ class LLMService: return content, "online" except Exception as exc: # pragma: no cover self._error = str(exc) - logger.warning("LLM request failed, fallback used: %s", exc) + logger.warning("LLM request failed: %s", exc) - return f"收到：{text}。这是本地兜底回复。", "fallback" + return "", "error" async def reply(self, text: str) -> str: content, _ = await self.reply_with_meta(text) diff --git a/services/tts.py b/services/tts.py index 44ed33c..71cafa8 100644 --- a/services/tts.py +++ b/services/tts.py @@ -2,22 +2,49 @@ from __future__ import annotations import logging import re +from pathlib import Path from typing import Optional import numpy as np -from config import settings +from config import kokoro_bundle_dir, settings logger = logging.getLogger(__name__) +def _kokoro_root() -> Path: + p = kokoro_bundle_dir().resolve() + if not p.is_dir(): + raise FileNotFoundError(f"Kokoro model directory missing: {p}") + cfg = p / "config.json" + w0 = p / "kokoro-v1_0.pth" + w1 = p / "kokoro-v1_1-zh.pth" + weight = w0 if w0.is_file() else (w1 if w1.is_file() else None) + if not cfg.is_file() or weight is None: + raise FileNotFoundError(f"Kokoro bundle incomplete (need config.json and weight .pth): {p}") + return p + + class TTSService: def __init__(self) -> None: self._pipeline = None + self._kokoro_root: Optional[Path] = None self._ready = False self._attempted = False self._init_error: Optional[str] = None + def _voice_pt_path(self) -> Path: + root = self._kokoro_root + if root is None: + raise RuntimeError("Kokoro pipeline not initialized") + name = (settings.tts_voice or "").strip() + if not name: + raise ValueError("TTS_VOICE must be set (voice name, e.g. zf_xiaoxiao)") + p = root / "voices" / f"{name}.pt" + if not p.is_file(): + raise FileNotFoundError(f"Voice file missing: {p}") + return p + def _ensure_loaded(self) -> None: if self._ready or self._attempted: return @@ -25,11 +52,24 @@ class TTSService: try: import kokoro - self._pipeline = kokoro.KPipeline(lang_code="z") + root = _kokoro_root() + w0 = root / "kokoro-v1_0.pth" + w1 = root / "kokoro-v1_1-zh.pth" + weight = w0 if w0.is_file() else w1 + hf_repo_id = ( + "hexgrad/Kokoro-82M-v1.1-zh" if weight.name.startswith("kokoro-v1_1") else "hexgrad/Kokoro-82M" + ) + kmodel = kokoro.KModel( + repo_id=hf_repo_id, + config=str(root / "config.json"), + model=str(weight), + ) + self._pipeline = kokoro.KPipeline(lang_code="z", model=kmodel, repo_id=str(root)) + self._kokoro_root = root self._ready = True except Exception as exc: # pragma: no cover self._init_error = str(exc) - logger.warning("TTS fallback mode: %s", exc) + logger.error("TTS init failed: %s", exc) @staticmethod def _normalize_text(text: str) -> str: @@ -62,20 +102,24 @@ class TTSService: if not text: return np.zeros(1, dtype=np.float32), 24000 self._ensure_loaded() - if self._ready and self._pipeline is not None: - chunks = [] - for _, _, audio in self._pipeline( - text, - voice=settings.tts_voice, - speed=max(0.8, min(1.1, settings.tts_speed)), - split_pattern=r"\n+", - ): - chunks.append(self._apply_edge_fade(np.asarray(audio, dtype=np.float32), 24000)) - if chunks: - return np.concatenate(chunks), 24000 - - # 0.5s silence fallback for flow verification - return np.zeros(12000, dtype=np.float32), 24000 + if not self._ready or self._pipeline is None: + return np.zeros(1, dtype=np.float32), 24000 + try: + voice_path = str(self._voice_pt_path()) + except (OSError, ValueError, RuntimeError) as exc: + logger.error("TTS voice: %s", exc) + return np.zeros(1, dtype=np.float32), 24000 + chunks = [] + for _, _, audio in self._pipeline( + text, + voice=voice_path, + speed=max(0.8, min(1.1, settings.tts_speed)), + split_pattern=r"\n+", + ): + chunks.append(self._apply_edge_fade(np.asarray(audio, dtype=np.float32), 24000)) + if chunks: + return np.concatenate(chunks), 24000 + return np.zeros(1, dtype=np.float32), 24000 @property def health(self) -> dict: diff --git a/services/vad.py b/services/vad.py index de75e55..4fb3394 100644 --- a/services/vad.py +++ b/services/vad.py @@ -2,6 +2,8 @@ from __future__ import annotations from typing import Optional +from config import vad_bundle_path + class VADService: def __init__(self) -> None: @@ -15,10 +17,15 @@ class VADService: if self._ready or self._attempted: return self._attempted = True + path = vad_bundle_path().resolve() + if not path.is_file(): + self._error = f"missing Silero VAD weights: {path}" + return try: - from silero_vad import VADIterator, load_silero_vad + from silero_vad import VADIterator + from silero_vad.utils_vad import init_jit_model - self._model = load_silero_vad() + self._model = init_jit_model(str(path)) self._iterator = VADIterator( self._model, threshold=0.65, diff --git a/web/app.js b/web/app.js index f8899a3..d2c311e 100644 --- a/web/app.js +++ b/web/app.js @@ -567,12 +567,16 @@ function renderLoop() { } function updateMetrics(data) { - mPeers.textContent = String(data.audio_clients ?? data.peers ?? 0); + mPeers.textContent = String(data.audio_clients ?? 0); mBusy.textContent = data.pipeline_busy ? "是" : "否"; mLatency.textContent = `${data.last_latency_ms ?? 0} ms`; mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`; mLlmLatency.textContent = `${data.last_llm_latency_ms ?? 0} ms`; - mLlmSource.textContent = data.last_llm_source === "online" ? "在线" : "兜底"; + { + const s = data.last_llm_source; + mLlmSource.textContent = + s === "online" ? "在线" : s === "error" ? "失败" : s === "skipped" ? "跳过" : s === "unavailable" ? "不可用" : s || "--"; + } mLlmHistory.textContent = String(data?.llm?.history_items ?? 0); mTtsLatency.textContent = `${data.last_tts_latency_ms ?? 0} ms`; mTtsFirst.textContent = `${data.last_tts_first_chunk_ms ?? 0} ms`; @@ -584,7 +588,7 @@ function updateMetrics(data) { mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--"; mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0); - setDot(dConn, (data.audio_clients ?? data.peers ?? 0) > 0 ? "ok" : "warn"); + setDot(dConn, (data.audio_clients ?? 0) > 0 ? "ok" : "warn"); setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn"); setDot(dAsr, data?.asr?.ready ? "ok" : "warn"); setDot(dLlm, data?.llm?.ready ? "ok" : "bad"); diff --git a/webrtc/__init__.py b/webrtc/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/webrtc/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/webrtc/__pycache__/__init__.cpython-310.pyc b/webrtc/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 45d1b7b..0000000 Binary files a/webrtc/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/webrtc/__pycache__/__init__.cpython-312.pyc b/webrtc/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 4deb15a..0000000 Binary files a/webrtc/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/webrtc/__pycache__/tracks.cpython-310.pyc b/webrtc/__pycache__/tracks.cpython-310.pyc deleted file mode 100644 index 0240db7..0000000 Binary files a/webrtc/__pycache__/tracks.cpython-310.pyc and /dev/null differ diff --git a/webrtc/__pycache__/tracks.cpython-312.pyc b/webrtc/__pycache__/tracks.cpython-312.pyc deleted file mode 100644 index f748168..0000000 Binary files a/webrtc/__pycache__/tracks.cpython-312.pyc and /dev/null differ diff --git a/webrtc/audio_track.py b/webrtc/audio_track.py deleted file mode 100644 index b38f2fc..0000000 --- a/webrtc/audio_track.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from webrtc.tracks import AudioBus, AvatarAudioTrack - -__all__ = ["AvatarAudioTrack", "build_audio_track"] - - -def build_audio_track(audio_bus: AudioBus, sample_rate: int = 48000, frame_ms: int = 20) -> AvatarAudioTrack: - return AvatarAudioTrack(audio_bus=audio_bus, sample_rate=sample_rate, frame_ms=frame_ms) diff --git a/webrtc/gateway.py b/webrtc/gateway.py deleted file mode 100644 index 00fd135..0000000 --- a/webrtc/gateway.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - -from aiortc import RTCPeerConnection -from fastapi import FastAPI - -from webrtc.audio_track import build_audio_track -from webrtc.tracks import AudioBus - - -def attach_webrtc_tracks( - pc: RTCPeerConnection, - avatar_service, - arbitrator, - audio_bus: AudioBus, -) -> None: - _ = avatar_service - _ = arbitrator - pc.addTrack(build_audio_track(audio_bus)) - - -def register_gateway_routes(app: FastAPI) -> None: - # Gateway endpoints are currently implemented in `main.py`. - # This function exists to keep project structure aligned with task list. - _ = app diff --git a/webrtc/tracks.py b/webrtc/tracks.py deleted file mode 100644 index 1af68d9..0000000 --- a/webrtc/tracks.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import asyncio -from fractions import Fraction -from collections import deque - -import numpy as np -from aiortc import MediaStreamTrack -from av import AudioFrame - - -class AudioBus: - def __init__(self, sample_rate: int = 48000) -> None: - self.sample_rate = sample_rate - self._chunks: deque[np.ndarray] = deque() - self._lock = asyncio.Lock() - self._level = 0.0 - - async def enqueue(self, pcm_int16_mono: np.ndarray) -> None: - async with self._lock: - if pcm_int16_mono.ndim != 1: - pcm_int16_mono = pcm_int16_mono.reshape(-1) - chunk = pcm_int16_mono.astype(np.int16, copy=False) - self._chunks.append(chunk) - if chunk.size > 0: - rms = float(np.sqrt(np.mean((chunk.astype(np.float32) / 32768.0) ** 2))) - self._level = 0.85 * self._level + 0.15 * min(1.0, rms * 10.0) - - async def read(self, samples: int) -> np.ndarray: - out = np.zeros(samples, dtype=np.int16) - async with self._lock: - i = 0 - while i < samples and self._chunks: - head = self._chunks[0] - n = min(samples - i, head.shape[0]) - out[i : i + n] = head[:n] - i += n - if n == head.shape[0]: - self._chunks.popleft() - else: - self._chunks[0] = head[n:] - self._level = 0.92 * self._level - return out - - async def level(self) -> float: - async with self._lock: - return float(self._level) - - async def clear(self) -> None: - async with self._lock: - self._chunks.clear() - self._level = 0.0 - - -class AvatarAudioTrack(MediaStreamTrack): - kind = "audio" - - def __init__(self, audio_bus: AudioBus, sample_rate: int = 48000, frame_ms: int = 20) -> None: - super().__init__() - self.audio_bus = audio_bus - self.sample_rate = sample_rate - self.samples_per_frame = int(sample_rate * frame_ms / 1000) - self._pts = 0 - self._time_base = Fraction(1, sample_rate) - - async def recv(self) -> AudioFrame: - await asyncio.sleep(self.samples_per_frame / self.sample_rate) - mono = await self.audio_bus.read(self.samples_per_frame) - samples = mono.reshape(1, -1) - frame = AudioFrame(format="s16", layout="mono", samples=self.samples_per_frame) - frame.planes[0].update(samples.tobytes()) - frame.sample_rate = self.sample_rate - frame.pts = self._pts - frame.time_base = self._time_base - self._pts += self.samples_per_frame - return frame diff --git a/webrtc/video_track.py b/webrtc/video_track.py deleted file mode 100644 index 54fc003..0000000 --- a/webrtc/video_track.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -__all__: list[str] = [] - - -def build_video_track(*_args, **_kwargs): - raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")