大功告成

This commit is contained in:
xsl
2026-03-27 17:10:41 +08:00
parent 22a5f31847
commit f24de38e94
30 changed files with 2310 additions and 404 deletions
Binary file not shown.
Binary file not shown.
+4 -7
View File
@@ -3,21 +3,18 @@ from __future__ import annotations
from aiortc import RTCPeerConnection
from fastapi import FastAPI
from config import settings
from core.state_machine import Arbitrator
from services.avatar import AvatarService
from webrtc.audio_track import build_audio_track
from webrtc.tracks import AudioBus
from webrtc.video_track import build_video_track
def attach_webrtc_tracks(
pc: RTCPeerConnection,
avatar_service: AvatarService,
arbitrator: Arbitrator,
avatar_service,
arbitrator,
audio_bus: AudioBus,
) -> None:
pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
_ = avatar_service
_ = arbitrator
pc.addTrack(build_audio_track(audio_bus))
+1 -39
View File
@@ -6,10 +6,7 @@ from collections import deque
import numpy as np
from aiortc import MediaStreamTrack
from av import AudioFrame, VideoFrame
from core.state_machine import Arbitrator, SessionState
from services.avatar import AvatarService
from av import AudioFrame
class AudioBus:
@@ -55,41 +52,6 @@ class AudioBus:
self._level = 0.0
class AvatarVideoTrack(MediaStreamTrack):
kind = "video"
def __init__(
self,
avatar_service: AvatarService,
arbitrator: Arbitrator,
audio_bus: AudioBus,
fps: int = 25,
) -> None:
super().__init__()
self.avatar_service = avatar_service
self.arbitrator = arbitrator
self.audio_bus = audio_bus
self.fps = fps
self._pts = 0
self._time_base = Fraction(1, 90000)
async def recv(self) -> VideoFrame:
await asyncio.sleep(1 / self.fps)
generated = await self.avatar_service.pop_generated_frame()
if generated is not None:
arr = generated
else:
level = await self.audio_bus.level()
speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
frame = VideoFrame.from_ndarray(arr, format="bgr24")
frame.pts = self._pts
frame.time_base = self._time_base
self._pts += int(90000 / self.fps)
return frame
class AvatarAudioTrack(MediaStreamTrack):
kind = "audio"
+3 -17
View File
@@ -1,21 +1,7 @@
from __future__ import annotations
from core.state_machine import Arbitrator
from services.avatar import AvatarService
from webrtc.tracks import AudioBus, AvatarVideoTrack
__all__ = ["AvatarVideoTrack", "build_video_track"]
__all__: list[str] = []
def build_video_track(
avatar_service: AvatarService,
arbitrator: Arbitrator,
audio_bus: AudioBus,
fps: int = 25,
) -> AvatarVideoTrack:
return AvatarVideoTrack(
avatar_service=avatar_service,
arbitrator=arbitrator,
audio_bus=audio_bus,
fps=fps,
)
def build_video_track(*_args, **_kwargs):
raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")