大功告成
This commit is contained in:
Binary file not shown.
Binary file not shown.
+4
-7
@@ -3,21 +3,18 @@ from __future__ import annotations
|
||||
from aiortc import RTCPeerConnection
|
||||
from fastapi import FastAPI
|
||||
|
||||
from config import settings
|
||||
from core.state_machine import Arbitrator
|
||||
from services.avatar import AvatarService
|
||||
from webrtc.audio_track import build_audio_track
|
||||
from webrtc.tracks import AudioBus
|
||||
from webrtc.video_track import build_video_track
|
||||
|
||||
|
||||
def attach_webrtc_tracks(
|
||||
pc: RTCPeerConnection,
|
||||
avatar_service: AvatarService,
|
||||
arbitrator: Arbitrator,
|
||||
avatar_service,
|
||||
arbitrator,
|
||||
audio_bus: AudioBus,
|
||||
) -> None:
|
||||
pc.addTrack(build_video_track(avatar_service, arbitrator, audio_bus, settings.avatar_fps))
|
||||
_ = avatar_service
|
||||
_ = arbitrator
|
||||
pc.addTrack(build_audio_track(audio_bus))
|
||||
|
||||
|
||||
|
||||
+1
-39
@@ -6,10 +6,7 @@ from collections import deque
|
||||
|
||||
import numpy as np
|
||||
from aiortc import MediaStreamTrack
|
||||
from av import AudioFrame, VideoFrame
|
||||
|
||||
from core.state_machine import Arbitrator, SessionState
|
||||
from services.avatar import AvatarService
|
||||
from av import AudioFrame
|
||||
|
||||
|
||||
class AudioBus:
|
||||
@@ -55,41 +52,6 @@ class AudioBus:
|
||||
self._level = 0.0
|
||||
|
||||
|
||||
class AvatarVideoTrack(MediaStreamTrack):
|
||||
kind = "video"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
avatar_service: AvatarService,
|
||||
arbitrator: Arbitrator,
|
||||
audio_bus: AudioBus,
|
||||
fps: int = 25,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.avatar_service = avatar_service
|
||||
self.arbitrator = arbitrator
|
||||
self.audio_bus = audio_bus
|
||||
self.fps = fps
|
||||
self._pts = 0
|
||||
self._time_base = Fraction(1, 90000)
|
||||
|
||||
async def recv(self) -> VideoFrame:
|
||||
await asyncio.sleep(1 / self.fps)
|
||||
generated = await self.avatar_service.pop_generated_frame()
|
||||
if generated is not None:
|
||||
arr = generated
|
||||
else:
|
||||
level = await self.audio_bus.level()
|
||||
speaking = self.arbitrator.state == SessionState.AVATAR_SPEAKING or level > 0.02
|
||||
arr = await self.avatar_service.render_frame(mouth_open=level, speaking=speaking)
|
||||
|
||||
frame = VideoFrame.from_ndarray(arr, format="bgr24")
|
||||
frame.pts = self._pts
|
||||
frame.time_base = self._time_base
|
||||
self._pts += int(90000 / self.fps)
|
||||
return frame
|
||||
|
||||
|
||||
class AvatarAudioTrack(MediaStreamTrack):
|
||||
kind = "audio"
|
||||
|
||||
|
||||
+3
-17
@@ -1,21 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.state_machine import Arbitrator
|
||||
from services.avatar import AvatarService
|
||||
from webrtc.tracks import AudioBus, AvatarVideoTrack
|
||||
|
||||
__all__ = ["AvatarVideoTrack", "build_video_track"]
|
||||
__all__: list[str] = []
|
||||
|
||||
|
||||
def build_video_track(
|
||||
avatar_service: AvatarService,
|
||||
arbitrator: Arbitrator,
|
||||
audio_bus: AudioBus,
|
||||
fps: int = 25,
|
||||
) -> AvatarVideoTrack:
|
||||
return AvatarVideoTrack(
|
||||
avatar_service=avatar_service,
|
||||
arbitrator=arbitrator,
|
||||
audio_bus=audio_bus,
|
||||
fps=fps,
|
||||
)
|
||||
def build_video_track(*_args, **_kwargs):
|
||||
raise RuntimeError("Server-side video tracks are disabled in 3D avatar mode.")
|
||||
|
||||
Reference in New Issue
Block a user