save code

scripts/smoke_test.py

@@ -35,34 +35,82 @@ async def _write_avatar_video(
     avatar: AvatarService, audio: np.ndarray, sr: int, out_path: str, fps: int
 ) -> tuple[int, float]:
     fourcc = cv2.VideoWriter_fourcc(*"mp4v")
-    writer = cv2.VideoWriter(out_path, fourcc, float(fps), (avatar.width, avatar.height))
+    width = 720
+    height = 720
+    writer = cv2.VideoWriter(out_path, fourcc, float(fps), (width, height))
     if not writer.isOpened():
         raise RuntimeError(f"failed to open video writer: {out_path}")
 
-    duration_s = float(audio.shape[0]) / float(sr) if sr > 0 else 0.0
-    total_frames = max(1, int(math.ceil(duration_s * fps)))
-    samples_per_frame = max(1, int(sr / fps))
+    payload = await avatar.build_controls_from_audio(audio, sr, text="smoke-test")
+    frames = payload.get("frames", [])
+    duration_ms = int(payload.get("duration_ms", 0))
 
-    for i in range(total_frames):
-        s = i * samples_per_frame
-        e = min(audio.shape[0], (i + 1) * samples_per_frame)
-        seg = audio[s:e]
-        if seg.size == 0:
-            level = 0.0
-        else:
-            rms = float(np.sqrt(np.mean(seg.astype(np.float32) ** 2)))
-            level = float(np.clip(rms * 10.0, 0.0, 1.0))
-        frame = await avatar.render_frame(mouth_open=level, speaking=True)
+    for frame_payload in frames:
+        controls = frame_payload.get("controls", {})
+        frame = _render_debug_frame(width, height, controls, speaking=bool(frame_payload.get("speaking", False)))
         writer.write(frame)
 
     # add 0.4s idle tail for easy visual check
     tail_frames = max(1, int(0.4 * fps))
     for _ in range(tail_frames):
-        frame = await avatar.render_frame(mouth_open=0.0, speaking=False)
+        frame = _render_debug_frame(width, height, {}, speaking=False)
         writer.write(frame)
 
     writer.release()
-    return total_frames + tail_frames, duration_s + 0.4
+    return len(frames) + tail_frames, (duration_ms / 1000.0) + 0.4
+
+
+def _render_debug_frame(width: int, height: int, controls: dict, speaking: bool) -> np.ndarray:
+    jaw_open = float(np.clip(controls.get("jawOpen", 0.0), 0.0, 1.0))
+    mouth_pucker = float(np.clip(controls.get("mouthPucker", 0.0), 0.0, 1.0))
+    viseme_aa = float(np.clip(controls.get("viseme_aa", jaw_open), 0.0, 1.0))
+    viseme_ee = float(np.clip(controls.get("viseme_ee", 0.0), 0.0, 1.0))
+    viseme_oh = float(np.clip(controls.get("viseme_oh", 0.0), 0.0, 1.0))
+    head_yaw = float(np.clip(controls.get("headYaw", 0.0), -1.0, 1.0))
+    head_pitch = float(np.clip(controls.get("headPitch", 0.0), -1.0, 1.0))
+    head_roll = float(np.clip(controls.get("headRoll", 0.0), -1.0, 1.0))
+
+    frame = np.zeros((height, width, 3), dtype=np.uint8)
+    frame[:] = (16, 12, 10)
+
+    center_x = width // 2 + int(head_yaw * 48)
+    center_y = height // 2 + int(head_pitch * 42) - 24
+    head_radius = 190
+
+    cv2.circle(frame, (center_x, center_y), head_radius + 26, (42, 28, 18), -1, lineType=cv2.LINE_AA)
+    cv2.circle(frame, (center_x, center_y), head_radius, (198, 214, 238), -1, lineType=cv2.LINE_AA)
+
+    eye_y = center_y - 44
+    eye_offset = 64
+    cv2.circle(frame, (center_x - eye_offset, eye_y), 12, (26, 31, 43), -1, lineType=cv2.LINE_AA)
+    cv2.circle(frame, (center_x + eye_offset, eye_y), 12, (26, 31, 43), -1, lineType=cv2.LINE_AA)
+
+    brow_dx = int(head_roll * 28)
+    cv2.line(frame, (center_x - 100 - brow_dx, eye_y - 34), (center_x - 42 - brow_dx, eye_y - 40), (48, 58, 74), 6, lineType=cv2.LINE_AA)
+    cv2.line(frame, (center_x + 42 - brow_dx, eye_y - 40), (center_x + 100 - brow_dx, eye_y - 34), (48, 58, 74), 6, lineType=cv2.LINE_AA)
+
+    mouth_center = (center_x, center_y + 86)
+    mouth_width = int(64 + viseme_oh * 36 + mouth_pucker * 18)
+    mouth_height = int(12 + jaw_open * 98 + viseme_aa * 18)
+    lip_color = (96, 54, 130) if speaking else (88, 74, 108)
+    inner_color = (36, 22, 54)
+    cv2.ellipse(frame, mouth_center, (mouth_width, max(6, mouth_height)), 0, 0, 360, lip_color, -1, lineType=cv2.LINE_AA)
+    cv2.ellipse(frame, mouth_center, (max(12, mouth_width - 16), max(4, mouth_height - 8)), 0, 0, 360, inner_color, -1, lineType=cv2.LINE_AA)
+
+    ee_width = int(max(20, 86 - viseme_ee * 46))
+    cv2.line(frame, (mouth_center[0] - ee_width, mouth_center[1]), (mouth_center[0] + ee_width, mouth_center[1]), (220, 224, 232), 3, lineType=cv2.LINE_AA)
+
+    metrics = [
+        f"jawOpen={jaw_open:.2f}",
+        f"viseme_aa={viseme_aa:.2f}",
+        f"viseme_ee={viseme_ee:.2f}",
+        f"viseme_oh={viseme_oh:.2f}",
+        f"headYaw={head_yaw:.2f}",
+    ]
+    for index, text in enumerate(metrics):
+        cv2.putText(frame, text, (28, 42 + index * 28), cv2.FONT_HERSHEY_SIMPLEX, 0.72, (210, 220, 235), 2, lineType=cv2.LINE_AA)
+
+    return frame
 
 
 async def main() -> None: