大功告成

2026-03-27 17:10:41 +08:00
parent 22a5f31847
commit f24de38e94
30 changed files with 2310 additions and 404 deletions
@@ -1,3 +1,8 @@
+import * as THREE from "three";
+import { OrbitControls } from "three/addons/controls/OrbitControls.js";
+import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
+import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm";
+
 const statusEl = document.getElementById("status");
 const outputEl = document.getElementById("output");
 const healthBtn = document.getElementById("healthBtn");
@@ -5,11 +10,25 @@ const offerBtn = document.getElementById("offerBtn");
 const resetBtn = document.getElementById("resetBtn");
 const sendBtn = document.getElementById("sendBtn");
 const chatInput = document.getElementById("chatInput");
-const remoteVideo = document.getElementById("remoteVideo");
-remoteVideo.muted = false;
-remoteVideo.volume = 1.0;
+const remoteAudio = document.getElementById("remoteAudio");
+const modelFile = document.getElementById("modelFile");
+const modelUrl = document.getElementById("modelUrl");
+const loadModelBtn = document.getElementById("loadModelBtn");
+const resetViewBtn = document.getElementById("resetViewBtn");
+const avatarCanvas = document.getElementById("avatarCanvas");
+const modelStatus = document.getElementById("modelStatus");
+const modelName = document.getElementById("modelName");
+const animationBuffer = document.getElementById("animationBuffer");
+
+remoteAudio.muted = false;
+remoteAudio.volume = 1.0;
+
 const messagesEl = document.getElementById("messages");
 const connBadge = document.getElementById("connBadge");
+const animationConn = document.getElementById("animationConn");
+const animationDriver = document.getElementById("animationDriver");
+const animationFrames = document.getElementById("animationFrames");
+const animationPreview = document.getElementById("animationPreview");
 const mPeers = document.getElementById("mPeers");
 const mBusy = document.getElementById("mBusy");
 const mLatency = document.getElementById("mLatency");
@@ -23,6 +42,9 @@ const mTts = document.getElementById("mTts");
 const mBargeIn = document.getElementById("mBargeIn");
 const mInputMode = document.getElementById("mInputMode");
 const mVadCount = document.getElementById("mVadCount");
+const mAnimClients = document.getElementById("mAnimClients");
+const mAnimDriver = document.getElementById("mAnimDriver");
+const mAnimFrames = document.getElementById("mAnimFrames");
 const dConn = document.getElementById("dConn");
 const dVad = document.getElementById("dVad");
 const dAsr = document.getElementById("dAsr");
@@ -32,11 +54,15 @@ const dTts = document.getElementById("dTts");
 let pc = null;
 let evt = null;
 let subtitleWs = null;
+let animationWs = null;
 let localMicStream = null;
-let lastSeenRun = 0;
-let lastBusy = false;
 let aiStreamingEl = null;
+
 const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
+const ANIMATION_BUFFER_MS = 120;
+const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
+const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
+const CLOCK = new THREE.Clock();

 const stateMap = {
  "SessionState.IDLE": "空闲",
@@ -49,6 +75,56 @@ const stateMap = {
  avatar_speaking: "数字人说话",
 };

+const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true });
+renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
+renderer.outputColorSpace = THREE.SRGBColorSpace;
+
+const scene = new THREE.Scene();
+scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5);
+
+const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100);
+camera.position.set(0, 1.45, 2.65);
+
+const controls = new OrbitControls(camera, avatarCanvas);
+controls.target.set(0, 1.15, 0);
+controls.enableDamping = true;
+controls.minDistance = 1.3;
+controls.maxDistance = 4.5;
+controls.maxPolarAngle = Math.PI * 0.58;
+controls.minPolarAngle = Math.PI * 0.22;
+
+const avatarState = {
+  currentRoot: null,
+  currentVrm: null,
+  morphBindings: new Map(),
+  pendingFrames: [],
+  scheduleCursorMs: performance.now(),
+  lastAppliedSeq: -1,
+  activeControls: {
+    jawOpen: 0,
+    viseme_aa: 0,
+    viseme_ee: 0,
+    viseme_oh: 0,
+    mouthPucker: 0,
+    headYaw: 0,
+    headPitch: 0,
+    headRoll: 0,
+  },
+  headBone: null,
+  neckBone: null,
+  idleRig: null,
+  debugRig: null,
+  debugJaw: null,
+  debugMouth: null,
+};
+
+const gltfLoader = new GLTFLoader();
+gltfLoader.register((parser) => new VRMLoaderPlugin(parser));
+gltfLoader.crossOrigin = "anonymous";
+
+const TMP_EULER = new THREE.Euler();
+const TMP_QUATERNION = new THREE.Quaternion();
+
 function fmtTime() {
  return new Date().toLocaleTimeString("zh-CN", { hour12: false });
 }
@@ -82,6 +158,132 @@ function setDot(el, kind) {
  el.className = `dot ${kind}`;
 }

+function initStage() {
+  scene.background = new THREE.Color(0x0b1118);
+
+  const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65);
+  hemiLight.position.set(0, 2.6, 0);
+  scene.add(hemiLight);
+
+  const keyLight = new THREE.DirectionalLight(0xffffff, 1.7);
+  keyLight.position.set(1.4, 2.2, 2.0);
+  scene.add(keyLight);
+
+  const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8);
+  rimLight.position.set(-2.0, 1.4, -1.5);
+  scene.add(rimLight);
+
+  const floor = new THREE.Mesh(
+    new THREE.CircleGeometry(2.8, 48),
+    new THREE.MeshStandardMaterial({
+      color: 0x101b28,
+      roughness: 0.94,
+      metalness: 0.08,
+      transparent: true,
+      opacity: 0.9,
+    }),
+  );
+  floor.rotation.x = -Math.PI / 2;
+  floor.position.y = -0.02;
+  scene.add(floor);
+
+  const halo = new THREE.Mesh(
+    new THREE.RingGeometry(1.2, 1.95, 64),
+    new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }),
+  );
+  halo.rotation.x = -Math.PI / 2;
+  halo.position.y = 0.01;
+  scene.add(halo);
+
+  createDebugRig();
+  resizeStage();
+  renderLoop();
+}
+
+function createDebugRig() {
+  const group = new THREE.Group();
+
+  const bustMaterial = new THREE.MeshStandardMaterial({
+    color: 0xc7d2e0,
+    roughness: 0.72,
+    metalness: 0.05,
+  });
+  const accentMaterial = new THREE.MeshStandardMaterial({
+    color: 0x0f172a,
+    roughness: 0.35,
+    metalness: 0.18,
+  });
+  const lipMaterial = new THREE.MeshStandardMaterial({
+    color: 0x8b425f,
+    roughness: 0.56,
+    metalness: 0.02,
+  });
+
+  const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial);
+  torso.position.set(0, 0.42, 0);
+  group.add(torso);
+
+  const neck = new THREE.Group();
+  neck.position.set(0, 0.95, 0);
+  group.add(neck);
+
+  const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial);
+  head.scale.set(0.92, 1.03, 0.92);
+  neck.add(head);
+
+  const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial);
+  hair.position.set(0, 0.06, 0);
+  neck.add(hair);
+
+  const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
+  leftEye.position.set(-0.1, 0.03, 0.28);
+  neck.add(leftEye);
+
+  const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
+  rightEye.position.set(0.1, 0.03, 0.28);
+  neck.add(rightEye);
+
+  const jawPivot = new THREE.Group();
+  jawPivot.position.set(0, -0.06, 0.15);
+  neck.add(jawPivot);
+
+  const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial);
+  jaw.position.set(0, -0.08, 0.02);
+  jawPivot.add(jaw);
+
+  const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial);
+  mouth.position.set(0, -0.02, 0.31);
+  neck.add(mouth);
+
+  scene.add(group);
+  avatarState.currentRoot = group;
+  avatarState.debugRig = group;
+  avatarState.debugJaw = jawPivot;
+  avatarState.debugMouth = mouth;
+  avatarState.headBone = neck;
+  avatarState.neckBone = neck;
+}
+
+function resizeStage() {
+  const rect = avatarCanvas.getBoundingClientRect();
+  if (!rect.width || !rect.height) return;
+  renderer.setSize(rect.width, rect.height, false);
+  camera.aspect = rect.width / rect.height;
+  camera.updateProjectionMatrix();
+}
+
+function renderLoop() {
+  requestAnimationFrame(renderLoop);
+  const delta = CLOCK.getDelta();
+  const elapsed = CLOCK.getElapsedTime();
+  controls.update();
+  flushAnimationQueue();
+  applyProceduralIdlePose(elapsed);
+  applyAnimationControls(avatarState.activeControls);
+  avatarState.currentVrm?.update?.(delta);
+  renderer.render(scene, camera);
+}
+
 function updateMetrics(data) {
  mPeers.textContent = String(data.peers ?? 0);
  mBusy.textContent = data.pipeline_busy ? "是" : "否";
@@ -96,6 +298,9 @@ function updateMetrics(data) {
  mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`;
  mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none";
  mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`;
+  mAnimClients.textContent = String(data.animation_clients ?? 0);
+  mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--";
+  mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0);

  setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn");
  setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
@@ -104,6 +309,394 @@ function updateMetrics(data) {
  setDot(dTts, data?.tts?.ready ? "ok" : "warn");
 }

+function queueAnimationFrames(payload) {
+  const frames = Array.isArray(payload.frames) ? payload.frames : [];
+  const now = performance.now();
+  const frameWindowMs = (1000 / Math.max(1, payload.fps || 25));
+
+  if (payload.type === "animation_reset" || payload.type === "animation_state") {
+    avatarState.pendingFrames = [];
+    avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS;
+    avatarState.activeControls = {
+      jawOpen: 0,
+      viseme_aa: 0,
+      viseme_ee: 0,
+      viseme_oh: 0,
+      mouthPucker: 0,
+      headYaw: 0,
+      headPitch: 0,
+      headRoll: 0,
+    };
+  }
+
+  const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs);
+  frames.forEach((frame, index) => {
+    const fallbackTime = index * frameWindowMs;
+    avatarState.pendingFrames.push({
+      dueAt: baseTime + (frame.time_ms ?? fallbackTime),
+      seq: frame.seq ?? index,
+      controls: frame.controls || {},
+    });
+  });
+
+  avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq);
+  avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs);
+  animationBuffer.textContent = String(avatarState.pendingFrames.length);
+}
+
+function flushAnimationQueue() {
+  const now = performance.now();
+  let frameToApply = null;
+  while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) {
+    frameToApply = avatarState.pendingFrames.shift();
+  }
+
+  if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) {
+    avatarState.lastAppliedSeq = frameToApply.seq;
+    avatarState.activeControls = {
+      ...avatarState.activeControls,
+      ...frameToApply.controls,
+    };
+  }
+
+  animationBuffer.textContent = String(avatarState.pendingFrames.length);
+}
+
+function applyAnimationControls(controls) {
+  const safeControls = controls || {};
+  const jawOpen = clampControl(safeControls.jawOpen);
+  const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen);
+  const visemeEe = clampControl(safeControls.viseme_ee);
+  const visemeOh = clampControl(safeControls.viseme_oh);
+  const mouthPucker = clampControl(safeControls.mouthPucker);
+  const headYaw = clampSigned(safeControls.headYaw, 0.35);
+  const headPitch = clampSigned(safeControls.headPitch, 0.3);
+  const headRoll = clampSigned(safeControls.headRoll, 0.24);
+  const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0;
+  const t = performance.now() * 0.001;
+
+  if (avatarState.currentVrm?.expressionManager) {
+    setExpressionValue("aa", visemeAa);
+    setExpressionValue("ee", visemeEe);
+    setExpressionValue("oh", visemeOh);
+    setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker));
+  }
+
+  if (avatarState.morphBindings.size > 0) {
+    setMorphValue("jawOpen", jawOpen);
+    setMorphValue("viseme_aa", visemeAa);
+    setMorphValue("viseme_ee", visemeEe);
+    setMorphValue("viseme_oh", visemeOh);
+    setMorphValue("mouthPucker", mouthPucker);
+  }
+
+  // For plain head meshes (no VRM expressions and no morph targets),
+  // amplify head motion to keep speaking visually obvious.
+  const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09;
+  const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12;
+  const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04;
+
+  applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied);
+  if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) {
+    applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4);
+  }
+
+  if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
+    const baseScale = 1.0 + jawOpen * 0.03;
+    avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale);
+  }
+
+  if (avatarState.debugJaw) {
+    avatarState.debugJaw.rotation.x = jawOpen * 0.5;
+    avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015;
+  }
+  if (avatarState.debugMouth) {
+    avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45;
+    avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8;
+    avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01;
+  }
+}
+
+function setExpressionValue(name, value) {
+  try {
+    avatarState.currentVrm.expressionManager.setValue(name, value);
+  } catch (_) {
+    // no-op: expression name may not exist on the current model
+  }
+}
+
+function setMorphValue(controlName, value) {
+  const bindings = avatarState.morphBindings.get(controlName) || [];
+  bindings.forEach((binding) => {
+    binding.mesh.morphTargetInfluences[binding.index] = value;
+  });
+}
+
+function clampControl(value) {
+  return Math.min(1, Math.max(0, Number(value) || 0));
+}
+
+function clampSigned(value, limit) {
+  return Math.min(limit, Math.max(-limit, Number(value) || 0));
+}
+
+function resetView() {
+  camera.position.set(0, 1.45, 2.65);
+  controls.target.set(0, 1.15, 0);
+  controls.update();
+}
+
+function resetAvatarPose() {
+  avatarState.activeControls = {
+    jawOpen: 0,
+    viseme_aa: 0,
+    viseme_ee: 0,
+    viseme_oh: 0,
+    mouthPucker: 0,
+    headYaw: 0,
+    headPitch: 0,
+    headRoll: 0,
+  };
+  applyAnimationControls(avatarState.activeControls);
+}
+
+function detachCurrentAvatar() {
+  if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
+    scene.remove(avatarState.currentRoot);
+  }
+  avatarState.pendingFrames = [];
+  avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+  avatarState.lastAppliedSeq = -1;
+  avatarState.idleRig = null;
+  avatarState.currentRoot = avatarState.debugRig;
+  avatarState.currentVrm = null;
+  avatarState.morphBindings.clear();
+  avatarState.headBone = avatarState.debugRig.children[1];
+  avatarState.neckBone = avatarState.debugRig.children[1];
+  avatarState.debugRig.visible = true;
+  modelName.textContent = "debug-avatar";
+}
+
+async function loadModelFromUrl(url) {
+  if (!url) return false;
+  modelStatus.textContent = "模型加载中...";
+  try {
+    const gltf = await gltfLoader.loadAsync(url);
+    attachLoadedAsset(gltf, url.split("/").pop() || url);
+    modelStatus.textContent = "模型已加载";
+    return true;
+  } catch (error) {
+    console.error(error);
+    modelStatus.textContent = "模型加载失败，已回退调试头像";
+    addMessage("system", `模型加载失败：${error?.message || error}`);
+    detachCurrentAvatar();
+    return false;
+  }
+}
+
+async function loadDefaultAvatarModel() {
+  modelUrl.value = DEFAULT_BODY_MODEL_URL;
+  const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
+  if (!ok) {
+    modelUrl.value = DEFAULT_VRM_FALLBACK_URL;
+    await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL);
+  }
+}
+
+async function loadModelFromFile(file) {
+  if (!file) return;
+  const url = URL.createObjectURL(file);
+  modelStatus.textContent = "本地模型加载中...";
+  try {
+    const gltf = await gltfLoader.loadAsync(url);
+    attachLoadedAsset(gltf, file.name);
+    modelStatus.textContent = "本地模型已加载";
+  } catch (error) {
+    console.error(error);
+    modelStatus.textContent = "本地模型加载失败，已回退调试头像";
+    addMessage("system", `本地模型加载失败：${error?.message || error}`);
+    detachCurrentAvatar();
+  } finally {
+    URL.revokeObjectURL(url);
+  }
+}
+
+function attachLoadedAsset(gltf, label) {
+  detachCurrentAvatar();
+
+  const vrm = gltf.userData?.vrm || null;
+  const root = vrm?.scene || gltf.scene;
+  if (!root) {
+    throw new Error("模型中未找到可渲染的 scene");
+  }
+
+  if (vrm) {
+    VRMUtils.removeUnnecessaryVertices(gltf.scene);
+    VRMUtils.combineSkeletons(gltf.scene);
+    VRMUtils.combineMorphs(vrm);
+    VRMUtils.rotateVRM0(vrm);
+  }
+
+  root.traverse((obj) => {
+    obj.frustumCulled = false;
+  });
+
+  root.position.set(0, 0, 0);
+  root.rotation.set(0, 0, 0);
+  root.scale.setScalar(1);
+  scene.add(root);
+
+  avatarState.currentRoot = root;
+  avatarState.currentVrm = vrm;
+  avatarState.debugRig.visible = false;
+  avatarState.morphBindings = collectMorphBindings(root);
+  avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1];
+  avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone;
+  avatarState.idleRig = buildIdleRig(vrm, root);
+  avatarState.pendingFrames = [];
+  avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+  avatarState.lastAppliedSeq = -1;
+
+  fitCameraToObject(root);
+  resetAvatarPose();
+  modelName.textContent = label;
+}
+
+function fitCameraToObject(root) {
+  const box = new THREE.Box3().setFromObject(root);
+  const center = new THREE.Vector3();
+  const size = new THREE.Vector3();
+  box.getCenter(center);
+  box.getSize(size);
+
+  const radius = Math.max(size.x, size.y, size.z, 1.2);
+  controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0));
+  camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9);
+  controls.update();
+}
+
+function resolveHeadBone(vrm, root) {
+  return (
+    vrm?.humanoid?.getNormalizedBoneNode?.("head") ||
+    vrm?.humanoid?.getRawBoneNode?.("head") ||
+    root.getObjectByName("Head") ||
+    root.getObjectByName("head") ||
+    root
+  );
+}
+
+function resolveNeckBone(vrm, root) {
+  return (
+    vrm?.humanoid?.getNormalizedBoneNode?.("neck") ||
+    vrm?.humanoid?.getRawBoneNode?.("neck") ||
+    root.getObjectByName("Neck") ||
+    root.getObjectByName("neck") ||
+    resolveHeadBone(vrm, root)
+  );
+}
+
+function collectMorphBindings(root) {
+  const bindings = new Map();
+  const aliases = {
+    jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"],
+    viseme_aa: ["viseme_aa", "aa", "a", "moutha"],
+    viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"],
+    viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"],
+    mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"],
+  };
+
+  root.traverse((node) => {
+    if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return;
+    const entries = Object.entries(node.morphTargetDictionary);
+    Object.entries(aliases).forEach(([controlName, names]) => {
+      entries.forEach(([targetName, index]) => {
+        const normalized = targetName.toLowerCase();
+        if (!names.includes(normalized)) return;
+        if (!bindings.has(controlName)) bindings.set(controlName, []);
+        bindings.get(controlName).push({ mesh: node, index });
+      });
+    });
+  });
+  return bindings;
+}
+
+function buildIdleRig(vrm, root) {
+  const lookup = {
+    hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]),
+    spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]),
+    chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]),
+    upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]),
+    neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]),
+    head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]),
+    leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]),
+    rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]),
+    leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]),
+    rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]),
+    leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]),
+    rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]),
+    leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]),
+    rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]),
+  };
+
+  const base = new Map();
+  Object.values(lookup).forEach((bone) => {
+    if (bone && !base.has(bone)) {
+      base.set(bone, bone.quaternion.clone());
+    }
+  });
+
+  return { bones: lookup, base };
+}
+
+function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) {
+  return (
+    vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) ||
+    vrm?.humanoid?.getRawBoneNode?.(humanoidName) ||
+    fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) ||
+    null
+  );
+}
+
+function applyProceduralIdlePose(time) {
+  const rig = avatarState.idleRig;
+  if (!rig) return;
+
+  const breathe = Math.sin(time * 1.9) * 0.018;
+  const sway = Math.sin(time * 1.15) * 0.05;
+  const armDrift = Math.sin(time * 1.55) * 0.07;
+  const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06;
+  const handDrift = Math.sin(time * 2.3) * 0.04;
+
+  applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04);
+  applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06);
+  applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08);
+  applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1);
+
+  applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18);
+  applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18);
+  applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0);
+  applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0);
+  applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12);
+  applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12);
+  applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05);
+  applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05);
+}
+
+function applyBoneRotation(bone, x = 0, y = 0, z = 0) {
+  if (!bone) return;
+  const baseQuaternion = avatarState.idleRig?.base?.get(bone);
+  if (baseQuaternion) {
+    bone.quaternion.copy(baseQuaternion);
+    TMP_EULER.set(x, y, z, "XYZ");
+    TMP_QUATERNION.setFromEuler(TMP_EULER);
+    bone.quaternion.multiply(TMP_QUATERNION);
+    return;
+  }
+  bone.rotation.x = x;
+  bone.rotation.y = y;
+  bone.rotation.z = z;
+}
+
 async function checkHealth() {
  const res = await fetch("/health");
  const data = await res.json();
@@ -123,8 +716,6 @@ function subscribeEvents() {
      outputEl.textContent = JSON.stringify(data, null, 2);
      sendBtn.disabled = !!data.pipeline_busy;
      updateMetrics(data);
-      lastBusy = !!data.pipeline_busy;
-      if (typeof data.pipeline_runs === "number" && data.pipeline_runs > lastSeenRun) lastSeenRun = data.pipeline_runs;
    } catch (_) {
      // no-op
    }
@@ -160,6 +751,36 @@ function subscribeSubtitles() {
  };
 }

+function subscribeAnimation() {
+  if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return;
+  const scheme = location.protocol === "https:" ? "wss" : "ws";
+  animationConn.textContent = "连接中";
+  animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`);
+  animationWs.onopen = () => {
+    animationConn.textContent = "已连接";
+  };
+  animationWs.onmessage = (event) => {
+    try {
+      const data = JSON.parse(event.data);
+      if (data.type === "ping") return;
+      if (data.driver) {
+        animationDriver.textContent = data.driver;
+      }
+      const frameCount = data.frame_count ?? data.frames?.length ?? 0;
+      animationFrames.textContent = String(frameCount);
+      const preview = data.frames?.[0]?.controls || null;
+      animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2);
+      queueAnimationFrames(data);
+    } catch (_) {
+      // no-op
+    }
+  };
+  animationWs.onclose = () => {
+    animationConn.textContent = "已断开";
+    setTimeout(subscribeAnimation, 1000);
+  };
+}
+
 async function connectWebRTC() {
  if (pc) {
    outputEl.textContent = "WebRTC 已连接或正在连接";
@@ -175,21 +796,22 @@ async function connectWebRTC() {
  });
  const currentPc = pc;
  let seenConnected = false;
+
  currentPc.onconnectionstatechange = () => {
    const st = currentPc.connectionState;
    if (st === "connected") {
      seenConnected = true;
      setConnBadge("已连接", "ok");
      localStorage.setItem(AUTO_RECONNECT_KEY, "1");
-    }
-    else if (st === "connecting") setConnBadge("连接中...", "warn");
-    else if (st === "failed" || st === "disconnected" || st === "closed") {
+    } else if (st === "connecting") {
+      setConnBadge("连接中...", "warn");
+    } else if (st === "failed" || st === "disconnected" || st === "closed") {
      setConnBadge(`连接${st}`, "warn");
      if (seenConnected && (st === "disconnected" || st === "failed")) {
        addMessage("system", "连接已断开，可能被新的客户端连接接管。");
      }
      if (localMicStream) {
-        localMicStream.getTracks().forEach((t) => t.stop());
+        localMicStream.getTracks().forEach((track) => track.stop());
        localMicStream = null;
      }
      if (pc === currentPc) {
@@ -198,13 +820,12 @@ async function connectWebRTC() {
      offerBtn.disabled = false;
    }
  };
-  // Explicitly request remote media m-lines so server can attach tracks.
-  pc.addTransceiver("video", { direction: "recvonly" });
+
  pc.ontrack = (event) => {
    if (event.streams && event.streams[0]) {
-      remoteVideo.srcObject = event.streams[0];
-      remoteVideo.play().catch(() => {
-        addMessage("system", "远端音视频已到达，如未自动播放请点击视频区域后重试。");
+      remoteAudio.srcObject = event.streams[0];
+      remoteAudio.play().catch(() => {
+        addMessage("system", "远端语音已到达，如未自动播放请先与页面交互后重试。");
      });
    }
  };
@@ -220,12 +841,8 @@ async function connectWebRTC() {
    offerBtn.disabled = false;
    throw err;
  }
-  stream.getTracks().forEach((track) => pc.addTrack(track, stream));
-  const sendAudioTrack = stream.getAudioTracks()[0];
-  if (sendAudioTrack) {
-    sendAudioTrack.enabled = true;
-  }

+  stream.getTracks().forEach((track) => pc.addTrack(track, stream));
  const offer = await pc.createOffer();
  await pc.setLocalDescription(offer);

@@ -240,11 +857,7 @@ async function connectWebRTC() {
  const answer = await res.json();
  await pc.setRemoteDescription(answer);
  statusEl.textContent = "状态：WebRTC 已连接";
-  outputEl.textContent = JSON.stringify(
-    { connected: true, iceConnectionState: pc.iceConnectionState },
-    null,
-    2
-  );
+  outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2);
  setConnBadge("已连接", "ok");
  addMessage("system", "WebRTC 已连接，可开始对话。");
  if (answer.replaced_previous) {
@@ -256,8 +869,7 @@ async function connectWebRTC() {

 async function sendText() {
  const text = (chatInput.value || "").trim();
-  if (!text) return;
-  if (sendBtn.disabled) return;
+  if (!text || sendBtn.disabled) return;
  sendBtn.disabled = true;
  const res = await fetch("/chat/text", {
    method: "POST",
@@ -277,6 +889,9 @@ async function sendText() {
 async function resetChat() {
  await fetch("/chat/reset", { method: "POST" });
  messagesEl.innerHTML = "";
+  avatarState.pendingFrames = [];
+  avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+  resetAvatarPose();
  addMessage("system", "会话已重置。");
  await checkHealth();
 }
@@ -284,24 +899,56 @@ async function resetChat() {
 function tryAutoReconnect() {
  if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
  setTimeout(() => {
-    if (!pc) connectWebRTC().catch(() => {
-      // no-op: user can reconnect manually if permission/policy blocks auto flow
-    });
+    if (!pc) {
+      connectWebRTC().catch(() => {
+        // no-op
+      });
+    }
  }, 300);
 }

+function handleModelFileChange(event) {
+  const [file] = event.target.files || [];
+  if (!file) return;
+  loadModelFromFile(file);
+}
+
+function handleModelUrlLoad() {
+  const url = modelUrl.value.trim();
+  if (!url) return;
+  loadModelFromUrl(url);
+}
+
+new ResizeObserver(() => resizeStage()).observe(avatarCanvas);
+window.addEventListener("resize", resizeStage);
+
 healthBtn.addEventListener("click", checkHealth);
 offerBtn.addEventListener("click", connectWebRTC);
 resetBtn.addEventListener("click", resetChat);
 sendBtn.addEventListener("click", sendText);
-chatInput.addEventListener("keydown", (e) => {
-  if (e.key === "Enter") {
-    e.preventDefault();
+loadModelBtn.addEventListener("click", handleModelUrlLoad);
+resetViewBtn.addEventListener("click", resetView);
+modelFile.addEventListener("change", handleModelFileChange);
+chatInput.addEventListener("keydown", (event) => {
+  if (event.key === "Enter") {
+    event.preventDefault();
    sendText();
  }
 });
+modelUrl.addEventListener("keydown", (event) => {
+  if (event.key === "Enter") {
+    event.preventDefault();
+    handleModelUrlLoad();
+  }
+});
+
+initStage();
+resetView();
+resetAvatarPose();
+loadDefaultAvatarModel();
 checkHealth();
 subscribeEvents();
 subscribeSubtitles();
+subscribeAnimation();
 setConnBadge("未连接", "neutral");
 tryAutoReconnect();