import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"; const statusEl = document.getElementById("status"); const outputEl = document.getElementById("output"); const healthBtn = document.getElementById("healthBtn"); const offerBtn = document.getElementById("offerBtn"); const resetBtn = document.getElementById("resetBtn"); const sendBtn = document.getElementById("sendBtn"); const chatInput = document.getElementById("chatInput"); const remoteAudio = document.getElementById("remoteAudio"); const modelFile = document.getElementById("modelFile"); const modelUrl = document.getElementById("modelUrl"); const loadModelBtn = document.getElementById("loadModelBtn"); const resetViewBtn = document.getElementById("resetViewBtn"); const avatarCanvas = document.getElementById("avatarCanvas"); const modelStatus = document.getElementById("modelStatus"); const modelName = document.getElementById("modelName"); const animationBuffer = document.getElementById("animationBuffer"); remoteAudio.muted = false; remoteAudio.volume = 1.0; const messagesEl = document.getElementById("messages"); const connBadge = document.getElementById("connBadge"); const animationConn = document.getElementById("animationConn"); const animationDriver = document.getElementById("animationDriver"); const animationFrames = document.getElementById("animationFrames"); const animationPreview = document.getElementById("animationPreview"); const mPeers = document.getElementById("mPeers"); const mBusy = document.getElementById("mBusy"); const mLatency = document.getElementById("mLatency"); const mAsrLatency = document.getElementById("mAsrLatency"); const mLlmLatency = document.getElementById("mLlmLatency"); const mLlmSource = document.getElementById("mLlmSource"); const mLlmHistory = document.getElementById("mLlmHistory"); const mTtsLatency = document.getElementById("mTtsLatency"); const mTtsFirst = document.getElementById("mTtsFirst"); const mTts = document.getElementById("mTts"); const mBargeIn = document.getElementById("mBargeIn"); const mInputMode = document.getElementById("mInputMode"); const mVadCount = document.getElementById("mVadCount"); const mAnimClients = document.getElementById("mAnimClients"); const mAnimDriver = document.getElementById("mAnimDriver"); const mAnimFrames = document.getElementById("mAnimFrames"); const dConn = document.getElementById("dConn"); const dVad = document.getElementById("dVad"); const dAsr = document.getElementById("dAsr"); const dLlm = document.getElementById("dLlm"); const dTts = document.getElementById("dTts"); let pc = null; let evt = null; let subtitleWs = null; let animationWs = null; let localMicStream = null; let aiStreamingEl = null; const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect"; const ANIMATION_BUFFER_MS = 120; const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm"; const CLOCK = new THREE.Clock(); const stateMap = { "SessionState.IDLE": "空闲", "SessionState.USER_SPEAKING": "用户说话", "SessionState.THINKING": "思考中", "SessionState.AVATAR_SPEAKING": "数字人说话", idle: "空闲", user_speaking: "用户说话", thinking: "思考中", avatar_speaking: "数字人说话", }; const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true }); renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); renderer.outputColorSpace = THREE.SRGBColorSpace; const scene = new THREE.Scene(); scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5); const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100); camera.position.set(0, 1.45, 2.65); const controls = new OrbitControls(camera, avatarCanvas); controls.target.set(0, 1.15, 0); controls.enableDamping = true; controls.minDistance = 1.3; controls.maxDistance = 4.5; controls.maxPolarAngle = Math.PI * 0.58; controls.minPolarAngle = Math.PI * 0.22; const avatarState = { currentRoot: null, currentVrm: null, morphBindings: new Map(), pendingFrames: [], scheduleCursorMs: performance.now(), lastAppliedSeq: -1, activeControls: { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }, headBone: null, neckBone: null, idleRig: null, debugRig: null, debugJaw: null, debugMouth: null, }; const gltfLoader = new GLTFLoader(); gltfLoader.register((parser) => new VRMLoaderPlugin(parser)); gltfLoader.crossOrigin = "anonymous"; const TMP_EULER = new THREE.Euler(); const TMP_QUATERNION = new THREE.Quaternion(); function fmtTime() { return new Date().toLocaleTimeString("zh-CN", { hour12: false }); } function toCnState(raw) { return stateMap[raw] || raw; } function addMessage(role, text) { const row = document.createElement("div"); row.className = "msg-row"; const div = document.createElement("div"); div.className = `msg ${role}`; div.textContent = text; const time = document.createElement("div"); time.className = `msg-time ${role}`; time.textContent = fmtTime(); row.appendChild(div); row.appendChild(time); messagesEl.appendChild(row); messagesEl.scrollTop = messagesEl.scrollHeight; return div; } function setConnBadge(label, kind = "neutral") { connBadge.textContent = label; connBadge.className = `badge ${kind}`; } function setDot(el, kind) { el.className = `dot ${kind}`; } function initStage() { scene.background = new THREE.Color(0x0b1118); const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65); hemiLight.position.set(0, 2.6, 0); scene.add(hemiLight); const keyLight = new THREE.DirectionalLight(0xffffff, 1.7); keyLight.position.set(1.4, 2.2, 2.0); scene.add(keyLight); const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8); rimLight.position.set(-2.0, 1.4, -1.5); scene.add(rimLight); const floor = new THREE.Mesh( new THREE.CircleGeometry(2.8, 48), new THREE.MeshStandardMaterial({ color: 0x101b28, roughness: 0.94, metalness: 0.08, transparent: true, opacity: 0.9, }), ); floor.rotation.x = -Math.PI / 2; floor.position.y = -0.02; scene.add(floor); const halo = new THREE.Mesh( new THREE.RingGeometry(1.2, 1.95, 64), new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }), ); halo.rotation.x = -Math.PI / 2; halo.position.y = 0.01; scene.add(halo); createDebugRig(); resizeStage(); renderLoop(); } function createDebugRig() { const group = new THREE.Group(); const bustMaterial = new THREE.MeshStandardMaterial({ color: 0xc7d2e0, roughness: 0.72, metalness: 0.05, }); const accentMaterial = new THREE.MeshStandardMaterial({ color: 0x0f172a, roughness: 0.35, metalness: 0.18, }); const lipMaterial = new THREE.MeshStandardMaterial({ color: 0x8b425f, roughness: 0.56, metalness: 0.02, }); const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial); torso.position.set(0, 0.42, 0); group.add(torso); const neck = new THREE.Group(); neck.position.set(0, 0.95, 0); group.add(neck); const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial); head.scale.set(0.92, 1.03, 0.92); neck.add(head); const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial); hair.position.set(0, 0.06, 0); neck.add(hair); const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); leftEye.position.set(-0.1, 0.03, 0.28); neck.add(leftEye); const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); rightEye.position.set(0.1, 0.03, 0.28); neck.add(rightEye); const jawPivot = new THREE.Group(); jawPivot.position.set(0, -0.06, 0.15); neck.add(jawPivot); const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial); jaw.position.set(0, -0.08, 0.02); jawPivot.add(jaw); const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial); mouth.position.set(0, -0.02, 0.31); neck.add(mouth); scene.add(group); avatarState.currentRoot = group; avatarState.debugRig = group; avatarState.debugJaw = jawPivot; avatarState.debugMouth = mouth; avatarState.headBone = neck; avatarState.neckBone = neck; } function resizeStage() { const rect = avatarCanvas.getBoundingClientRect(); if (!rect.width || !rect.height) return; renderer.setSize(rect.width, rect.height, false); camera.aspect = rect.width / rect.height; camera.updateProjectionMatrix(); } function renderLoop() { requestAnimationFrame(renderLoop); const delta = CLOCK.getDelta(); const elapsed = CLOCK.getElapsedTime(); controls.update(); flushAnimationQueue(); applyProceduralIdlePose(elapsed); applyAnimationControls(avatarState.activeControls); avatarState.currentVrm?.update?.(delta); renderer.render(scene, camera); } function updateMetrics(data) { mPeers.textContent = String(data.peers ?? 0); mBusy.textContent = data.pipeline_busy ? "是" : "否"; mLatency.textContent = `${data.last_latency_ms ?? 0} ms`; mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`; mLlmLatency.textContent = `${data.last_llm_latency_ms ?? 0} ms`; mLlmSource.textContent = data.last_llm_source === "online" ? "在线" : "兜底"; mLlmHistory.textContent = String(data?.llm?.history_items ?? 0); mTtsLatency.textContent = `${data.last_tts_latency_ms ?? 0} ms`; mTtsFirst.textContent = `${data.last_tts_first_chunk_ms ?? 0} ms`; mTts.textContent = data?.tts?.ready ? "就绪" : "未就绪"; mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`; mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none"; mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`; mAnimClients.textContent = String(data.animation_clients ?? 0); mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--"; mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0); setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn"); setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn"); setDot(dAsr, data?.asr?.ready ? "ok" : "warn"); setDot(dLlm, data?.llm?.ready ? "ok" : "bad"); setDot(dTts, data?.tts?.ready ? "ok" : "warn"); } function queueAnimationFrames(payload) { const frames = Array.isArray(payload.frames) ? payload.frames : []; const now = performance.now(); const frameWindowMs = (1000 / Math.max(1, payload.fps || 25)); if (payload.type === "animation_reset" || payload.type === "animation_state") { avatarState.pendingFrames = []; avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS; avatarState.activeControls = { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }; } const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs); frames.forEach((frame, index) => { const fallbackTime = index * frameWindowMs; avatarState.pendingFrames.push({ dueAt: baseTime + (frame.time_ms ?? fallbackTime), seq: frame.seq ?? index, controls: frame.controls || {}, }); }); avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq); avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs); animationBuffer.textContent = String(avatarState.pendingFrames.length); } function flushAnimationQueue() { const now = performance.now(); let frameToApply = null; while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) { frameToApply = avatarState.pendingFrames.shift(); } if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) { avatarState.lastAppliedSeq = frameToApply.seq; avatarState.activeControls = { ...avatarState.activeControls, ...frameToApply.controls, }; } animationBuffer.textContent = String(avatarState.pendingFrames.length); } function applyAnimationControls(controls) { const safeControls = controls || {}; const jawOpen = clampControl(safeControls.jawOpen); const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen); const visemeEe = clampControl(safeControls.viseme_ee); const visemeOh = clampControl(safeControls.viseme_oh); const mouthPucker = clampControl(safeControls.mouthPucker); const headYaw = clampSigned(safeControls.headYaw, 0.35); const headPitch = clampSigned(safeControls.headPitch, 0.3); const headRoll = clampSigned(safeControls.headRoll, 0.24); const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0; const t = performance.now() * 0.001; if (avatarState.currentVrm?.expressionManager) { setExpressionValue("aa", visemeAa); setExpressionValue("ee", visemeEe); setExpressionValue("oh", visemeOh); setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker)); } if (avatarState.morphBindings.size > 0) { setMorphValue("jawOpen", jawOpen); setMorphValue("viseme_aa", visemeAa); setMorphValue("viseme_ee", visemeEe); setMorphValue("viseme_oh", visemeOh); setMorphValue("mouthPucker", mouthPucker); } // For plain head meshes (no VRM expressions and no morph targets), // amplify head motion to keep speaking visually obvious. const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09; const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12; const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04; applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied); if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) { applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4); } if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { const baseScale = 1.0 + jawOpen * 0.03; avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale); } if (avatarState.debugJaw) { avatarState.debugJaw.rotation.x = jawOpen * 0.5; avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015; } if (avatarState.debugMouth) { avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45; avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8; avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01; } } function setExpressionValue(name, value) { try { avatarState.currentVrm.expressionManager.setValue(name, value); } catch (_) { // no-op: expression name may not exist on the current model } } function setMorphValue(controlName, value) { const bindings = avatarState.morphBindings.get(controlName) || []; bindings.forEach((binding) => { binding.mesh.morphTargetInfluences[binding.index] = value; }); } function clampControl(value) { return Math.min(1, Math.max(0, Number(value) || 0)); } function clampSigned(value, limit) { return Math.min(limit, Math.max(-limit, Number(value) || 0)); } function resetView() { camera.position.set(0, 1.45, 2.65); controls.target.set(0, 1.15, 0); controls.update(); } function resetAvatarPose() { avatarState.activeControls = { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }; applyAnimationControls(avatarState.activeControls); } function detachCurrentAvatar() { if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { scene.remove(avatarState.currentRoot); } avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; avatarState.lastAppliedSeq = -1; avatarState.idleRig = null; avatarState.currentRoot = avatarState.debugRig; avatarState.currentVrm = null; avatarState.morphBindings.clear(); avatarState.headBone = avatarState.debugRig.children[1]; avatarState.neckBone = avatarState.debugRig.children[1]; avatarState.debugRig.visible = true; modelName.textContent = "debug-avatar"; } async function loadModelFromUrl(url) { if (!url) return false; modelStatus.textContent = "模型加载中..."; try { const gltf = await gltfLoader.loadAsync(url); attachLoadedAsset(gltf, url.split("/").pop() || url); modelStatus.textContent = "模型已加载"; return true; } catch (error) { console.error(error); modelStatus.textContent = "模型加载失败,已回退调试头像"; addMessage("system", `模型加载失败:${error?.message || error}`); detachCurrentAvatar(); return false; } } async function loadDefaultAvatarModel() { modelUrl.value = DEFAULT_BODY_MODEL_URL; const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL); if (!ok) { modelUrl.value = DEFAULT_VRM_FALLBACK_URL; await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL); } } async function loadModelFromFile(file) { if (!file) return; const url = URL.createObjectURL(file); modelStatus.textContent = "本地模型加载中..."; try { const gltf = await gltfLoader.loadAsync(url); attachLoadedAsset(gltf, file.name); modelStatus.textContent = "本地模型已加载"; } catch (error) { console.error(error); modelStatus.textContent = "本地模型加载失败,已回退调试头像"; addMessage("system", `本地模型加载失败:${error?.message || error}`); detachCurrentAvatar(); } finally { URL.revokeObjectURL(url); } } function attachLoadedAsset(gltf, label) { detachCurrentAvatar(); const vrm = gltf.userData?.vrm || null; const root = vrm?.scene || gltf.scene; if (!root) { throw new Error("模型中未找到可渲染的 scene"); } if (vrm) { VRMUtils.removeUnnecessaryVertices(gltf.scene); VRMUtils.combineSkeletons(gltf.scene); VRMUtils.combineMorphs(vrm); VRMUtils.rotateVRM0(vrm); } root.traverse((obj) => { obj.frustumCulled = false; }); root.position.set(0, 0, 0); root.rotation.set(0, 0, 0); root.scale.setScalar(1); scene.add(root); avatarState.currentRoot = root; avatarState.currentVrm = vrm; avatarState.debugRig.visible = false; avatarState.morphBindings = collectMorphBindings(root); avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1]; avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone; avatarState.idleRig = buildIdleRig(vrm, root); avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; avatarState.lastAppliedSeq = -1; fitCameraToObject(root); resetAvatarPose(); modelName.textContent = label; } function fitCameraToObject(root) { const box = new THREE.Box3().setFromObject(root); const center = new THREE.Vector3(); const size = new THREE.Vector3(); box.getCenter(center); box.getSize(size); const radius = Math.max(size.x, size.y, size.z, 1.2); controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0)); camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9); controls.update(); } function resolveHeadBone(vrm, root) { return ( vrm?.humanoid?.getNormalizedBoneNode?.("head") || vrm?.humanoid?.getRawBoneNode?.("head") || root.getObjectByName("Head") || root.getObjectByName("head") || root ); } function resolveNeckBone(vrm, root) { return ( vrm?.humanoid?.getNormalizedBoneNode?.("neck") || vrm?.humanoid?.getRawBoneNode?.("neck") || root.getObjectByName("Neck") || root.getObjectByName("neck") || resolveHeadBone(vrm, root) ); } function collectMorphBindings(root) { const bindings = new Map(); const aliases = { jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"], viseme_aa: ["viseme_aa", "aa", "a", "moutha"], viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"], viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"], mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"], }; root.traverse((node) => { if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return; const entries = Object.entries(node.morphTargetDictionary); Object.entries(aliases).forEach(([controlName, names]) => { entries.forEach(([targetName, index]) => { const normalized = targetName.toLowerCase(); if (!names.includes(normalized)) return; if (!bindings.has(controlName)) bindings.set(controlName, []); bindings.get(controlName).push({ mesh: node, index }); }); }); }); return bindings; } function buildIdleRig(vrm, root) { const lookup = { hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]), spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]), chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]), upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]), neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]), head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]), leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]), rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]), leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]), rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]), leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]), rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]), leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]), rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]), }; const base = new Map(); Object.values(lookup).forEach((bone) => { if (bone && !base.has(bone)) { base.set(bone, bone.quaternion.clone()); } }); return { bones: lookup, base }; } function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) { return ( vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) || vrm?.humanoid?.getRawBoneNode?.(humanoidName) || fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) || null ); } function applyProceduralIdlePose(time) { const rig = avatarState.idleRig; if (!rig) return; const breathe = Math.sin(time * 1.9) * 0.018; const sway = Math.sin(time * 1.15) * 0.05; const armDrift = Math.sin(time * 1.55) * 0.07; const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06; const handDrift = Math.sin(time * 2.3) * 0.04; applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04); applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06); applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08); applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1); applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18); applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18); applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0); applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0); applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12); applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12); applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05); applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05); } function applyBoneRotation(bone, x = 0, y = 0, z = 0) { if (!bone) return; const baseQuaternion = avatarState.idleRig?.base?.get(bone); if (baseQuaternion) { bone.quaternion.copy(baseQuaternion); TMP_EULER.set(x, y, z, "XYZ"); TMP_QUATERNION.setFromEuler(TMP_EULER); bone.quaternion.multiply(TMP_QUATERNION); return; } bone.rotation.x = x; bone.rotation.y = y; bone.rotation.z = z; } async function checkHealth() { const res = await fetch("/health"); const data = await res.json(); statusEl.textContent = `状态:${toCnState(data.state)}`; outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); } function subscribeEvents() { if (evt) return; evt = new EventSource("/events"); evt.onmessage = (event) => { try { const data = JSON.parse(event.data); statusEl.textContent = `状态:${toCnState(data.state)}`; outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); } catch (_) { // no-op } }; } function subscribeSubtitles() { if (subtitleWs && (subtitleWs.readyState === WebSocket.OPEN || subtitleWs.readyState === WebSocket.CONNECTING)) return; const scheme = location.protocol === "https:" ? "wss" : "ws"; subtitleWs = new WebSocket(`${scheme}://${location.host}/ws/subtitles`); subtitleWs.onmessage = (event) => { try { const data = JSON.parse(event.data); if (data.role === "user") { aiStreamingEl = null; addMessage(data.role, data.text || ""); } else if (data.role === "ai") { if (data.partial) { if (!aiStreamingEl) aiStreamingEl = addMessage("ai", ""); aiStreamingEl.textContent = `${aiStreamingEl.textContent}${data.text || ""}`; if (data.final) aiStreamingEl = null; } else { aiStreamingEl = null; addMessage("ai", data.text || ""); } } } catch (_) { // no-op } }; subtitleWs.onclose = () => { setTimeout(subscribeSubtitles, 1000); }; } function subscribeAnimation() { if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return; const scheme = location.protocol === "https:" ? "wss" : "ws"; animationConn.textContent = "连接中"; animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`); animationWs.onopen = () => { animationConn.textContent = "已连接"; }; animationWs.onmessage = (event) => { try { const data = JSON.parse(event.data); if (data.type === "ping") return; if (data.driver) { animationDriver.textContent = data.driver; } const frameCount = data.frame_count ?? data.frames?.length ?? 0; animationFrames.textContent = String(frameCount); const preview = data.frames?.[0]?.controls || null; animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2); queueAnimationFrames(data); } catch (_) { // no-op } }; animationWs.onclose = () => { animationConn.textContent = "已断开"; setTimeout(subscribeAnimation, 1000); }; } async function connectWebRTC() { if (pc) { outputEl.textContent = "WebRTC 已连接或正在连接"; return; } setConnBadge("连接中...", "warn"); offerBtn.disabled = true; const metaRes = await fetch("/meta"); const meta = await metaRes.json(); pc = new RTCPeerConnection({ iceServers: [{ urls: [meta.stun_url || "stun:stun.l.google.com:19302"] }], }); const currentPc = pc; let seenConnected = false; currentPc.onconnectionstatechange = () => { const st = currentPc.connectionState; if (st === "connected") { seenConnected = true; setConnBadge("已连接", "ok"); localStorage.setItem(AUTO_RECONNECT_KEY, "1"); } else if (st === "connecting") { setConnBadge("连接中...", "warn"); } else if (st === "failed" || st === "disconnected" || st === "closed") { setConnBadge(`连接${st}`, "warn"); if (seenConnected && (st === "disconnected" || st === "failed")) { addMessage("system", "连接已断开,可能被新的客户端连接接管。"); } if (localMicStream) { localMicStream.getTracks().forEach((track) => track.stop()); localMicStream = null; } if (pc === currentPc) { pc = null; } offerBtn.disabled = false; } }; pc.ontrack = (event) => { if (event.streams && event.streams[0]) { remoteAudio.srcObject = event.streams[0]; remoteAudio.play().catch(() => { addMessage("system", "远端语音已到达,如未自动播放请先与页面交互后重试。"); }); } }; let stream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); localMicStream = stream; } catch (err) { statusEl.textContent = "状态:麦克风权限失败"; outputEl.textContent = `getUserMedia 失败:${err?.message || err}\n提示:跨机器访问请使用 HTTPS 或浏览器安全例外。`; setConnBadge("麦克风失败", "warn"); offerBtn.disabled = false; throw err; } stream.getTracks().forEach((track) => pc.addTrack(track, stream)); const offer = await pc.createOffer(); await pc.setLocalDescription(offer); const res = await fetch("/webrtc/offer", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sdp: pc.localDescription.sdp, type: pc.localDescription.type, }), }); const answer = await res.json(); await pc.setRemoteDescription(answer); statusEl.textContent = "状态:WebRTC 已连接"; outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2); setConnBadge("已连接", "ok"); addMessage("system", "WebRTC 已连接,可开始对话。"); if (answer.replaced_previous) { addMessage("system", "检测到已有旧连接,已自动断开旧连接并由当前页面接管。"); } subscribeEvents(); } async function sendText() { const text = (chatInput.value || "").trim(); if (!text || sendBtn.disabled) return; sendBtn.disabled = true; const res = await fetch("/chat/text", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), }); const data = await res.json(); outputEl.textContent = JSON.stringify(data, null, 2); if (data.ok) { chatInput.value = ""; } else if (data.message) { addMessage("system", `发送失败:${data.message}`); } sendBtn.disabled = false; } async function resetChat() { await fetch("/chat/reset", { method: "POST" }); messagesEl.innerHTML = ""; avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS; resetAvatarPose(); addMessage("system", "会话已重置。"); await checkHealth(); } function tryAutoReconnect() { if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return; setTimeout(() => { if (!pc) { connectWebRTC().catch(() => { // no-op }); } }, 300); } function handleModelFileChange(event) { const [file] = event.target.files || []; if (!file) return; loadModelFromFile(file); } function handleModelUrlLoad() { const url = modelUrl.value.trim(); if (!url) return; loadModelFromUrl(url); } new ResizeObserver(() => resizeStage()).observe(avatarCanvas); window.addEventListener("resize", resizeStage); healthBtn.addEventListener("click", checkHealth); offerBtn.addEventListener("click", connectWebRTC); resetBtn.addEventListener("click", resetChat); sendBtn.addEventListener("click", sendText); loadModelBtn.addEventListener("click", handleModelUrlLoad); resetViewBtn.addEventListener("click", resetView); modelFile.addEventListener("change", handleModelFileChange); chatInput.addEventListener("keydown", (event) => { if (event.key === "Enter") { event.preventDefault(); sendText(); } }); modelUrl.addEventListener("keydown", (event) => { if (event.key === "Enter") { event.preventDefault(); handleModelUrlLoad(); } }); initStage(); resetView(); resetAvatarPose(); loadDefaultAvatarModel(); checkHealth(); subscribeEvents(); subscribeSubtitles(); subscribeAnimation(); setConnBadge("未连接", "neutral"); tryAutoReconnect();