import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"; const statusEl = document.getElementById("status"); const outputEl = document.getElementById("output"); const healthBtn = document.getElementById("healthBtn"); const offerBtn = document.getElementById("offerBtn"); const resetBtn = document.getElementById("resetBtn"); const sendBtn = document.getElementById("sendBtn"); const chatInput = document.getElementById("chatInput"); const remoteAudio = document.getElementById("remoteAudio"); const modelFile = document.getElementById("modelFile"); const modelUrl = document.getElementById("modelUrl"); const loadModelBtn = document.getElementById("loadModelBtn"); const resetViewBtn = document.getElementById("resetViewBtn"); const avatarCanvas = document.getElementById("avatarCanvas"); const modelStatus = document.getElementById("modelStatus"); const modelName = document.getElementById("modelName"); const animationBuffer = document.getElementById("animationBuffer"); remoteAudio.muted = false; remoteAudio.volume = 1.0; const messagesEl = document.getElementById("messages"); const connBadge = document.getElementById("connBadge"); const animationConn = document.getElementById("animationConn"); const animationDriver = document.getElementById("animationDriver"); const animationFrames = document.getElementById("animationFrames"); const animationPreview = document.getElementById("animationPreview"); const mPeers = document.getElementById("mPeers"); const mBusy = document.getElementById("mBusy"); const mLatency = document.getElementById("mLatency"); const mAsrLatency = document.getElementById("mAsrLatency"); const mLlmLatency = document.getElementById("mLlmLatency"); const mLlmSource = document.getElementById("mLlmSource"); const mLlmHistory = document.getElementById("mLlmHistory"); const mTtsLatency = document.getElementById("mTtsLatency"); const mTtsFirst = document.getElementById("mTtsFirst"); const mTts = document.getElementById("mTts"); const mBargeIn = document.getElementById("mBargeIn"); const mInputMode = document.getElementById("mInputMode"); const mVadCount = document.getElementById("mVadCount"); const mAnimClients = document.getElementById("mAnimClients"); const mAnimDriver = document.getElementById("mAnimDriver"); const mAnimFrames = document.getElementById("mAnimFrames"); const dConn = document.getElementById("dConn"); const dVad = document.getElementById("dVad"); const dAsr = document.getElementById("dAsr"); const dLlm = document.getElementById("dLlm"); const dTts = document.getElementById("dTts"); const OFFER_BTN_DEFAULT_TEXT = offerBtn?.textContent || "连接语音通道"; let evt = null; let subtitleWs = null; let animationWs = null; let audioWs = null; let localMicStream = null; let aiStreamingEl = null; let mediaAudioContext = null; let micSourceNode = null; let micProcessorNode = null; let micSinkGain = null; let audioPlaybackCursor = 0; const audioPlaybackSources = new Set(); const recentSubtitleEvents = new Map(); let reconnectOnGestureArmed = false; let pageGestureBootstrapArmed = false; const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect"; const ANIMATION_BUFFER_MS = 120; const AUDIO_PLAYBACK_BUFFER_MS = 25; const AV_SYNC_AUDIO_LEAD_MS = 60; const MOUTH_DRIVE_LIMITS = { expression: { aa: 0.58, ee: 0.40, oh: 0.48, ouFromOh: 0.16, pucker: 0.26, }, morph: { jawOpen: 0.42, visemeAa: 0.36, visemeEe: 0.34, visemeOh: 0.38, mouthPucker: 0.22, }, debugJaw: 0.52, }; const DEFAULT_BODY_MODEL_URL = (modelUrl?.value || "").trim() || "/web/models/VRM1_Constraint_Twist_Sample.vrm"; const CLOCK = new THREE.Clock(); const stateMap = { "SessionState.IDLE": "空闲", "SessionState.USER_SPEAKING": "用户说话", "SessionState.THINKING": "思考中", "SessionState.AVATAR_SPEAKING": "数字人说话", idle: "空闲", user_speaking: "用户说话", thinking: "思考中", avatar_speaking: "数字人说话", }; const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true }); renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); renderer.outputColorSpace = THREE.SRGBColorSpace; const scene = new THREE.Scene(); scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5); const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100); camera.position.set(0, 1.45, 2.65); const controls = new OrbitControls(camera, avatarCanvas); controls.target.set(0, 1.15, 0); controls.enableDamping = true; controls.minDistance = 1.3; controls.maxDistance = 4.5; controls.maxPolarAngle = Math.PI * 0.58; controls.minPolarAngle = Math.PI * 0.22; const avatarState = { currentRoot: null, currentVrm: null, morphBindings: new Map(), pendingFrames: [], scheduleCursorMs: performance.now(), lastAppliedSeq: -1, activeControls: { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }, headBone: null, neckBone: null, idleRig: null, debugRig: null, debugJaw: null, debugMouth: null, debugLeftEye: null, debugRightEye: null, blink: { value: 0, phase: "idle", phaseStartedAtMs: performance.now(), nextAtMs: performance.now() + 1800, closeMs: 90, holdMs: 50, openMs: 120, pendingDoubleBlink: false, }, }; const gltfLoader = new GLTFLoader(); gltfLoader.register((parser) => new VRMLoaderPlugin(parser)); gltfLoader.crossOrigin = "anonymous"; const TMP_EULER = new THREE.Euler(); const TMP_QUATERNION = new THREE.Quaternion(); function fmtTime() { return new Date().toLocaleTimeString("zh-CN", { hour12: false }); } function toCnState(raw) { return stateMap[raw] || raw; } function addMessage(role, text) { const row = document.createElement("div"); row.className = "msg-row"; const div = document.createElement("div"); div.className = `msg ${role}`; div.textContent = text; const time = document.createElement("div"); time.className = `msg-time ${role}`; time.textContent = fmtTime(); row.appendChild(div); row.appendChild(time); messagesEl.appendChild(row); messagesEl.scrollTop = messagesEl.scrollHeight; return div; } function shouldRenderSubtitleEvent(data) { const key = JSON.stringify({ role: data?.role || "", text: data?.text || "", partial: Boolean(data?.partial), final: Boolean(data?.final), source: data?.source || "", }); const now = Date.now(); const lastSeenAt = recentSubtitleEvents.get(key) || 0; recentSubtitleEvents.set(key, now); for (const [entryKey, seenAt] of recentSubtitleEvents.entries()) { if (now - seenAt > 1500) { recentSubtitleEvents.delete(entryKey); } } return now - lastSeenAt > 350; } function setConnBadge(label, kind = "neutral") { connBadge.textContent = label; connBadge.className = `badge ${kind}`; } function setOfferButtonState(mode = "default") { if (!offerBtn) return; if (mode === "resume") { offerBtn.textContent = "点击恢复语音"; return; } offerBtn.textContent = OFFER_BTN_DEFAULT_TEXT; } function setDot(el, kind) { el.className = `dot ${kind}`; } function syncConnBadgeFromState(health = null) { if (audioWs?.readyState === WebSocket.OPEN) { if (!health || (health.audio_clients ?? 0) > 0) { setConnBadge("已连接", "ok"); return; } } if (audioWs?.readyState === WebSocket.CONNECTING) { setConnBadge("连接中...", "warn"); return; } if (health && (health.audio_clients ?? 0) > 0) { setConnBadge("已连接", "ok"); return; } setConnBadge("未连接", "neutral"); } function disarmReconnectOnGesture() { if (!reconnectOnGestureArmed) return; window.removeEventListener("pointerdown", handleReconnectGesture, true); window.removeEventListener("keydown", handleReconnectGesture, true); reconnectOnGestureArmed = false; setOfferButtonState(); } function armReconnectOnGesture() { if (reconnectOnGestureArmed) return; reconnectOnGestureArmed = true; statusEl.textContent = "状态:等待恢复语音"; outputEl.textContent = "浏览器拦截了刷新后的自动语音恢复。点击页面任意位置,或点“连接语音通道”,即可恢复麦克风和语音播放。"; setConnBadge("等待点击恢复", "warn"); setOfferButtonState("resume"); window.addEventListener("pointerdown", handleReconnectGesture, true); window.addEventListener("keydown", handleReconnectGesture, true); } async function handleReconnectGesture() { try { await ensureVoiceReadyFromGesture(); } catch (error) { console.error(error); armReconnectOnGesture(); } } function disarmPageGestureBootstrap() { if (!pageGestureBootstrapArmed) return; window.removeEventListener("pointerdown", handlePageGestureBootstrap, true); window.removeEventListener("keydown", handlePageGestureBootstrap, true); pageGestureBootstrapArmed = false; if (!reconnectOnGestureArmed) { setOfferButtonState(); } } function armPageGestureBootstrap() { if (pageGestureBootstrapArmed) return; pageGestureBootstrapArmed = true; setOfferButtonState("resume"); window.addEventListener("pointerdown", handlePageGestureBootstrap, true); window.addEventListener("keydown", handlePageGestureBootstrap, true); } async function ensureVoiceReadyFromGesture() { await ensureMediaAudioContext(); if (audioWs?.readyState === WebSocket.CONNECTING) { return; } if (audioWs?.readyState === WebSocket.OPEN) { await startMicCapture(); disarmReconnectOnGesture(); disarmPageGestureBootstrap(); syncConnBadgeFromState(); return; } await connectAudio({ silentReadyMessage: true }); disarmReconnectOnGesture(); disarmPageGestureBootstrap(); } async function handlePageGestureBootstrap() { if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") { disarmPageGestureBootstrap(); return; } try { await ensureVoiceReadyFromGesture(); } catch (error) { console.error(error); } } function wsBaseUrl(path) { const scheme = location.protocol === "https:" ? "wss" : "ws"; return `${scheme}://${location.host}${path}`; } async function ensureMediaAudioContext() { const AudioContextClass = window.AudioContext || window.webkitAudioContext; if (!AudioContextClass) { throw new Error("当前浏览器不支持 Web Audio API"); } if (!mediaAudioContext) { mediaAudioContext = new AudioContextClass(); } if (mediaAudioContext.state === "suspended") { await mediaAudioContext.resume(); } if (!Number.isFinite(audioPlaybackCursor) || audioPlaybackCursor <= 0) { audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds(); } return mediaAudioContext; } function resetAudioPlayback() { if (mediaAudioContext) { audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds(); } Array.from(audioPlaybackSources).forEach((source) => { try { source.stop(); } catch (_) { // no-op } }); audioPlaybackSources.clear(); } async function queueServerAudio(arrayBuffer) { const context = await ensureMediaAudioContext(); const decoded = await context.decodeAudioData(arrayBuffer.slice(0)); const source = context.createBufferSource(); source.buffer = decoded; source.connect(context.destination); const startAt = Math.max(context.currentTime + getAudioPlaybackBufferSeconds(), audioPlaybackCursor); source.start(startAt); audioPlaybackCursor = startAt + decoded.duration; audioPlaybackSources.add(source); source.onended = () => { audioPlaybackSources.delete(source); }; } function stopMicCapture() { if (micSourceNode) { micSourceNode.disconnect(); micSourceNode = null; } if (micProcessorNode) { micProcessorNode.disconnect(); micProcessorNode.onaudioprocess = null; micProcessorNode = null; } if (micSinkGain) { micSinkGain.disconnect(); micSinkGain = null; } if (localMicStream) { localMicStream.getTracks().forEach((track) => track.stop()); localMicStream = null; } } async function startMicCapture() { const context = await ensureMediaAudioContext(); if (localMicStream && micSourceNode && micProcessorNode) { return context; } const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, }, video: false, }); localMicStream = stream; micSourceNode = context.createMediaStreamSource(stream); micProcessorNode = context.createScriptProcessor(4096, 1, 1); micSinkGain = context.createGain(); micSinkGain.gain.value = 0; micProcessorNode.onaudioprocess = (event) => { if (!audioWs || audioWs.readyState !== WebSocket.OPEN) { return; } const input = event.inputBuffer.getChannelData(0); const pcm = new Int16Array(input.length); for (let i = 0; i < input.length; i += 1) { const sample = Math.max(-1, Math.min(1, input[i] || 0)); pcm[i] = sample < 0 ? sample * 32768 : sample * 32767; } audioWs.send(pcm.buffer); }; micSourceNode.connect(micProcessorNode); micProcessorNode.connect(micSinkGain); micSinkGain.connect(context.destination); return context; } function initStage() { scene.background = new THREE.Color(0x0b1118); const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65); hemiLight.position.set(0, 2.6, 0); scene.add(hemiLight); const keyLight = new THREE.DirectionalLight(0xffffff, 1.7); keyLight.position.set(1.4, 2.2, 2.0); scene.add(keyLight); const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8); rimLight.position.set(-2.0, 1.4, -1.5); scene.add(rimLight); const floor = new THREE.Mesh( new THREE.CircleGeometry(2.8, 48), new THREE.MeshStandardMaterial({ color: 0x101b28, roughness: 0.94, metalness: 0.08, transparent: true, opacity: 0.9, }), ); floor.rotation.x = -Math.PI / 2; floor.position.y = -0.02; scene.add(floor); const halo = new THREE.Mesh( new THREE.RingGeometry(1.2, 1.95, 64), new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }), ); halo.rotation.x = -Math.PI / 2; halo.position.y = 0.01; scene.add(halo); createDebugRig(); resizeStage(); renderLoop(); } function createDebugRig() { const group = new THREE.Group(); const bustMaterial = new THREE.MeshStandardMaterial({ color: 0xc7d2e0, roughness: 0.72, metalness: 0.05, }); const accentMaterial = new THREE.MeshStandardMaterial({ color: 0x0f172a, roughness: 0.35, metalness: 0.18, }); const lipMaterial = new THREE.MeshStandardMaterial({ color: 0x8b425f, roughness: 0.56, metalness: 0.02, }); const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial); torso.position.set(0, 0.42, 0); group.add(torso); const neck = new THREE.Group(); neck.position.set(0, 0.95, 0); group.add(neck); const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial); head.scale.set(0.92, 1.03, 0.92); neck.add(head); const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial); hair.position.set(0, 0.06, 0); neck.add(hair); const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); leftEye.position.set(-0.1, 0.03, 0.28); neck.add(leftEye); const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial); rightEye.position.set(0.1, 0.03, 0.28); neck.add(rightEye); const jawPivot = new THREE.Group(); jawPivot.position.set(0, -0.06, 0.15); neck.add(jawPivot); const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial); jaw.position.set(0, -0.08, 0.02); jawPivot.add(jaw); const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial); mouth.position.set(0, -0.02, 0.31); neck.add(mouth); scene.add(group); avatarState.currentRoot = group; avatarState.debugRig = group; avatarState.debugJaw = jawPivot; avatarState.debugMouth = mouth; avatarState.debugLeftEye = leftEye; avatarState.debugRightEye = rightEye; avatarState.headBone = neck; avatarState.neckBone = neck; } function resizeStage() { const rect = avatarCanvas.getBoundingClientRect(); if (!rect.width || !rect.height) return; renderer.setSize(rect.width, rect.height, false); camera.aspect = rect.width / rect.height; camera.updateProjectionMatrix(); } function renderLoop() { requestAnimationFrame(renderLoop); const delta = CLOCK.getDelta(); const elapsed = CLOCK.getElapsedTime(); controls.update(); flushAnimationQueue(); updateBlinkState(performance.now()); applyProceduralIdlePose(elapsed); applyAnimationControls(avatarState.activeControls); avatarState.currentVrm?.update?.(delta); renderer.render(scene, camera); } function updateMetrics(data) { mPeers.textContent = String(data.audio_clients ?? data.peers ?? 0); mBusy.textContent = data.pipeline_busy ? "是" : "否"; mLatency.textContent = `${data.last_latency_ms ?? 0} ms`; mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`; mLlmLatency.textContent = `${data.last_llm_latency_ms ?? 0} ms`; mLlmSource.textContent = data.last_llm_source === "online" ? "在线" : "兜底"; mLlmHistory.textContent = String(data?.llm?.history_items ?? 0); mTtsLatency.textContent = `${data.last_tts_latency_ms ?? 0} ms`; mTtsFirst.textContent = `${data.last_tts_first_chunk_ms ?? 0} ms`; mTts.textContent = data?.tts?.ready ? "就绪" : "未就绪"; mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`; mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none"; mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`; mAnimClients.textContent = String(data.animation_clients ?? 0); mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--"; mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0); setDot(dConn, (data.audio_clients ?? data.peers ?? 0) > 0 ? "ok" : "warn"); setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn"); setDot(dAsr, data?.asr?.ready ? "ok" : "warn"); setDot(dLlm, data?.llm?.ready ? "ok" : "bad"); setDot(dTts, data?.tts?.ready ? "ok" : "warn"); syncConnBadgeFromState(data); } function queueAnimationFrames(payload) { const frames = Array.isArray(payload.frames) ? payload.frames : []; const now = performance.now(); const frameWindowMs = (1000 / Math.max(1, payload.fps || 25)); const animationLeadMs = getAnimationLeadMs(); if (payload.type === "animation_reset" || payload.type === "animation_state") { avatarState.pendingFrames = []; avatarState.scheduleCursorMs = now + animationLeadMs; avatarState.activeControls = { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }; } const baseTime = Math.max(now + animationLeadMs, avatarState.scheduleCursorMs); frames.forEach((frame, index) => { const fallbackTime = index * frameWindowMs; avatarState.pendingFrames.push({ dueAt: baseTime + (frame.time_ms ?? fallbackTime), seq: frame.seq ?? index, controls: frame.controls || {}, }); }); avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq); avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs); animationBuffer.textContent = String(avatarState.pendingFrames.length); } function flushAnimationQueue() { const now = performance.now(); let frameToApply = null; while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) { frameToApply = avatarState.pendingFrames.shift(); } if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) { avatarState.lastAppliedSeq = frameToApply.seq; avatarState.activeControls = { ...avatarState.activeControls, ...frameToApply.controls, }; } animationBuffer.textContent = String(avatarState.pendingFrames.length); } function applyAnimationControls(controls) { const safeControls = controls || {}; const jawOpen = clampControl(safeControls.jawOpen); const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen); const visemeEe = clampControl(safeControls.viseme_ee); const visemeOh = clampControl(safeControls.viseme_oh); const mouthPucker = clampControl(safeControls.mouthPucker); const headYaw = clampSigned(safeControls.headYaw, 0.35); const headPitch = clampSigned(safeControls.headPitch, 0.3); const headRoll = clampSigned(safeControls.headRoll, 0.24); const useExpressions = Boolean(avatarState.currentVrm?.expressionManager); const useMorphTargets = !useExpressions && avatarState.morphBindings.size > 0; const hasFacialRig = useExpressions || useMorphTargets; const mouthDrive = buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker); const t = performance.now() * 0.001; if (useExpressions) { setExpressionValue("aa", mouthDrive.expression.aa); setExpressionValue("ee", mouthDrive.expression.ee); setExpressionValue("oh", mouthDrive.expression.oh); setExpressionValue("ou", mouthDrive.expression.ou); } if (useMorphTargets) { setMorphValue("jawOpen", mouthDrive.morph.jawOpen); setMorphValue("viseme_aa", mouthDrive.morph.viseme_aa); setMorphValue("viseme_ee", mouthDrive.morph.viseme_ee); setMorphValue("viseme_oh", mouthDrive.morph.viseme_oh); setMorphValue("mouthPucker", mouthDrive.morph.mouthPucker); } applyBlinkControls(avatarState.blink?.value || 0); // For plain head meshes (no VRM expressions and no morph targets), // amplify head motion to keep speaking visually obvious. const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09; const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12; const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04; applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied); if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) { applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4); } if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { const baseScale = 1.0 + mouthDrive.debugJaw * 0.03; avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale); } if (avatarState.debugJaw) { avatarState.debugJaw.rotation.x = mouthDrive.debugJaw * 0.5; avatarState.debugJaw.position.y = -0.06 - mouthDrive.debugJaw * 0.015; } if (avatarState.debugMouth) { avatarState.debugMouth.scale.x = 1.0 + mouthDrive.morph.mouthPucker * 0.45; avatarState.debugMouth.scale.y = 1.0 + mouthDrive.debugJaw * 2.6; avatarState.debugMouth.position.z = 0.31 + mouthDrive.morph.mouthPucker * 0.01; } } function setExpressionValue(name, value) { try { avatarState.currentVrm.expressionManager.setValue(name, value); } catch (_) { // no-op: expression name may not exist on the current model } } function setMorphValue(controlName, value) { const bindings = avatarState.morphBindings.get(controlName) || []; bindings.forEach((binding) => { binding.mesh.morphTargetInfluences[binding.index] = value; }); } function buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker) { const mouthOpen = Math.max(jawOpen * 0.9, visemeAa); return { expression: { aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.expression.aa), ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.expression.ee), oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.expression.oh), ou: clampControl( Math.max( visemeOh * MOUTH_DRIVE_LIMITS.expression.ouFromOh, mouthPucker * MOUTH_DRIVE_LIMITS.expression.pucker ) ), }, morph: { jawOpen: clampControl(jawOpen * MOUTH_DRIVE_LIMITS.morph.jawOpen), viseme_aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.morph.visemeAa), viseme_ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.morph.visemeEe), viseme_oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.morph.visemeOh), mouthPucker: clampControl(mouthPucker * MOUTH_DRIVE_LIMITS.morph.mouthPucker), }, debugJaw: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.debugJaw), }; } function clampControl(value) { return Math.min(1, Math.max(0, Number(value) || 0)); } function getAudioPlaybackBufferSeconds() { return AUDIO_PLAYBACK_BUFFER_MS / 1000; } function getAnimationLeadMs() { return ANIMATION_BUFFER_MS + AV_SYNC_AUDIO_LEAD_MS; } function clampSigned(value, limit) { return Math.min(limit, Math.max(-limit, Number(value) || 0)); } function resetView() { camera.position.set(0, 1.45, 2.65); controls.target.set(0, 1.15, 0); controls.update(); } function resetAvatarPose() { avatarState.activeControls = { jawOpen: 0, viseme_aa: 0, viseme_ee: 0, viseme_oh: 0, mouthPucker: 0, headYaw: 0, headPitch: 0, headRoll: 0, }; applyAnimationControls(avatarState.activeControls); } function detachCurrentAvatar() { if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) { scene.remove(avatarState.currentRoot); } avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs(); avatarState.lastAppliedSeq = -1; avatarState.idleRig = null; avatarState.currentRoot = avatarState.debugRig; avatarState.currentVrm = null; avatarState.morphBindings.clear(); avatarState.headBone = avatarState.debugRig.children[1]; avatarState.neckBone = avatarState.debugRig.children[1]; avatarState.debugRig.visible = true; avatarState.debugLeftEye = avatarState.debugLeftEye || avatarState.debugRig.children[1]?.children?.[2] || null; avatarState.debugRightEye = avatarState.debugRightEye || avatarState.debugRig.children[1]?.children?.[3] || null; modelName.textContent = "debug-avatar"; } async function loadModelFromUrl(url) { if (!url) return false; modelStatus.textContent = "模型加载中..."; try { const gltf = await gltfLoader.loadAsync(url); attachLoadedAsset(gltf, url.split("/").pop() || url); modelStatus.textContent = "模型已加载"; return true; } catch (error) { console.error(error); modelStatus.textContent = "模型加载失败,已回退调试头像"; addMessage("system", `模型加载失败:${error?.message || error}`); detachCurrentAvatar(); return false; } } async function loadDefaultAvatarModel() { modelUrl.value = DEFAULT_BODY_MODEL_URL; await loadModelFromUrl(DEFAULT_BODY_MODEL_URL); } async function loadModelFromFile(file) { if (!file) return; const url = URL.createObjectURL(file); modelStatus.textContent = "本地模型加载中..."; try { const gltf = await gltfLoader.loadAsync(url); attachLoadedAsset(gltf, file.name); modelStatus.textContent = "本地模型已加载"; } catch (error) { console.error(error); modelStatus.textContent = "本地模型加载失败,已回退调试头像"; addMessage("system", `本地模型加载失败:${error?.message || error}`); detachCurrentAvatar(); } finally { URL.revokeObjectURL(url); } } function attachLoadedAsset(gltf, label) { detachCurrentAvatar(); const vrm = gltf.userData?.vrm || null; const root = vrm?.scene || gltf.scene; if (!root) { throw new Error("模型中未找到可渲染的 scene"); } if (vrm) { VRMUtils.removeUnnecessaryVertices(gltf.scene); VRMUtils.combineSkeletons(gltf.scene); VRMUtils.combineMorphs(vrm); VRMUtils.rotateVRM0(vrm); } root.traverse((obj) => { obj.frustumCulled = false; }); root.position.set(0, 0, 0); root.rotation.set(0, 0, 0); root.scale.setScalar(1); scene.add(root); avatarState.currentRoot = root; avatarState.currentVrm = vrm; avatarState.debugRig.visible = false; avatarState.morphBindings = collectMorphBindings(root); avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1]; avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone; avatarState.idleRig = buildIdleRig(vrm, root); avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs(); avatarState.lastAppliedSeq = -1; fitCameraToObject(root); resetAvatarPose(); modelName.textContent = label; } function fitCameraToObject(root) { const box = new THREE.Box3().setFromObject(root); const center = new THREE.Vector3(); const size = new THREE.Vector3(); box.getCenter(center); box.getSize(size); const radius = Math.max(size.x, size.y, size.z, 1.2); controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0)); camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9); controls.update(); } function resolveHeadBone(vrm, root) { return ( vrm?.humanoid?.getNormalizedBoneNode?.("head") || vrm?.humanoid?.getRawBoneNode?.("head") || root.getObjectByName("Head") || root.getObjectByName("head") || root ); } function resolveNeckBone(vrm, root) { return ( vrm?.humanoid?.getNormalizedBoneNode?.("neck") || vrm?.humanoid?.getRawBoneNode?.("neck") || root.getObjectByName("Neck") || root.getObjectByName("neck") || resolveHeadBone(vrm, root) ); } function collectMorphBindings(root) { const bindings = new Map(); const aliases = { jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open"], viseme_aa: ["viseme_aa", "aa", "a", "moutha"], viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"], viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"], mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"], blink: ["blink", "eyeblink", "eye_blink", "blinkboth"], blinkLeft: ["blinkleft", "blink_left", "eyeblinkleft", "eyelid_l", "eyeblink_l"], blinkRight: ["blinkright", "blink_right", "eyeblinkright", "eyelid_r", "eyeblink_r"], }; root.traverse((node) => { if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return; const entries = Object.entries(node.morphTargetDictionary); Object.entries(aliases).forEach(([controlName, names]) => { entries.forEach(([targetName, index]) => { const normalized = targetName.toLowerCase(); if (!names.includes(normalized)) return; if (!bindings.has(controlName)) bindings.set(controlName, []); bindings.get(controlName).push({ mesh: node, index }); }); }); }); return bindings; } function buildIdleRig(vrm, root) { const lookup = { hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]), spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]), chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]), upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]), neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]), head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]), leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]), rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]), leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]), rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]), leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]), rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]), leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]), rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]), }; const base = new Map(); Object.values(lookup).forEach((bone) => { if (bone && !base.has(bone)) { base.set(bone, bone.quaternion.clone()); } }); return { bones: lookup, base }; } function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) { return ( vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) || vrm?.humanoid?.getRawBoneNode?.(humanoidName) || fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) || null ); } function applyProceduralIdlePose(time) { const rig = avatarState.idleRig; if (!rig) return; const breathe = Math.sin(time * 1.9) * 0.018; const sway = Math.sin(time * 1.15) * 0.05; const armDrift = Math.sin(time * 1.55) * 0.07; const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06; const handDrift = Math.sin(time * 2.3) * 0.04; applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04); applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06); applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08); applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1); applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18); applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18); applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0); applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0); applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12); applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12); applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05); applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05); } function scheduleNextBlink(nowMs, preferSoon = false) { const quickBlink = Math.random() < 0.18; const minDelay = preferSoon ? 240 : 1800; const maxDelay = preferSoon ? 520 : 5200; avatarState.blink.nextAtMs = nowMs + minDelay + Math.random() * (maxDelay - minDelay); avatarState.blink.pendingDoubleBlink = !preferSoon && quickBlink; } function updateBlinkState(nowMs) { const blink = avatarState.blink; if (!blink) return; if (blink.phase === "idle") { blink.value = 0; if (nowMs >= blink.nextAtMs) { blink.phase = "closing"; blink.phaseStartedAtMs = nowMs; } return; } if (blink.phase === "closing") { const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.closeMs); blink.value = progress; if (progress >= 1) { blink.phase = "hold"; blink.phaseStartedAtMs = nowMs; blink.value = 1; } return; } if (blink.phase === "hold") { blink.value = 1; if (nowMs - blink.phaseStartedAtMs >= blink.holdMs) { blink.phase = "opening"; blink.phaseStartedAtMs = nowMs; } return; } if (blink.phase === "opening") { const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.openMs); blink.value = 1 - progress; if (progress >= 1) { blink.phase = "idle"; blink.phaseStartedAtMs = nowMs; blink.value = 0; scheduleNextBlink(nowMs, blink.pendingDoubleBlink); } } } function applyBlinkControls(blinkValue) { const clamped = clampControl(blinkValue); if (avatarState.currentVrm?.expressionManager) { setExpressionValue("blink", clamped); setExpressionValue("blinkLeft", clamped); setExpressionValue("blinkRight", clamped); } setMorphValue("blink", clamped); setMorphValue("blinkLeft", clamped); setMorphValue("blinkRight", clamped); const eyeScaleY = Math.max(0.08, 1 - clamped * 0.92); const eyeScaleZ = 1 + clamped * 0.35; if (avatarState.debugLeftEye) { avatarState.debugLeftEye.scale.set(1, eyeScaleY, eyeScaleZ); } if (avatarState.debugRightEye) { avatarState.debugRightEye.scale.set(1, eyeScaleY, eyeScaleZ); } } function applyBoneRotation(bone, x = 0, y = 0, z = 0) { if (!bone) return; const baseQuaternion = avatarState.idleRig?.base?.get(bone); if (baseQuaternion) { bone.quaternion.copy(baseQuaternion); TMP_EULER.set(x, y, z, "XYZ"); TMP_QUATERNION.setFromEuler(TMP_EULER); bone.quaternion.multiply(TMP_QUATERNION); return; } bone.rotation.x = x; bone.rotation.y = y; bone.rotation.z = z; } async function checkHealth() { const res = await fetch("/health"); const data = await res.json(); statusEl.textContent = `状态:${toCnState(data.state)}`; outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); } function subscribeEvents() { if (evt) return; evt = new EventSource("/events"); evt.onmessage = (event) => { try { const data = JSON.parse(event.data); statusEl.textContent = `状态:${toCnState(data.state)}`; outputEl.textContent = JSON.stringify(data, null, 2); sendBtn.disabled = !!data.pipeline_busy; updateMetrics(data); } catch (_) { // no-op } }; } function subscribeSubtitles() { if (subtitleWs && (subtitleWs.readyState === WebSocket.OPEN || subtitleWs.readyState === WebSocket.CONNECTING)) return; subtitleWs = new WebSocket(wsBaseUrl("/ws/subtitles")); subtitleWs.onmessage = (event) => { try { const data = JSON.parse(event.data); if (!shouldRenderSubtitleEvent(data)) { return; } if (data.role === "user") { aiStreamingEl = null; addMessage(data.role, data.text || ""); } else if (data.role === "ai") { if (data.partial) { if (!aiStreamingEl) aiStreamingEl = addMessage("ai", ""); aiStreamingEl.textContent = `${aiStreamingEl.textContent}${data.text || ""}`; if (data.final) aiStreamingEl = null; } else { aiStreamingEl = null; addMessage("ai", data.text || ""); } } } catch (_) { // no-op } }; subtitleWs.onclose = (event) => { if (subtitleWs && subtitleWs.readyState === WebSocket.CLOSED) { subtitleWs = null; } if (event.code === 1012) { addMessage("system", "字幕通道已被新客户端接管。当前页面停止接收字幕。"); return; } setTimeout(subscribeSubtitles, 1000); }; } function subscribeAnimation() { if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return; animationConn.textContent = "连接中"; animationWs = new WebSocket(wsBaseUrl("/ws/animation")); animationWs.onopen = () => { animationConn.textContent = "已连接"; }; animationWs.onmessage = (event) => { try { const data = JSON.parse(event.data); if (data.type === "ping") return; if (data.driver) { animationDriver.textContent = data.driver; } const frameCount = data.frame_count ?? data.frames?.length ?? 0; animationFrames.textContent = String(frameCount); const preview = data.frames?.[0]?.controls || null; animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2); queueAnimationFrames(data); } catch (_) { // no-op } }; animationWs.onclose = (event) => { if (animationWs && animationWs.readyState === WebSocket.CLOSED) { animationWs = null; } animationConn.textContent = "已断开"; if (event.code === 1012) { animationConn.textContent = "已被新客户端接管"; return; } setTimeout(subscribeAnimation, 1000); }; } async function connectAudio(options = {}) { const { silentReadyMessage = false } = options; if (audioWs && audioWs.readyState === WebSocket.OPEN) { disarmReconnectOnGesture(); outputEl.textContent = "语音通道已连接"; return; } if (audioWs && audioWs.readyState === WebSocket.CONNECTING) { outputEl.textContent = "语音通道正在连接"; await new Promise((resolve, reject) => { const startedAt = Date.now(); const timer = window.setInterval(() => { if (!audioWs) { window.clearInterval(timer); reject(new Error("语音通道不存在")); return; } if (audioWs.readyState === WebSocket.OPEN) { window.clearInterval(timer); resolve(); return; } if (audioWs.readyState === WebSocket.CLOSING || audioWs.readyState === WebSocket.CLOSED) { window.clearInterval(timer); reject(new Error("语音通道连接已关闭")); return; } if (Date.now() - startedAt > 5000) { window.clearInterval(timer); reject(new Error("语音通道连接超时")); } }, 100); }); return; } setConnBadge("连接中...", "warn"); offerBtn.disabled = true; let context; try { context = await startMicCapture(); } catch (err) { statusEl.textContent = "状态:麦克风权限失败"; outputEl.textContent = `getUserMedia 失败:${err?.message || err}\n提示:跨机器访问请使用 HTTPS 或浏览器安全例外。`; setConnBadge("麦克风失败", "warn"); offerBtn.disabled = false; throw err; } const ws = new WebSocket(wsBaseUrl("/ws/audio")); ws.binaryType = "arraybuffer"; audioWs = ws; await new Promise((resolve, reject) => { let settled = false; ws.onopen = () => { if (audioWs !== ws) { try { ws.close(1000, "stale-open"); } catch (_) { // no-op } return; } ws.send( JSON.stringify({ type: "audio_config", sample_rate: context.sampleRate, channels: 1, format: "pcm_s16le", }), ); statusEl.textContent = "状态:语音通道已连接"; outputEl.textContent = JSON.stringify({ connected: true, transport: "websocket_audio" }, null, 2); setConnBadge("已连接", "ok"); localStorage.setItem(AUTO_RECONNECT_KEY, "1"); disarmReconnectOnGesture(); offerBtn.disabled = false; if (!silentReadyMessage) { addMessage("system", "语音通道已连接,可开始对话。"); } subscribeEvents(); settled = true; resolve(); }; ws.onmessage = (event) => { if (typeof event.data === "string") { try { const data = JSON.parse(event.data); if (data.type === "ping" || data.type === "pong" || data.type === "audio_ready") { return; } if (data.type === "audio_reset") { resetAudioPlayback(); return; } } catch (_) { // no-op } return; } queueServerAudio(event.data).catch((error) => { console.error(error); addMessage("system", `语音播放失败:${error?.message || error}`); }); }; ws.onerror = (event) => { console.error(event); if (!settled) { settled = true; offerBtn.disabled = false; setConnBadge("连接失败", "warn"); reject(new Error("语音通道连接失败")); } }; ws.onclose = (event) => { if (audioWs !== ws) { if (!settled) { settled = true; resolve(); } return; } resetAudioPlayback(); stopMicCapture(); if (audioWs === ws) { audioWs = null; } syncConnBadgeFromState(); offerBtn.disabled = false; if (event.code !== 1000 && event.code !== 1001 && event.code !== 1012) { addMessage("system", "语音通道已断开,请重新连接。"); } if (!settled) { settled = true; reject(new Error("语音通道在建立前已关闭")); } }; }); } async function sendText() { const text = (chatInput.value || "").trim(); if (!text || sendBtn.disabled) return; if (!audioWs || audioWs.readyState !== WebSocket.OPEN) { try { await connectAudio({ silentReadyMessage: true }); } catch (_) { addMessage("system", "语音通道未连接,本次仅保证文字链路可用。"); } } sendBtn.disabled = true; const res = await fetch("/chat/text", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), }); const data = await res.json(); outputEl.textContent = JSON.stringify(data, null, 2); if (data.ok) { chatInput.value = ""; } else if (data.message) { addMessage("system", `发送失败:${data.message}`); } sendBtn.disabled = false; } async function resetChat() { await fetch("/chat/reset", { method: "POST" }); messagesEl.innerHTML = ""; avatarState.pendingFrames = []; avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs(); resetAudioPlayback(); resetAvatarPose(); addMessage("system", "会话已重置。"); await checkHealth(); } function tryAutoReconnect() { if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return; armPageGestureBootstrap(); setTimeout(() => { if (!audioWs) { connectAudio({ silentReadyMessage: true }).catch(() => { armReconnectOnGesture(); }); } }, 300); } function handleModelFileChange(event) { const [file] = event.target.files || []; if (!file) return; loadModelFromFile(file); } function handleModelUrlLoad() { const url = modelUrl.value.trim(); if (!url) return; loadModelFromUrl(url); } new ResizeObserver(() => resizeStage()).observe(avatarCanvas); window.addEventListener("resize", resizeStage); healthBtn.addEventListener("click", checkHealth); offerBtn.addEventListener("click", () => { connectAudio().catch((error) => { console.error(error); }); }); resetBtn.addEventListener("click", resetChat); sendBtn.addEventListener("click", sendText); loadModelBtn.addEventListener("click", handleModelUrlLoad); resetViewBtn.addEventListener("click", resetView); modelFile.addEventListener("change", handleModelFileChange); chatInput.addEventListener("keydown", (event) => { if (event.key === "Enter") { event.preventDefault(); sendText(); } }); modelUrl.addEventListener("keydown", (event) => { if (event.key === "Enter") { event.preventDefault(); handleModelUrlLoad(); } }); initStage(); resetView(); resetAvatarPose(); loadDefaultAvatarModel(); checkHealth(); subscribeEvents(); subscribeSubtitles(); subscribeAnimation(); setConnBadge("未连接", "neutral"); tryAutoReconnect();