1447 lines
46 KiB
JavaScript
1447 lines
46 KiB
JavaScript
import * as THREE from "three";
|
|
import { OrbitControls } from "three/addons/controls/OrbitControls.js";
|
|
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
|
|
import { VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm";
|
|
|
|
const statusEl = document.getElementById("status");
|
|
const outputEl = document.getElementById("output");
|
|
const healthBtn = document.getElementById("healthBtn");
|
|
const offerBtn = document.getElementById("offerBtn");
|
|
const resetBtn = document.getElementById("resetBtn");
|
|
const sendBtn = document.getElementById("sendBtn");
|
|
const chatInput = document.getElementById("chatInput");
|
|
const remoteAudio = document.getElementById("remoteAudio");
|
|
const modelFile = document.getElementById("modelFile");
|
|
const modelUrl = document.getElementById("modelUrl");
|
|
const loadModelBtn = document.getElementById("loadModelBtn");
|
|
const resetViewBtn = document.getElementById("resetViewBtn");
|
|
const avatarCanvas = document.getElementById("avatarCanvas");
|
|
const modelStatus = document.getElementById("modelStatus");
|
|
const modelName = document.getElementById("modelName");
|
|
const animationBuffer = document.getElementById("animationBuffer");
|
|
|
|
remoteAudio.muted = false;
|
|
remoteAudio.volume = 1.0;
|
|
|
|
const messagesEl = document.getElementById("messages");
|
|
const connBadge = document.getElementById("connBadge");
|
|
const animationConn = document.getElementById("animationConn");
|
|
const animationDriver = document.getElementById("animationDriver");
|
|
const animationFrames = document.getElementById("animationFrames");
|
|
const animationPreview = document.getElementById("animationPreview");
|
|
const mPeers = document.getElementById("mPeers");
|
|
const mBusy = document.getElementById("mBusy");
|
|
const mLatency = document.getElementById("mLatency");
|
|
const mAsrLatency = document.getElementById("mAsrLatency");
|
|
const mLlmLatency = document.getElementById("mLlmLatency");
|
|
const mLlmSource = document.getElementById("mLlmSource");
|
|
const mLlmHistory = document.getElementById("mLlmHistory");
|
|
const mTtsLatency = document.getElementById("mTtsLatency");
|
|
const mTtsFirst = document.getElementById("mTtsFirst");
|
|
const mTts = document.getElementById("mTts");
|
|
const mBargeIn = document.getElementById("mBargeIn");
|
|
const mInputMode = document.getElementById("mInputMode");
|
|
const mVadCount = document.getElementById("mVadCount");
|
|
const mAnimClients = document.getElementById("mAnimClients");
|
|
const mAnimDriver = document.getElementById("mAnimDriver");
|
|
const mAnimFrames = document.getElementById("mAnimFrames");
|
|
const dConn = document.getElementById("dConn");
|
|
const dVad = document.getElementById("dVad");
|
|
const dAsr = document.getElementById("dAsr");
|
|
const dLlm = document.getElementById("dLlm");
|
|
const dTts = document.getElementById("dTts");
|
|
const OFFER_BTN_DEFAULT_TEXT = offerBtn?.textContent || "连接语音通道";
|
|
|
|
let evt = null;
|
|
let subtitleWs = null;
|
|
let animationWs = null;
|
|
let audioWs = null;
|
|
let localMicStream = null;
|
|
let aiStreamingEl = null;
|
|
let mediaAudioContext = null;
|
|
let micSourceNode = null;
|
|
let micProcessorNode = null;
|
|
let micSinkGain = null;
|
|
let audioPlaybackCursor = 0;
|
|
const audioPlaybackSources = new Set();
|
|
const recentSubtitleEvents = new Map();
|
|
let reconnectOnGestureArmed = false;
|
|
let pageGestureBootstrapArmed = false;
|
|
|
|
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
|
|
const ANIMATION_BUFFER_MS = 120;
|
|
const AUDIO_PLAYBACK_BUFFER_MS = 25;
|
|
const AV_SYNC_AUDIO_LEAD_MS = 60;
|
|
const MOUTH_DRIVE_LIMITS = {
|
|
expression: {
|
|
aa: 0.58,
|
|
ee: 0.40,
|
|
oh: 0.48,
|
|
ouFromOh: 0.16,
|
|
pucker: 0.26,
|
|
},
|
|
morph: {
|
|
jawOpen: 0.42,
|
|
visemeAa: 0.36,
|
|
visemeEe: 0.34,
|
|
visemeOh: 0.38,
|
|
mouthPucker: 0.22,
|
|
},
|
|
debugJaw: 0.52,
|
|
};
|
|
const DEFAULT_BODY_MODEL_URL =
|
|
(modelUrl?.value || "").trim() ||
|
|
"/web/models/VRM1_Constraint_Twist_Sample.vrm";
|
|
const CLOCK = new THREE.Clock();
|
|
|
|
const stateMap = {
|
|
"SessionState.IDLE": "空闲",
|
|
"SessionState.USER_SPEAKING": "用户说话",
|
|
"SessionState.THINKING": "思考中",
|
|
"SessionState.AVATAR_SPEAKING": "数字人说话",
|
|
idle: "空闲",
|
|
user_speaking: "用户说话",
|
|
thinking: "思考中",
|
|
avatar_speaking: "数字人说话",
|
|
};
|
|
|
|
const renderer = new THREE.WebGLRenderer({ canvas: avatarCanvas, antialias: true, alpha: true });
|
|
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
|
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
|
|
|
const scene = new THREE.Scene();
|
|
scene.fog = new THREE.Fog(0x0a1017, 5.5, 11.5);
|
|
|
|
const camera = new THREE.PerspectiveCamera(30, 16 / 9, 0.1, 100);
|
|
camera.position.set(0, 1.45, 2.65);
|
|
|
|
const controls = new OrbitControls(camera, avatarCanvas);
|
|
controls.target.set(0, 1.15, 0);
|
|
controls.enableDamping = true;
|
|
controls.minDistance = 1.3;
|
|
controls.maxDistance = 4.5;
|
|
controls.maxPolarAngle = Math.PI * 0.58;
|
|
controls.minPolarAngle = Math.PI * 0.22;
|
|
|
|
const avatarState = {
|
|
currentRoot: null,
|
|
currentVrm: null,
|
|
morphBindings: new Map(),
|
|
pendingFrames: [],
|
|
scheduleCursorMs: performance.now(),
|
|
lastAppliedSeq: -1,
|
|
activeControls: {
|
|
jawOpen: 0,
|
|
viseme_aa: 0,
|
|
viseme_ee: 0,
|
|
viseme_oh: 0,
|
|
mouthPucker: 0,
|
|
headYaw: 0,
|
|
headPitch: 0,
|
|
headRoll: 0,
|
|
},
|
|
headBone: null,
|
|
neckBone: null,
|
|
idleRig: null,
|
|
debugRig: null,
|
|
debugJaw: null,
|
|
debugMouth: null,
|
|
debugLeftEye: null,
|
|
debugRightEye: null,
|
|
blink: {
|
|
value: 0,
|
|
phase: "idle",
|
|
phaseStartedAtMs: performance.now(),
|
|
nextAtMs: performance.now() + 1800,
|
|
closeMs: 90,
|
|
holdMs: 50,
|
|
openMs: 120,
|
|
pendingDoubleBlink: false,
|
|
},
|
|
};
|
|
|
|
const gltfLoader = new GLTFLoader();
|
|
gltfLoader.register((parser) => new VRMLoaderPlugin(parser));
|
|
gltfLoader.crossOrigin = "anonymous";
|
|
|
|
const TMP_EULER = new THREE.Euler();
|
|
const TMP_QUATERNION = new THREE.Quaternion();
|
|
|
|
function fmtTime() {
|
|
return new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
}
|
|
|
|
function toCnState(raw) {
|
|
return stateMap[raw] || raw;
|
|
}
|
|
|
|
function addMessage(role, text) {
|
|
const row = document.createElement("div");
|
|
row.className = "msg-row";
|
|
const div = document.createElement("div");
|
|
div.className = `msg ${role}`;
|
|
div.textContent = text;
|
|
const time = document.createElement("div");
|
|
time.className = `msg-time ${role}`;
|
|
time.textContent = fmtTime();
|
|
row.appendChild(div);
|
|
row.appendChild(time);
|
|
messagesEl.appendChild(row);
|
|
messagesEl.scrollTop = messagesEl.scrollHeight;
|
|
return div;
|
|
}
|
|
|
|
function shouldRenderSubtitleEvent(data) {
|
|
const key = JSON.stringify({
|
|
role: data?.role || "",
|
|
text: data?.text || "",
|
|
partial: Boolean(data?.partial),
|
|
final: Boolean(data?.final),
|
|
source: data?.source || "",
|
|
});
|
|
const now = Date.now();
|
|
const lastSeenAt = recentSubtitleEvents.get(key) || 0;
|
|
recentSubtitleEvents.set(key, now);
|
|
|
|
for (const [entryKey, seenAt] of recentSubtitleEvents.entries()) {
|
|
if (now - seenAt > 1500) {
|
|
recentSubtitleEvents.delete(entryKey);
|
|
}
|
|
}
|
|
|
|
return now - lastSeenAt > 350;
|
|
}
|
|
|
|
function setConnBadge(label, kind = "neutral") {
|
|
connBadge.textContent = label;
|
|
connBadge.className = `badge ${kind}`;
|
|
}
|
|
|
|
function setOfferButtonState(mode = "default") {
|
|
if (!offerBtn) return;
|
|
if (mode === "resume") {
|
|
offerBtn.textContent = "点击恢复语音";
|
|
return;
|
|
}
|
|
offerBtn.textContent = OFFER_BTN_DEFAULT_TEXT;
|
|
}
|
|
|
|
function setDot(el, kind) {
|
|
el.className = `dot ${kind}`;
|
|
}
|
|
|
|
function syncConnBadgeFromState(health = null) {
|
|
if (audioWs?.readyState === WebSocket.OPEN) {
|
|
if (!health || (health.audio_clients ?? 0) > 0) {
|
|
setConnBadge("已连接", "ok");
|
|
return;
|
|
}
|
|
}
|
|
if (audioWs?.readyState === WebSocket.CONNECTING) {
|
|
setConnBadge("连接中...", "warn");
|
|
return;
|
|
}
|
|
if (health && (health.audio_clients ?? 0) > 0) {
|
|
setConnBadge("已连接", "ok");
|
|
return;
|
|
}
|
|
setConnBadge("未连接", "neutral");
|
|
}
|
|
|
|
function disarmReconnectOnGesture() {
|
|
if (!reconnectOnGestureArmed) return;
|
|
window.removeEventListener("pointerdown", handleReconnectGesture, true);
|
|
window.removeEventListener("keydown", handleReconnectGesture, true);
|
|
reconnectOnGestureArmed = false;
|
|
setOfferButtonState();
|
|
}
|
|
|
|
function armReconnectOnGesture() {
|
|
if (reconnectOnGestureArmed) return;
|
|
reconnectOnGestureArmed = true;
|
|
statusEl.textContent = "状态：等待恢复语音";
|
|
outputEl.textContent = "浏览器拦截了刷新后的自动语音恢复。点击页面任意位置，或点“连接语音通道”，即可恢复麦克风和语音播放。";
|
|
setConnBadge("等待点击恢复", "warn");
|
|
setOfferButtonState("resume");
|
|
window.addEventListener("pointerdown", handleReconnectGesture, true);
|
|
window.addEventListener("keydown", handleReconnectGesture, true);
|
|
}
|
|
|
|
async function handleReconnectGesture() {
|
|
try {
|
|
await ensureVoiceReadyFromGesture();
|
|
} catch (error) {
|
|
console.error(error);
|
|
armReconnectOnGesture();
|
|
}
|
|
}
|
|
|
|
function disarmPageGestureBootstrap() {
|
|
if (!pageGestureBootstrapArmed) return;
|
|
window.removeEventListener("pointerdown", handlePageGestureBootstrap, true);
|
|
window.removeEventListener("keydown", handlePageGestureBootstrap, true);
|
|
pageGestureBootstrapArmed = false;
|
|
if (!reconnectOnGestureArmed) {
|
|
setOfferButtonState();
|
|
}
|
|
}
|
|
|
|
function armPageGestureBootstrap() {
|
|
if (pageGestureBootstrapArmed) return;
|
|
pageGestureBootstrapArmed = true;
|
|
setOfferButtonState("resume");
|
|
window.addEventListener("pointerdown", handlePageGestureBootstrap, true);
|
|
window.addEventListener("keydown", handlePageGestureBootstrap, true);
|
|
}
|
|
|
|
async function ensureVoiceReadyFromGesture() {
|
|
await ensureMediaAudioContext();
|
|
|
|
if (audioWs?.readyState === WebSocket.CONNECTING) {
|
|
return;
|
|
}
|
|
|
|
if (audioWs?.readyState === WebSocket.OPEN) {
|
|
await startMicCapture();
|
|
disarmReconnectOnGesture();
|
|
disarmPageGestureBootstrap();
|
|
syncConnBadgeFromState();
|
|
return;
|
|
}
|
|
|
|
await connectAudio({ silentReadyMessage: true });
|
|
disarmReconnectOnGesture();
|
|
disarmPageGestureBootstrap();
|
|
}
|
|
|
|
async function handlePageGestureBootstrap() {
|
|
if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") {
|
|
disarmPageGestureBootstrap();
|
|
return;
|
|
}
|
|
try {
|
|
await ensureVoiceReadyFromGesture();
|
|
} catch (error) {
|
|
console.error(error);
|
|
}
|
|
}
|
|
|
|
function wsBaseUrl(path) {
|
|
const scheme = location.protocol === "https:" ? "wss" : "ws";
|
|
return `${scheme}://${location.host}${path}`;
|
|
}
|
|
|
|
async function ensureMediaAudioContext() {
|
|
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
|
|
if (!AudioContextClass) {
|
|
throw new Error("当前浏览器不支持 Web Audio API");
|
|
}
|
|
if (!mediaAudioContext) {
|
|
mediaAudioContext = new AudioContextClass();
|
|
}
|
|
if (mediaAudioContext.state === "suspended") {
|
|
await mediaAudioContext.resume();
|
|
}
|
|
if (!Number.isFinite(audioPlaybackCursor) || audioPlaybackCursor <= 0) {
|
|
audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
|
|
}
|
|
return mediaAudioContext;
|
|
}
|
|
|
|
function resetAudioPlayback() {
|
|
if (mediaAudioContext) {
|
|
audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
|
|
}
|
|
Array.from(audioPlaybackSources).forEach((source) => {
|
|
try {
|
|
source.stop();
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
});
|
|
audioPlaybackSources.clear();
|
|
}
|
|
|
|
async function queueServerAudio(arrayBuffer) {
|
|
const context = await ensureMediaAudioContext();
|
|
const decoded = await context.decodeAudioData(arrayBuffer.slice(0));
|
|
const source = context.createBufferSource();
|
|
source.buffer = decoded;
|
|
source.connect(context.destination);
|
|
const startAt = Math.max(context.currentTime + getAudioPlaybackBufferSeconds(), audioPlaybackCursor);
|
|
source.start(startAt);
|
|
audioPlaybackCursor = startAt + decoded.duration;
|
|
audioPlaybackSources.add(source);
|
|
source.onended = () => {
|
|
audioPlaybackSources.delete(source);
|
|
};
|
|
}
|
|
|
|
function stopMicCapture() {
|
|
if (micSourceNode) {
|
|
micSourceNode.disconnect();
|
|
micSourceNode = null;
|
|
}
|
|
if (micProcessorNode) {
|
|
micProcessorNode.disconnect();
|
|
micProcessorNode.onaudioprocess = null;
|
|
micProcessorNode = null;
|
|
}
|
|
if (micSinkGain) {
|
|
micSinkGain.disconnect();
|
|
micSinkGain = null;
|
|
}
|
|
if (localMicStream) {
|
|
localMicStream.getTracks().forEach((track) => track.stop());
|
|
localMicStream = null;
|
|
}
|
|
}
|
|
|
|
async function startMicCapture() {
|
|
const context = await ensureMediaAudioContext();
|
|
if (localMicStream && micSourceNode && micProcessorNode) {
|
|
return context;
|
|
}
|
|
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
audio: {
|
|
echoCancellation: true,
|
|
noiseSuppression: true,
|
|
autoGainControl: true,
|
|
},
|
|
video: false,
|
|
});
|
|
|
|
localMicStream = stream;
|
|
micSourceNode = context.createMediaStreamSource(stream);
|
|
micProcessorNode = context.createScriptProcessor(4096, 1, 1);
|
|
micSinkGain = context.createGain();
|
|
micSinkGain.gain.value = 0;
|
|
|
|
micProcessorNode.onaudioprocess = (event) => {
|
|
if (!audioWs || audioWs.readyState !== WebSocket.OPEN) {
|
|
return;
|
|
}
|
|
const input = event.inputBuffer.getChannelData(0);
|
|
const pcm = new Int16Array(input.length);
|
|
for (let i = 0; i < input.length; i += 1) {
|
|
const sample = Math.max(-1, Math.min(1, input[i] || 0));
|
|
pcm[i] = sample < 0 ? sample * 32768 : sample * 32767;
|
|
}
|
|
audioWs.send(pcm.buffer);
|
|
};
|
|
|
|
micSourceNode.connect(micProcessorNode);
|
|
micProcessorNode.connect(micSinkGain);
|
|
micSinkGain.connect(context.destination);
|
|
return context;
|
|
}
|
|
|
|
function initStage() {
|
|
scene.background = new THREE.Color(0x0b1118);
|
|
|
|
const hemiLight = new THREE.HemisphereLight(0xb8d7ff, 0x142334, 1.65);
|
|
hemiLight.position.set(0, 2.6, 0);
|
|
scene.add(hemiLight);
|
|
|
|
const keyLight = new THREE.DirectionalLight(0xffffff, 1.7);
|
|
keyLight.position.set(1.4, 2.2, 2.0);
|
|
scene.add(keyLight);
|
|
|
|
const rimLight = new THREE.DirectionalLight(0x67c9ff, 0.8);
|
|
rimLight.position.set(-2.0, 1.4, -1.5);
|
|
scene.add(rimLight);
|
|
|
|
const floor = new THREE.Mesh(
|
|
new THREE.CircleGeometry(2.8, 48),
|
|
new THREE.MeshStandardMaterial({
|
|
color: 0x101b28,
|
|
roughness: 0.94,
|
|
metalness: 0.08,
|
|
transparent: true,
|
|
opacity: 0.9,
|
|
}),
|
|
);
|
|
floor.rotation.x = -Math.PI / 2;
|
|
floor.position.y = -0.02;
|
|
scene.add(floor);
|
|
|
|
const halo = new THREE.Mesh(
|
|
new THREE.RingGeometry(1.2, 1.95, 64),
|
|
new THREE.MeshBasicMaterial({ color: 0x153a62, transparent: true, opacity: 0.28, side: THREE.DoubleSide }),
|
|
);
|
|
halo.rotation.x = -Math.PI / 2;
|
|
halo.position.y = 0.01;
|
|
scene.add(halo);
|
|
|
|
createDebugRig();
|
|
resizeStage();
|
|
renderLoop();
|
|
}
|
|
|
|
function createDebugRig() {
|
|
const group = new THREE.Group();
|
|
|
|
const bustMaterial = new THREE.MeshStandardMaterial({
|
|
color: 0xc7d2e0,
|
|
roughness: 0.72,
|
|
metalness: 0.05,
|
|
});
|
|
const accentMaterial = new THREE.MeshStandardMaterial({
|
|
color: 0x0f172a,
|
|
roughness: 0.35,
|
|
metalness: 0.18,
|
|
});
|
|
const lipMaterial = new THREE.MeshStandardMaterial({
|
|
color: 0x8b425f,
|
|
roughness: 0.56,
|
|
metalness: 0.02,
|
|
});
|
|
|
|
const torso = new THREE.Mesh(new THREE.CapsuleGeometry(0.34, 0.72, 8, 14), bustMaterial);
|
|
torso.position.set(0, 0.42, 0);
|
|
group.add(torso);
|
|
|
|
const neck = new THREE.Group();
|
|
neck.position.set(0, 0.95, 0);
|
|
group.add(neck);
|
|
|
|
const head = new THREE.Mesh(new THREE.SphereGeometry(0.34, 32, 32), bustMaterial);
|
|
head.scale.set(0.92, 1.03, 0.92);
|
|
neck.add(head);
|
|
|
|
const hair = new THREE.Mesh(new THREE.SphereGeometry(0.35, 32, 32, 0, Math.PI * 2, 0, Math.PI * 0.58), accentMaterial);
|
|
hair.position.set(0, 0.06, 0);
|
|
neck.add(hair);
|
|
|
|
const leftEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
|
|
leftEye.position.set(-0.1, 0.03, 0.28);
|
|
neck.add(leftEye);
|
|
|
|
const rightEye = new THREE.Mesh(new THREE.SphereGeometry(0.028, 16, 16), accentMaterial);
|
|
rightEye.position.set(0.1, 0.03, 0.28);
|
|
neck.add(rightEye);
|
|
|
|
const jawPivot = new THREE.Group();
|
|
jawPivot.position.set(0, -0.06, 0.15);
|
|
neck.add(jawPivot);
|
|
|
|
const jaw = new THREE.Mesh(new THREE.BoxGeometry(0.24, 0.08, 0.16), bustMaterial);
|
|
jaw.position.set(0, -0.08, 0.02);
|
|
jawPivot.add(jaw);
|
|
|
|
const mouth = new THREE.Mesh(new THREE.BoxGeometry(0.18, 0.025, 0.02), lipMaterial);
|
|
mouth.position.set(0, -0.02, 0.31);
|
|
neck.add(mouth);
|
|
|
|
scene.add(group);
|
|
avatarState.currentRoot = group;
|
|
avatarState.debugRig = group;
|
|
avatarState.debugJaw = jawPivot;
|
|
avatarState.debugMouth = mouth;
|
|
avatarState.debugLeftEye = leftEye;
|
|
avatarState.debugRightEye = rightEye;
|
|
avatarState.headBone = neck;
|
|
avatarState.neckBone = neck;
|
|
}
|
|
|
|
function resizeStage() {
|
|
const rect = avatarCanvas.getBoundingClientRect();
|
|
if (!rect.width || !rect.height) return;
|
|
renderer.setSize(rect.width, rect.height, false);
|
|
camera.aspect = rect.width / rect.height;
|
|
camera.updateProjectionMatrix();
|
|
}
|
|
|
|
function renderLoop() {
|
|
requestAnimationFrame(renderLoop);
|
|
const delta = CLOCK.getDelta();
|
|
const elapsed = CLOCK.getElapsedTime();
|
|
controls.update();
|
|
flushAnimationQueue();
|
|
updateBlinkState(performance.now());
|
|
applyProceduralIdlePose(elapsed);
|
|
applyAnimationControls(avatarState.activeControls);
|
|
avatarState.currentVrm?.update?.(delta);
|
|
renderer.render(scene, camera);
|
|
}
|
|
|
|
function updateMetrics(data) {
|
|
mPeers.textContent = String(data.audio_clients ?? 0);
|
|
mBusy.textContent = data.pipeline_busy ? "是" : "否";
|
|
mLatency.textContent = `${data.last_latency_ms ?? 0} ms`;
|
|
mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`;
|
|
mLlmLatency.textContent = `${data.last_llm_latency_ms ?? 0} ms`;
|
|
{
|
|
const s = data.last_llm_source;
|
|
mLlmSource.textContent =
|
|
s === "online" ? "在线" : s === "error" ? "失败" : s === "skipped" ? "跳过" : s === "unavailable" ? "不可用" : s || "--";
|
|
}
|
|
mLlmHistory.textContent = String(data?.llm?.history_items ?? 0);
|
|
mTtsLatency.textContent = `${data.last_tts_latency_ms ?? 0} ms`;
|
|
mTtsFirst.textContent = `${data.last_tts_first_chunk_ms ?? 0} ms`;
|
|
mTts.textContent = data?.tts?.ready ? "就绪" : "未就绪";
|
|
mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`;
|
|
mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none";
|
|
mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`;
|
|
mAnimClients.textContent = String(data.animation_clients ?? 0);
|
|
mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--";
|
|
mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0);
|
|
|
|
setDot(dConn, (data.audio_clients ?? 0) > 0 ? "ok" : "warn");
|
|
setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
|
|
setDot(dAsr, data?.asr?.ready ? "ok" : "warn");
|
|
setDot(dLlm, data?.llm?.ready ? "ok" : "bad");
|
|
setDot(dTts, data?.tts?.ready ? "ok" : "warn");
|
|
syncConnBadgeFromState(data);
|
|
}
|
|
|
|
function queueAnimationFrames(payload) {
|
|
const frames = Array.isArray(payload.frames) ? payload.frames : [];
|
|
const now = performance.now();
|
|
const frameWindowMs = (1000 / Math.max(1, payload.fps || 25));
|
|
const animationLeadMs = getAnimationLeadMs();
|
|
|
|
if (payload.type === "animation_reset" || payload.type === "animation_state") {
|
|
avatarState.pendingFrames = [];
|
|
avatarState.scheduleCursorMs = now + animationLeadMs;
|
|
avatarState.activeControls = {
|
|
jawOpen: 0,
|
|
viseme_aa: 0,
|
|
viseme_ee: 0,
|
|
viseme_oh: 0,
|
|
mouthPucker: 0,
|
|
headYaw: 0,
|
|
headPitch: 0,
|
|
headRoll: 0,
|
|
};
|
|
}
|
|
|
|
const baseTime = Math.max(now + animationLeadMs, avatarState.scheduleCursorMs);
|
|
frames.forEach((frame, index) => {
|
|
const fallbackTime = index * frameWindowMs;
|
|
avatarState.pendingFrames.push({
|
|
dueAt: baseTime + (frame.time_ms ?? fallbackTime),
|
|
seq: frame.seq ?? index,
|
|
controls: frame.controls || {},
|
|
});
|
|
});
|
|
|
|
avatarState.pendingFrames.sort((a, b) => a.dueAt - b.dueAt || a.seq - b.seq);
|
|
avatarState.scheduleCursorMs = baseTime + (payload.duration_ms ?? frames.length * frameWindowMs);
|
|
animationBuffer.textContent = String(avatarState.pendingFrames.length);
|
|
}
|
|
|
|
function flushAnimationQueue() {
|
|
const now = performance.now();
|
|
let frameToApply = null;
|
|
while (avatarState.pendingFrames.length > 0 && avatarState.pendingFrames[0].dueAt <= now) {
|
|
frameToApply = avatarState.pendingFrames.shift();
|
|
}
|
|
|
|
if (frameToApply && frameToApply.seq !== avatarState.lastAppliedSeq) {
|
|
avatarState.lastAppliedSeq = frameToApply.seq;
|
|
avatarState.activeControls = {
|
|
...avatarState.activeControls,
|
|
...frameToApply.controls,
|
|
};
|
|
}
|
|
|
|
animationBuffer.textContent = String(avatarState.pendingFrames.length);
|
|
}
|
|
|
|
function applyAnimationControls(controls) {
|
|
const safeControls = controls || {};
|
|
const jawOpen = clampControl(safeControls.jawOpen);
|
|
const visemeAa = clampControl(safeControls.viseme_aa ?? jawOpen);
|
|
const visemeEe = clampControl(safeControls.viseme_ee);
|
|
const visemeOh = clampControl(safeControls.viseme_oh);
|
|
const mouthPucker = clampControl(safeControls.mouthPucker);
|
|
const headYaw = clampSigned(safeControls.headYaw, 0.35);
|
|
const headPitch = clampSigned(safeControls.headPitch, 0.3);
|
|
const headRoll = clampSigned(safeControls.headRoll, 0.24);
|
|
const useExpressions = Boolean(avatarState.currentVrm?.expressionManager);
|
|
const useMorphTargets = !useExpressions && avatarState.morphBindings.size > 0;
|
|
const hasFacialRig = useExpressions || useMorphTargets;
|
|
const mouthDrive = buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker);
|
|
const t = performance.now() * 0.001;
|
|
|
|
if (useExpressions) {
|
|
setExpressionValue("aa", mouthDrive.expression.aa);
|
|
setExpressionValue("ee", mouthDrive.expression.ee);
|
|
setExpressionValue("oh", mouthDrive.expression.oh);
|
|
setExpressionValue("ou", mouthDrive.expression.ou);
|
|
}
|
|
|
|
if (useMorphTargets) {
|
|
setMorphValue("jawOpen", mouthDrive.morph.jawOpen);
|
|
setMorphValue("viseme_aa", mouthDrive.morph.viseme_aa);
|
|
setMorphValue("viseme_ee", mouthDrive.morph.viseme_ee);
|
|
setMorphValue("viseme_oh", mouthDrive.morph.viseme_oh);
|
|
setMorphValue("mouthPucker", mouthDrive.morph.mouthPucker);
|
|
}
|
|
|
|
applyBlinkControls(avatarState.blink?.value || 0);
|
|
|
|
// For plain head meshes (no VRM expressions and no morph targets),
|
|
// amplify head motion to keep speaking visually obvious.
|
|
const yawApplied = hasFacialRig ? headYaw * 1.8 : headYaw * 10.0 + Math.sin(t * 5.0) * jawOpen * 0.09;
|
|
const pitchApplied = hasFacialRig ? headPitch * 1.6 : headPitch * 10.0 + jawOpen * 0.12;
|
|
const rollApplied = hasFacialRig ? headRoll * 1.6 : headRoll * 8.0 + Math.sin(t * 4.0) * jawOpen * 0.04;
|
|
|
|
applyBoneRotation(avatarState.headBone, pitchApplied, yawApplied, rollApplied);
|
|
if (avatarState.neckBone && avatarState.neckBone !== avatarState.headBone) {
|
|
applyBoneRotation(avatarState.neckBone, pitchApplied * 0.4, yawApplied * 0.5, rollApplied * 0.4);
|
|
}
|
|
|
|
if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
|
|
const baseScale = 1.0 + mouthDrive.debugJaw * 0.03;
|
|
avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale);
|
|
}
|
|
|
|
if (avatarState.debugJaw) {
|
|
avatarState.debugJaw.rotation.x = mouthDrive.debugJaw * 0.5;
|
|
avatarState.debugJaw.position.y = -0.06 - mouthDrive.debugJaw * 0.015;
|
|
}
|
|
if (avatarState.debugMouth) {
|
|
avatarState.debugMouth.scale.x = 1.0 + mouthDrive.morph.mouthPucker * 0.45;
|
|
avatarState.debugMouth.scale.y = 1.0 + mouthDrive.debugJaw * 2.6;
|
|
avatarState.debugMouth.position.z = 0.31 + mouthDrive.morph.mouthPucker * 0.01;
|
|
}
|
|
}
|
|
|
|
function setExpressionValue(name, value) {
|
|
try {
|
|
avatarState.currentVrm.expressionManager.setValue(name, value);
|
|
} catch (_) {
|
|
// no-op: expression name may not exist on the current model
|
|
}
|
|
}
|
|
|
|
function setMorphValue(controlName, value) {
|
|
const bindings = avatarState.morphBindings.get(controlName) || [];
|
|
bindings.forEach((binding) => {
|
|
binding.mesh.morphTargetInfluences[binding.index] = value;
|
|
});
|
|
}
|
|
|
|
function buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker) {
|
|
const mouthOpen = Math.max(jawOpen * 0.9, visemeAa);
|
|
return {
|
|
expression: {
|
|
aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.expression.aa),
|
|
ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.expression.ee),
|
|
oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.expression.oh),
|
|
ou: clampControl(
|
|
Math.max(
|
|
visemeOh * MOUTH_DRIVE_LIMITS.expression.ouFromOh,
|
|
mouthPucker * MOUTH_DRIVE_LIMITS.expression.pucker
|
|
)
|
|
),
|
|
},
|
|
morph: {
|
|
jawOpen: clampControl(jawOpen * MOUTH_DRIVE_LIMITS.morph.jawOpen),
|
|
viseme_aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.morph.visemeAa),
|
|
viseme_ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.morph.visemeEe),
|
|
viseme_oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.morph.visemeOh),
|
|
mouthPucker: clampControl(mouthPucker * MOUTH_DRIVE_LIMITS.morph.mouthPucker),
|
|
},
|
|
debugJaw: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.debugJaw),
|
|
};
|
|
}
|
|
|
|
function clampControl(value) {
|
|
return Math.min(1, Math.max(0, Number(value) || 0));
|
|
}
|
|
|
|
function getAudioPlaybackBufferSeconds() {
|
|
return AUDIO_PLAYBACK_BUFFER_MS / 1000;
|
|
}
|
|
|
|
function getAnimationLeadMs() {
|
|
return ANIMATION_BUFFER_MS + AV_SYNC_AUDIO_LEAD_MS;
|
|
}
|
|
|
|
function clampSigned(value, limit) {
|
|
return Math.min(limit, Math.max(-limit, Number(value) || 0));
|
|
}
|
|
|
|
function resetView() {
|
|
camera.position.set(0, 1.45, 2.65);
|
|
controls.target.set(0, 1.15, 0);
|
|
controls.update();
|
|
}
|
|
|
|
function resetAvatarPose() {
|
|
avatarState.activeControls = {
|
|
jawOpen: 0,
|
|
viseme_aa: 0,
|
|
viseme_ee: 0,
|
|
viseme_oh: 0,
|
|
mouthPucker: 0,
|
|
headYaw: 0,
|
|
headPitch: 0,
|
|
headRoll: 0,
|
|
};
|
|
applyAnimationControls(avatarState.activeControls);
|
|
}
|
|
|
|
function detachCurrentAvatar() {
|
|
if (avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
|
|
scene.remove(avatarState.currentRoot);
|
|
}
|
|
avatarState.pendingFrames = [];
|
|
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
|
|
avatarState.lastAppliedSeq = -1;
|
|
avatarState.idleRig = null;
|
|
avatarState.currentRoot = avatarState.debugRig;
|
|
avatarState.currentVrm = null;
|
|
avatarState.morphBindings.clear();
|
|
avatarState.headBone = avatarState.debugRig.children[1];
|
|
avatarState.neckBone = avatarState.debugRig.children[1];
|
|
avatarState.debugRig.visible = true;
|
|
avatarState.debugLeftEye = avatarState.debugLeftEye || avatarState.debugRig.children[1]?.children?.[2] || null;
|
|
avatarState.debugRightEye = avatarState.debugRightEye || avatarState.debugRig.children[1]?.children?.[3] || null;
|
|
modelName.textContent = "debug-avatar";
|
|
}
|
|
|
|
async function loadModelFromUrl(url) {
|
|
if (!url) return false;
|
|
modelStatus.textContent = "模型加载中...";
|
|
try {
|
|
const gltf = await gltfLoader.loadAsync(url);
|
|
attachLoadedAsset(gltf, url.split("/").pop() || url);
|
|
modelStatus.textContent = "模型已加载";
|
|
return true;
|
|
} catch (error) {
|
|
console.error(error);
|
|
modelStatus.textContent = "模型加载失败，已回退调试头像";
|
|
addMessage("system", `模型加载失败：${error?.message || error}`);
|
|
detachCurrentAvatar();
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function loadDefaultAvatarModel() {
|
|
modelUrl.value = DEFAULT_BODY_MODEL_URL;
|
|
await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
|
|
}
|
|
|
|
async function loadModelFromFile(file) {
|
|
if (!file) return;
|
|
const url = URL.createObjectURL(file);
|
|
modelStatus.textContent = "本地模型加载中...";
|
|
try {
|
|
const gltf = await gltfLoader.loadAsync(url);
|
|
attachLoadedAsset(gltf, file.name);
|
|
modelStatus.textContent = "本地模型已加载";
|
|
} catch (error) {
|
|
console.error(error);
|
|
modelStatus.textContent = "本地模型加载失败，已回退调试头像";
|
|
addMessage("system", `本地模型加载失败：${error?.message || error}`);
|
|
detachCurrentAvatar();
|
|
} finally {
|
|
URL.revokeObjectURL(url);
|
|
}
|
|
}
|
|
|
|
function attachLoadedAsset(gltf, label) {
|
|
detachCurrentAvatar();
|
|
|
|
const vrm = gltf.userData?.vrm || null;
|
|
const root = vrm?.scene || gltf.scene;
|
|
if (!root) {
|
|
throw new Error("模型中未找到可渲染的 scene");
|
|
}
|
|
|
|
if (vrm) {
|
|
VRMUtils.removeUnnecessaryVertices(gltf.scene);
|
|
VRMUtils.combineSkeletons(gltf.scene);
|
|
VRMUtils.combineMorphs(vrm);
|
|
VRMUtils.rotateVRM0(vrm);
|
|
}
|
|
|
|
root.traverse((obj) => {
|
|
obj.frustumCulled = false;
|
|
});
|
|
|
|
root.position.set(0, 0, 0);
|
|
root.rotation.set(0, 0, 0);
|
|
root.scale.setScalar(1);
|
|
scene.add(root);
|
|
|
|
avatarState.currentRoot = root;
|
|
avatarState.currentVrm = vrm;
|
|
avatarState.debugRig.visible = false;
|
|
avatarState.morphBindings = collectMorphBindings(root);
|
|
avatarState.headBone = resolveHeadBone(vrm, root) || avatarState.debugRig.children[1];
|
|
avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone;
|
|
avatarState.idleRig = buildIdleRig(vrm, root);
|
|
avatarState.pendingFrames = [];
|
|
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
|
|
avatarState.lastAppliedSeq = -1;
|
|
|
|
fitCameraToObject(root);
|
|
resetAvatarPose();
|
|
modelName.textContent = label;
|
|
}
|
|
|
|
function fitCameraToObject(root) {
|
|
const box = new THREE.Box3().setFromObject(root);
|
|
const center = new THREE.Vector3();
|
|
const size = new THREE.Vector3();
|
|
box.getCenter(center);
|
|
box.getSize(size);
|
|
|
|
const radius = Math.max(size.x, size.y, size.z, 1.2);
|
|
controls.target.copy(center).add(new THREE.Vector3(0, size.y * 0.1, 0));
|
|
camera.position.set(center.x, center.y + size.y * 0.1, center.z + radius * 1.9);
|
|
controls.update();
|
|
}
|
|
|
|
function resolveHeadBone(vrm, root) {
|
|
return (
|
|
vrm?.humanoid?.getNormalizedBoneNode?.("head") ||
|
|
vrm?.humanoid?.getRawBoneNode?.("head") ||
|
|
root.getObjectByName("Head") ||
|
|
root.getObjectByName("head") ||
|
|
root
|
|
);
|
|
}
|
|
|
|
function resolveNeckBone(vrm, root) {
|
|
return (
|
|
vrm?.humanoid?.getNormalizedBoneNode?.("neck") ||
|
|
vrm?.humanoid?.getRawBoneNode?.("neck") ||
|
|
root.getObjectByName("Neck") ||
|
|
root.getObjectByName("neck") ||
|
|
resolveHeadBone(vrm, root)
|
|
);
|
|
}
|
|
|
|
function collectMorphBindings(root) {
|
|
const bindings = new Map();
|
|
const aliases = {
|
|
jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open"],
|
|
viseme_aa: ["viseme_aa", "aa", "a", "moutha"],
|
|
viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"],
|
|
viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"],
|
|
mouthPucker: ["mouthpucker", "pucker", "ou", "kiss"],
|
|
blink: ["blink", "eyeblink", "eye_blink", "blinkboth"],
|
|
blinkLeft: ["blinkleft", "blink_left", "eyeblinkleft", "eyelid_l", "eyeblink_l"],
|
|
blinkRight: ["blinkright", "blink_right", "eyeblinkright", "eyelid_r", "eyeblink_r"],
|
|
};
|
|
|
|
root.traverse((node) => {
|
|
if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return;
|
|
const entries = Object.entries(node.morphTargetDictionary);
|
|
Object.entries(aliases).forEach(([controlName, names]) => {
|
|
entries.forEach(([targetName, index]) => {
|
|
const normalized = targetName.toLowerCase();
|
|
if (!names.includes(normalized)) return;
|
|
if (!bindings.has(controlName)) bindings.set(controlName, []);
|
|
bindings.get(controlName).push({ mesh: node, index });
|
|
});
|
|
});
|
|
});
|
|
return bindings;
|
|
}
|
|
|
|
function buildIdleRig(vrm, root) {
|
|
const lookup = {
|
|
hips: resolveHumanoidBone(vrm, root, "hips", ["Hips", "hips"]),
|
|
spine: resolveHumanoidBone(vrm, root, "spine", ["Spine", "spine"]),
|
|
chest: resolveHumanoidBone(vrm, root, "chest", ["Chest", "chest", "UpperChest", "upperChest"]),
|
|
upperChest: resolveHumanoidBone(vrm, root, "upperChest", ["UpperChest", "upperChest", "Chest", "chest"]),
|
|
neck: resolveHumanoidBone(vrm, root, "neck", ["Neck", "neck"]),
|
|
head: resolveHumanoidBone(vrm, root, "head", ["Head", "head"]),
|
|
leftShoulder: resolveHumanoidBone(vrm, root, "leftShoulder", ["LeftShoulder", "leftShoulder", "Shoulder_L"]),
|
|
rightShoulder: resolveHumanoidBone(vrm, root, "rightShoulder", ["RightShoulder", "rightShoulder", "Shoulder_R"]),
|
|
leftUpperArm: resolveHumanoidBone(vrm, root, "leftUpperArm", ["LeftUpperArm", "leftUpperArm", "Arm_L"]),
|
|
rightUpperArm: resolveHumanoidBone(vrm, root, "rightUpperArm", ["RightUpperArm", "rightUpperArm", "Arm_R"]),
|
|
leftLowerArm: resolveHumanoidBone(vrm, root, "leftLowerArm", ["LeftLowerArm", "leftLowerArm", "ForeArm_L"]),
|
|
rightLowerArm: resolveHumanoidBone(vrm, root, "rightLowerArm", ["RightLowerArm", "rightLowerArm", "ForeArm_R"]),
|
|
leftHand: resolveHumanoidBone(vrm, root, "leftHand", ["LeftHand", "leftHand", "Hand_L"]),
|
|
rightHand: resolveHumanoidBone(vrm, root, "rightHand", ["RightHand", "rightHand", "Hand_R"]),
|
|
};
|
|
|
|
const base = new Map();
|
|
Object.values(lookup).forEach((bone) => {
|
|
if (bone && !base.has(bone)) {
|
|
base.set(bone, bone.quaternion.clone());
|
|
}
|
|
});
|
|
|
|
return { bones: lookup, base };
|
|
}
|
|
|
|
function resolveHumanoidBone(vrm, root, humanoidName, fallbacks) {
|
|
return (
|
|
vrm?.humanoid?.getNormalizedBoneNode?.(humanoidName) ||
|
|
vrm?.humanoid?.getRawBoneNode?.(humanoidName) ||
|
|
fallbacks.map((name) => root.getObjectByName(name)).find(Boolean) ||
|
|
null
|
|
);
|
|
}
|
|
|
|
function applyProceduralIdlePose(time) {
|
|
const rig = avatarState.idleRig;
|
|
if (!rig) return;
|
|
|
|
const breathe = Math.sin(time * 1.9) * 0.018;
|
|
const sway = Math.sin(time * 1.15) * 0.05;
|
|
const armDrift = Math.sin(time * 1.55) * 0.07;
|
|
const forearmDrift = Math.sin(time * 1.55 + 0.8) * 0.06;
|
|
const handDrift = Math.sin(time * 2.3) * 0.04;
|
|
|
|
applyBoneRotation(rig.bones.hips, 0, sway * 0.08, sway * 0.04);
|
|
applyBoneRotation(rig.bones.spine, breathe, sway * 0.18, sway * 0.06);
|
|
applyBoneRotation(rig.bones.chest, breathe * 1.4, sway * 0.24, sway * 0.08);
|
|
applyBoneRotation(rig.bones.upperChest, breathe * 1.6, sway * 0.28, sway * 0.1);
|
|
|
|
applyBoneRotation(rig.bones.leftShoulder, 0.04, 0.03, -0.18);
|
|
applyBoneRotation(rig.bones.rightShoulder, 0.04, -0.03, 0.18);
|
|
applyBoneRotation(rig.bones.leftUpperArm, 0.12 + armDrift * 0.08, 0.03, -1.0);
|
|
applyBoneRotation(rig.bones.rightUpperArm, 0.12 + armDrift * 0.08, -0.03, 1.0);
|
|
applyBoneRotation(rig.bones.leftLowerArm, -0.14 + forearmDrift * 0.08, 0, -0.12);
|
|
applyBoneRotation(rig.bones.rightLowerArm, -0.14 + forearmDrift * 0.08, 0, 0.12);
|
|
applyBoneRotation(rig.bones.leftHand, handDrift * 0.2, 0, -0.05);
|
|
applyBoneRotation(rig.bones.rightHand, handDrift * 0.2, 0, 0.05);
|
|
}
|
|
|
|
function scheduleNextBlink(nowMs, preferSoon = false) {
|
|
const quickBlink = Math.random() < 0.18;
|
|
const minDelay = preferSoon ? 240 : 1800;
|
|
const maxDelay = preferSoon ? 520 : 5200;
|
|
avatarState.blink.nextAtMs = nowMs + minDelay + Math.random() * (maxDelay - minDelay);
|
|
avatarState.blink.pendingDoubleBlink = !preferSoon && quickBlink;
|
|
}
|
|
|
|
function updateBlinkState(nowMs) {
|
|
const blink = avatarState.blink;
|
|
if (!blink) return;
|
|
|
|
if (blink.phase === "idle") {
|
|
blink.value = 0;
|
|
if (nowMs >= blink.nextAtMs) {
|
|
blink.phase = "closing";
|
|
blink.phaseStartedAtMs = nowMs;
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (blink.phase === "closing") {
|
|
const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.closeMs);
|
|
blink.value = progress;
|
|
if (progress >= 1) {
|
|
blink.phase = "hold";
|
|
blink.phaseStartedAtMs = nowMs;
|
|
blink.value = 1;
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (blink.phase === "hold") {
|
|
blink.value = 1;
|
|
if (nowMs - blink.phaseStartedAtMs >= blink.holdMs) {
|
|
blink.phase = "opening";
|
|
blink.phaseStartedAtMs = nowMs;
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (blink.phase === "opening") {
|
|
const progress = Math.min(1, (nowMs - blink.phaseStartedAtMs) / blink.openMs);
|
|
blink.value = 1 - progress;
|
|
if (progress >= 1) {
|
|
blink.phase = "idle";
|
|
blink.phaseStartedAtMs = nowMs;
|
|
blink.value = 0;
|
|
scheduleNextBlink(nowMs, blink.pendingDoubleBlink);
|
|
}
|
|
}
|
|
}
|
|
|
|
function applyBlinkControls(blinkValue) {
|
|
const clamped = clampControl(blinkValue);
|
|
|
|
if (avatarState.currentVrm?.expressionManager) {
|
|
setExpressionValue("blink", clamped);
|
|
setExpressionValue("blinkLeft", clamped);
|
|
setExpressionValue("blinkRight", clamped);
|
|
}
|
|
|
|
setMorphValue("blink", clamped);
|
|
setMorphValue("blinkLeft", clamped);
|
|
setMorphValue("blinkRight", clamped);
|
|
|
|
const eyeScaleY = Math.max(0.08, 1 - clamped * 0.92);
|
|
const eyeScaleZ = 1 + clamped * 0.35;
|
|
if (avatarState.debugLeftEye) {
|
|
avatarState.debugLeftEye.scale.set(1, eyeScaleY, eyeScaleZ);
|
|
}
|
|
if (avatarState.debugRightEye) {
|
|
avatarState.debugRightEye.scale.set(1, eyeScaleY, eyeScaleZ);
|
|
}
|
|
}
|
|
|
|
function applyBoneRotation(bone, x = 0, y = 0, z = 0) {
|
|
if (!bone) return;
|
|
const baseQuaternion = avatarState.idleRig?.base?.get(bone);
|
|
if (baseQuaternion) {
|
|
bone.quaternion.copy(baseQuaternion);
|
|
TMP_EULER.set(x, y, z, "XYZ");
|
|
TMP_QUATERNION.setFromEuler(TMP_EULER);
|
|
bone.quaternion.multiply(TMP_QUATERNION);
|
|
return;
|
|
}
|
|
bone.rotation.x = x;
|
|
bone.rotation.y = y;
|
|
bone.rotation.z = z;
|
|
}
|
|
|
|
async function checkHealth() {
|
|
const res = await fetch("/health");
|
|
const data = await res.json();
|
|
statusEl.textContent = `状态：${toCnState(data.state)}`;
|
|
outputEl.textContent = JSON.stringify(data, null, 2);
|
|
sendBtn.disabled = !!data.pipeline_busy;
|
|
updateMetrics(data);
|
|
}
|
|
|
|
function subscribeEvents() {
|
|
if (evt) return;
|
|
evt = new EventSource("/events");
|
|
evt.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
statusEl.textContent = `状态：${toCnState(data.state)}`;
|
|
outputEl.textContent = JSON.stringify(data, null, 2);
|
|
sendBtn.disabled = !!data.pipeline_busy;
|
|
updateMetrics(data);
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
};
|
|
}
|
|
|
|
function subscribeSubtitles() {
|
|
if (subtitleWs && (subtitleWs.readyState === WebSocket.OPEN || subtitleWs.readyState === WebSocket.CONNECTING)) return;
|
|
subtitleWs = new WebSocket(wsBaseUrl("/ws/subtitles"));
|
|
subtitleWs.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
if (!shouldRenderSubtitleEvent(data)) {
|
|
return;
|
|
}
|
|
if (data.role === "user") {
|
|
aiStreamingEl = null;
|
|
addMessage(data.role, data.text || "");
|
|
} else if (data.role === "ai") {
|
|
if (data.partial) {
|
|
if (!aiStreamingEl) aiStreamingEl = addMessage("ai", "");
|
|
aiStreamingEl.textContent = `${aiStreamingEl.textContent}${data.text || ""}`;
|
|
if (data.final) aiStreamingEl = null;
|
|
} else {
|
|
aiStreamingEl = null;
|
|
addMessage("ai", data.text || "");
|
|
}
|
|
}
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
};
|
|
subtitleWs.onclose = (event) => {
|
|
if (subtitleWs && subtitleWs.readyState === WebSocket.CLOSED) {
|
|
subtitleWs = null;
|
|
}
|
|
if (event.code === 1012) {
|
|
addMessage("system", "字幕通道已被新客户端接管。当前页面停止接收字幕。");
|
|
return;
|
|
}
|
|
setTimeout(subscribeSubtitles, 1000);
|
|
};
|
|
}
|
|
|
|
function subscribeAnimation() {
|
|
if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return;
|
|
animationConn.textContent = "连接中";
|
|
animationWs = new WebSocket(wsBaseUrl("/ws/animation"));
|
|
animationWs.onopen = () => {
|
|
animationConn.textContent = "已连接";
|
|
};
|
|
animationWs.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
if (data.type === "ping") return;
|
|
if (data.driver) {
|
|
animationDriver.textContent = data.driver;
|
|
}
|
|
const frameCount = data.frame_count ?? data.frames?.length ?? 0;
|
|
animationFrames.textContent = String(frameCount);
|
|
const preview = data.frames?.[0]?.controls || null;
|
|
animationPreview.textContent = preview ? JSON.stringify(preview, null, 2) : JSON.stringify(data, null, 2);
|
|
queueAnimationFrames(data);
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
};
|
|
animationWs.onclose = (event) => {
|
|
if (animationWs && animationWs.readyState === WebSocket.CLOSED) {
|
|
animationWs = null;
|
|
}
|
|
animationConn.textContent = "已断开";
|
|
if (event.code === 1012) {
|
|
animationConn.textContent = "已被新客户端接管";
|
|
return;
|
|
}
|
|
setTimeout(subscribeAnimation, 1000);
|
|
};
|
|
}
|
|
|
|
async function connectAudio(options = {}) {
|
|
const { silentReadyMessage = false } = options;
|
|
if (audioWs && audioWs.readyState === WebSocket.OPEN) {
|
|
disarmReconnectOnGesture();
|
|
outputEl.textContent = "语音通道已连接";
|
|
return;
|
|
}
|
|
if (audioWs && audioWs.readyState === WebSocket.CONNECTING) {
|
|
outputEl.textContent = "语音通道正在连接";
|
|
await new Promise((resolve, reject) => {
|
|
const startedAt = Date.now();
|
|
const timer = window.setInterval(() => {
|
|
if (!audioWs) {
|
|
window.clearInterval(timer);
|
|
reject(new Error("语音通道不存在"));
|
|
return;
|
|
}
|
|
if (audioWs.readyState === WebSocket.OPEN) {
|
|
window.clearInterval(timer);
|
|
resolve();
|
|
return;
|
|
}
|
|
if (audioWs.readyState === WebSocket.CLOSING || audioWs.readyState === WebSocket.CLOSED) {
|
|
window.clearInterval(timer);
|
|
reject(new Error("语音通道连接已关闭"));
|
|
return;
|
|
}
|
|
if (Date.now() - startedAt > 5000) {
|
|
window.clearInterval(timer);
|
|
reject(new Error("语音通道连接超时"));
|
|
}
|
|
}, 100);
|
|
});
|
|
return;
|
|
}
|
|
|
|
setConnBadge("连接中...", "warn");
|
|
offerBtn.disabled = true;
|
|
|
|
let context;
|
|
try {
|
|
context = await startMicCapture();
|
|
} catch (err) {
|
|
statusEl.textContent = "状态：麦克风权限失败";
|
|
outputEl.textContent = `getUserMedia 失败：${err?.message || err}\n提示：跨机器访问请使用 HTTPS 或浏览器安全例外。`;
|
|
setConnBadge("麦克风失败", "warn");
|
|
offerBtn.disabled = false;
|
|
throw err;
|
|
}
|
|
|
|
const ws = new WebSocket(wsBaseUrl("/ws/audio"));
|
|
ws.binaryType = "arraybuffer";
|
|
audioWs = ws;
|
|
|
|
await new Promise((resolve, reject) => {
|
|
let settled = false;
|
|
|
|
ws.onopen = () => {
|
|
if (audioWs !== ws) {
|
|
try {
|
|
ws.close(1000, "stale-open");
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
return;
|
|
}
|
|
ws.send(
|
|
JSON.stringify({
|
|
type: "audio_config",
|
|
sample_rate: context.sampleRate,
|
|
channels: 1,
|
|
format: "pcm_s16le",
|
|
}),
|
|
);
|
|
statusEl.textContent = "状态：语音通道已连接";
|
|
outputEl.textContent = JSON.stringify({ connected: true, transport: "websocket_audio" }, null, 2);
|
|
setConnBadge("已连接", "ok");
|
|
localStorage.setItem(AUTO_RECONNECT_KEY, "1");
|
|
disarmReconnectOnGesture();
|
|
offerBtn.disabled = false;
|
|
if (!silentReadyMessage) {
|
|
addMessage("system", "语音通道已连接，可开始对话。");
|
|
}
|
|
subscribeEvents();
|
|
settled = true;
|
|
resolve();
|
|
};
|
|
|
|
ws.onmessage = (event) => {
|
|
if (typeof event.data === "string") {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
if (data.type === "ping" || data.type === "pong" || data.type === "audio_ready") {
|
|
return;
|
|
}
|
|
if (data.type === "audio_reset") {
|
|
resetAudioPlayback();
|
|
return;
|
|
}
|
|
} catch (_) {
|
|
// no-op
|
|
}
|
|
return;
|
|
}
|
|
|
|
queueServerAudio(event.data).catch((error) => {
|
|
console.error(error);
|
|
addMessage("system", `语音播放失败：${error?.message || error}`);
|
|
});
|
|
};
|
|
|
|
ws.onerror = (event) => {
|
|
console.error(event);
|
|
if (!settled) {
|
|
settled = true;
|
|
offerBtn.disabled = false;
|
|
setConnBadge("连接失败", "warn");
|
|
reject(new Error("语音通道连接失败"));
|
|
}
|
|
};
|
|
|
|
ws.onclose = (event) => {
|
|
if (audioWs !== ws) {
|
|
if (!settled) {
|
|
settled = true;
|
|
resolve();
|
|
}
|
|
return;
|
|
}
|
|
resetAudioPlayback();
|
|
stopMicCapture();
|
|
if (audioWs === ws) {
|
|
audioWs = null;
|
|
}
|
|
syncConnBadgeFromState();
|
|
offerBtn.disabled = false;
|
|
if (event.code !== 1000 && event.code !== 1001 && event.code !== 1012) {
|
|
addMessage("system", "语音通道已断开，请重新连接。");
|
|
}
|
|
if (!settled) {
|
|
settled = true;
|
|
reject(new Error("语音通道在建立前已关闭"));
|
|
}
|
|
};
|
|
});
|
|
}
|
|
|
|
async function sendText() {
|
|
const text = (chatInput.value || "").trim();
|
|
if (!text || sendBtn.disabled) return;
|
|
if (!audioWs || audioWs.readyState !== WebSocket.OPEN) {
|
|
try {
|
|
await connectAudio({ silentReadyMessage: true });
|
|
} catch (_) {
|
|
addMessage("system", "语音通道未连接，本次仅保证文字链路可用。");
|
|
}
|
|
}
|
|
sendBtn.disabled = true;
|
|
const res = await fetch("/chat/text", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ text }),
|
|
});
|
|
const data = await res.json();
|
|
outputEl.textContent = JSON.stringify(data, null, 2);
|
|
if (data.ok) {
|
|
chatInput.value = "";
|
|
} else if (data.message) {
|
|
addMessage("system", `发送失败：${data.message}`);
|
|
}
|
|
sendBtn.disabled = false;
|
|
}
|
|
|
|
async function resetChat() {
|
|
await fetch("/chat/reset", { method: "POST" });
|
|
messagesEl.innerHTML = "";
|
|
avatarState.pendingFrames = [];
|
|
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
|
|
resetAudioPlayback();
|
|
resetAvatarPose();
|
|
addMessage("system", "会话已重置。");
|
|
await checkHealth();
|
|
}
|
|
|
|
function tryAutoReconnect() {
|
|
if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
|
|
armPageGestureBootstrap();
|
|
setTimeout(() => {
|
|
if (!audioWs) {
|
|
connectAudio({ silentReadyMessage: true }).catch(() => {
|
|
armReconnectOnGesture();
|
|
});
|
|
}
|
|
}, 300);
|
|
}
|
|
|
|
function handleModelFileChange(event) {
|
|
const [file] = event.target.files || [];
|
|
if (!file) return;
|
|
loadModelFromFile(file);
|
|
}
|
|
|
|
function handleModelUrlLoad() {
|
|
const url = modelUrl.value.trim();
|
|
if (!url) return;
|
|
loadModelFromUrl(url);
|
|
}
|
|
|
|
new ResizeObserver(() => resizeStage()).observe(avatarCanvas);
|
|
window.addEventListener("resize", resizeStage);
|
|
|
|
healthBtn.addEventListener("click", checkHealth);
|
|
offerBtn.addEventListener("click", () => {
|
|
connectAudio().catch((error) => {
|
|
console.error(error);
|
|
});
|
|
});
|
|
resetBtn.addEventListener("click", resetChat);
|
|
sendBtn.addEventListener("click", sendText);
|
|
loadModelBtn.addEventListener("click", handleModelUrlLoad);
|
|
resetViewBtn.addEventListener("click", resetView);
|
|
modelFile.addEventListener("change", handleModelFileChange);
|
|
chatInput.addEventListener("keydown", (event) => {
|
|
if (event.key === "Enter") {
|
|
event.preventDefault();
|
|
sendText();
|
|
}
|
|
});
|
|
modelUrl.addEventListener("keydown", (event) => {
|
|
if (event.key === "Enter") {
|
|
event.preventDefault();
|
|
handleModelUrlLoad();
|
|
}
|
|
});
|
|
|
|
initStage();
|
|
resetView();
|
|
resetAvatarPose();
|
|
loadDefaultAvatarModel();
|
|
checkHealth();
|
|
subscribeEvents();
|
|
subscribeSubtitles();
|
|
subscribeAnimation();
|
|
setConnBadge("未连接", "neutral");
|
|
tryAutoReconnect();
|