diff --git a/.run/start-test.log b/.run/start-test.log
index b9c6f2b..d55e8a9 100644
--- a/.run/start-test.log
+++ b/.run/start-test.log
@@ -336,3 +336,6 @@ INFO:root:Loading ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--Se
WARNING:services.asr:ASR running in degraded mode on cpu
0%|[34m [0m| 0/1 [00:00, ?it/s]
100%|[34m██████████[0m| 1/1 [00:00<00:00, 2.32it/s]
{'load_data': '0.000', 'extract_feat': '0.009', 'forward': '0.431', 'batch_size': '1', 'rtf': '0.028'}, : 100%|[34m██████████[0m| 1/1 [00:00<00:00, 2.32it/s]
rtf_avg: 0.028: 100%|[34m██████████[0m| 1/1 [00:00<00:00, 2.32it/s]
rtf_avg: 0.028: 100%|[34m██████████[0m| 1/1 [00:00<00:00, 2.32it/s]
INFO:httpx:HTTP Request: POST https://api.deepseek.com/v1/chat/completions "HTTP/1.1 200 OK"
+INFO: connection closed
+INFO: connection closed
+INFO: connection closed
diff --git a/.run/visual-chat-8018.log b/.run/visual-chat-8018.log
new file mode 100644
index 0000000..fec0d2c
--- /dev/null
+++ b/.run/visual-chat-8018.log
@@ -0,0 +1,34 @@
+INFO: Started server process [3446]
+INFO: Waiting for application startup.
+INFO: Application startup complete.
+INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
+INFO: 127.0.0.1:59440 - "HEAD /web/vendor/three/three.core.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:59450 - "HEAD /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:42148 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:42162 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:42164 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET / HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/style.css HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/app.js?v=20260330a HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.module.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/vendor/three/examples/jsm/controls/OrbitControls.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/loaders/GLTFLoader.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51616 - "GET /web/vendor/%40pixiv/three-vrm/three-vrm.module.min.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51614 - "GET /health HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51568 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51616 - "GET /events HTTP/1.1" 200 OK
+INFO: 127.0.0.1:51614 - "GET /favicon.ico HTTP/1.1" 404 Not Found
+INFO: 127.0.0.1:51680 - "WebSocket /ws/subtitles" [accepted]
+INFO: connection open
+INFO: 127.0.0.1:51682 - "WebSocket /ws/animation" [accepted]
+INFO: connection open
+INFO: 127.0.0.1:53334 - "WebSocket /ws/audio" [accepted]
+INFO: connection open
+INFO: connection closed
+INFO: 127.0.0.1:53342 - "WebSocket /ws/audio" [accepted]
+INFO: connection open
+INFO: connection closed
+INFO: connection closed
+INFO: connection closed
diff --git a/.run/visual-chat-8018.pid b/.run/visual-chat-8018.pid
new file mode 100644
index 0000000..6cf44d4
--- /dev/null
+++ b/.run/visual-chat-8018.pid
@@ -0,0 +1 @@
+3446
diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc
index c716853..9928b06 100644
Binary files a/__pycache__/main.cpython-312.pyc and b/__pycache__/main.cpython-312.pyc differ
diff --git a/web/app.js b/web/app.js
index 4abffea..f8899a3 100644
--- a/web/app.js
+++ b/web/app.js
@@ -70,10 +70,28 @@ let pageGestureBootstrapArmed = false;
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
const ANIMATION_BUFFER_MS = 120;
+const AUDIO_PLAYBACK_BUFFER_MS = 25;
+const AV_SYNC_AUDIO_LEAD_MS = 60;
+const MOUTH_DRIVE_LIMITS = {
+ expression: {
+ aa: 0.58,
+ ee: 0.40,
+ oh: 0.48,
+ ouFromOh: 0.16,
+ pucker: 0.26,
+ },
+ morph: {
+ jawOpen: 0.42,
+ visemeAa: 0.36,
+ visemeEe: 0.34,
+ visemeOh: 0.38,
+ mouthPucker: 0.22,
+ },
+ debugJaw: 0.52,
+};
const DEFAULT_BODY_MODEL_URL =
(modelUrl?.value || "").trim() ||
"/web/models/VRM1_Constraint_Twist_Sample.vrm";
-const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
const CLOCK = new THREE.Clock();
const stateMap = {
@@ -325,14 +343,14 @@ async function ensureMediaAudioContext() {
await mediaAudioContext.resume();
}
if (!Number.isFinite(audioPlaybackCursor) || audioPlaybackCursor <= 0) {
- audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
+ audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
}
return mediaAudioContext;
}
function resetAudioPlayback() {
if (mediaAudioContext) {
- audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
+ audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
}
Array.from(audioPlaybackSources).forEach((source) => {
try {
@@ -350,7 +368,7 @@ async function queueServerAudio(arrayBuffer) {
const source = context.createBufferSource();
source.buffer = decoded;
source.connect(context.destination);
- const startAt = Math.max(context.currentTime + 0.04, audioPlaybackCursor);
+ const startAt = Math.max(context.currentTime + getAudioPlaybackBufferSeconds(), audioPlaybackCursor);
source.start(startAt);
audioPlaybackCursor = startAt + decoded.duration;
audioPlaybackSources.add(source);
@@ -578,10 +596,11 @@ function queueAnimationFrames(payload) {
const frames = Array.isArray(payload.frames) ? payload.frames : [];
const now = performance.now();
const frameWindowMs = (1000 / Math.max(1, payload.fps || 25));
+ const animationLeadMs = getAnimationLeadMs();
if (payload.type === "animation_reset" || payload.type === "animation_state") {
avatarState.pendingFrames = [];
- avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS;
+ avatarState.scheduleCursorMs = now + animationLeadMs;
avatarState.activeControls = {
jawOpen: 0,
viseme_aa: 0,
@@ -594,7 +613,7 @@ function queueAnimationFrames(payload) {
};
}
- const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs);
+ const baseTime = Math.max(now + animationLeadMs, avatarState.scheduleCursorMs);
frames.forEach((frame, index) => {
const fallbackTime = index * frameWindowMs;
avatarState.pendingFrames.push({
@@ -637,22 +656,25 @@ function applyAnimationControls(controls) {
const headYaw = clampSigned(safeControls.headYaw, 0.35);
const headPitch = clampSigned(safeControls.headPitch, 0.3);
const headRoll = clampSigned(safeControls.headRoll, 0.24);
- const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0;
+ const useExpressions = Boolean(avatarState.currentVrm?.expressionManager);
+ const useMorphTargets = !useExpressions && avatarState.morphBindings.size > 0;
+ const hasFacialRig = useExpressions || useMorphTargets;
+ const mouthDrive = buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker);
const t = performance.now() * 0.001;
- if (avatarState.currentVrm?.expressionManager) {
- setExpressionValue("aa", visemeAa);
- setExpressionValue("ee", visemeEe);
- setExpressionValue("oh", visemeOh);
- setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker));
+ if (useExpressions) {
+ setExpressionValue("aa", mouthDrive.expression.aa);
+ setExpressionValue("ee", mouthDrive.expression.ee);
+ setExpressionValue("oh", mouthDrive.expression.oh);
+ setExpressionValue("ou", mouthDrive.expression.ou);
}
- if (avatarState.morphBindings.size > 0) {
- setMorphValue("jawOpen", jawOpen);
- setMorphValue("viseme_aa", visemeAa);
- setMorphValue("viseme_ee", visemeEe);
- setMorphValue("viseme_oh", visemeOh);
- setMorphValue("mouthPucker", mouthPucker);
+ if (useMorphTargets) {
+ setMorphValue("jawOpen", mouthDrive.morph.jawOpen);
+ setMorphValue("viseme_aa", mouthDrive.morph.viseme_aa);
+ setMorphValue("viseme_ee", mouthDrive.morph.viseme_ee);
+ setMorphValue("viseme_oh", mouthDrive.morph.viseme_oh);
+ setMorphValue("mouthPucker", mouthDrive.morph.mouthPucker);
}
applyBlinkControls(avatarState.blink?.value || 0);
@@ -669,18 +691,18 @@ function applyAnimationControls(controls) {
}
if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
- const baseScale = 1.0 + jawOpen * 0.03;
+ const baseScale = 1.0 + mouthDrive.debugJaw * 0.03;
avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale);
}
if (avatarState.debugJaw) {
- avatarState.debugJaw.rotation.x = jawOpen * 0.5;
- avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015;
+ avatarState.debugJaw.rotation.x = mouthDrive.debugJaw * 0.5;
+ avatarState.debugJaw.position.y = -0.06 - mouthDrive.debugJaw * 0.015;
}
if (avatarState.debugMouth) {
- avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45;
- avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8;
- avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01;
+ avatarState.debugMouth.scale.x = 1.0 + mouthDrive.morph.mouthPucker * 0.45;
+ avatarState.debugMouth.scale.y = 1.0 + mouthDrive.debugJaw * 2.6;
+ avatarState.debugMouth.position.z = 0.31 + mouthDrive.morph.mouthPucker * 0.01;
}
}
@@ -699,10 +721,43 @@ function setMorphValue(controlName, value) {
});
}
+function buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker) {
+ const mouthOpen = Math.max(jawOpen * 0.9, visemeAa);
+ return {
+ expression: {
+ aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.expression.aa),
+ ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.expression.ee),
+ oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.expression.oh),
+ ou: clampControl(
+ Math.max(
+ visemeOh * MOUTH_DRIVE_LIMITS.expression.ouFromOh,
+ mouthPucker * MOUTH_DRIVE_LIMITS.expression.pucker
+ )
+ ),
+ },
+ morph: {
+ jawOpen: clampControl(jawOpen * MOUTH_DRIVE_LIMITS.morph.jawOpen),
+ viseme_aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.morph.visemeAa),
+ viseme_ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.morph.visemeEe),
+ viseme_oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.morph.visemeOh),
+ mouthPucker: clampControl(mouthPucker * MOUTH_DRIVE_LIMITS.morph.mouthPucker),
+ },
+ debugJaw: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.debugJaw),
+ };
+}
+
function clampControl(value) {
return Math.min(1, Math.max(0, Number(value) || 0));
}
+function getAudioPlaybackBufferSeconds() {
+ return AUDIO_PLAYBACK_BUFFER_MS / 1000;
+}
+
+function getAnimationLeadMs() {
+ return ANIMATION_BUFFER_MS + AV_SYNC_AUDIO_LEAD_MS;
+}
+
function clampSigned(value, limit) {
return Math.min(limit, Math.max(-limit, Number(value) || 0));
}
@@ -732,7 +787,7 @@ function detachCurrentAvatar() {
scene.remove(avatarState.currentRoot);
}
avatarState.pendingFrames = [];
- avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+ avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
avatarState.lastAppliedSeq = -1;
avatarState.idleRig = null;
avatarState.currentRoot = avatarState.debugRig;
@@ -765,11 +820,7 @@ async function loadModelFromUrl(url) {
async function loadDefaultAvatarModel() {
modelUrl.value = DEFAULT_BODY_MODEL_URL;
- const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
- if (!ok) {
- modelUrl.value = DEFAULT_VRM_FALLBACK_URL;
- await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL);
- }
+ await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
}
async function loadModelFromFile(file) {
@@ -823,7 +874,7 @@ function attachLoadedAsset(gltf, label) {
avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone;
avatarState.idleRig = buildIdleRig(vrm, root);
avatarState.pendingFrames = [];
- avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+ avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
avatarState.lastAppliedSeq = -1;
fitCameraToObject(root);
@@ -867,7 +918,7 @@ function resolveNeckBone(vrm, root) {
function collectMorphBindings(root) {
const bindings = new Map();
const aliases = {
- jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"],
+ jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open"],
viseme_aa: ["viseme_aa", "aa", "a", "moutha"],
viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"],
viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"],
@@ -1321,7 +1372,7 @@ async function resetChat() {
await fetch("/chat/reset", { method: "POST" });
messagesEl.innerHTML = "";
avatarState.pendingFrames = [];
- avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+ avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
resetAudioPlayback();
resetAvatarPose();
addMessage("system", "会话已重置。");
diff --git a/web/index.html b/web/index.html
index 6e9f87a..24cd911 100644
--- a/web/index.html
+++ b/web/index.html
@@ -90,13 +90,12 @@
-
-
+