xsl/product
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save code

Browse Source
This commit is contained in:
xsl
2026-03-30 21:37:46 +08:00
parent 66c742864c
commit 760cf864eb
12 changed files with 86318 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.run/start-test.log
+3
View File
@@ -336,3 +336,6 @@ INFO:root:Loading ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--Se
INFO:root:ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt
INFO:root:scope_map: ['module.', 'None']
INFO:root:excludes: None
INFO:root:Loading ckpt: /home/xsl/.cache/huggingface/hub/models--FunAudioLLM--SenseVoiceSmall/snapshots/3eb3b4eeffc2f2dde6051b853983753db33e35c3/model.pt, status: <All keys matched successfully>
WARNING:services.asr:ASR running in degraded mode on cpu
.run/visual-chat-8018.log
+34
View File
@@ -0,0 +1,34 @@
INFO: Started server process [3446]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on https://0.0.0.0:8018 (Press CTRL+C to quit)
INFO: 127.0.0.1:59440 - "HEAD /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:59450 - "HEAD /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42148 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42162 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:42164 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/style.css HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/app.js?v=20260330a HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.module.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/examples/jsm/controls/OrbitControls.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/loaders/GLTFLoader.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51616 - "GET /web/vendor/%40pixiv/three-vrm/three-vrm.module.min.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/vendor/three/three.core.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /health HTTP/1.1" 200 OK
INFO: 127.0.0.1:51568 - "GET /web/models/VRM1_Constraint_Twist_Sample.vrm HTTP/1.1" 200 OK
INFO: 127.0.0.1:51616 - "GET /events HTTP/1.1" 200 OK
INFO: 127.0.0.1:51614 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 127.0.0.1:51680 - "WebSocket /ws/subtitles" [accepted]
INFO: connection open
INFO: 127.0.0.1:51682 - "WebSocket /ws/animation" [accepted]
INFO: connection open
INFO: 127.0.0.1:53334 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: connection closed
INFO: 127.0.0.1:53342 - "WebSocket /ws/audio" [accepted]
INFO: connection open
INFO: connection closed
INFO: connection closed
INFO: connection closed
.run/visual-chat-8018.pid
+1
View File
@@ -0,0 +1 @@
3446
__pycache__/main.cpython-312.pyc
BIN
View File
Binary file not shown.
web/app.js
+84 -33
View File
@@ -70,10 +70,28 @@ let pageGestureBootstrapArmed = false;
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
const ANIMATION_BUFFER_MS = 120;
const AUDIO_PLAYBACK_BUFFER_MS = 25;
const AV_SYNC_AUDIO_LEAD_MS = 60;
const MOUTH_DRIVE_LIMITS = {
expression: {
aa: 0.58,
ee: 0.40,
oh: 0.48,
ouFromOh: 0.16,
pucker: 0.26,
},
morph: {
jawOpen: 0.42,
visemeAa: 0.36,
visemeEe: 0.34,
visemeOh: 0.38,
mouthPucker: 0.22,
},
debugJaw: 0.52,
};
const DEFAULT_BODY_MODEL_URL =
(modelUrl?.value || "").trim() ||
"/web/models/VRM1_Constraint_Twist_Sample.vrm";
const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
const CLOCK = new THREE.Clock();
const stateMap = {
@@ -325,14 +343,14 @@ async function ensureMediaAudioContext() {
await mediaAudioContext.resume();
}
if (!Number.isFinite(audioPlaybackCursor) || audioPlaybackCursor <= 0) {
audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
}
return mediaAudioContext;
}
function resetAudioPlayback() {
if (mediaAudioContext) {
audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
audioPlaybackCursor = mediaAudioContext.currentTime + getAudioPlaybackBufferSeconds();
}
Array.from(audioPlaybackSources).forEach((source) => {
try {
@@ -350,7 +368,7 @@ async function queueServerAudio(arrayBuffer) {
const source = context.createBufferSource();
source.buffer = decoded;
source.connect(context.destination);
const startAt = Math.max(context.currentTime + 0.04, audioPlaybackCursor);
const startAt = Math.max(context.currentTime + getAudioPlaybackBufferSeconds(), audioPlaybackCursor);
source.start(startAt);
audioPlaybackCursor = startAt + decoded.duration;
audioPlaybackSources.add(source);
@@ -578,10 +596,11 @@ function queueAnimationFrames(payload) {
const frames = Array.isArray(payload.frames) ? payload.frames : [];
const now = performance.now();
const frameWindowMs = (1000 / Math.max(1, payload.fps || 25));
const animationLeadMs = getAnimationLeadMs();
if (payload.type === "animation_reset" || payload.type === "animation_state") {
avatarState.pendingFrames = [];
avatarState.scheduleCursorMs = now + ANIMATION_BUFFER_MS;
avatarState.scheduleCursorMs = now + animationLeadMs;
avatarState.activeControls = {
jawOpen: 0,
viseme_aa: 0,
@@ -594,7 +613,7 @@ function queueAnimationFrames(payload) {
};
}
const baseTime = Math.max(now + ANIMATION_BUFFER_MS, avatarState.scheduleCursorMs);
const baseTime = Math.max(now + animationLeadMs, avatarState.scheduleCursorMs);
frames.forEach((frame, index) => {
const fallbackTime = index * frameWindowMs;
avatarState.pendingFrames.push({
@@ -637,22 +656,25 @@ function applyAnimationControls(controls) {
const headYaw = clampSigned(safeControls.headYaw, 0.35);
const headPitch = clampSigned(safeControls.headPitch, 0.3);
const headRoll = clampSigned(safeControls.headRoll, 0.24);
const hasFacialRig = Boolean(avatarState.currentVrm?.expressionManager) || avatarState.morphBindings.size > 0;
const useExpressions = Boolean(avatarState.currentVrm?.expressionManager);
const useMorphTargets = !useExpressions && avatarState.morphBindings.size > 0;
const hasFacialRig = useExpressions || useMorphTargets;
const mouthDrive = buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker);
const t = performance.now() * 0.001;
if (avatarState.currentVrm?.expressionManager) {
setExpressionValue("aa", visemeAa);
setExpressionValue("ee", visemeEe);
setExpressionValue("oh", visemeOh);
setExpressionValue("ou", Math.max(visemeOh * 0.35, mouthPucker));
if (useExpressions) {
setExpressionValue("aa", mouthDrive.expression.aa);
setExpressionValue("ee", mouthDrive.expression.ee);
setExpressionValue("oh", mouthDrive.expression.oh);
setExpressionValue("ou", mouthDrive.expression.ou);
}
if (avatarState.morphBindings.size > 0) {
setMorphValue("jawOpen", jawOpen);
setMorphValue("viseme_aa", visemeAa);
setMorphValue("viseme_ee", visemeEe);
setMorphValue("viseme_oh", visemeOh);
setMorphValue("mouthPucker", mouthPucker);
if (useMorphTargets) {
setMorphValue("jawOpen", mouthDrive.morph.jawOpen);
setMorphValue("viseme_aa", mouthDrive.morph.viseme_aa);
setMorphValue("viseme_ee", mouthDrive.morph.viseme_ee);
setMorphValue("viseme_oh", mouthDrive.morph.viseme_oh);
setMorphValue("mouthPucker", mouthDrive.morph.mouthPucker);
}
applyBlinkControls(avatarState.blink?.value || 0);
@@ -669,18 +691,18 @@ function applyAnimationControls(controls) {
}
if (!hasFacialRig && avatarState.currentRoot && avatarState.currentRoot !== avatarState.debugRig) {
const baseScale = 1.0 + jawOpen * 0.03;
const baseScale = 1.0 + mouthDrive.debugJaw * 0.03;
avatarState.currentRoot.scale.set(baseScale, baseScale, baseScale);
}
if (avatarState.debugJaw) {
avatarState.debugJaw.rotation.x = jawOpen * 0.5;
avatarState.debugJaw.position.y = -0.06 - jawOpen * 0.015;
avatarState.debugJaw.rotation.x = mouthDrive.debugJaw * 0.5;
avatarState.debugJaw.position.y = -0.06 - mouthDrive.debugJaw * 0.015;
}
if (avatarState.debugMouth) {
avatarState.debugMouth.scale.x = 1.0 + mouthPucker * 0.45;
avatarState.debugMouth.scale.y = 1.0 + jawOpen * 3.8;
avatarState.debugMouth.position.z = 0.31 + mouthPucker * 0.01;
avatarState.debugMouth.scale.x = 1.0 + mouthDrive.morph.mouthPucker * 0.45;
avatarState.debugMouth.scale.y = 1.0 + mouthDrive.debugJaw * 2.6;
avatarState.debugMouth.position.z = 0.31 + mouthDrive.morph.mouthPucker * 0.01;
}
}
@@ -699,10 +721,43 @@ function setMorphValue(controlName, value) {
});
}
function buildMouthDrive(jawOpen, visemeAa, visemeEe, visemeOh, mouthPucker) {
const mouthOpen = Math.max(jawOpen * 0.9, visemeAa);
return {
expression: {
aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.expression.aa),
ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.expression.ee),
oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.expression.oh),
ou: clampControl(
Math.max(
visemeOh * MOUTH_DRIVE_LIMITS.expression.ouFromOh,
mouthPucker * MOUTH_DRIVE_LIMITS.expression.pucker
)
),
},
morph: {
jawOpen: clampControl(jawOpen * MOUTH_DRIVE_LIMITS.morph.jawOpen),
viseme_aa: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.morph.visemeAa),
viseme_ee: clampControl(visemeEe * MOUTH_DRIVE_LIMITS.morph.visemeEe),
viseme_oh: clampControl(visemeOh * MOUTH_DRIVE_LIMITS.morph.visemeOh),
mouthPucker: clampControl(mouthPucker * MOUTH_DRIVE_LIMITS.morph.mouthPucker),
},
debugJaw: clampControl(mouthOpen * MOUTH_DRIVE_LIMITS.debugJaw),
};
}
function clampControl(value) {
return Math.min(1, Math.max(0, Number(value) || 0));
}
function getAudioPlaybackBufferSeconds() {
return AUDIO_PLAYBACK_BUFFER_MS / 1000;
}
function getAnimationLeadMs() {
return ANIMATION_BUFFER_MS + AV_SYNC_AUDIO_LEAD_MS;
}
function clampSigned(value, limit) {
return Math.min(limit, Math.max(-limit, Number(value) || 0));
}
@@ -732,7 +787,7 @@ function detachCurrentAvatar() {
scene.remove(avatarState.currentRoot);
}
avatarState.pendingFrames = [];
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
avatarState.lastAppliedSeq = -1;
avatarState.idleRig = null;
avatarState.currentRoot = avatarState.debugRig;
@@ -765,11 +820,7 @@ async function loadModelFromUrl(url) {
async function loadDefaultAvatarModel() {
modelUrl.value = DEFAULT_BODY_MODEL_URL;
const ok = await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
if (!ok) {
modelUrl.value = DEFAULT_VRM_FALLBACK_URL;
await loadModelFromUrl(DEFAULT_VRM_FALLBACK_URL);
}
await loadModelFromUrl(DEFAULT_BODY_MODEL_URL);
}
async function loadModelFromFile(file) {
@@ -823,7 +874,7 @@ function attachLoadedAsset(gltf, label) {
avatarState.neckBone = resolveNeckBone(vrm, root) || avatarState.headBone;
avatarState.idleRig = buildIdleRig(vrm, root);
avatarState.pendingFrames = [];
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
avatarState.lastAppliedSeq = -1;
fitCameraToObject(root);
@@ -867,7 +918,7 @@ function resolveNeckBone(vrm, root) {
function collectMorphBindings(root) {
const bindings = new Map();
const aliases = {
jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open", "aa", "viseme_aa"],
jawOpen: ["jawopen", "jaw_open", "mouthopen", "mouth_open"],
viseme_aa: ["viseme_aa", "aa", "a", "moutha"],
viseme_ee: ["viseme_ee", "ee", "ih", "i", "mouthee"],
viseme_oh: ["viseme_oh", "oh", "ou", "o", "mouthoh"],
@@ -1321,7 +1372,7 @@ async function resetChat() {
await fetch("/chat/reset", { method: "POST" });
messagesEl.innerHTML = "";
avatarState.pendingFrames = [];
avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
avatarState.scheduleCursorMs = performance.now() + getAnimationLeadMs();
resetAudioPlayback();
resetAvatarPose();
addMessage("system", "会话已重置。");
web/index.html
+4 -5
View File
@@ -90,13 +90,12 @@
<script type="importmap">
{
"imports": {
"three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
"@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.5.1/lib/three-vrm.module.min.js"
"three": "/web/vendor/three/three.module.js",
"three/addons/": "/web/vendor/three/examples/jsm/",
"@pixiv/three-vrm": "/web/vendor/@pixiv/three-vrm/three-vrm.module.min.js"
}
}
</script>
<script type="module" src="/web/app.js?v=20260329m"></script>
<script type="module" src="/web/app.js?v=20260329q"></script>
<script type="module" src="/web/app.js?v=20260330a"></script>
</body>
</html>
web/vendor/@pixiv/three-vrm/three-vrm.module.min.js
Vendored
+983
View File
File diff suppressed because one or more lines are too long
web/vendor/three/examples/jsm/controls/OrbitControls.js
Vendored
+1860
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/examples/jsm/loaders/GLTFLoader.js
Vendored
+4890
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/examples/jsm/utils/BufferGeometryUtils.js
Vendored
+1435
View File
File diff suppressed because it is too large Load Diff
web/vendor/three/three.core.js
Vendored
+58773
View File
File diff suppressed because one or more lines are too long
web/vendor/three/three.module.js
Vendored
+18251
View File
File diff suppressed because one or more lines are too long