save code

2026-03-29 23:01:39 +08:00
parent 36e838caec
commit 4dcae11c1b
30 changed files with 1094 additions and 203 deletions
@@ -50,17 +50,28 @@ const dVad = document.getElementById("dVad");
 const dAsr = document.getElementById("dAsr");
 const dLlm = document.getElementById("dLlm");
 const dTts = document.getElementById("dTts");
+const OFFER_BTN_DEFAULT_TEXT = offerBtn?.textContent || "连接语音通道";

-let pc = null;
 let evt = null;
 let subtitleWs = null;
 let animationWs = null;
+let audioWs = null;
 let localMicStream = null;
 let aiStreamingEl = null;
+let mediaAudioContext = null;
+let micSourceNode = null;
+let micProcessorNode = null;
+let micSinkGain = null;
+let audioPlaybackCursor = 0;
+const audioPlaybackSources = new Set();
+let reconnectOnGestureArmed = false;
+let pageGestureBootstrapArmed = false;

 const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
 const ANIMATION_BUFFER_MS = 120;
-const DEFAULT_BODY_MODEL_URL = "https://cdn.jsdelivr.net/gh/pixiv/three-vrm@release/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
+const DEFAULT_BODY_MODEL_URL =
+  (modelUrl?.value || "").trim() ||
+  "/web/models/VRM1_Constraint_Twist_Sample.vrm";
 const DEFAULT_VRM_FALLBACK_URL = "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm";
 const CLOCK = new THREE.Clock();

@@ -154,10 +165,226 @@ function setConnBadge(label, kind = "neutral") {
  connBadge.className = `badge ${kind}`;
 }

+function setOfferButtonState(mode = "default") {
+  if (!offerBtn) return;
+  if (mode === "resume") {
+    offerBtn.textContent = "点击恢复语音";
+    return;
+  }
+  offerBtn.textContent = OFFER_BTN_DEFAULT_TEXT;
+}
+
 function setDot(el, kind) {
  el.className = `dot ${kind}`;
 }

+function syncConnBadgeFromState(health = null) {
+  if (audioWs?.readyState === WebSocket.OPEN) {
+    if (!health || (health.audio_clients ?? 0) > 0) {
+      setConnBadge("已连接", "ok");
+      return;
+    }
+  }
+  if (audioWs?.readyState === WebSocket.CONNECTING) {
+    setConnBadge("连接中...", "warn");
+    return;
+  }
+  if (health && (health.audio_clients ?? 0) > 0) {
+    setConnBadge("已连接", "ok");
+    return;
+  }
+  setConnBadge("未连接", "neutral");
+}
+
+function disarmReconnectOnGesture() {
+  if (!reconnectOnGestureArmed) return;
+  window.removeEventListener("pointerdown", handleReconnectGesture, true);
+  window.removeEventListener("keydown", handleReconnectGesture, true);
+  reconnectOnGestureArmed = false;
+  setOfferButtonState();
+}
+
+function armReconnectOnGesture() {
+  if (reconnectOnGestureArmed) return;
+  reconnectOnGestureArmed = true;
+  statusEl.textContent = "状态：等待恢复语音";
+  outputEl.textContent = "浏览器拦截了刷新后的自动语音恢复。点击页面任意位置，或点“连接语音通道”，即可恢复麦克风和语音播放。";
+  setConnBadge("等待点击恢复", "warn");
+  setOfferButtonState("resume");
+  window.addEventListener("pointerdown", handleReconnectGesture, true);
+  window.addEventListener("keydown", handleReconnectGesture, true);
+}
+
+async function handleReconnectGesture() {
+  try {
+    await ensureVoiceReadyFromGesture();
+  } catch (error) {
+    console.error(error);
+    armReconnectOnGesture();
+  }
+}
+
+function disarmPageGestureBootstrap() {
+  if (!pageGestureBootstrapArmed) return;
+  window.removeEventListener("pointerdown", handlePageGestureBootstrap, true);
+  window.removeEventListener("keydown", handlePageGestureBootstrap, true);
+  pageGestureBootstrapArmed = false;
+  if (!reconnectOnGestureArmed) {
+    setOfferButtonState();
+  }
+}
+
+function armPageGestureBootstrap() {
+  if (pageGestureBootstrapArmed) return;
+  pageGestureBootstrapArmed = true;
+  setOfferButtonState("resume");
+  window.addEventListener("pointerdown", handlePageGestureBootstrap, true);
+  window.addEventListener("keydown", handlePageGestureBootstrap, true);
+}
+
+async function ensureVoiceReadyFromGesture() {
+  await ensureMediaAudioContext();
+
+  if (audioWs?.readyState === WebSocket.CONNECTING) {
+    return;
+  }
+
+  if (audioWs?.readyState === WebSocket.OPEN) {
+    await startMicCapture();
+    disarmReconnectOnGesture();
+    disarmPageGestureBootstrap();
+    syncConnBadgeFromState();
+    return;
+  }
+
+  await connectAudio({ silentReadyMessage: true });
+  disarmReconnectOnGesture();
+  disarmPageGestureBootstrap();
+}
+
+async function handlePageGestureBootstrap() {
+  if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") {
+    disarmPageGestureBootstrap();
+    return;
+  }
+  try {
+    await ensureVoiceReadyFromGesture();
+  } catch (error) {
+    console.error(error);
+  }
+}
+
+function wsBaseUrl(path) {
+  const scheme = location.protocol === "https:" ? "wss" : "ws";
+  return `${scheme}://${location.host}${path}`;
+}
+
+async function ensureMediaAudioContext() {
+  const AudioContextClass = window.AudioContext || window.webkitAudioContext;
+  if (!AudioContextClass) {
+    throw new Error("当前浏览器不支持 Web Audio API");
+  }
+  if (!mediaAudioContext) {
+    mediaAudioContext = new AudioContextClass();
+  }
+  if (mediaAudioContext.state === "suspended") {
+    await mediaAudioContext.resume();
+  }
+  if (!Number.isFinite(audioPlaybackCursor) || audioPlaybackCursor <= 0) {
+    audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
+  }
+  return mediaAudioContext;
+}
+
+function resetAudioPlayback() {
+  if (mediaAudioContext) {
+    audioPlaybackCursor = mediaAudioContext.currentTime + 0.04;
+  }
+  Array.from(audioPlaybackSources).forEach((source) => {
+    try {
+      source.stop();
+    } catch (_) {
+      // no-op
+    }
+  });
+  audioPlaybackSources.clear();
+}
+
+async function queueServerAudio(arrayBuffer) {
+  const context = await ensureMediaAudioContext();
+  const decoded = await context.decodeAudioData(arrayBuffer.slice(0));
+  const source = context.createBufferSource();
+  source.buffer = decoded;
+  source.connect(context.destination);
+  const startAt = Math.max(context.currentTime + 0.04, audioPlaybackCursor);
+  source.start(startAt);
+  audioPlaybackCursor = startAt + decoded.duration;
+  audioPlaybackSources.add(source);
+  source.onended = () => {
+    audioPlaybackSources.delete(source);
+  };
+}
+
+function stopMicCapture() {
+  if (micSourceNode) {
+    micSourceNode.disconnect();
+    micSourceNode = null;
+  }
+  if (micProcessorNode) {
+    micProcessorNode.disconnect();
+    micProcessorNode.onaudioprocess = null;
+    micProcessorNode = null;
+  }
+  if (micSinkGain) {
+    micSinkGain.disconnect();
+    micSinkGain = null;
+  }
+  if (localMicStream) {
+    localMicStream.getTracks().forEach((track) => track.stop());
+    localMicStream = null;
+  }
+}
+
+async function startMicCapture() {
+  const context = await ensureMediaAudioContext();
+  if (localMicStream && micSourceNode && micProcessorNode) {
+    return context;
+  }
+
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+    video: false,
+  });
+
+  localMicStream = stream;
+  micSourceNode = context.createMediaStreamSource(stream);
+  micProcessorNode = context.createScriptProcessor(4096, 1, 1);
+  micSinkGain = context.createGain();
+  micSinkGain.gain.value = 0;
+
+  micProcessorNode.onaudioprocess = (event) => {
+    if (!audioWs || audioWs.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    const input = event.inputBuffer.getChannelData(0);
+    const pcm = new Int16Array(input.length);
+    for (let i = 0; i < input.length; i += 1) {
+      const sample = Math.max(-1, Math.min(1, input[i] || 0));
+      pcm[i] = sample < 0 ? sample * 32768 : sample * 32767;
+    }
+    audioWs.send(pcm.buffer);
+  };
+
+  micSourceNode.connect(micProcessorNode);
+  micProcessorNode.connect(micSinkGain);
+  micSinkGain.connect(context.destination);
+  return context;
+}
+
 function initStage() {
  scene.background = new THREE.Color(0x0b1118);

@@ -285,7 +512,7 @@ function renderLoop() {
 }

 function updateMetrics(data) {
-  mPeers.textContent = String(data.peers ?? 0);
+  mPeers.textContent = String(data.audio_clients ?? data.peers ?? 0);
  mBusy.textContent = data.pipeline_busy ? "是" : "否";
  mLatency.textContent = `${data.last_latency_ms ?? 0} ms`;
  mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`;
@@ -302,11 +529,12 @@ function updateMetrics(data) {
  mAnimDriver.textContent = data?.avatar?.driver || data.last_animation_mode || "--";
  mAnimFrames.textContent = String(data.last_animation_frame_count ?? data?.avatar?.last_frame_count ?? 0);

-  setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn");
+  setDot(dConn, (data.audio_clients ?? data.peers ?? 0) > 0 ? "ok" : "warn");
  setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
  setDot(dAsr, data?.asr?.ready ? "ok" : "warn");
  setDot(dLlm, data?.llm?.ready ? "ok" : "bad");
  setDot(dTts, data?.tts?.ready ? "ok" : "warn");
+  syncConnBadgeFromState(data);
 }

 function queueAnimationFrames(payload) {
@@ -724,8 +952,7 @@ function subscribeEvents() {

 function subscribeSubtitles() {
  if (subtitleWs && (subtitleWs.readyState === WebSocket.OPEN || subtitleWs.readyState === WebSocket.CONNECTING)) return;
-  const scheme = location.protocol === "https:" ? "wss" : "ws";
-  subtitleWs = new WebSocket(`${scheme}://${location.host}/ws/subtitles`);
+  subtitleWs = new WebSocket(wsBaseUrl("/ws/subtitles"));
  subtitleWs.onmessage = (event) => {
    try {
      const data = JSON.parse(event.data);
@@ -753,9 +980,8 @@ function subscribeSubtitles() {

 function subscribeAnimation() {
  if (animationWs && (animationWs.readyState === WebSocket.OPEN || animationWs.readyState === WebSocket.CONNECTING)) return;
-  const scheme = location.protocol === "https:" ? "wss" : "ws";
  animationConn.textContent = "连接中";
-  animationWs = new WebSocket(`${scheme}://${location.host}/ws/animation`);
+  animationWs = new WebSocket(wsBaseUrl("/ws/animation"));
  animationWs.onopen = () => {
    animationConn.textContent = "已连接";
  };
@@ -781,59 +1007,48 @@ function subscribeAnimation() {
  };
 }

-async function connectWebRTC() {
-  if (pc) {
-    outputEl.textContent = "WebRTC 已连接或正在连接";
+async function connectAudio(options = {}) {
+  const { silentReadyMessage = false } = options;
+  if (audioWs && audioWs.readyState === WebSocket.OPEN) {
+    disarmReconnectOnGesture();
+    outputEl.textContent = "语音通道已连接";
    return;
  }
+  if (audioWs && audioWs.readyState === WebSocket.CONNECTING) {
+    outputEl.textContent = "语音通道正在连接";
+    await new Promise((resolve, reject) => {
+      const startedAt = Date.now();
+      const timer = window.setInterval(() => {
+        if (!audioWs) {
+          window.clearInterval(timer);
+          reject(new Error("语音通道不存在"));
+          return;
+        }
+        if (audioWs.readyState === WebSocket.OPEN) {
+          window.clearInterval(timer);
+          resolve();
+          return;
+        }
+        if (audioWs.readyState === WebSocket.CLOSING || audioWs.readyState === WebSocket.CLOSED) {
+          window.clearInterval(timer);
+          reject(new Error("语音通道连接已关闭"));
+          return;
+        }
+        if (Date.now() - startedAt > 5000) {
+          window.clearInterval(timer);
+          reject(new Error("语音通道连接超时"));
+        }
+      }, 100);
+    });
+    return;
+  }
+
  setConnBadge("连接中...", "warn");
  offerBtn.disabled = true;

-  const metaRes = await fetch("/meta");
-  const meta = await metaRes.json();
-  pc = new RTCPeerConnection({
-    iceServers: [{ urls: [meta.stun_url || "stun:stun.l.google.com:19302"] }],
-  });
-  const currentPc = pc;
-  let seenConnected = false;
-
-  currentPc.onconnectionstatechange = () => {
-    const st = currentPc.connectionState;
-    if (st === "connected") {
-      seenConnected = true;
-      setConnBadge("已连接", "ok");
-      localStorage.setItem(AUTO_RECONNECT_KEY, "1");
-    } else if (st === "connecting") {
-      setConnBadge("连接中...", "warn");
-    } else if (st === "failed" || st === "disconnected" || st === "closed") {
-      setConnBadge(`连接${st}`, "warn");
-      if (seenConnected && (st === "disconnected" || st === "failed")) {
-        addMessage("system", "连接已断开，可能被新的客户端连接接管。");
-      }
-      if (localMicStream) {
-        localMicStream.getTracks().forEach((track) => track.stop());
-        localMicStream = null;
-      }
-      if (pc === currentPc) {
-        pc = null;
-      }
-      offerBtn.disabled = false;
-    }
-  };
-
-  pc.ontrack = (event) => {
-    if (event.streams && event.streams[0]) {
-      remoteAudio.srcObject = event.streams[0];
-      remoteAudio.play().catch(() => {
-        addMessage("system", "远端语音已到达，如未自动播放请先与页面交互后重试。");
-      });
-    }
-  };
-
-  let stream;
+  let context;
  try {
-    stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
-    localMicStream = stream;
+    context = await startMicCapture();
  } catch (err) {
    statusEl.textContent = "状态：麦克风权限失败";
    outputEl.textContent = `getUserMedia 失败：${err?.message || err}\n提示：跨机器访问请使用 HTTPS 或浏览器安全例外。`;
@@ -842,34 +1057,113 @@ async function connectWebRTC() {
    throw err;
  }

-  stream.getTracks().forEach((track) => pc.addTrack(track, stream));
-  const offer = await pc.createOffer();
-  await pc.setLocalDescription(offer);
+  const ws = new WebSocket(wsBaseUrl("/ws/audio"));
+  ws.binaryType = "arraybuffer";
+  audioWs = ws;

-  const res = await fetch("/webrtc/offer", {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      sdp: pc.localDescription.sdp,
-      type: pc.localDescription.type,
-    }),
+  await new Promise((resolve, reject) => {
+    let settled = false;
+
+    ws.onopen = () => {
+      if (audioWs !== ws) {
+        try {
+          ws.close(1000, "stale-open");
+        } catch (_) {
+          // no-op
+        }
+        return;
+      }
+      ws.send(
+        JSON.stringify({
+          type: "audio_config",
+          sample_rate: context.sampleRate,
+          channels: 1,
+          format: "pcm_s16le",
+        }),
+      );
+      statusEl.textContent = "状态：语音通道已连接";
+      outputEl.textContent = JSON.stringify({ connected: true, transport: "websocket_audio" }, null, 2);
+      setConnBadge("已连接", "ok");
+      localStorage.setItem(AUTO_RECONNECT_KEY, "1");
+      disarmReconnectOnGesture();
+      offerBtn.disabled = false;
+      if (!silentReadyMessage) {
+        addMessage("system", "语音通道已连接，可开始对话。");
+      }
+      subscribeEvents();
+      settled = true;
+      resolve();
+    };
+
+    ws.onmessage = (event) => {
+      if (typeof event.data === "string") {
+        try {
+          const data = JSON.parse(event.data);
+          if (data.type === "ping" || data.type === "pong" || data.type === "audio_ready") {
+            return;
+          }
+          if (data.type === "audio_reset") {
+            resetAudioPlayback();
+            return;
+          }
+        } catch (_) {
+          // no-op
+        }
+        return;
+      }
+
+      queueServerAudio(event.data).catch((error) => {
+        console.error(error);
+        addMessage("system", `语音播放失败：${error?.message || error}`);
+      });
+    };
+
+    ws.onerror = (event) => {
+      console.error(event);
+      if (!settled) {
+        settled = true;
+        offerBtn.disabled = false;
+        setConnBadge("连接失败", "warn");
+        reject(new Error("语音通道连接失败"));
+      }
+    };
+
+    ws.onclose = (event) => {
+      if (audioWs !== ws) {
+        if (!settled) {
+          settled = true;
+          resolve();
+        }
+        return;
+      }
+      resetAudioPlayback();
+      stopMicCapture();
+      if (audioWs === ws) {
+        audioWs = null;
+      }
+      syncConnBadgeFromState();
+      offerBtn.disabled = false;
+      if (event.code !== 1000 && event.code !== 1001 && event.code !== 1012) {
+        addMessage("system", "语音通道已断开，请重新连接。");
+      }
+      if (!settled) {
+        settled = true;
+        reject(new Error("语音通道在建立前已关闭"));
+      }
+    };
  });
-  const answer = await res.json();
-  await pc.setRemoteDescription(answer);
-  statusEl.textContent = "状态：WebRTC 已连接";
-  outputEl.textContent = JSON.stringify({ connected: true, iceConnectionState: pc.iceConnectionState }, null, 2);
-  setConnBadge("已连接", "ok");
-  addMessage("system", "WebRTC 已连接，可开始对话。");
-  if (answer.replaced_previous) {
-    addMessage("system", "检测到已有旧连接，已自动断开旧连接并由当前页面接管。");
-  }
-
-  subscribeEvents();
 }

 async function sendText() {
  const text = (chatInput.value || "").trim();
  if (!text || sendBtn.disabled) return;
+  if (!audioWs || audioWs.readyState !== WebSocket.OPEN) {
+    try {
+      await connectAudio({ silentReadyMessage: true });
+    } catch (_) {
+      addMessage("system", "语音通道未连接，本次仅保证文字链路可用。");
+    }
+  }
  sendBtn.disabled = true;
  const res = await fetch("/chat/text", {
    method: "POST",
@@ -891,6 +1185,7 @@ async function resetChat() {
  messagesEl.innerHTML = "";
  avatarState.pendingFrames = [];
  avatarState.scheduleCursorMs = performance.now() + ANIMATION_BUFFER_MS;
+  resetAudioPlayback();
  resetAvatarPose();
  addMessage("system", "会话已重置。");
  await checkHealth();
@@ -898,10 +1193,11 @@ async function resetChat() {

 function tryAutoReconnect() {
  if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
+  armPageGestureBootstrap();
  setTimeout(() => {
-    if (!pc) {
-      connectWebRTC().catch(() => {
-        // no-op
+    if (!audioWs) {
+      connectAudio({ silentReadyMessage: true }).catch(() => {
+        armReconnectOnGesture();
      });
    }
  }, 300);
@@ -923,7 +1219,11 @@ new ResizeObserver(() => resizeStage()).observe(avatarCanvas);
 window.addEventListener("resize", resizeStage);

 healthBtn.addEventListener("click", checkHealth);
-offerBtn.addEventListener("click", connectWebRTC);
+offerBtn.addEventListener("click", () => {
+  connectAudio().catch((error) => {
+    console.error(error);
+  });
+});
 resetBtn.addEventListener("click", resetChat);
 sendBtn.addEventListener("click", sendText);
 loadModelBtn.addEventListener("click", handleModelUrlLoad);
@@ -18,7 +18,7 @@
      <div class="stage-toolbar">
        <label class="file-pill" for="modelFile">载入本地 VRM/GLB</label>
        <input id="modelFile" type="file" accept=".vrm,.glb,.gltf" />
-        <input id="modelUrl" type="text" placeholder="输入 VRM/GLB/GLTF 地址，例如官方示例 VRM 或 /web/models/avatar.vrm" />
+        <input id="modelUrl" type="text" value="/web/models/VRM1_Constraint_Twist_Sample.vrm" placeholder="输入 VRM/GLB/GLTF 地址，例如官方示例 VRM 或 /web/models/avatar.vrm" />
        <button id="loadModelBtn">加载模型 URL</button>
        <button id="resetViewBtn">重置视角</button>
      </div>
@@ -48,7 +48,7 @@
    <audio id="remoteAudio" autoplay></audio>
    <div class="actions">
      <button id="healthBtn">检查服务状态</button>
-      <button id="offerBtn">连接 WebRTC</button>
+      <button id="offerBtn">连接语音通道</button>
      <button id="resetBtn">清空会话</button>
      <span id="connBadge" class="badge neutral">未连接</span>
    </div>
@@ -58,14 +58,14 @@
    </div>
    <div id="messages" class="messages"></div>
    <div class="diag" id="diag">
-      <div class="diag-item"><span class="dot" id="dConn"></span><span>WebRTC连接</span></div>
+      <div class="diag-item"><span class="dot" id="dConn"></span><span>语音通道</span></div>
      <div class="diag-item"><span class="dot" id="dVad"></span><span>VAD触发</span></div>
      <div class="diag-item"><span class="dot" id="dAsr"></span><span>ASR</span></div>
      <div class="diag-item"><span class="dot" id="dLlm"></span><span>LLM</span></div>
      <div class="diag-item"><span class="dot" id="dTts"></span><span>TTS</span></div>
    </div>
    <div class="metrics" id="metrics">
-      <div class="metric"><span>连接数</span><strong id="mPeers">0</strong></div>
+      <div class="metric"><span>语音通道</span><strong id="mPeers">0</strong></div>
      <div class="metric"><span>处理中</span><strong id="mBusy">否</strong></div>
      <div class="metric"><span>总耗时</span><strong id="mLatency">0 ms</strong></div>
      <div class="metric"><span>ASR耗时</span><strong id="mAsrLatency">0 ms</strong></div>
@@ -96,6 +96,6 @@
      }
    }
  </script>
-  <script type="module" src="/web/app.js?v=20260327d"></script>
+  <script type="module" src="/web/app.js?v=20260329k"></script>
 </body>
 </html>