This commit is contained in:
xsl
2026-03-27 10:49:34 +08:00
commit 22a5f31847
56 changed files with 5145 additions and 0 deletions
+307
View File
@@ -0,0 +1,307 @@
const statusEl = document.getElementById("status");
const outputEl = document.getElementById("output");
const healthBtn = document.getElementById("healthBtn");
const offerBtn = document.getElementById("offerBtn");
const resetBtn = document.getElementById("resetBtn");
const sendBtn = document.getElementById("sendBtn");
const chatInput = document.getElementById("chatInput");
const remoteVideo = document.getElementById("remoteVideo");
remoteVideo.muted = false;
remoteVideo.volume = 1.0;
const messagesEl = document.getElementById("messages");
const connBadge = document.getElementById("connBadge");
const mPeers = document.getElementById("mPeers");
const mBusy = document.getElementById("mBusy");
const mLatency = document.getElementById("mLatency");
const mAsrLatency = document.getElementById("mAsrLatency");
const mLlmLatency = document.getElementById("mLlmLatency");
const mLlmSource = document.getElementById("mLlmSource");
const mLlmHistory = document.getElementById("mLlmHistory");
const mTtsLatency = document.getElementById("mTtsLatency");
const mTtsFirst = document.getElementById("mTtsFirst");
const mTts = document.getElementById("mTts");
const mBargeIn = document.getElementById("mBargeIn");
const mInputMode = document.getElementById("mInputMode");
const mVadCount = document.getElementById("mVadCount");
const dConn = document.getElementById("dConn");
const dVad = document.getElementById("dVad");
const dAsr = document.getElementById("dAsr");
const dLlm = document.getElementById("dLlm");
const dTts = document.getElementById("dTts");
let pc = null;
let evt = null;
let subtitleWs = null;
let localMicStream = null;
let lastSeenRun = 0;
let lastBusy = false;
let aiStreamingEl = null;
const AUTO_RECONNECT_KEY = "visual_chat_auto_reconnect";
const stateMap = {
"SessionState.IDLE": "空闲",
"SessionState.USER_SPEAKING": "用户说话",
"SessionState.THINKING": "思考中",
"SessionState.AVATAR_SPEAKING": "数字人说话",
idle: "空闲",
user_speaking: "用户说话",
thinking: "思考中",
avatar_speaking: "数字人说话",
};
function fmtTime() {
return new Date().toLocaleTimeString("zh-CN", { hour12: false });
}
function toCnState(raw) {
return stateMap[raw] || raw;
}
function addMessage(role, text) {
const row = document.createElement("div");
row.className = "msg-row";
const div = document.createElement("div");
div.className = `msg ${role}`;
div.textContent = text;
const time = document.createElement("div");
time.className = `msg-time ${role}`;
time.textContent = fmtTime();
row.appendChild(div);
row.appendChild(time);
messagesEl.appendChild(row);
messagesEl.scrollTop = messagesEl.scrollHeight;
return div;
}
function setConnBadge(label, kind = "neutral") {
connBadge.textContent = label;
connBadge.className = `badge ${kind}`;
}
function setDot(el, kind) {
el.className = `dot ${kind}`;
}
function updateMetrics(data) {
mPeers.textContent = String(data.peers ?? 0);
mBusy.textContent = data.pipeline_busy ? "是" : "否";
mLatency.textContent = `${data.last_latency_ms ?? 0} ms`;
mAsrLatency.textContent = `${data.last_asr_latency_ms ?? 0} ms`;
mLlmLatency.textContent = `${data.last_llm_latency_ms ?? 0} ms`;
mLlmSource.textContent = data.last_llm_source === "online" ? "在线" : "兜底";
mLlmHistory.textContent = String(data?.llm?.history_items ?? 0);
mTtsLatency.textContent = `${data.last_tts_latency_ms ?? 0} ms`;
mTtsFirst.textContent = `${data.last_tts_first_chunk_ms ?? 0} ms`;
mTts.textContent = data?.tts?.ready ? "就绪" : "未就绪";
mBargeIn.textContent = `${data.last_barge_in_ms ?? 0} ms (${data.barge_in_count ?? 0})`;
mInputMode.textContent = data.last_input_mode === "voice" ? "语音" : data.last_input_mode === "text" ? "文本" : "none";
mVadCount.textContent = `${data.vad_start_count ?? 0}/${data.vad_end_count ?? 0}`;
setDot(dConn, (data.peers ?? 0) > 0 ? "ok" : "warn");
setDot(dVad, (data.vad_start_count ?? 0) > 0 ? "ok" : "warn");
setDot(dAsr, data?.asr?.ready ? "ok" : "warn");
setDot(dLlm, data?.llm?.ready ? "ok" : "bad");
setDot(dTts, data?.tts?.ready ? "ok" : "warn");
}
async function checkHealth() {
const res = await fetch("/health");
const data = await res.json();
statusEl.textContent = `状态:${toCnState(data.state)}`;
outputEl.textContent = JSON.stringify(data, null, 2);
sendBtn.disabled = !!data.pipeline_busy;
updateMetrics(data);
}
function subscribeEvents() {
if (evt) return;
evt = new EventSource("/events");
evt.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
statusEl.textContent = `状态:${toCnState(data.state)}`;
outputEl.textContent = JSON.stringify(data, null, 2);
sendBtn.disabled = !!data.pipeline_busy;
updateMetrics(data);
lastBusy = !!data.pipeline_busy;
if (typeof data.pipeline_runs === "number" && data.pipeline_runs > lastSeenRun) lastSeenRun = data.pipeline_runs;
} catch (_) {
// no-op
}
};
}
function subscribeSubtitles() {
if (subtitleWs && (subtitleWs.readyState === WebSocket.OPEN || subtitleWs.readyState === WebSocket.CONNECTING)) return;
const scheme = location.protocol === "https:" ? "wss" : "ws";
subtitleWs = new WebSocket(`${scheme}://${location.host}/ws/subtitles`);
subtitleWs.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.role === "user") {
aiStreamingEl = null;
addMessage(data.role, data.text || "");
} else if (data.role === "ai") {
if (data.partial) {
if (!aiStreamingEl) aiStreamingEl = addMessage("ai", "");
aiStreamingEl.textContent = `${aiStreamingEl.textContent}${data.text || ""}`;
if (data.final) aiStreamingEl = null;
} else {
aiStreamingEl = null;
addMessage("ai", data.text || "");
}
}
} catch (_) {
// no-op
}
};
subtitleWs.onclose = () => {
setTimeout(subscribeSubtitles, 1000);
};
}
async function connectWebRTC() {
if (pc) {
outputEl.textContent = "WebRTC 已连接或正在连接";
return;
}
setConnBadge("连接中...", "warn");
offerBtn.disabled = true;
const metaRes = await fetch("/meta");
const meta = await metaRes.json();
pc = new RTCPeerConnection({
iceServers: [{ urls: [meta.stun_url || "stun:stun.l.google.com:19302"] }],
});
const currentPc = pc;
let seenConnected = false;
currentPc.onconnectionstatechange = () => {
const st = currentPc.connectionState;
if (st === "connected") {
seenConnected = true;
setConnBadge("已连接", "ok");
localStorage.setItem(AUTO_RECONNECT_KEY, "1");
}
else if (st === "connecting") setConnBadge("连接中...", "warn");
else if (st === "failed" || st === "disconnected" || st === "closed") {
setConnBadge(`连接${st}`, "warn");
if (seenConnected && (st === "disconnected" || st === "failed")) {
addMessage("system", "连接已断开,可能被新的客户端连接接管。");
}
if (localMicStream) {
localMicStream.getTracks().forEach((t) => t.stop());
localMicStream = null;
}
if (pc === currentPc) {
pc = null;
}
offerBtn.disabled = false;
}
};
// Explicitly request remote media m-lines so server can attach tracks.
pc.addTransceiver("video", { direction: "recvonly" });
pc.ontrack = (event) => {
if (event.streams && event.streams[0]) {
remoteVideo.srcObject = event.streams[0];
remoteVideo.play().catch(() => {
addMessage("system", "远端音视频已到达,如未自动播放请点击视频区域后重试。");
});
}
};
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
localMicStream = stream;
} catch (err) {
statusEl.textContent = "状态:麦克风权限失败";
outputEl.textContent = `getUserMedia 失败:${err?.message || err}\n提示:跨机器访问请使用 HTTPS 或浏览器安全例外。`;
setConnBadge("麦克风失败", "warn");
offerBtn.disabled = false;
throw err;
}
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
const sendAudioTrack = stream.getAudioTracks()[0];
if (sendAudioTrack) {
sendAudioTrack.enabled = true;
}
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
const res = await fetch("/webrtc/offer", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
sdp: pc.localDescription.sdp,
type: pc.localDescription.type,
}),
});
const answer = await res.json();
await pc.setRemoteDescription(answer);
statusEl.textContent = "状态:WebRTC 已连接";
outputEl.textContent = JSON.stringify(
{ connected: true, iceConnectionState: pc.iceConnectionState },
null,
2
);
setConnBadge("已连接", "ok");
addMessage("system", "WebRTC 已连接,可开始对话。");
if (answer.replaced_previous) {
addMessage("system", "检测到已有旧连接,已自动断开旧连接并由当前页面接管。");
}
subscribeEvents();
}
async function sendText() {
const text = (chatInput.value || "").trim();
if (!text) return;
if (sendBtn.disabled) return;
sendBtn.disabled = true;
const res = await fetch("/chat/text", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ text }),
});
const data = await res.json();
outputEl.textContent = JSON.stringify(data, null, 2);
if (data.ok) {
chatInput.value = "";
} else if (data.message) {
addMessage("system", `发送失败:${data.message}`);
}
sendBtn.disabled = false;
}
async function resetChat() {
await fetch("/chat/reset", { method: "POST" });
messagesEl.innerHTML = "";
addMessage("system", "会话已重置。");
await checkHealth();
}
function tryAutoReconnect() {
if (localStorage.getItem(AUTO_RECONNECT_KEY) !== "1") return;
setTimeout(() => {
if (!pc) connectWebRTC().catch(() => {
// no-op: user can reconnect manually if permission/policy blocks auto flow
});
}, 300);
}
healthBtn.addEventListener("click", checkHealth);
offerBtn.addEventListener("click", connectWebRTC);
resetBtn.addEventListener("click", resetChat);
sendBtn.addEventListener("click", sendText);
chatInput.addEventListener("keydown", (e) => {
if (e.key === "Enter") {
e.preventDefault();
sendText();
}
});
checkHealth();
subscribeEvents();
subscribeSubtitles();
setConnBadge("未连接", "neutral");
tryAutoReconnect();
+56
View File
@@ -0,0 +1,56 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Visual Chat</title>
<link rel="stylesheet" href="/web/style.css" />
</head>
<body>
<main class="wrap">
<h1>可视化语音聊天系统</h1>
<p id="status">状态:启动中</p>
<div class="video-box">
<video id="remoteVideo" autoplay playsinline></video>
</div>
<div class="actions">
<button id="healthBtn">检查服务状态</button>
<button id="offerBtn">连接 WebRTC</button>
<button id="resetBtn">清空会话</button>
<span id="connBadge" class="badge neutral">未连接</span>
</div>
<div class="chat-box">
<input id="chatInput" type="text" placeholder="输入文字,按发送走文本对话流程" />
<button id="sendBtn">发送</button>
</div>
<div id="messages" class="messages"></div>
<div class="diag" id="diag">
<div class="diag-item"><span class="dot" id="dConn"></span><span>WebRTC连接</span></div>
<div class="diag-item"><span class="dot" id="dVad"></span><span>VAD触发</span></div>
<div class="diag-item"><span class="dot" id="dAsr"></span><span>ASR</span></div>
<div class="diag-item"><span class="dot" id="dLlm"></span><span>LLM</span></div>
<div class="diag-item"><span class="dot" id="dTts"></span><span>TTS</span></div>
</div>
<div class="metrics" id="metrics">
<div class="metric"><span>连接数</span><strong id="mPeers">0</strong></div>
<div class="metric"><span>处理中</span><strong id="mBusy"></strong></div>
<div class="metric"><span>总耗时</span><strong id="mLatency">0 ms</strong></div>
<div class="metric"><span>ASR耗时</span><strong id="mAsrLatency">0 ms</strong></div>
<div class="metric"><span>LLM耗时</span><strong id="mLlmLatency">0 ms</strong></div>
<div class="metric"><span>LLM来源</span><strong id="mLlmSource">unknown</strong></div>
<div class="metric"><span>上下文条目</span><strong id="mLlmHistory">0</strong></div>
<div class="metric"><span>TTS耗时</span><strong id="mTtsLatency">0 ms</strong></div>
<div class="metric"><span>TTS首包</span><strong id="mTtsFirst">0 ms</strong></div>
<div class="metric"><span>TTS</span><strong id="mTts">未就绪</strong></div>
<div class="metric"><span>打断耗时</span><strong id="mBargeIn">0 ms</strong></div>
<div class="metric"><span>输入模式</span><strong id="mInputMode">none</strong></div>
<div class="metric"><span>VAD触发</span><strong id="mVadCount">0/0</strong></div>
</div>
<div class="panel">
<h3>运行状态</h3>
<pre id="output"></pre>
</div>
</main>
<script src="/web/app.js"></script>
</body>
</html>
+239
View File
@@ -0,0 +1,239 @@
body {
margin: 0;
font-family: sans-serif;
background: #0f1115;
color: #f2f2f2;
}
.wrap {
max-width: 900px;
margin: 24px auto;
padding: 0 16px;
}
.video-box {
width: 100%;
aspect-ratio: 16 / 9;
border: 1px solid #333;
border-radius: 10px;
background: #1a1f27;
overflow: hidden;
}
video {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
}
.actions {
margin-top: 16px;
display: flex;
align-items: center;
gap: 8px;
}
.chat-box {
margin-top: 12px;
display: flex;
gap: 8px;
}
.chat-box input {
flex: 1;
background: #111827;
color: #f3f4f6;
border: 1px solid #374151;
border-radius: 8px;
padding: 10px;
}
.messages {
margin-top: 14px;
min-height: 180px;
max-height: 360px;
overflow: auto;
border: 1px solid #1f2937;
border-radius: 10px;
padding: 10px;
background: #0b1220;
display: flex;
flex-direction: column;
gap: 8px;
}
.diag {
margin-top: 12px;
border: 1px solid #1f2937;
border-radius: 10px;
background: #0b0d12;
padding: 10px;
display: flex;
gap: 16px;
flex-wrap: wrap;
}
.diag-item {
display: inline-flex;
align-items: center;
gap: 6px;
color: #d1d5db;
font-size: 13px;
}
.dot {
width: 10px;
height: 10px;
border-radius: 999px;
display: inline-block;
background: #4b5563;
box-shadow: 0 0 0 1px #374151 inset;
}
.dot.ok {
background: #22c55e;
box-shadow: 0 0 0 1px #166534 inset;
}
.dot.warn {
background: #f59e0b;
box-shadow: 0 0 0 1px #92400e inset;
}
.dot.bad {
background: #ef4444;
box-shadow: 0 0 0 1px #991b1b inset;
}
.msg {
max-width: 80%;
padding: 8px 10px;
border-radius: 10px;
line-height: 1.4;
white-space: pre-wrap;
}
.msg-row {
display: flex;
flex-direction: column;
gap: 4px;
}
.msg.user {
align-self: flex-end;
background: #1d4ed8;
}
.msg.ai {
align-self: flex-start;
background: #374151;
}
.panel {
margin-top: 14px;
}
.metrics {
margin-top: 12px;
display: grid;
grid-template-columns: repeat(4, minmax(120px, 1fr));
gap: 8px;
}
.metric {
border: 1px solid #1f2937;
background: #0b0d12;
border-radius: 8px;
padding: 8px 10px;
display: flex;
flex-direction: column;
gap: 4px;
}
.metric span {
color: #9ca3af;
font-size: 12px;
}
.metric strong {
font-size: 14px;
}
.panel h3 {
margin: 0 0 8px;
font-size: 14px;
color: #9ca3af;
}
button {
background: #2563eb;
color: white;
border: 0;
border-radius: 8px;
padding: 8px 12px;
cursor: pointer;
}
button:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.badge {
font-size: 12px;
padding: 4px 8px;
border-radius: 999px;
border: 1px solid transparent;
}
.badge.neutral {
background: #111827;
border-color: #374151;
color: #d1d5db;
}
.badge.ok {
background: #052e16;
border-color: #166534;
color: #86efac;
}
.badge.warn {
background: #3f1d0a;
border-color: #92400e;
color: #fdba74;
}
.msg.system {
align-self: center;
background: #1f2937;
color: #d1d5db;
font-size: 12px;
}
.msg-time {
font-size: 11px;
color: #9ca3af;
}
.msg-time.user {
align-self: flex-end;
}
.msg-time.ai {
align-self: flex-start;
}
.msg-time.system {
align-self: center;
}
pre {
margin-top: 0;
background: #0b0d12;
border: 1px solid #222;
border-radius: 8px;
padding: 12px;
overflow: auto;
}