From 624fec8a23a17526c5157122fb21513c4762a8b5 Mon Sep 17 00:00:00 2001 From: xsl Date: Tue, 1 Sep 2026 21:59:45 +0800 Subject: [PATCH] =?UTF-8?q?harness:=20=E5=85=A8=E6=8B=9F=E7=9C=9Flive?= =?UTF-8?q?=E8=AF=95=E7=8E=A9=E7=AE=A1=E7=BA=BF=EF=BC=88CDP=E7=9C=9F?= =?UTF-8?q?=E6=9C=BA=E9=A9=B1=E5=8A=A8chromium+WebAudio=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E9=87=87=E9=9B=86+FunASR/SenseVoice=E5=A2=9E=E9=87=8F=E8=BD=AC?= =?UTF-8?q?=E5=86=99=EF=BC=8C=E5=88=A4=E6=97=81=E7=99=BDOK/TRUNCATED/MISMA?= =?UTF-8?q?TCH/SILENCE=EF=BC=9B=E6=AF=8F=E6=B8=B8=E6=88=8F=E7=8B=AC?= =?UTF-8?q?=E7=AB=8B=E6=8E=A7=E5=88=B6=E7=AB=AF=E5=8F=A38123-8128=EF=BC=89?= =?UTF-8?q?+=20serve.sh=E5=8E=BB=E7=A1=AC=E7=BC=96=E7=A0=81=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=20+=20gitignore=E5=B1=8F=E8=94=BDlive/out=E4=BA=A7?= =?UTF-8?q?=E7=89=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + web/harness/live/README.md | 104 ++++++ web/harness/live/cdp.mjs | 207 +++++++++++ web/harness/live/control.mjs | 548 ++++++++++++++++++++++++++++++ web/harness/live/inject.js | 251 ++++++++++++++ web/harness/live/listen_server.py | 176 ++++++++++ web/serve.sh | 10 +- 7 files changed, 1297 insertions(+), 1 deletion(-) create mode 100644 web/harness/live/README.md create mode 100644 web/harness/live/cdp.mjs create mode 100644 web/harness/live/control.mjs create mode 100644 web/harness/live/inject.js create mode 100644 web/harness/live/listen_server.py diff --git a/.gitignore b/.gitignore index 1bbc63be..8b2693e1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ tools/samples/ .gpu.lock # Web 服务器运行时数据（用户/PIN/会话/进度，勿入库） web/server/data/ +# 实时玩家 agent 产物（录音/事件/转写/chromium profile） +web/harness/live/out/ # ---- 生成中间产物（可由 gen/ 脚本复现） ---- app/src/*/gen/wav/ diff --git a/web/harness/live/README.md b/web/harness/live/README.md new file mode 100644 index 00000000..bc393363 --- /dev/null +++ b/web/harness/live/README.md @@ -0,0 +1,104 @@ +# live/ — 全拟真玩家 agent 基础设施 + +用**真实无头 Chromium** 黑盒驱动任意一款 SoundGame（纯音频 HTML5 游戏）， +录制 Web Audio **真实混音输出**，用本地 **SenseVoice ASR**「听懂」旁白是否 +完整播完/被截断，外部 agent 通过 HTTP 逐步玩游戏（每步根据听到的内容决策）。 + +零依赖（Node ≥22 全局 WebSocket/fetch + 一个 Python 文件用 funasr）， +不改 `web/engine/` 与游戏代码，全部探针经 CDP `Page.addScriptToEvaluateOnNewDocument` +预注入（patch 原型）。 + +## 组成 + +| 文件 | 职责 | +|---|---| +| `cdp.mjs` | 极简 CDP 客户端：spawn chromium（`--headless=new --remote-debugging-port=0`），WebSocket 多路复用，`Target.createTarget`+`attachToTarget(flatten)` 后用 sessionId 发 `Page.*`/`Runtime.evaluate`/`Input.dispatchTouchEvent` | +| `inject.js` | 预注入探针（纯文本，控制端读文件注入）：fetch/decodeAudioData/getChannelData/ABSN start/stop/connect/vibrate/console 七类 patch；混音经 tap GainNode → `MediaRecorder`(opus) → base64 chunks | +| `control.mjs` | HTTP 控制服务器（端口 8123）：会话/手势/事件/转写/落盘；每 500ms drain 页面 `__live`（chunks 追加 `mix.webm`，事件换算到录音时基） | +| `listen_server.py` | SenseVoice 常驻进程（stdin/stdout JSON Lines）：ffmpeg 解码 webm → 16k wav，按 voice 事件窗口切片转写 + astats 能量判定 + difflib 相似度 | + +voice 检测原理：引擎 voice 层走 SoundTouch 管线（`ScriptProcessorNode` 从 +AudioBuffer 抽读），**不经** `AudioBufferSourceNode.start`。探针 patch +`AudioBuffer.prototype.getChannelData`——引擎里只有 SoundTouch 的 +`WebAudioBufferSource.extract` 对 voice buffer 调它——以此检出 voice 段的 +开始/时长，用 `ScriptProcessorNode.disconnect`（引擎 `_killVoice` 必经） +检出结束。bgm/sfx 走 `AudioBufferSourceNode.start/stop`，直接挂钩。 + +时间轴对齐：事件时间戳是 `performance.now()`（ms）；`__live.recStartT` 是 +recorder 启动当刻的同源时间戳；`ta = (t - recStartT)/1000` 即录音时间轴秒 +（误差 ≈ AudioContext resume 延迟，几十 ms，ASR 窗口 ±0.3s 余量吸收）。 + +## 判定逻辑（截断双保险） + +- **事件层**：`vend.ta < t0 + dur/tempo - 0.15s` → 引擎级截断（tempo=设置语速， + 长按快进窗口按 1.7 倍速修正预期结束时刻）。 +- **ASR 层**：窗口 `[t0-0.15, t1+0.35]` 切片转写，与 `voice_text.json` 期望文本 + 做 `difflib.SequenceMatcher`（去 `<|...|>` 标签/标点/小写归一化），`<0.6` → 内容不完整/错配。 +- 窗口 RMS < -45dB → `SILENCE`；无 voice 事件的区间（间隙探针）有能量 → + `SFX_BGM_ONLY`，无能量 → `SILENCE`。 +- verdict 优先级：`TRUNCATED`（事件层）> `SILENCE` > `MISMATCH` > `OK`。 + +## 用法 + +前置：游戏服务器在 8001（`PORT=8001 node web/server/server.js`）。 + +```bash +node web/harness/live/control.mjs # 8123；首次启动加载 SenseVoice 约 10-20s +``` + +### curl 示例会话 + +```bash +# 1. 开会话（settings 接受友好名 normal/low/off 或引擎枚举值 1.0/0.3/0） +curl -X POST localhost:8123/session -H 'content-type: application/json' -d '{ + "game": "rpg", + "settings": {"voice":"male","speed":"normal","bgm":"normal","guide":"full"} +}' + +# 2. 第一个动作必须是 tap（解锁启动遮罩，否则全部无声） +curl -X POST localhost:8123/gesture -H 'content-type: application/json' -d '{"type":"tap"}' + +# 3. 循环：听 → 决策 → 动手势 +curl "localhost:8123/listen" # 自上次以来新判定的段落 +curl "localhost:8123/events?since=0" # 增量原始事件（下次用返回的 next 当 since） +curl -X POST localhost:8123/gesture -H 'content-type: application/json' \ + -d '{"type":"swiperight"}' # tap/doubletap/swipeup/swipedown/swipeleft/swiperight/longpress + # x,y 缺省屏幕中心；swipe 默认 200px/300ms + +# 4. 复盘与收尾 +curl "localhost:8123/transcript" # 合并时间轴全量（JSON） +curl -X POST localhost:8123/stop # 落盘 4 产物，关页签；服务器保持运行 +``` + +`/listen` 返回行： + +```json +{"t0": 53.68, "t1": 55.56, "voiceId": "desc_path", + "expected": "林间小路。风穿过树叶，脚下的落叶沙沙作响。", + "heard": "林间小路。", "similarity": 0.3636, "verdict": "TRUNCATED"} +``` + +产物落到 `web/harness/live/out/-/`：`mix.webm`（真实混音）、 +`events.jsonl`（全事件，含录音时基 `ta`）、`transcript.md`（人读时间轴）、 +`verdicts.json`（全部 voice 段 + 判定）。 + +## 已知限制 + +- **短句（≤6 字）ASR 同音误识**会把相似度拉到 0.6 以下 → 偶发 `MISMATCH` + 假阳性（例：「前进，去岔路口」被听成「前进却差了」）。源资产直转是对的， + 问题出在混音背景（bgm/ambience）下的短切片；verdict 是「ASR 层」判定， + 事件层 `TRUNCATED`/`SILENCE` 不受影响。需要高置信时以 `similarity` 数值 + 而非仅 verdict 做决策。 +- 单会话：同时只开一个游戏页签，`POST /session` 会顶掉前一个（不落盘）。 +- 录音 t=0 与事件时基的对齐误差 ≈ AudioContext resume 延迟（实测几十 ms）， + 不影响 ±0.3s 余量的窗口切分。 +- MediaRecorder 不可用时探针降级为只记事件不录音（`__live.recState='unavailable'`， + 事件流里有 `recfail`）；此时 `/listen` 的 ASR 层不可用。 +- 间隙探针只覆盖相邻 voice 段之间与首段之前的 ≥1.5s 区间；末尾开放区间不探测。 +- chromium profile 写在 `out/.profiles/`，浏览器随控制服务器退出时清理。 + +## 环境 + +- `chromium`（snap）在 PATH；`node` ≥22；`ffmpeg` 在 PATH。 +- `python3` 需 funasr + torch（SenseVoiceSmall 模型走 modelscope 缓存）。 +- 环境变量：`LIVE_PORT`（默认 8123）、`GAME_BASE`（默认 `http://localhost:8001`）。 diff --git a/web/harness/live/cdp.mjs b/web/harness/live/cdp.mjs new file mode 100644 index 00000000..29ac2de8 --- /dev/null +++ b/web/harness/live/cdp.mjs @@ -0,0 +1,207 @@ +/** + * cdp.mjs — 极简 Chrome DevTools Protocol 客户端（零依赖，Node >= 22 全局 WebSocket）。 + * + * launch()：spawn 无头 Chromium（--remote-debugging-port=0），从 stderr 解析 + * `DevTools listening on ws://…`，连浏览器级 WebSocket。 + * browser.newPage()：Target.createTarget + attachToTarget(flatten) → Page 句柄， + * 之后所有命令带 sessionId 发送。 + * + * 用法： + * const browser = await launch(); + * const page = await browser.newPage({ width: 390, height: 844 }); + * await page.addInitScript(source); + * await page.navigate('http://localhost:8001/rpg/'); + * const v = await page.eval('1+1'); + * await page.tap(195, 422); + * await browser.dispose(); + */ +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const CDP_TIMEOUT_MS = 30000; + +/** 极简 CDP 连接：id 多路复用 + method 事件回调。 */ +class Connection { + constructor(ws) { + this.ws = ws; + this.nextId = 1; + this.pending = new Map(); // id -> {resolve, reject, timer} + this.eventHandlers = []; // [{method, fn}]，method 为 null 时收全部 + ws.addEventListener('message', (e) => { + let msg; + try { msg = JSON.parse(e.data); } catch (_) { return; } + if (msg.id !== undefined) { + const p = this.pending.get(msg.id); + if (!p) return; + this.pending.delete(msg.id); + clearTimeout(p.timer); + if (msg.error) p.reject(new Error(`${msg.error.message} (code ${msg.error.code})`)); + else p.resolve(msg.result); + } else if (msg.method) { + for (const h of this.eventHandlers) { + if (h.method === null || h.method === msg.method) { + try { h.fn(msg.params, msg.sessionId || null); } catch (_) {} + } + } + } + }); + ws.addEventListener('close', () => { + for (const [, p] of this.pending) { clearTimeout(p.timer); p.reject(new Error('CDP websocket closed')); } + this.pending.clear(); + }); + } + + static async connect(wsUrl) { + const ws = new WebSocket(wsUrl); + await new Promise((resolve, reject) => { + ws.addEventListener('open', resolve, { once: true }); + ws.addEventListener('error', () => reject(new Error(`WebSocket 连接失败: ${wsUrl}`)), { once: true }); + }); + return new Connection(ws); + } + + /** @param {string} method @param {object} [params] @param {string} [sessionId] */ + send(method, params = {}, sessionId = undefined, timeoutMs = CDP_TIMEOUT_MS) { + const id = this.nextId++; + const msg = { id, method, params }; + if (sessionId) msg.sessionId = sessionId; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`CDP 超时: ${method}`)); + }, timeoutMs); + this.pending.set(id, { resolve, reject, timer }); + this.ws.send(JSON.stringify(msg)); + }); + } + + on(method, fn) { this.eventHandlers.push({ method, fn }); } + + close() { try { this.ws.close(); } catch (_) {} } +} + +/** 页面句柄：所有命令自动带 sessionId。 */ +export class Page { + constructor(conn, sessionId, targetId) { + this.conn = conn; + this.sessionId = sessionId; + this.targetId = targetId; + } + + send(method, params = {}, timeoutMs) { return this.conn.send(method, params, this.sessionId, timeoutMs); } + + /** 页面脚本之前执行（每次导航都注入）。 */ + async addInitScript(source) { + const r = await this.send('Page.addScriptToEvaluateOnNewDocument', { source }); + return r.identifier; + } + + async navigate(url) { return this.send('Page.navigate', { url }); } + + /** Runtime.evaluate，returnByValue；可选 awaitPromise。异常会 reject。 */ + async eval(expression, { awaitPromise = false, timeoutMs } = {}) { + const r = await this.send('Runtime.evaluate', { + expression, returnByValue: true, awaitPromise, + }, timeoutMs); + if (r.exceptionDetails) { + const d = r.exceptionDetails; + throw new Error(`页面内求值异常: ${d.text} ${(d.exception && d.exception.description) || ''}`); + } + return r.result ? r.result.value : undefined; + } + + /** 视口与触摸仿真（在 navigate 前设置）。 */ + async setViewport(width, height, { deviceScaleFactor = 2, mobile = true } = {}) { + await this.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor, mobile }); + await this.send('Emulation.setTouchEmulationEnabled', { enabled: true, maxTouchPoints: 1 }); + } + + /** 原始触摸事件派发。type: touchStart|touchMove|touchEnd|touchCancel */ + async dispatchTouch(type, touchPoints = []) { + await this.send('Input.dispatchTouchEvent', { type, touchPoints }); + } + + async close() { + await this.conn.send('Target.closeTarget', { targetId: this.targetId }).catch(() => {}); + } +} + +export class Browser { + constructor(proc, conn, profileDir) { + this.proc = proc; + this.conn = conn; + this.profileDir = profileDir; + } + + /** 开新页签并挂 session。 */ + async newPage({ width = 390, height = 844 } = {}) { + const { targetId } = await this.conn.send('Target.createTarget', { url: 'about:blank' }); + const { sessionId } = await this.conn.send('Target.attachToTarget', { targetId, flatten: true }); + const page = new Page(this.conn, sessionId, targetId); + await page.send('Page.enable'); + await page.send('Runtime.enable'); + if (width && height) await page.setViewport(width, height); + return page; + } + + on(method, fn) { this.conn.on(method, fn); } + + /** 关浏览器 + 清 profile 目录。 */ + async dispose() { + try { this.conn.close(); } catch (_) {} + if (this.proc && !this.proc.killed) { + this.proc.kill('SIGKILL'); + } + if (this.profileDir) { + try { fs.rmSync(this.profileDir, { recursive: true, force: true }); } catch (_) {} + } + } +} + +/** + * spawn 无头 Chromium 并连上 CDP。 + * @param {object} [opts] + * @param {string} [opts.executable='chromium'] + * @param {string} [opts.profileBase] 用户数据目录的父目录（snap 对 $HOME 内非隐藏路径可写） + * @returns {Promise} + */ +export async function launch(opts = {}) { + const executable = opts.executable || 'chromium'; + const profileBase = opts.profileBase || fs.mkdtempSync(path.join(os.tmpdir(), 'live-cdp-')); + fs.mkdirSync(profileBase, { recursive: true }); + const profileDir = fs.mkdtempSync(path.join(profileBase, 'profile-')); + + const args = [ + '--headless=new', + '--remote-debugging-port=0', + `--user-data-dir=${profileDir}`, + '--no-first-run', + '--disable-gpu', + '--mute-audio', + '--autoplay-policy=no-user-gesture-required', + 'about:blank', + ]; + const proc = spawn(executable, args, { stdio: ['ignore', 'ignore', 'pipe'] }); + + const wsUrl = await new Promise((resolve, reject) => { + let buf = ''; + const timer = setTimeout(() => reject(new Error(`等待 DevTools ws 地址超时。stderr 摘录:\n${buf.slice(-2000)}`)), 20000); + proc.stderr.on('data', (d) => { + buf += d.toString(); + const m = buf.match(/DevTools listening on (ws:\/\/\S+)/); + if (m) { clearTimeout(timer); resolve(m[1]); } + }); + proc.on('exit', (code) => { clearTimeout(timer); reject(new Error(`chromium 提前退出 (code ${code})。stderr 摘录:\n${buf.slice(-2000)}`)); }); + proc.on('error', (e) => { clearTimeout(timer); reject(e); }); + }); + + const conn = await Connection.connect(wsUrl); + const browser = new Browser(proc, conn, profileDir); + // 忽略浏览器级噪声事件即可；需要时 browser.on(...) 订阅 + return browser; +} + +/** sleep 工具（手势编排用）。 */ +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); diff --git a/web/harness/live/control.mjs b/web/harness/live/control.mjs new file mode 100644 index 00000000..734e8537 --- /dev/null +++ b/web/harness/live/control.mjs @@ -0,0 +1,548 @@ +#!/usr/bin/env node +/** + * control.mjs — 全拟真玩家 agent HTTP 控制服务器（Node 零依赖，端口 8123）。 + * + * 外部 agent 通过 HTTP 逐步玩任意一款 SoundGame： + * POST /session {game, settings?, viewport?} 开新页（预注入探针 + 设置种子） + * POST /gesture {type, x?, y?} 真实触摸手势（tap/doubletap/swipe四向/longpress） + * GET /events?since=N 增量事件（voice/fetch/震动/console/手势…） + * GET /listen 增量转写判定（OK/TRUNCATED/MISMATCH/SILENCE/SFX_BGM_ONLY） + * GET /transcript 合并时间轴全量（复盘用） + * POST /stop 收尾落盘（mix.webm/events.jsonl/transcript.md/verdicts.json） + * + * 架构：chromium(CDP 真实输入) → 页面探针 inject.js（事件 + MediaRecorder 混音） + * → 本服务每 500ms drain（chunks 追加 mix.webm，事件换算到录音时基） + * → /listen 把闭合 voice 段交给 listen_server.py（SenseVoice ASR + 相似度） + * + * 截断双保险：① 事件层：vend 时刻 < 预期自然结束(t0 + dur/tempo，含长按快进修正) - 0.15s； + * ② ASR 层：heard vs expected 相似度 < 0.6 → MISMATCH。 + * + * 运行：node web/harness/live/control.mjs （需游戏服务器已在 8001 端口运行） + */ +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import url from 'node:url'; +import { spawn } from 'node:child_process'; +import { launch, sleep } from './cdp.mjs'; + +const HERE = path.dirname(url.fileURLToPath(import.meta.url)); +const OUT_BASE = path.join(HERE, 'out'); +const INJECT_SRC = fs.readFileSync(path.join(HERE, 'inject.js'), 'utf8'); +const VOICE_TEXT = JSON.parse(fs.readFileSync(path.join(HERE, '..', 'voice_text.json'), 'utf8')); + +const PORT = Number(process.env.LIVE_PORT) || 8123; +const GAME_BASE = process.env.GAME_BASE || 'http://localhost:8001'; +const DRAIN_MS = 500; +const SIM_THRESHOLD = 0.6; +const TRUNC_SLACK_S = 0.15; +const BOOST_RATE = 1.7; // 引擎长按快进倍速（audio.js LONGPRESS_BOOST） + +// 设置友好名 → 引擎枚举值（engine/settings.js VALID） +const SPEED_MAP = { normal: 1.0, fast: 1.35, faster: 1.7, max: 2.2 }; +const BGM_MAP = { normal: 1.0, low: 0.3, off: 0 }; +function mapSettings(raw = {}) { + const out = {}; + if (raw.voice !== undefined) out.voice = raw.voice; // male|female + if (raw.guide !== undefined) out.guide = raw.guide; // full|standard|off + if (raw.speed !== undefined) { + out.speed = typeof raw.speed === 'number' ? raw.speed : SPEED_MAP[raw.speed] ?? 1.0; + } + if (raw.bgm !== undefined) { + out.bgm = typeof raw.bgm === 'number' ? raw.bgm : BGM_MAP[raw.bgm] ?? 1.0; + } + return out; +} + +const round3 = (x) => (x == null ? null : Math.round(x * 1000) / 1000); +const voiceIdOf = (u) => { + const base = String(u || '').split('/').pop() || ''; + const dot = base.lastIndexOf('.'); + return dot > 0 ? base.slice(0, dot) : base; +}; + +// ---------------------------------------------------------------- 会话状态（单会话） +const S = { + browser: null, + page: null, + game: null, + settings: null, + baseSpeed: 1.0, + viewport: { width: 390, height: 844 }, + outdir: null, + mixPath: null, + eventsPath: null, + recStartT: null, // 页面 performance.now() ms，录音 t=0 对齐点 + events: [], // 全事件，已加 ta（录音时基秒） + segs: [], // voice 段 + boosts: [], // [{on, off}] 秒（录音时基） + rows: [], // /listen 全部判定行 + listenCursor: 0, + gapProbes: [], // [{t0, t1, done}] + lastAudioDur: 0, + drainTimer: null, + listenProc: null, // {proc, pending:Map, nextId, readyPromise} +}; + +// ---------------------------------------------------------------- 浏览器 / python 子进程 +async function ensureBrowser() { + if (S.browser) return S.browser; + const profileBase = path.join(OUT_BASE, '.profiles'); + fs.mkdirSync(profileBase, { recursive: true }); + S.browser = await launch({ profileBase }); + return S.browser; +} + +function ensureListen() { + if (S.listenProc) return S.listenProc.readyPromise; + const proc = spawn('python3', [path.join(HERE, 'listen_server.py')], { + stdio: ['pipe', 'pipe', 'inherit'], + }); + const pending = new Map(); + let nextId = 1; + let buf = ''; + let readyResolve; + const readyPromise = new Promise((res) => { readyResolve = res; }); + proc.stdout.on('data', (d) => { + buf += d.toString(); + let i; + while ((i = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, i).trim(); + buf = buf.slice(i + 1); + if (!line) continue; + let msg; + try { msg = JSON.parse(line); } catch (_) { continue; } + if (msg.ready) { readyResolve(); continue; } + const p = pending.get(msg.id); + if (p) { pending.delete(msg.id); p(msg); } + } + }); + proc.on('exit', (code) => { + console.error(`[control] listen_server 退出 (code ${code})`); + S.listenProc = null; + }); + const req = (obj) => new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, resolve); + proc.stdin.write(JSON.stringify({ id, ...obj }) + '\n'); + setTimeout(() => { if (pending.delete(id)) reject(new Error('listen_server 响应超时')); }, 120000); + }); + S.listenProc = { proc, req, readyPromise }; + return readyPromise; +} + +// ---------------------------------------------------------------- drain：chunk 落盘 + 事件换算 +function drainEval() { + if (!S.page) return Promise.resolve(null); + return S.page.eval(`(() => { + const L = window.__live; + if (!L) return null; + const c = L.chunks.splice(0); + const e = L.events.splice(0); + return { c, e, r: L.recStartT, state: L.recState }; + })()`).catch(() => null); +} + +function processEvent(ev) { + const ta = S.recStartT != null ? round3((ev.t - S.recStartT) / 1000) : null; + const rec = { ...ev, ta }; + S.events.push(rec); + if (S.eventsPath) fs.appendFileSync(S.eventsPath, JSON.stringify(rec) + '\n'); + + if (ev.ev === 'recstart') { /* recStartT 已在 drain 里更新 */ } + else if (ev.ev === 'boost') { + if (ev.on) S.boosts.push({ on: ta, off: null }); + else { + const last = [...S.boosts].reverse().find((b) => b.off == null); + if (last) last.off = ta; + } + } else if (ev.ev === 'vstart') { + const seg = { + idx: S.segs.length, voiceId: voiceIdOf(ev.url), url: ev.url, + t0: ta, t1: null, dur: ev.dur, expEnd: null, truncated: null, + reason: null, result: null, sent: false, + }; + S.segs.push(seg); + } else if (ev.ev === 'vend') { + // 关闭最近一个同 url 的未闭合段 + for (let i = S.segs.length - 1; i >= 0; i--) { + const s = S.segs[i]; + if (s.t1 == null && s.url === ev.url) { + s.t1 = ta; + s.reason = ev.reason || null; + s.expEnd = expectedEnd(s.t0, s.dur); + s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S; + break; + } + } + } +} + +async function drain() { + const r = await drainEval(); + if (!r) return; + if (r.r != null) S.recStartT = r.r; + if (r.c && r.c.length && S.mixPath) { + fs.appendFileSync(S.mixPath, Buffer.concat(r.c.map((b) => Buffer.from(b, 'base64')))); + } + if (r.e) for (const ev of r.e) processEvent(ev); +} + +// ------------------------------------------------- voice 预期自然结束时刻（含长按快进修正） +function rateAt(t) { + for (const b of S.boosts) { + if (b.on != null && t >= b.on && (b.off == null || t < b.off)) return BOOST_RATE; + } + return S.baseSpeed; +} +function expectedEnd(t0, dur) { + if (t0 == null || !dur) return null; + let pos = 0; + let t = t0; + for (let guard = 0; guard < 1000 && pos < dur - 1e-6; guard++) { + const r = rateAt(t); + let next = Infinity; + for (const b of S.boosts) { + if (b.on != null && b.on > t && b.on < next) next = b.on; + if (b.off != null && b.off > t && b.off < next) next = b.off; + } + const need = (dur - pos) / r; + if (t + need <= next) return round3(t + need); + pos += (next - t) * r; + t = next; + } + return round3(t + (dur - pos) / rateAt(t)); +} + +// ---------------------------------------------------------------- /listen 判定 +async function buildListenRows() { + // 未发送的闭合 voice 段 + const voices = []; + for (const s of S.segs) { + if (s.t1 == null || s.sent) continue; + s.sent = true; // pending 的会在响应里翻回 false + voices.push({ + seg: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1, + expected: (VOICE_TEXT[S.game] || {})[s.voiceId] || null, + }); + } + // 间隙探针：相邻 voice 段之间 / 录音起点到第一段，>1.5s 且未探过 + const closed = S.segs.filter((s) => s.t1 != null).sort((a, b) => a.t0 - b.t0); + const bounds = []; + if (closed.length) { + bounds.push([0, closed[0].t0]); + for (let i = 1; i < closed.length; i++) bounds.push([closed[i - 1].t1, closed[i].t0]); + } + for (const [a, b] of bounds) { + if (b - a < 1.5) continue; + if (S.gapProbes.some((g) => Math.abs(g.t0 - a) < 0.5 && Math.abs(g.t1 - b) < 0.5)) continue; + const g = { t0: a, t1: b, done: false }; + S.gapProbes.push(g); + voices.push({ seg: `gap-${S.gapProbes.length}`, voiceId: null, t0: a, t1: b, expected: null }); + } + if (!voices.length) return []; + + await ensureListen(); + let resp; + try { + resp = await S.listenProc.req({ cmd: 'transcribe', webm: S.mixPath, voices }); + if (!resp.ok) throw new Error(resp.error || 'unknown'); + } catch (e) { + // 失败回滚：下次 /listen 重试同批段与探针 + for (const v of voices) { + if (typeof v.seg === 'number') { const s = S.segs[v.seg]; if (s) s.sent = false; } + } + S.gapProbes = S.gapProbes.filter((g) => g.done); + throw new Error(`listen_server: ${e.message}`); + } + S.lastAudioDur = resp.audioDur; + + const newRows = []; + for (const row of resp.rows) { + if (row.status === 'pending') { + // 音频还没录到：段翻回未发送，间隙探针撤销 + if (typeof row.seg === 'number') { + const s = S.segs[row.seg]; + if (s) s.sent = false; + } else { + S.gapProbes = S.gapProbes.filter((g) => g.done || !(Math.abs(g.t0 - (voices.find((v) => v.seg === row.seg) || {}).t0) < 0.01)); + } + continue; + } + if (typeof row.seg === 'number') { + const s = S.segs[row.seg]; + if (!s || s.result) continue; + const verdict = s.truncated ? 'TRUNCATED' + : row.silent ? 'SILENCE' + : (row.similarity != null && row.similarity < SIM_THRESHOLD) ? 'MISMATCH' + : 'OK'; + s.result = { ...row, verdict }; + newRows.push({ + t0: row.t0, t1: row.t1, voiceId: row.voiceId, expected: row.expected, + heard: row.heard, similarity: row.similarity, verdict, + }); + } else { + // 间隙探针：有声音但没 voice = SFX/BGM；无声 = SILENCE + const g = S.gapProbes.find((x) => !x.done && Math.abs(x.t0 - row.t0) < 0.01); + if (g) g.done = true; + newRows.push({ + t0: row.t0, t1: row.t1, voiceId: null, expected: null, + heard: '', similarity: null, verdict: row.silent ? 'SILENCE' : 'SFX_BGM_ONLY', + }); + } + } + return newRows; +} + +// ---------------------------------------------------------------- 手势编排 +async function gesture({ type, x, y }) { + const cx = x ?? Math.round(S.viewport.width / 2); + const cy = y ?? Math.round(S.viewport.height / 2); + const pt = (px, py) => [{ x: px, y: py, id: 1 }]; + const tapOnce = async (px, py) => { + await S.page.dispatchTouch('touchStart', pt(px, py)); + await sleep(60); + await S.page.dispatchTouch('touchEnd', []); + }; + // 先把动作本身记进页面时间轴（t=手势开始） + await S.page.eval(`window.__live && window.__live.events.push({ev:'gesture', type:${JSON.stringify(type)}, x:${cx}, y:${cy}, t:performance.now()})`).catch(() => {}); + switch (type) { + case 'tap': + await tapOnce(cx, cy); + break; + case 'doubletap': + await tapOnce(cx, cy); + await sleep(120); + await tapOnce(cx, cy); + break; + case 'longpress': + await S.page.dispatchTouch('touchStart', pt(cx, cy)); + await sleep(600); + await S.page.dispatchTouch('touchEnd', []); + break; + case 'swipeup': case 'swipedown': case 'swipeleft': case 'swiperight': { + const d = { swipeup: [0, -200], swipedown: [0, 200], swipeleft: [-200, 0], swiperight: [200, 0] }[type]; + await S.page.dispatchTouch('touchStart', pt(cx, cy)); + const steps = 10; // 200px / 300ms ≈ 667px/s（阈值 400px/s） + for (let i = 1; i <= steps; i++) { + await S.page.dispatchTouch('touchMove', pt(cx + (d[0] * i) / steps, cy + (d[1] * i) / steps)); + await sleep(30); + } + await S.page.dispatchTouch('touchEnd', []); + break; + } + default: + throw new Error(`未知手势类型: ${type}`); + } + await sleep(400); // 给游戏响应时间（单击回调本身有 ~300ms 双击窗口延迟） + await drain(); +} + +// ---------------------------------------------------------------- HTTP 端点 +let opChain = Promise.resolve(); +const locked = (fn) => { + const p = opChain.then(fn); + opChain = p.catch(() => {}); + return p; +}; + +async function endSession({ finalize }) { + if (!S.page) return null; + if (S.drainTimer) { clearInterval(S.drainTimer); S.drainTimer = null; } + if (finalize) { + // 停录音、拿回尾部 chunk、最后 drain + await S.page.eval('window.__live ? window.__live.stopRec() : Promise.resolve()', { awaitPromise: true }).catch(() => {}); + await sleep(300); + await drain(); + // 未闭合的 voice 段按当前时刻封闭（记为 reason:'session-end'） + for (const s of S.segs) { + if (s.t1 == null) { + s.t1 = S.lastAudioDur || s.t0; + s.reason = 'session-end'; + s.expEnd = expectedEnd(s.t0, s.dur); + s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S; + } + } + } + const page = S.page; + S.page = null; + await page.close().catch(() => {}); + return page; +} + +async function writeArtifacts() { + const files = []; + const w = (name, content) => { + const p = path.join(S.outdir, name); + fs.writeFileSync(p, content); + files.push(p); + }; + // verdicts.json + w('verdicts.json', JSON.stringify({ + game: S.game, settings: S.settings, outdir: S.outdir, + segs: S.segs.map((s) => ({ + idx: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1, dur: s.dur, + expEnd: s.expEnd, truncated: s.truncated, reason: s.reason, + verdict: s.result ? s.result.verdict : null, + heard: s.result ? s.result.heard : null, + similarity: s.result ? s.result.similarity : null, + })), + rows: S.rows, + }, null, 1)); + // transcript.md：手势 + voice + 判定 + 震动 + console 错误的合并时间轴 + const lines = [`# ${S.game} 实时会话 transcript`, '']; + const items = []; + for (const e of S.events) { + const t = e.ta; + if (t == null) continue; + if (e.ev === 'gesture') items.push([t, `**GESTURE** ${e.type} @(${e.x},${e.y})`]); + else if (e.ev === 'vstart') items.push([t, `VOICE ▶ ${voiceIdOf(e.url)} (dur ${e.dur}s)`]); + else if (e.ev === 'vend') items.push([t, `VOICE ■ ${voiceIdOf(e.url)} (${e.reason})`]); + else if (e.ev === 'start' && e.layer !== 'voice') items.push([t, `${(e.layer || 'sfx').toUpperCase()} ♪ ${e.url ? voiceIdOf(e.url) : '?'}${e.loop ? ' (loop)' : ''}`]); + else if (e.ev === 'vibrate') items.push([t, `VIBRATE ${e.pattern}`]); + else if (e.ev === 'asset404') items.push([t, `404 ${e.url}`]); + else if (e.ev === 'console.error') items.push([t, `CONSOLE.ERROR ${e.msg}`]); + else if (e.ev === 'pageerror' || e.ev === 'unhandledrejection') items.push([t, `PAGEERROR ${e.msg}`]); + else if (e.ev === 'recstart') items.push([t, `REC 开始 (${e.mime})`]); + } + for (const r of S.rows) { + const head = r.voiceId ? `VERDICT ${r.voiceId}: **${r.verdict}**` : `VERDICT (无语音区间): **${r.verdict}**`; + const detail = r.voiceId + ? ` sim=${r.similarity}\n - 期望: ${r.expected}\n - 听到: ${r.heard}` + : ''; + items.push([r.t1, `${head}${detail}`]); + } + items.sort((a, b) => a[0] - b[0]); + for (const [t, text] of items) lines.push(`- \`[${t.toFixed(2).padStart(8)}s]\` ${text}`); + w('transcript.md', lines.join('\n') + '\n'); + if (S.mixPath && fs.existsSync(S.mixPath)) files.push(S.mixPath); + if (S.eventsPath && fs.existsSync(S.eventsPath)) files.push(S.eventsPath); + return files; +} + +async function route(method, pathName, query, body) { + if (method === 'GET' && pathName === '/') { + return { ok: true, game: S.game, pageOpen: !!S.page, events: S.events.length, rows: S.rows.length }; + } + + if (method === 'POST' && pathName === '/session') { + return locked(async () => { + const game = body && body.game; + if (!game || !VOICE_TEXT[game]) throw httpError(400, `未知游戏: ${game}（可选 ${Object.keys(VOICE_TEXT).join('|')}）`); + await endSession({ finalize: false }); + await ensureBrowser(); + S.game = game; + S.settings = mapSettings(body.settings || {}); + S.baseSpeed = S.settings.speed ?? 1.0; + S.viewport = { width: body.viewport?.width || 390, height: body.viewport?.height || 844 }; + const ts = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14); + S.outdir = path.join(OUT_BASE, `${game}-${ts}`); + fs.mkdirSync(S.outdir, { recursive: true }); + S.mixPath = path.join(S.outdir, 'mix.webm'); + S.eventsPath = path.join(S.outdir, 'events.jsonl'); + fs.writeFileSync(S.mixPath, ''); + fs.writeFileSync(S.eventsPath, ''); + S.recStartT = null; + S.events = []; S.segs = []; S.boosts = []; S.rows = []; + S.listenCursor = 0; S.gapProbes = []; S.lastAudioDur = 0; + + S.page = await S.browser.newPage(S.viewport); + const seed = `try{localStorage.setItem('soundgame.settings', ${JSON.stringify(JSON.stringify(S.settings))});}catch(e){}\n`; + await S.page.addInitScript(seed + INJECT_SRC); + await S.page.navigate(`${GAME_BASE}/${game}/`); + S.drainTimer = setInterval(() => { drain().catch(() => {}); }, DRAIN_MS); + return { ok: true, game, settings: S.settings, viewport: S.viewport, outdir: S.outdir }; + }); + } + + if (method === 'POST' && pathName === '/gesture') { + return locked(async () => { + if (!S.page) throw httpError(409, '没有打开的会话（先 POST /session）'); + await gesture(body || {}); + return { ok: true }; + }); + } + + if (method === 'GET' && pathName === '/events') { + const since = Math.max(0, Number(query.since) || 0); + await drain(); + return { events: S.events.slice(since), next: S.events.length }; + } + + if (method === 'GET' && pathName === '/listen') { + return locked(async () => { + if (!S.page) throw httpError(409, '没有打开的会话（先 POST /session）'); + await drain(); + const newRows = await buildListenRows(); + S.rows.push(...newRows); + const out = S.rows.slice(S.listenCursor); + S.listenCursor = S.rows.length; + return { audioDur: S.lastAudioDur, rows: out }; + }); + } + + if (method === 'GET' && pathName === '/transcript') { + await drain(); + return { + game: S.game, outdir: S.outdir, audioDur: S.lastAudioDur, + events: S.events, + segs: S.segs.map((s) => ({ ...s, sent: undefined })), + rows: S.rows, + }; + } + + if (method === 'POST' && pathName === '/stop') { + return locked(async () => { + if (!S.page) throw httpError(409, '没有打开的会话'); + await endSession({ finalize: true }); + const files = await writeArtifacts(); + return { ok: true, outdir: S.outdir, files, segs: S.segs.length, rows: S.rows.length }; + }); + } + + throw httpError(404, `${method} ${pathName} 不存在`); +} + +function httpError(status, message) { + const e = new Error(message); + e.status = status; + return e; +} + +const server = http.createServer(async (req, res) => { + const u = new URL(req.url, 'http://x'); + let body = null; + if (req.method === 'POST') { + const chunks = []; + for await (const c of req) chunks.push(c); + const raw = Buffer.concat(chunks).toString('utf8'); + if (raw.trim()) { + try { body = JSON.parse(raw); } catch (_) { + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ ok: false, error: '请求体不是合法 JSON' })); + return; + } + } + } + try { + const out = await route(req.method, u.pathname, Object.fromEntries(u.searchParams), body); + res.writeHead(200, { 'content-type': 'application/json; charset=utf-8' }); + res.end(JSON.stringify(out)); + } catch (e) { + res.writeHead(e.status || 500, { 'content-type': 'application/json; charset=utf-8' }); + res.end(JSON.stringify({ ok: false, error: e.message })); + } +}); + +server.listen(PORT, () => { + console.log(`[control] 全拟真玩家 agent 控制服务器: http://localhost:${PORT}/`); + console.log(`[control] 游戏服务器: ${GAME_BASE} 产物目录: ${OUT_BASE}`); + ensureListen().then(() => console.log('[control] listen_server (SenseVoice) 就绪')).catch((e) => console.error('[control] listen_server 启动失败:', e)); +}); + +process.on('SIGINT', async () => { + await endSession({ finalize: false }).catch(() => {}); + if (S.browser) await S.browser.dispose().catch(() => {}); + if (S.listenProc) S.listenProc.proc.kill(); + process.exit(0); +}); diff --git a/web/harness/live/inject.js b/web/harness/live/inject.js new file mode 100644 index 00000000..9addd149 --- /dev/null +++ b/web/harness/live/inject.js @@ -0,0 +1,251 @@ +/** + * inject.js — 全拟真玩家 agent 页面探针（黑盒，不改引擎/游戏代码）。 + * + * 由控制端通过 Page.addScriptToEvaluateOnNewDocument 注入，在页面任何脚本 + * 之前运行。职责： + * 1. fetch 包装：记录 audio/ 资产请求与 404；ArrayBuffer → 来源 URL 关联。 + * 2. decodeAudioData 包装：AudioBuffer → 来源 URL 关联（WeakMap）。 + * 3. AudioBufferSourceNode start/stop 包装：bgm/sfx 播放事件（voice 不经此路径， + * voice 走 SoundTouch ScriptProcessorNode 管线，见第 4 条）。 + * 4. AudioBuffer.prototype.getChannelData 包装：SoundTouch 的 + * WebAudioBufferSource.extract 是引擎里唯一对 voice buffer 调 + * getChannelData 的地方 → 用「voice buffer 被抽读」检出 voice 段 + * （vstart/vend，带 url 与 buffer 时长）。ScriptProcessorNode 的 + * disconnect 作为 voice 被 kill（顶掉/stopVoice/自然结束）的信号。 + * 5. AudioNode.connect 包装：connect(ctx.destination) 改道经过 tap + * GainNode，同时接真 destination 与 MediaStreamDestination，用 + * MediaRecorder 录真实混音 → base64 chunk 存 __live.chunks。 + * 6. navigator.vibrate / console.error / window error / 长按快进事件记录。 + * + * 所有事件 {t} 用 performance.now()（毫秒）。录音起点 __live.recStartT + * 是 recorder.start() 当刻的 performance.now()；控制端用 + * audio_t = (t - recStartT)/1000 把事件换算到录音时间轴（误差 ≈ AudioContext + * resume 延迟，几十 ms，ASR 窗口 ±0.3s 余量可吸收）。 + * + * 暴露：window.__live = { events, chunks, recStartT, recState, stopRec() } + */ +(() => { + if (window.__live) return; + + const now = () => performance.now(); + const events = []; + const chunks = []; + /** @type {null|(() => Promise)} */ + let stopRecFn = null; + + const live = { + events, + chunks, + recStartT: null, // recorder.start() 当刻 performance.now()（ms） + recState: 'none', // none | recording | stopped | unavailable + stopRec: () => (stopRecFn ? stopRecFn() : Promise.resolve()), + }; + window.__live = live; + + const push = (obj) => { obj.t = now(); events.push(obj); }; + + // ---------- console / error ---------- + const origErr = console.error.bind(console); + console.error = (...args) => { + try { push({ ev: 'console.error', msg: args.map((a) => String(a)).join(' ').slice(0, 600) }); } catch (_) {} + origErr(...args); + }; + const origWarn = console.warn.bind(console); + console.warn = (...args) => { + try { push({ ev: 'console.warn', msg: args.map((a) => String(a)).join(' ').slice(0, 600) }); } catch (_) {} + origWarn(...args); + }; + window.addEventListener('error', (e) => push({ ev: 'pageerror', msg: String(e.message || e).slice(0, 600) })); + window.addEventListener('unhandledrejection', (e) => push({ ev: 'unhandledrejection', msg: String(e.reason).slice(0, 600) })); + + // ---------- vibrate / 长按快进 ---------- + if (navigator.vibrate) { + const ov = navigator.vibrate.bind(navigator); + navigator.vibrate = (p) => { + try { push({ ev: 'vibrate', pattern: JSON.stringify(p) }); } catch (_) {} + return ov(p); + }; + } + window.addEventListener('soundgame:longpress', () => push({ ev: 'boost', on: true })); + window.addEventListener('soundgame:longpressend', () => push({ ev: 'boost', on: false })); + + // ---------- fetch：audio/ 资产请求记录 + ArrayBuffer→URL ---------- + const abUrl = new WeakMap(); + try { + const origFetch = window.fetch.bind(window); + window.fetch = async (input, init) => { + const url = typeof input === 'string' ? input : (input && input.url) || String(input); + const t0 = now(); + const resp = await origFetch(input, init); + try { + if (url.includes('audio/')) { + push({ ev: 'fetch', url, status: resp.status, ok: resp.ok, ms: Math.round(now() - t0) }); + if (!resp.ok) push({ ev: 'asset404', url, status: resp.status }); + const origAB = resp.arrayBuffer.bind(resp); + resp.arrayBuffer = async () => { + const ab = await origAB(); + abUrl.set(ab, url); + return ab; + }; + } + } catch (_) {} + return resp; + }; + } catch (e) { push({ ev: 'probe_error', where: 'fetch', msg: String(e) }); } + + // ---------- decodeAudioData：AudioBuffer→URL ---------- + const bufUrl = new WeakMap(); + const layerOf = (url) => (url.includes('/voice/') ? 'voice' : url.includes('/bgm/') ? 'bgm' : 'sfx'); + try { + const protos = []; + if (window.BaseAudioContext) protos.push(window.BaseAudioContext.prototype); + if (window.AudioContext && !protos.includes(window.AudioContext.prototype)) protos.push(window.AudioContext.prototype); + for (const proto of protos) { + const origDec = proto.decodeAudioData; + proto.decodeAudioData = function (ab, ...rest) { + const src = abUrl.get(ab) || null; + const p = origDec.call(this, ab, ...rest); + if (p && typeof p.then === 'function') { + p.then((buf) => { if (src) bufUrl.set(buf, src); }).catch(() => {}); + } + return p; + }; + } + } catch (e) { push({ ev: 'probe_error', where: 'decodeAudioData', msg: String(e) }); } + + // ---------- voice 段检测（getChannelData 抽读 + ScriptProcessorNode disconnect） ---------- + // SoundTouch 内部 FIFO 攒批抽读：同一句内 getChannelData 间隔可达 ~0.5s， + // 所以段关闭主要靠 voice 节点 disconnect（引擎 killVoice 必走），间隙判定只作兜底。 + const VOICE_GAP_MS = 900; // 抽读间隔超过此值 → 上一段已结束（兜底） + let seg = null; // {url, t0(ms), lastAct(ms), dur(s)} + const closeSeg = (tEnd, reason) => { + if (!seg) return; + events.push({ ev: 'vend', url: seg.url, t: tEnd, reason }); + seg = null; + }; + try { + const origGcd = AudioBuffer.prototype.getChannelData; + AudioBuffer.prototype.getChannelData = function (ch) { + const url = bufUrl.get(this); + if (url && layerOf(url) === 'voice') { + const t = now(); + if (!seg || seg.url !== url || t - seg.lastAct > VOICE_GAP_MS) { + if (seg) closeSeg(seg.lastAct, 'preempt'); + seg = { url, t0: t, lastAct: t, dur: this.duration }; + events.push({ ev: 'vstart', url, t, dur: this.duration }); + } else { + seg.lastAct = t; + } + } + return origGcd.call(this, ch); + }; + + const AC = window.AudioContext || window.webkitAudioContext; + if (AC) { + const voiceNodes = new WeakSet(); // SoundTouch 管线创建的 ScriptProcessorNode + const origCSP = AC.prototype.createScriptProcessor; + AC.prototype.createScriptProcessor = function (...a) { + const n = origCSP.apply(this, a); + voiceNodes.add(n); + return n; + }; + const origDisc = AudioNode.prototype.disconnect; + AudioNode.prototype.disconnect = function (...a) { + // voice 节点断开 = 引擎 killVoice（自然结束/被顶掉/stopVoice 都走这里）。 + // 玩家实际听到的尾音就到此刻（FIFO 里未播部分被丢弃），用 now() 而非 lastAct。 + if (voiceNodes.has(this) && seg) closeSeg(now(), 'kill'); + return origDisc.apply(this, a); + }; + } + // 兜底清扫：长静默后关闭未结段 + setInterval(() => { if (seg && now() - seg.lastAct > VOICE_GAP_MS) closeSeg(seg.lastAct, 'end'); }, 200); + } catch (e) { push({ ev: 'probe_error', where: 'voice-detect', msg: String(e) }); } + + // ---------- AudioBufferSourceNode：bgm/sfx 播放事件 ---------- + try { + let sid = 0; + const sidOf = new WeakMap(); + const origStart = AudioBufferSourceNode.prototype.start; + AudioBufferSourceNode.prototype.start = function (when, ...rest) { + try { + const url = this.buffer ? bufUrl.get(this.buffer) || null : null; + const id = ++sid; + sidOf.set(this, id); + events.push({ + ev: 'start', t: now(), sid: id, url, layer: url ? layerOf(url) : null, + dur: this.buffer ? this.buffer.duration : null, + rate: this.playbackRate ? this.playbackRate.value : 1, + loop: !!this.loop, when: when || 0, + }); + } catch (_) {} + return origStart.call(this, when, ...rest); + }; + const origStop = AudioBufferSourceNode.prototype.stop; + AudioBufferSourceNode.prototype.stop = function (when) { + try { events.push({ ev: 'stop', t: now(), sid: sidOf.get(this) || null, when: when || 0 }); } catch (_) {} + return origStop.call(this, when); + }; + } catch (e) { push({ ev: 'probe_error', where: 'absn', msg: String(e) }); } + + // ---------- connect 包装：destination 前插 tap → MediaRecorder 录混音 ---------- + const abToB64 = (ab) => { + const u8 = new Uint8Array(ab); + let s = ''; + for (let i = 0; i < u8.length; i += 8192) s += String.fromCharCode.apply(null, u8.subarray(i, i + 8192)); + return btoa(s); + }; + try { + const origConnect = AudioNode.prototype.connect; + const taps = new WeakMap(); // destination -> tap GainNode + const startRec = (ctx, tap) => { + try { + if (typeof MediaRecorder === 'undefined') { + live.recState = 'unavailable'; + push({ ev: 'recfail', msg: 'MediaRecorder undefined（降级：只记事件不录音）' }); + return; + } + const msDest = ctx.createMediaStreamDestination(); + origConnect.call(tap, msDest); + const rec = new MediaRecorder(msDest.stream, { mimeType: 'audio/webm;codecs=opus' }); + let chain = Promise.resolve(); // 保证 chunk 顺序 + rec.ondataavailable = (e) => { + if (e.data && e.data.size) { + chain = chain.then(() => e.data.arrayBuffer()).then((ab) => { chunks.push(abToB64(ab)); }).catch(() => {}); + } + }; + rec.onerror = (e) => push({ ev: 'recfail', msg: String(e.error || e) }); + rec.start(1000); // 1s timeslice + live.recStartT = now(); + live.recState = 'recording'; + push({ ev: 'recstart', mime: rec.mimeType }); + stopRecFn = () => new Promise((res) => { + try { + rec.onstop = () => { chain.then(res); }; + rec.stop(); + } catch (_) { res(); } + live.recState = 'stopped'; + push({ ev: 'recstop' }); + }); + } catch (e) { + live.recState = 'unavailable'; + push({ ev: 'recfail', msg: String(e) }); + } + }; + AudioNode.prototype.connect = function (dest, ...rest) { + // 只改道「直连 ctx.destination」的边；AudioParam 形式的 connect 不动 + if (dest && dest.context && dest.context.destination === dest) { + let tap = taps.get(dest); + if (!tap) { + tap = dest.context.createGain(); + origConnect.call(tap, dest); + taps.set(dest, tap); + startRec(dest.context, tap); + } + return origConnect.call(this, tap, ...rest); + } + return origConnect.call(this, dest, ...rest); + }; + } catch (e) { push({ ev: 'probe_error', where: 'connect', msg: String(e) }); } + + push({ ev: 'probe_ready' }); +})(); diff --git a/web/harness/live/listen_server.py b/web/harness/live/listen_server.py new file mode 100644 index 00000000..6abecc20 --- /dev/null +++ b/web/harness/live/listen_server.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +""" +listen_server.py — SenseVoice ASR 常驻服务（stdin/stdout JSON Lines 协议）。 + +由 control.mjs 以子进程拉起，模型只加载一次。协议（每行一个 JSON）： + + 请求：{"id": N, "cmd": "ping"} + {"id": N, "cmd": "transcribe", "webm": "/abs/path/mix.webm", + "voices": [{"seg": 0, "voiceId": "intro_1"|null, + "t0": 秒, "t1": 秒, "expected": "台词全文"|null}]} + 响应：{"id": N, "ok": true, "audioDur": 秒, + "rows": [{"seg": 0, "status": "ok"|"pending", "voiceId", "t0", "t1", + "expected", "heard", "similarity", "rms", "silent"}]} + +行为： +- 每次 transcribe 用 ffmpeg 把 webm 从头解码成 16k mono wav（webm 可能仍在 + 增长，ffmpeg 读到当前末尾为止，尾部截断告警属正常）；audioDur 由 wav 字节数算出。 +- 每个 voice 窗口切 [t0-0.15, t1+0.35]（钳到 [0, audioDur]），先 astats 量能量： + RMS < -45dB → silent=true，不送 ASR；否则 SenseVoice 整段转写一次。 +- voiceId=null 的条目是「间隙探针」（判定 SFX_BGM_ONLY/SILENCE 用），只量能量不 ASR。 +- t0 超出当前音频末尾 → status="pending"（控制端下次重试）。 +- similarity：difflib.SequenceMatcher 作用于归一化文本（去 <|...|> 标签、 + 小写化、只留字母数字与 CJK 字符）。无 expected 或无 heard 时为 None/0.0。 + +杂散输出（funasr/tqdm/modelscope 的打印）全部改道 stderr，stdout 只走协议。 +""" +import sys +import os +import re +import json +import difflib +import subprocess +import tempfile + +REAL_OUT = sys.stdout +sys.stdout = sys.stderr # 第三方库的 print/progress 全部赶去 stderr + +TAG_RE = re.compile(r'<\|[^|]*\|>') +SILENCE_DB = -45.0 +SAMPLE_BYTES = 16000 * 2 # 16kHz mono s16 + + +def send(obj): + REAL_OUT.write(json.dumps(obj, ensure_ascii=False) + '\n') + REAL_OUT.flush() + + +def log(*args): + print('[listen]', *args, file=sys.stderr, flush=True) + + +def clean(text): + return TAG_RE.sub('', text or '').strip() + + +def norm(text): + t = clean(text).lower() + # isalnum 对中日韩字符为 True；标点/空白被去掉 + return ''.join(ch for ch in t if ch.isalnum()) + + +def similarity(heard, expected): + a, b = norm(heard), norm(expected) + if not a or not b: + return 0.0 + return difflib.SequenceMatcher(None, a, b).ratio() + + +TMP = tempfile.mkdtemp(prefix='listen-') + + +def decode_webm(webm_path, out_wav): + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', '-i', webm_path, + '-ar', '16000', '-ac', '1', '-f', 'wav', out_wav], + check=False) + + +def wav_duration(wav_path): + try: + return max(0.0, (os.path.getsize(wav_path) - 44) / SAMPLE_BYTES) + except OSError: + return 0.0 + + +def slice_wav(wav_path, t0, t1, out_path): + dur = max(0.05, t1 - t0) + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', '-ss', f'{t0:.3f}', '-i', wav_path, + '-t', f'{dur:.3f}', '-f', 'wav', out_path], + check=False) + + +def rms_db(wav_path): + p = subprocess.run(['ffmpeg', '-i', wav_path, '-af', 'astats', '-f', 'null', '-'], + capture_output=True, text=True) + m = re.search(r'RMS level dB:\s*(-?[\d.]+)', p.stderr or '') + return float(m.group(1)) if m else -90.0 + + +def handle_transcribe(req, model): + webm = req['webm'] + full = os.path.join(TMP, 'full.wav') + decode_webm(webm, full) + audio_dur = wav_duration(full) + rows = [] + for v in req.get('voices', []): + seg_id = v.get('seg') + t0 = float(v.get('t0', 0)) + t1 = float(v.get('t1', 0)) + if t0 > audio_dur - 0.05: + rows.append({'seg': seg_id, 'status': 'pending'}) + continue + a = max(0.0, t0 - 0.15) + b = min(t1 + 0.35, audio_dur) + sl = os.path.join(TMP, f'seg-{seg_id}.wav') + slice_wav(full, a, b, sl) + rms = rms_db(sl) + silent = rms < SILENCE_DB + heard = '' + voice_id = v.get('voiceId') + if not silent and voice_id: + try: + r = model.generate(input=sl, cache={}, language='zh', use_itn=True) + heard = clean(r[0]['text']) if r else '' + except Exception as e: # ASR 失败不致命，当作没听到 + log(f'asr error seg={seg_id}: {e}') + expected = v.get('expected') + if expected and heard: + sim = round(similarity(heard, expected), 4) + elif expected and not silent: + sim = 0.0 + else: + sim = None + rows.append({ + 'seg': seg_id, 'status': 'ok', 'voiceId': voice_id, + 't0': t0, 't1': t1, 'expected': expected, 'heard': heard, + 'similarity': sim, 'rms': rms, 'silent': silent, + }) + return {'ok': True, 'audioDur': round(audio_dur, 3), 'rows': rows} + + +def main(): + log('loading SenseVoiceSmall (cpu) ...') + from funasr import AutoModel + model = AutoModel(model='iic/SenseVoiceSmall', vad_model=None, + device='cpu', disable_update=True) + log('model ready') + send({'ready': True}) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except Exception: + send({'ok': False, 'error': 'bad json'}) + continue + rid = req.get('id') + cmd = req.get('cmd') + try: + if cmd == 'ping': + resp = {'ok': True} + elif cmd == 'transcribe': + resp = handle_transcribe(req, model) + else: + resp = {'ok': False, 'error': f'unknown cmd: {cmd}'} + except Exception as e: + resp = {'ok': False, 'error': str(e)} + resp['id'] = rid + send(resp) + + +if __name__ == '__main__': + main() diff --git a/web/serve.sh b/web/serve.sh index 81df4a3f..fa518d10 100755 --- a/web/serve.sh +++ b/web/serve.sh @@ -19,6 +19,14 @@ IP=$(hostname -I | awk '{print $1}') PORT=${PORT:-8000} +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# node 解析顺序：项目自带 tools/node → PATH 中的 node +if [ -x "$ROOT/tools/node/bin/node" ]; then + NODE="$ROOT/tools/node/bin/node" +else + NODE="$(command -v node)" +fi echo "================================================" echo " SoundGame Web 本地服务器（静态 + 用户系统 API）" @@ -30,4 +38,4 @@ echo " 手机需与电脑在同一 WiFi" echo " Ctrl+C 停止" echo "================================================" -exec /home/ubuntu/SoundGame/tools/node/bin/node /home/ubuntu/SoundGame/web/server/server.js +exec "$NODE" "$ROOT/web/server/server.js"