xsl
566241e05d
配对码与认领（B）： - POST /api/pair/start|join：6 位码 10 分钟单次有效、仅存内存零泄露面、5 连错 per-IP 封锁 10 分钟；join=新设备档案按桶 LWW 并入老档案+token 改绑（复用 claim-merge）；日志脱敏 - 大厅：生成配对码（speechSynthesis 逐位朗读+aria-live+重听）/从其他设备转入（one-time-code 输入框）/从本机移除/401 过期自愈引导/aria 补全/玩家名可听化 - api-smoke 12/12（配对全流程/频控/封锁/过期自愈/移除/digitsToCn）+ CDP 真浏览器 E2E 22/22 键盘语音双套资产（A，路线 a）： - 164 变体句（8 游戏 grep 实测，超出调研预估 153：含轻触/点两下/按住变体）×2 性别=328 生成 656 文件 36.9 分钟；触屏主资产零字节改动（PCM 逐样本比对后还原） - 引擎 _candidates 三跳路由 key→gender→平铺（inputMode 跟随）；缓存键 mode|gender|id；engine-input 新增 K9 路由断言（72 条） - check_all_assets 增 key 列（两性别一致且 ⊆ 主集）8 游戏全绿；voice_text.json 增 __key__ 表；live control.mjs key 会话对照键盘措辞验收（集成） - voice-ids 文档：8 游戏键盘变体小节由 KEY_LINES 程序化生成（与台词表逐字断言一致）；补齐 storyline/escape/simon 三个历史缺失文档；design README 增 new8 索引 - mock 适配：playtest persona 加 maxTouchPoints=1 保持触屏链断言语义零改动 回归：12 套件全绿（8 playtest + guide 122 + engine-voice 19 + engine-input 72 + api-smoke 12/12） 同批入库：批次 2/3 已完稿设计文档（17-abyss/18-detective/19-cipher 全套，20-22 在途）
609 lines
25 KiB
JavaScript
609 lines
25 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* control.mjs — 全拟真玩家 agent HTTP 控制服务器（Node 零依赖，端口 8123）。
|
||
*
|
||
* 外部 agent 通过 HTTP 逐步玩任意一款 SoundGame：
|
||
* POST /session {game, settings?, viewport?} 开新页（预注入探针 + 设置种子）
|
||
* POST /gesture {type, x?, y?} 真实触摸手势（tap/doubletap/swipe四向/longpress）
|
||
* POST /gesture {keyboard:true, key, holdMs?} 键盘手势（CDP Input.dispatchKeyEvent；
|
||
* key=ArrowLeft|…|KeyW…|Space|Enter，
|
||
* holdMs 默认 60，长按空格传 ≥450）
|
||
* POST /gesture {type, keyboard:true} 同款手势类型走键盘（tap→空格快按、
|
||
* doubletap→回车、longpress→空格按 600ms、
|
||
* swipeup→↑…）——会话剧本加 keyboard:true 即可复跑
|
||
* GET /events?since=N 增量事件（voice/fetch/震动/console/手势…）
|
||
* GET /listen 增量转写判定（OK/TRUNCATED/MISMATCH/SILENCE/SFX_BGM_ONLY）
|
||
* GET /transcript 合并时间轴全量（复盘用）
|
||
* POST /stop 收尾落盘（mix.webm/events.jsonl/transcript.md/verdicts.json）
|
||
*
|
||
* 架构：chromium(CDP 真实输入) → 页面探针 inject.js（事件 + MediaRecorder 混音）
|
||
* → 本服务每 500ms drain（chunks 追加 mix.webm，事件换算到录音时基）
|
||
* → /listen 把闭合 voice 段交给 listen_server.py（SenseVoice ASR + 相似度）
|
||
*
|
||
* 截断双保险：① 事件层：vend 时刻 < 预期自然结束(t0 + dur/tempo，含长按快进修正) - 0.15s；
|
||
* ② ASR 层：heard vs expected 相似度 < 0.6 → MISMATCH。
|
||
*
|
||
* 运行：node web/harness/live/control.mjs （需游戏服务器已在 8001 端口运行）
|
||
*/
|
||
import http from 'node:http';
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import url from 'node:url';
|
||
import { spawn } from 'node:child_process';
|
||
import { launch, sleep } from './cdp.mjs';
|
||
|
||
const HERE = path.dirname(url.fileURLToPath(import.meta.url));
|
||
const OUT_BASE = path.join(HERE, 'out');
|
||
const INJECT_SRC = fs.readFileSync(path.join(HERE, 'inject.js'), 'utf8');
|
||
const VOICE_TEXT = JSON.parse(fs.readFileSync(path.join(HERE, '..', 'voice_text.json'), 'utf8'));
|
||
|
||
const PORT = Number(process.env.LIVE_PORT) || 8123;
|
||
const GAME_BASE = process.env.GAME_BASE || 'http://localhost:8001';
|
||
const DRAIN_MS = 500;
|
||
const SIM_THRESHOLD = 0.6;
|
||
const TRUNC_SLACK_S = 0.15;
|
||
const BOOST_RATE = 1.7; // 引擎长按快进倍速（audio.js LONGPRESS_BOOST）
|
||
|
||
// 设置友好名 → 引擎枚举值（engine/settings.js VALID）
|
||
const SPEED_MAP = { normal: 1.0, fast: 1.35, faster: 1.7, max: 2.2 };
|
||
const BGM_MAP = { normal: 1.0, low: 0.3, off: 0 };
|
||
function mapSettings(raw = {}) {
|
||
const out = {};
|
||
if (raw.voice !== undefined) out.voice = raw.voice; // male|female
|
||
if (raw.guide !== undefined) out.guide = raw.guide; // full|standard|off
|
||
if (raw.input !== undefined) out.input = raw.input; // auto|touch|key（提示文案输入模式）
|
||
if (raw.speed !== undefined) {
|
||
out.speed = typeof raw.speed === 'number' ? raw.speed : SPEED_MAP[raw.speed] ?? 1.0;
|
||
}
|
||
if (raw.bgm !== undefined) {
|
||
out.bgm = typeof raw.bgm === 'number' ? raw.bgm : BGM_MAP[raw.bgm] ?? 1.0;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
const round3 = (x) => (x == null ? null : Math.round(x * 1000) / 1000);
|
||
const voiceIdOf = (u) => {
|
||
const base = String(u || '').split('/').pop() || '';
|
||
const dot = base.lastIndexOf('.');
|
||
return dot > 0 ? base.slice(0, dot) : base;
|
||
};
|
||
|
||
// ---------------------------------------------------------------- 会话状态（单会话）
|
||
const S = {
|
||
browser: null,
|
||
page: null,
|
||
game: null,
|
||
settings: null,
|
||
baseSpeed: 1.0,
|
||
viewport: { width: 390, height: 844 },
|
||
outdir: null,
|
||
mixPath: null,
|
||
eventsPath: null,
|
||
recStartT: null, // 页面 performance.now() ms，录音 t=0 对齐点
|
||
events: [], // 全事件，已加 ta（录音时基秒）
|
||
segs: [], // voice 段
|
||
boosts: [], // [{on, off}] 秒（录音时基）
|
||
rows: [], // /listen 全部判定行
|
||
listenCursor: 0,
|
||
gapProbes: [], // [{t0, t1, done}]
|
||
lastAudioDur: 0,
|
||
drainTimer: null,
|
||
listenProc: null, // {proc, pending:Map, nextId, readyPromise}
|
||
};
|
||
|
||
// ---------------------------------------------------------------- 浏览器 / python 子进程
|
||
async function ensureBrowser() {
|
||
if (S.browser) return S.browser;
|
||
const profileBase = path.join(OUT_BASE, '.profiles');
|
||
fs.mkdirSync(profileBase, { recursive: true });
|
||
S.browser = await launch({ profileBase });
|
||
return S.browser;
|
||
}
|
||
|
||
function ensureListen() {
|
||
if (S.listenProc) return S.listenProc.readyPromise;
|
||
const proc = spawn('python3', [path.join(HERE, 'listen_server.py')], {
|
||
stdio: ['pipe', 'pipe', 'inherit'],
|
||
});
|
||
const pending = new Map();
|
||
let nextId = 1;
|
||
let buf = '';
|
||
let readyResolve;
|
||
const readyPromise = new Promise((res) => { readyResolve = res; });
|
||
proc.stdout.on('data', (d) => {
|
||
buf += d.toString();
|
||
let i;
|
||
while ((i = buf.indexOf('\n')) >= 0) {
|
||
const line = buf.slice(0, i).trim();
|
||
buf = buf.slice(i + 1);
|
||
if (!line) continue;
|
||
let msg;
|
||
try { msg = JSON.parse(line); } catch (_) { continue; }
|
||
if (msg.ready) { readyResolve(); continue; }
|
||
const p = pending.get(msg.id);
|
||
if (p) { pending.delete(msg.id); p(msg); }
|
||
}
|
||
});
|
||
proc.on('exit', (code) => {
|
||
console.error(`[control] listen_server 退出 (code ${code})`);
|
||
S.listenProc = null;
|
||
});
|
||
const req = (obj) => new Promise((resolve, reject) => {
|
||
const id = nextId++;
|
||
pending.set(id, resolve);
|
||
proc.stdin.write(JSON.stringify({ id, ...obj }) + '\n');
|
||
setTimeout(() => { if (pending.delete(id)) reject(new Error('listen_server 响应超时')); }, 120000);
|
||
});
|
||
S.listenProc = { proc, req, readyPromise };
|
||
return readyPromise;
|
||
}
|
||
|
||
// ---------------------------------------------------------------- drain：chunk 落盘 + 事件换算
|
||
function drainEval() {
|
||
if (!S.page) return Promise.resolve(null);
|
||
return S.page.eval(`(() => {
|
||
const L = window.__live;
|
||
if (!L) return null;
|
||
const c = L.chunks.splice(0);
|
||
const e = L.events.splice(0);
|
||
return { c, e, r: L.recStartT, state: L.recState };
|
||
})()`).catch(() => null);
|
||
}
|
||
|
||
function processEvent(ev) {
|
||
const ta = S.recStartT != null ? round3((ev.t - S.recStartT) / 1000) : null;
|
||
const rec = { ...ev, ta };
|
||
S.events.push(rec);
|
||
if (S.eventsPath) fs.appendFileSync(S.eventsPath, JSON.stringify(rec) + '\n');
|
||
|
||
if (ev.ev === 'recstart') { /* recStartT 已在 drain 里更新 */ }
|
||
else if (ev.ev === 'boost') {
|
||
if (ev.on) S.boosts.push({ on: ta, off: null });
|
||
else {
|
||
const last = [...S.boosts].reverse().find((b) => b.off == null);
|
||
if (last) last.off = ta;
|
||
}
|
||
} else if (ev.ev === 'vstart') {
|
||
const seg = {
|
||
idx: S.segs.length, voiceId: voiceIdOf(ev.url), url: ev.url,
|
||
t0: ta, t1: null, dur: ev.dur, expEnd: null, truncated: null,
|
||
reason: null, result: null, sent: false,
|
||
};
|
||
S.segs.push(seg);
|
||
} else if (ev.ev === 'vend') {
|
||
// 关闭最近一个同 url 的未闭合段
|
||
for (let i = S.segs.length - 1; i >= 0; i--) {
|
||
const s = S.segs[i];
|
||
if (s.t1 == null && s.url === ev.url) {
|
||
s.t1 = ta;
|
||
s.reason = ev.reason || null;
|
||
s.expEnd = expectedEnd(s.t0, s.dur);
|
||
s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async function drain() {
|
||
const r = await drainEval();
|
||
if (!r) return;
|
||
if (r.r != null) S.recStartT = r.r;
|
||
if (r.c && r.c.length && S.mixPath) {
|
||
fs.appendFileSync(S.mixPath, Buffer.concat(r.c.map((b) => Buffer.from(b, 'base64'))));
|
||
}
|
||
if (r.e) for (const ev of r.e) processEvent(ev);
|
||
}
|
||
|
||
// ------------------------------------------------- voice 预期自然结束时刻（含长按快进修正）
|
||
function rateAt(t) {
|
||
for (const b of S.boosts) {
|
||
if (b.on != null && t >= b.on && (b.off == null || t < b.off)) return BOOST_RATE;
|
||
}
|
||
return S.baseSpeed;
|
||
}
|
||
function expectedEnd(t0, dur) {
|
||
if (t0 == null || !dur) return null;
|
||
let pos = 0;
|
||
let t = t0;
|
||
for (let guard = 0; guard < 1000 && pos < dur - 1e-6; guard++) {
|
||
const r = rateAt(t);
|
||
let next = Infinity;
|
||
for (const b of S.boosts) {
|
||
if (b.on != null && b.on > t && b.on < next) next = b.on;
|
||
if (b.off != null && b.off > t && b.off < next) next = b.off;
|
||
}
|
||
const need = (dur - pos) / r;
|
||
if (t + need <= next) return round3(t + need);
|
||
pos += (next - t) * r;
|
||
t = next;
|
||
}
|
||
return round3(t + (dur - pos) / rateAt(t));
|
||
}
|
||
|
||
// ---------------------------------------------------------------- /listen 判定
|
||
async function buildListenRows() {
|
||
// 未发送的闭合 voice 段
|
||
const voices = [];
|
||
for (const s of S.segs) {
|
||
if (s.t1 == null || s.sent) continue;
|
||
s.sent = true; // pending 的会在响应里翻回 false
|
||
voices.push({
|
||
seg: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1,
|
||
// key 变体会话（settings.input='key'）对照 __key__ 表，变体句按键盘措辞验收
|
||
expected: (S.settings?.input === 'key' && (VOICE_TEXT.__key__?.[S.game] || {})[s.voiceId])
|
||
|| (VOICE_TEXT[S.game] || {})[s.voiceId] || null,
|
||
});
|
||
}
|
||
// 间隙探针：相邻 voice 段之间 / 录音起点到第一段，>1.5s 且未探过
|
||
const closed = S.segs.filter((s) => s.t1 != null).sort((a, b) => a.t0 - b.t0);
|
||
const bounds = [];
|
||
if (closed.length) {
|
||
bounds.push([0, closed[0].t0]);
|
||
for (let i = 1; i < closed.length; i++) bounds.push([closed[i - 1].t1, closed[i].t0]);
|
||
}
|
||
for (const [a, b] of bounds) {
|
||
if (b - a < 1.5) continue;
|
||
if (S.gapProbes.some((g) => Math.abs(g.t0 - a) < 0.5 && Math.abs(g.t1 - b) < 0.5)) continue;
|
||
const g = { t0: a, t1: b, done: false };
|
||
S.gapProbes.push(g);
|
||
voices.push({ seg: `gap-${S.gapProbes.length}`, voiceId: null, t0: a, t1: b, expected: null });
|
||
}
|
||
if (!voices.length) return [];
|
||
|
||
await ensureListen();
|
||
let resp;
|
||
try {
|
||
resp = await S.listenProc.req({ cmd: 'transcribe', webm: S.mixPath, voices });
|
||
if (!resp.ok) throw new Error(resp.error || 'unknown');
|
||
} catch (e) {
|
||
// 失败回滚：下次 /listen 重试同批段与探针
|
||
for (const v of voices) {
|
||
if (typeof v.seg === 'number') { const s = S.segs[v.seg]; if (s) s.sent = false; }
|
||
}
|
||
S.gapProbes = S.gapProbes.filter((g) => g.done);
|
||
throw new Error(`listen_server: ${e.message}`);
|
||
}
|
||
S.lastAudioDur = resp.audioDur;
|
||
|
||
const newRows = [];
|
||
for (const row of resp.rows) {
|
||
if (row.status === 'pending') {
|
||
// 音频还没录到：段翻回未发送，间隙探针撤销
|
||
if (typeof row.seg === 'number') {
|
||
const s = S.segs[row.seg];
|
||
if (s) s.sent = false;
|
||
} else {
|
||
S.gapProbes = S.gapProbes.filter((g) => g.done || !(Math.abs(g.t0 - (voices.find((v) => v.seg === row.seg) || {}).t0) < 0.01));
|
||
}
|
||
continue;
|
||
}
|
||
if (typeof row.seg === 'number') {
|
||
const s = S.segs[row.seg];
|
||
if (!s || s.result) continue;
|
||
const verdict = s.truncated ? 'TRUNCATED'
|
||
: row.silent ? 'SILENCE'
|
||
: (row.similarity != null && row.similarity < SIM_THRESHOLD) ? 'MISMATCH'
|
||
: 'OK';
|
||
s.result = { ...row, verdict };
|
||
newRows.push({
|
||
t0: row.t0, t1: row.t1, voiceId: row.voiceId, expected: row.expected,
|
||
heard: row.heard, similarity: row.similarity, verdict,
|
||
});
|
||
} else {
|
||
// 间隙探针：有声音但没 voice = SFX/BGM；无声 = SILENCE
|
||
const g = S.gapProbes.find((x) => !x.done && Math.abs(x.t0 - row.t0) < 0.01);
|
||
if (g) g.done = true;
|
||
newRows.push({
|
||
t0: row.t0, t1: row.t1, voiceId: null, expected: null,
|
||
heard: '', similarity: null, verdict: row.silent ? 'SILENCE' : 'SFX_BGM_ONLY',
|
||
});
|
||
}
|
||
}
|
||
return newRows;
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 手势编排
|
||
// 键盘通道（engine/gesture.js 键位）：key 用 KeyboardEvent.code 值。
|
||
const KEY_DESC = {
|
||
ArrowUp: { code: 'ArrowUp', key: 'ArrowUp', vk: 38 },
|
||
ArrowDown: { code: 'ArrowDown', key: 'ArrowDown', vk: 40 },
|
||
ArrowLeft: { code: 'ArrowLeft', key: 'ArrowLeft', vk: 37 },
|
||
ArrowRight: { code: 'ArrowRight', key: 'ArrowRight', vk: 39 },
|
||
KeyW: { code: 'KeyW', key: 'w', vk: 87 },
|
||
KeyA: { code: 'KeyA', key: 'a', vk: 65 },
|
||
KeyS: { code: 'KeyS', key: 's', vk: 83 },
|
||
KeyD: { code: 'KeyD', key: 'd', vk: 68 },
|
||
Space: { code: 'Space', key: ' ', vk: 32, text: ' ' },
|
||
Enter: { code: 'Enter', key: 'Enter', vk: 13, text: '\r' },
|
||
};
|
||
// 触摸手势类型 → 键盘等价（gesture.js 键位表）：type 走键盘时按此换算
|
||
const TYPE_TO_KEY = {
|
||
tap: { key: 'Space', holdMs: 60 },
|
||
doubletap: { key: 'Enter', holdMs: 60 },
|
||
longpress: { key: 'Space', holdMs: 600 },
|
||
swipeup: { key: 'ArrowUp', holdMs: 60 },
|
||
swipedown: { key: 'ArrowDown', holdMs: 60 },
|
||
swipeleft: { key: 'ArrowLeft', holdMs: 60 },
|
||
swiperight: { key: 'ArrowRight', holdMs: 60 },
|
||
};
|
||
|
||
/** 键盘手势：CDP Input.dispatchKeyEvent（rawKeyDown 会让页面收不到 code 派生
|
||
* 的完整 KeyboardEvent 特性，用 keyDown/keyUp 对），比 Runtime.evaluate 派发
|
||
* 合成事件更拟真（走真实输入管线，含默认行为与焦点语义）。 */
|
||
async function keyGesture({ key, type, holdMs }) {
|
||
let desc = key ? KEY_DESC[key] : null;
|
||
let hold = holdMs;
|
||
if (!desc && type) {
|
||
const m = TYPE_TO_KEY[type];
|
||
if (!m) throw new Error(`未知键盘手势类型: ${type}`);
|
||
desc = KEY_DESC[m.key];
|
||
hold = hold ?? m.holdMs;
|
||
}
|
||
if (!desc) throw new Error(`未知按键: ${key}（可用 ${Object.keys(KEY_DESC).join('|')}）`);
|
||
hold = hold ?? 60;
|
||
const label = key || type;
|
||
// 先把动作本身记进页面时间轴（t=按键开始），附 keyboard/key 供审计区分通道
|
||
await S.page.eval(`window.__live && window.__live.events.push({ev:'gesture', type:${JSON.stringify(label)}, keyboard:true, key:${JSON.stringify(desc.code)}, t:performance.now()})`).catch(() => {});
|
||
const base = { key: desc.key, code: desc.code, windowsVirtualKeyCode: desc.vk, nativeVirtualKeyCode: desc.vk };
|
||
await S.page.send('Input.dispatchKeyEvent', { type: 'keyDown', ...base, ...(desc.text ? { text: desc.text, unmodifiedText: desc.text } : {}) });
|
||
await sleep(hold);
|
||
await S.page.send('Input.dispatchKeyEvent', { type: 'keyUp', ...base });
|
||
await sleep(400); // 与触摸手势一致：给游戏响应时间（空格单击有 ~300ms 双击窗延迟）
|
||
await drain();
|
||
}
|
||
|
||
async function gesture(body = {}) {
|
||
if (body.keyboard) return keyGesture(body); // 键盘通道：{"keyboard":true,"key":...} 或 {type,keyboard:true}
|
||
const { type, x, y } = body;
|
||
const cx = x ?? Math.round(S.viewport.width / 2);
|
||
const cy = y ?? Math.round(S.viewport.height / 2);
|
||
const pt = (px, py) => [{ x: px, y: py, id: 1 }];
|
||
const tapOnce = async (px, py) => {
|
||
await S.page.dispatchTouch('touchStart', pt(px, py));
|
||
await sleep(60);
|
||
await S.page.dispatchTouch('touchEnd', []);
|
||
};
|
||
// 先把动作本身记进页面时间轴（t=手势开始）
|
||
await S.page.eval(`window.__live && window.__live.events.push({ev:'gesture', type:${JSON.stringify(type)}, x:${cx}, y:${cy}, t:performance.now()})`).catch(() => {});
|
||
switch (type) {
|
||
case 'tap':
|
||
await tapOnce(cx, cy);
|
||
break;
|
||
case 'doubletap':
|
||
await tapOnce(cx, cy);
|
||
await sleep(120);
|
||
await tapOnce(cx, cy);
|
||
break;
|
||
case 'longpress':
|
||
await S.page.dispatchTouch('touchStart', pt(cx, cy));
|
||
await sleep(600);
|
||
await S.page.dispatchTouch('touchEnd', []);
|
||
break;
|
||
case 'swipeup': case 'swipedown': case 'swipeleft': case 'swiperight': {
|
||
const d = { swipeup: [0, -200], swipedown: [0, 200], swipeleft: [-200, 0], swiperight: [200, 0] }[type];
|
||
await S.page.dispatchTouch('touchStart', pt(cx, cy));
|
||
const steps = 10; // 200px / 300ms ≈ 667px/s（阈值 400px/s）
|
||
for (let i = 1; i <= steps; i++) {
|
||
await S.page.dispatchTouch('touchMove', pt(cx + (d[0] * i) / steps, cy + (d[1] * i) / steps));
|
||
await sleep(30);
|
||
}
|
||
await S.page.dispatchTouch('touchEnd', []);
|
||
break;
|
||
}
|
||
default:
|
||
throw new Error(`未知手势类型: ${type}`);
|
||
}
|
||
await sleep(400); // 给游戏响应时间（单击回调本身有 ~300ms 双击窗口延迟）
|
||
await drain();
|
||
}
|
||
|
||
// ---------------------------------------------------------------- HTTP 端点
|
||
let opChain = Promise.resolve();
|
||
const locked = (fn) => {
|
||
const p = opChain.then(fn);
|
||
opChain = p.catch(() => {});
|
||
return p;
|
||
};
|
||
|
||
async function endSession({ finalize }) {
|
||
if (!S.page) return null;
|
||
if (S.drainTimer) { clearInterval(S.drainTimer); S.drainTimer = null; }
|
||
if (finalize) {
|
||
// 停录音、拿回尾部 chunk、最后 drain
|
||
await S.page.eval('window.__live ? window.__live.stopRec() : Promise.resolve()', { awaitPromise: true }).catch(() => {});
|
||
await sleep(300);
|
||
await drain();
|
||
// 未闭合的 voice 段按当前时刻封闭（记为 reason:'session-end'）
|
||
for (const s of S.segs) {
|
||
if (s.t1 == null) {
|
||
s.t1 = S.lastAudioDur || s.t0;
|
||
s.reason = 'session-end';
|
||
s.expEnd = expectedEnd(s.t0, s.dur);
|
||
s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S;
|
||
}
|
||
}
|
||
}
|
||
const page = S.page;
|
||
S.page = null;
|
||
await page.close().catch(() => {});
|
||
return page;
|
||
}
|
||
|
||
async function writeArtifacts() {
|
||
const files = [];
|
||
const w = (name, content) => {
|
||
const p = path.join(S.outdir, name);
|
||
fs.writeFileSync(p, content);
|
||
files.push(p);
|
||
};
|
||
// verdicts.json
|
||
w('verdicts.json', JSON.stringify({
|
||
game: S.game, settings: S.settings, outdir: S.outdir,
|
||
segs: S.segs.map((s) => ({
|
||
idx: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1, dur: s.dur,
|
||
expEnd: s.expEnd, truncated: s.truncated, reason: s.reason,
|
||
verdict: s.result ? s.result.verdict : null,
|
||
heard: s.result ? s.result.heard : null,
|
||
similarity: s.result ? s.result.similarity : null,
|
||
})),
|
||
rows: S.rows,
|
||
}, null, 1));
|
||
// transcript.md：手势 + voice + 判定 + 震动 + console 错误的合并时间轴
|
||
const lines = [`# ${S.game} 实时会话 transcript`, ''];
|
||
const items = [];
|
||
for (const e of S.events) {
|
||
const t = e.ta;
|
||
if (t == null) continue;
|
||
if (e.ev === 'gesture') items.push([t, e.keyboard ? `**KEY** ${e.key || e.type}` : `**GESTURE** ${e.type} @(${e.x},${e.y})`]);
|
||
else if (e.ev === 'vstart') items.push([t, `VOICE ▶ ${voiceIdOf(e.url)} (dur ${e.dur}s)`]);
|
||
else if (e.ev === 'vend') items.push([t, `VOICE ■ ${voiceIdOf(e.url)} (${e.reason})`]);
|
||
else if (e.ev === 'start' && e.layer !== 'voice') items.push([t, `${(e.layer || 'sfx').toUpperCase()} ♪ ${e.url ? voiceIdOf(e.url) : '?'}${e.loop ? ' (loop)' : ''}`]);
|
||
else if (e.ev === 'vibrate') items.push([t, `VIBRATE ${e.pattern}`]);
|
||
else if (e.ev === 'asset404') items.push([t, `404 ${e.url}`]);
|
||
else if (e.ev === 'console.error') items.push([t, `CONSOLE.ERROR ${e.msg}`]);
|
||
else if (e.ev === 'pageerror' || e.ev === 'unhandledrejection') items.push([t, `PAGEERROR ${e.msg}`]);
|
||
else if (e.ev === 'recstart') items.push([t, `REC 开始 (${e.mime})`]);
|
||
}
|
||
for (const r of S.rows) {
|
||
const head = r.voiceId ? `VERDICT ${r.voiceId}: **${r.verdict}**` : `VERDICT (无语音区间): **${r.verdict}**`;
|
||
const detail = r.voiceId
|
||
? ` sim=${r.similarity}\n - 期望: ${r.expected}\n - 听到: ${r.heard}`
|
||
: '';
|
||
items.push([r.t1, `${head}${detail}`]);
|
||
}
|
||
items.sort((a, b) => a[0] - b[0]);
|
||
for (const [t, text] of items) lines.push(`- \`[${t.toFixed(2).padStart(8)}s]\` ${text}`);
|
||
w('transcript.md', lines.join('\n') + '\n');
|
||
if (S.mixPath && fs.existsSync(S.mixPath)) files.push(S.mixPath);
|
||
if (S.eventsPath && fs.existsSync(S.eventsPath)) files.push(S.eventsPath);
|
||
return files;
|
||
}
|
||
|
||
async function route(method, pathName, query, body) {
|
||
if (method === 'GET' && pathName === '/') {
|
||
return { ok: true, game: S.game, pageOpen: !!S.page, events: S.events.length, rows: S.rows.length };
|
||
}
|
||
|
||
if (method === 'POST' && pathName === '/session') {
|
||
return locked(async () => {
|
||
const game = body && body.game;
|
||
if (!game || !VOICE_TEXT[game]) throw httpError(400, `未知游戏: ${game}（可选 ${Object.keys(VOICE_TEXT).join('|')}）`);
|
||
await endSession({ finalize: false });
|
||
await ensureBrowser();
|
||
S.game = game;
|
||
S.settings = mapSettings(body.settings || {});
|
||
S.baseSpeed = S.settings.speed ?? 1.0;
|
||
S.viewport = { width: body.viewport?.width || 390, height: body.viewport?.height || 844 };
|
||
const ts = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
||
S.outdir = path.join(OUT_BASE, `${game}-${ts}`);
|
||
fs.mkdirSync(S.outdir, { recursive: true });
|
||
S.mixPath = path.join(S.outdir, 'mix.webm');
|
||
S.eventsPath = path.join(S.outdir, 'events.jsonl');
|
||
fs.writeFileSync(S.mixPath, '');
|
||
fs.writeFileSync(S.eventsPath, '');
|
||
S.recStartT = null;
|
||
S.events = []; S.segs = []; S.boosts = []; S.rows = [];
|
||
S.listenCursor = 0; S.gapProbes = []; S.lastAudioDur = 0;
|
||
|
||
S.page = await S.browser.newPage(S.viewport);
|
||
const seed = `try{localStorage.setItem('soundgame.settings', ${JSON.stringify(JSON.stringify(S.settings))});}catch(e){}\n`;
|
||
await S.page.addInitScript(seed + INJECT_SRC);
|
||
await S.page.navigate(`${GAME_BASE}/${game}/`);
|
||
S.drainTimer = setInterval(() => { drain().catch(() => {}); }, DRAIN_MS);
|
||
return { ok: true, game, settings: S.settings, viewport: S.viewport, outdir: S.outdir };
|
||
});
|
||
}
|
||
|
||
if (method === 'POST' && pathName === '/gesture') {
|
||
return locked(async () => {
|
||
if (!S.page) throw httpError(409, '没有打开的会话（先 POST /session）');
|
||
await gesture(body || {});
|
||
return { ok: true };
|
||
});
|
||
}
|
||
|
||
if (method === 'GET' && pathName === '/events') {
|
||
const since = Math.max(0, Number(query.since) || 0);
|
||
await drain();
|
||
return { events: S.events.slice(since), next: S.events.length };
|
||
}
|
||
|
||
if (method === 'GET' && pathName === '/listen') {
|
||
return locked(async () => {
|
||
if (!S.page) throw httpError(409, '没有打开的会话（先 POST /session）');
|
||
await drain();
|
||
const newRows = await buildListenRows();
|
||
S.rows.push(...newRows);
|
||
const out = S.rows.slice(S.listenCursor);
|
||
S.listenCursor = S.rows.length;
|
||
return { audioDur: S.lastAudioDur, rows: out };
|
||
});
|
||
}
|
||
|
||
if (method === 'GET' && pathName === '/transcript') {
|
||
await drain();
|
||
return {
|
||
game: S.game, outdir: S.outdir, audioDur: S.lastAudioDur,
|
||
events: S.events,
|
||
segs: S.segs.map((s) => ({ ...s, sent: undefined })),
|
||
rows: S.rows,
|
||
};
|
||
}
|
||
|
||
if (method === 'POST' && pathName === '/stop') {
|
||
return locked(async () => {
|
||
if (!S.page) throw httpError(409, '没有打开的会话');
|
||
await endSession({ finalize: true });
|
||
const files = await writeArtifacts();
|
||
return { ok: true, outdir: S.outdir, files, segs: S.segs.length, rows: S.rows.length };
|
||
});
|
||
}
|
||
|
||
throw httpError(404, `${method} ${pathName} 不存在`);
|
||
}
|
||
|
||
function httpError(status, message) {
|
||
const e = new Error(message);
|
||
e.status = status;
|
||
return e;
|
||
}
|
||
|
||
const server = http.createServer(async (req, res) => {
|
||
const u = new URL(req.url, 'http://x');
|
||
let body = null;
|
||
if (req.method === 'POST') {
|
||
const chunks = [];
|
||
for await (const c of req) chunks.push(c);
|
||
const raw = Buffer.concat(chunks).toString('utf8');
|
||
if (raw.trim()) {
|
||
try { body = JSON.parse(raw); } catch (_) {
|
||
res.writeHead(400, { 'content-type': 'application/json' });
|
||
res.end(JSON.stringify({ ok: false, error: '请求体不是合法 JSON' }));
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
try {
|
||
const out = await route(req.method, u.pathname, Object.fromEntries(u.searchParams), body);
|
||
res.writeHead(200, { 'content-type': 'application/json; charset=utf-8' });
|
||
res.end(JSON.stringify(out));
|
||
} catch (e) {
|
||
res.writeHead(e.status || 500, { 'content-type': 'application/json; charset=utf-8' });
|
||
res.end(JSON.stringify({ ok: false, error: e.message }));
|
||
}
|
||
});
|
||
|
||
server.listen(PORT, () => {
|
||
console.log(`[control] 全拟真玩家 agent 控制服务器: http://localhost:${PORT}/`);
|
||
console.log(`[control] 游戏服务器: ${GAME_BASE} 产物目录: ${OUT_BASE}`);
|
||
ensureListen().then(() => console.log('[control] listen_server (SenseVoice) 就绪')).catch((e) => console.error('[control] listen_server 启动失败:', e));
|
||
});
|
||
|
||
process.on('SIGINT', async () => {
|
||
await endSession({ finalize: false }).catch(() => {});
|
||
if (S.browser) await S.browser.dispose().catch(() => {});
|
||
if (S.listenProc) S.listenProc.proc.kill();
|
||
process.exit(0);
|
||
});
|