Files
SoundGame/web/harness/live/control.mjs
T

549 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* control.mjs — 全拟真玩家 agent HTTP 控制服务器(Node 零依赖,端口 8123)。
*
* 外部 agent 通过 HTTP 逐步玩任意一款 SoundGame:
* POST /session {game, settings?, viewport?} 开新页(预注入探针 + 设置种子)
* POST /gesture {type, x?, y?} 真实触摸手势(tap/doubletap/swipe四向/longpress)
* GET /events?since=N 增量事件(voice/fetch/震动/console/手势…)
* GET /listen 增量转写判定(OK/TRUNCATED/MISMATCH/SILENCE/SFX_BGM_ONLY)
* GET /transcript 合并时间轴全量(复盘用)
* POST /stop 收尾落盘(mix.webm/events.jsonl/transcript.md/verdicts.json)
*
* 架构:chromium(CDP 真实输入) → 页面探针 inject.js(事件 + MediaRecorder 混音)
* → 本服务每 500ms drain(chunks 追加 mix.webm,事件换算到录音时基)
* → /listen 把闭合 voice 段交给 listen_server.py(SenseVoice ASR + 相似度)
*
* 截断双保险:① 事件层:vend 时刻 < 预期自然结束(t0 + dur/tempo,含长按快进修正) - 0.15s;
* ② ASR 层:heard vs expected 相似度 < 0.6 → MISMATCH。
*
* 运行:node web/harness/live/control.mjs (需游戏服务器已在 8001 端口运行)
*/
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
import url from 'node:url';
import { spawn } from 'node:child_process';
import { launch, sleep } from './cdp.mjs';
const HERE = path.dirname(url.fileURLToPath(import.meta.url));
const OUT_BASE = path.join(HERE, 'out');
const INJECT_SRC = fs.readFileSync(path.join(HERE, 'inject.js'), 'utf8');
const VOICE_TEXT = JSON.parse(fs.readFileSync(path.join(HERE, '..', 'voice_text.json'), 'utf8'));
const PORT = Number(process.env.LIVE_PORT) || 8123;
const GAME_BASE = process.env.GAME_BASE || 'http://localhost:8001';
const DRAIN_MS = 500;
const SIM_THRESHOLD = 0.6;
const TRUNC_SLACK_S = 0.15;
const BOOST_RATE = 1.7; // 引擎长按快进倍速(audio.js LONGPRESS_BOOST)
// 设置友好名 → 引擎枚举值(engine/settings.js VALID)
const SPEED_MAP = { normal: 1.0, fast: 1.35, faster: 1.7, max: 2.2 };
const BGM_MAP = { normal: 1.0, low: 0.3, off: 0 };
function mapSettings(raw = {}) {
const out = {};
if (raw.voice !== undefined) out.voice = raw.voice; // male|female
if (raw.guide !== undefined) out.guide = raw.guide; // full|standard|off
if (raw.speed !== undefined) {
out.speed = typeof raw.speed === 'number' ? raw.speed : SPEED_MAP[raw.speed] ?? 1.0;
}
if (raw.bgm !== undefined) {
out.bgm = typeof raw.bgm === 'number' ? raw.bgm : BGM_MAP[raw.bgm] ?? 1.0;
}
return out;
}
const round3 = (x) => (x == null ? null : Math.round(x * 1000) / 1000);
const voiceIdOf = (u) => {
const base = String(u || '').split('/').pop() || '';
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(0, dot) : base;
};
// ---------------------------------------------------------------- 会话状态(单会话)
const S = {
browser: null,
page: null,
game: null,
settings: null,
baseSpeed: 1.0,
viewport: { width: 390, height: 844 },
outdir: null,
mixPath: null,
eventsPath: null,
recStartT: null, // 页面 performance.now() ms,录音 t=0 对齐点
events: [], // 全事件,已加 ta(录音时基秒)
segs: [], // voice 段
boosts: [], // [{on, off}] 秒(录音时基)
rows: [], // /listen 全部判定行
listenCursor: 0,
gapProbes: [], // [{t0, t1, done}]
lastAudioDur: 0,
drainTimer: null,
listenProc: null, // {proc, pending:Map, nextId, readyPromise}
};
// ---------------------------------------------------------------- 浏览器 / python 子进程
async function ensureBrowser() {
if (S.browser) return S.browser;
const profileBase = path.join(OUT_BASE, '.profiles');
fs.mkdirSync(profileBase, { recursive: true });
S.browser = await launch({ profileBase });
return S.browser;
}
function ensureListen() {
if (S.listenProc) return S.listenProc.readyPromise;
const proc = spawn('python3', [path.join(HERE, 'listen_server.py')], {
stdio: ['pipe', 'pipe', 'inherit'],
});
const pending = new Map();
let nextId = 1;
let buf = '';
let readyResolve;
const readyPromise = new Promise((res) => { readyResolve = res; });
proc.stdout.on('data', (d) => {
buf += d.toString();
let i;
while ((i = buf.indexOf('\n')) >= 0) {
const line = buf.slice(0, i).trim();
buf = buf.slice(i + 1);
if (!line) continue;
let msg;
try { msg = JSON.parse(line); } catch (_) { continue; }
if (msg.ready) { readyResolve(); continue; }
const p = pending.get(msg.id);
if (p) { pending.delete(msg.id); p(msg); }
}
});
proc.on('exit', (code) => {
console.error(`[control] listen_server 退出 (code ${code})`);
S.listenProc = null;
});
const req = (obj) => new Promise((resolve, reject) => {
const id = nextId++;
pending.set(id, resolve);
proc.stdin.write(JSON.stringify({ id, ...obj }) + '\n');
setTimeout(() => { if (pending.delete(id)) reject(new Error('listen_server 响应超时')); }, 120000);
});
S.listenProc = { proc, req, readyPromise };
return readyPromise;
}
// ---------------------------------------------------------------- drain:chunk 落盘 + 事件换算
function drainEval() {
if (!S.page) return Promise.resolve(null);
return S.page.eval(`(() => {
const L = window.__live;
if (!L) return null;
const c = L.chunks.splice(0);
const e = L.events.splice(0);
return { c, e, r: L.recStartT, state: L.recState };
})()`).catch(() => null);
}
function processEvent(ev) {
const ta = S.recStartT != null ? round3((ev.t - S.recStartT) / 1000) : null;
const rec = { ...ev, ta };
S.events.push(rec);
if (S.eventsPath) fs.appendFileSync(S.eventsPath, JSON.stringify(rec) + '\n');
if (ev.ev === 'recstart') { /* recStartT 已在 drain 里更新 */ }
else if (ev.ev === 'boost') {
if (ev.on) S.boosts.push({ on: ta, off: null });
else {
const last = [...S.boosts].reverse().find((b) => b.off == null);
if (last) last.off = ta;
}
} else if (ev.ev === 'vstart') {
const seg = {
idx: S.segs.length, voiceId: voiceIdOf(ev.url), url: ev.url,
t0: ta, t1: null, dur: ev.dur, expEnd: null, truncated: null,
reason: null, result: null, sent: false,
};
S.segs.push(seg);
} else if (ev.ev === 'vend') {
// 关闭最近一个同 url 的未闭合段
for (let i = S.segs.length - 1; i >= 0; i--) {
const s = S.segs[i];
if (s.t1 == null && s.url === ev.url) {
s.t1 = ta;
s.reason = ev.reason || null;
s.expEnd = expectedEnd(s.t0, s.dur);
s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S;
break;
}
}
}
}
async function drain() {
const r = await drainEval();
if (!r) return;
if (r.r != null) S.recStartT = r.r;
if (r.c && r.c.length && S.mixPath) {
fs.appendFileSync(S.mixPath, Buffer.concat(r.c.map((b) => Buffer.from(b, 'base64'))));
}
if (r.e) for (const ev of r.e) processEvent(ev);
}
// ------------------------------------------------- voice 预期自然结束时刻(含长按快进修正)
function rateAt(t) {
for (const b of S.boosts) {
if (b.on != null && t >= b.on && (b.off == null || t < b.off)) return BOOST_RATE;
}
return S.baseSpeed;
}
function expectedEnd(t0, dur) {
if (t0 == null || !dur) return null;
let pos = 0;
let t = t0;
for (let guard = 0; guard < 1000 && pos < dur - 1e-6; guard++) {
const r = rateAt(t);
let next = Infinity;
for (const b of S.boosts) {
if (b.on != null && b.on > t && b.on < next) next = b.on;
if (b.off != null && b.off > t && b.off < next) next = b.off;
}
const need = (dur - pos) / r;
if (t + need <= next) return round3(t + need);
pos += (next - t) * r;
t = next;
}
return round3(t + (dur - pos) / rateAt(t));
}
// ---------------------------------------------------------------- /listen 判定
async function buildListenRows() {
// 未发送的闭合 voice 段
const voices = [];
for (const s of S.segs) {
if (s.t1 == null || s.sent) continue;
s.sent = true; // pending 的会在响应里翻回 false
voices.push({
seg: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1,
expected: (VOICE_TEXT[S.game] || {})[s.voiceId] || null,
});
}
// 间隙探针:相邻 voice 段之间 / 录音起点到第一段,>1.5s 且未探过
const closed = S.segs.filter((s) => s.t1 != null).sort((a, b) => a.t0 - b.t0);
const bounds = [];
if (closed.length) {
bounds.push([0, closed[0].t0]);
for (let i = 1; i < closed.length; i++) bounds.push([closed[i - 1].t1, closed[i].t0]);
}
for (const [a, b] of bounds) {
if (b - a < 1.5) continue;
if (S.gapProbes.some((g) => Math.abs(g.t0 - a) < 0.5 && Math.abs(g.t1 - b) < 0.5)) continue;
const g = { t0: a, t1: b, done: false };
S.gapProbes.push(g);
voices.push({ seg: `gap-${S.gapProbes.length}`, voiceId: null, t0: a, t1: b, expected: null });
}
if (!voices.length) return [];
await ensureListen();
let resp;
try {
resp = await S.listenProc.req({ cmd: 'transcribe', webm: S.mixPath, voices });
if (!resp.ok) throw new Error(resp.error || 'unknown');
} catch (e) {
// 失败回滚:下次 /listen 重试同批段与探针
for (const v of voices) {
if (typeof v.seg === 'number') { const s = S.segs[v.seg]; if (s) s.sent = false; }
}
S.gapProbes = S.gapProbes.filter((g) => g.done);
throw new Error(`listen_server: ${e.message}`);
}
S.lastAudioDur = resp.audioDur;
const newRows = [];
for (const row of resp.rows) {
if (row.status === 'pending') {
// 音频还没录到:段翻回未发送,间隙探针撤销
if (typeof row.seg === 'number') {
const s = S.segs[row.seg];
if (s) s.sent = false;
} else {
S.gapProbes = S.gapProbes.filter((g) => g.done || !(Math.abs(g.t0 - (voices.find((v) => v.seg === row.seg) || {}).t0) < 0.01));
}
continue;
}
if (typeof row.seg === 'number') {
const s = S.segs[row.seg];
if (!s || s.result) continue;
const verdict = s.truncated ? 'TRUNCATED'
: row.silent ? 'SILENCE'
: (row.similarity != null && row.similarity < SIM_THRESHOLD) ? 'MISMATCH'
: 'OK';
s.result = { ...row, verdict };
newRows.push({
t0: row.t0, t1: row.t1, voiceId: row.voiceId, expected: row.expected,
heard: row.heard, similarity: row.similarity, verdict,
});
} else {
// 间隙探针:有声音但没 voice = SFX/BGM;无声 = SILENCE
const g = S.gapProbes.find((x) => !x.done && Math.abs(x.t0 - row.t0) < 0.01);
if (g) g.done = true;
newRows.push({
t0: row.t0, t1: row.t1, voiceId: null, expected: null,
heard: '', similarity: null, verdict: row.silent ? 'SILENCE' : 'SFX_BGM_ONLY',
});
}
}
return newRows;
}
// ---------------------------------------------------------------- 手势编排
async function gesture({ type, x, y }) {
const cx = x ?? Math.round(S.viewport.width / 2);
const cy = y ?? Math.round(S.viewport.height / 2);
const pt = (px, py) => [{ x: px, y: py, id: 1 }];
const tapOnce = async (px, py) => {
await S.page.dispatchTouch('touchStart', pt(px, py));
await sleep(60);
await S.page.dispatchTouch('touchEnd', []);
};
// 先把动作本身记进页面时间轴(t=手势开始)
await S.page.eval(`window.__live && window.__live.events.push({ev:'gesture', type:${JSON.stringify(type)}, x:${cx}, y:${cy}, t:performance.now()})`).catch(() => {});
switch (type) {
case 'tap':
await tapOnce(cx, cy);
break;
case 'doubletap':
await tapOnce(cx, cy);
await sleep(120);
await tapOnce(cx, cy);
break;
case 'longpress':
await S.page.dispatchTouch('touchStart', pt(cx, cy));
await sleep(600);
await S.page.dispatchTouch('touchEnd', []);
break;
case 'swipeup': case 'swipedown': case 'swipeleft': case 'swiperight': {
const d = { swipeup: [0, -200], swipedown: [0, 200], swipeleft: [-200, 0], swiperight: [200, 0] }[type];
await S.page.dispatchTouch('touchStart', pt(cx, cy));
const steps = 10; // 200px / 300ms ≈ 667px/s(阈值 400px/s)
for (let i = 1; i <= steps; i++) {
await S.page.dispatchTouch('touchMove', pt(cx + (d[0] * i) / steps, cy + (d[1] * i) / steps));
await sleep(30);
}
await S.page.dispatchTouch('touchEnd', []);
break;
}
default:
throw new Error(`未知手势类型: ${type}`);
}
await sleep(400); // 给游戏响应时间(单击回调本身有 ~300ms 双击窗口延迟)
await drain();
}
// ---------------------------------------------------------------- HTTP 端点
let opChain = Promise.resolve();
const locked = (fn) => {
const p = opChain.then(fn);
opChain = p.catch(() => {});
return p;
};
async function endSession({ finalize }) {
if (!S.page) return null;
if (S.drainTimer) { clearInterval(S.drainTimer); S.drainTimer = null; }
if (finalize) {
// 停录音、拿回尾部 chunk、最后 drain
await S.page.eval('window.__live ? window.__live.stopRec() : Promise.resolve()', { awaitPromise: true }).catch(() => {});
await sleep(300);
await drain();
// 未闭合的 voice 段按当前时刻封闭(记为 reason:'session-end')
for (const s of S.segs) {
if (s.t1 == null) {
s.t1 = S.lastAudioDur || s.t0;
s.reason = 'session-end';
s.expEnd = expectedEnd(s.t0, s.dur);
s.truncated = s.t1 < s.expEnd - TRUNC_SLACK_S;
}
}
}
const page = S.page;
S.page = null;
await page.close().catch(() => {});
return page;
}
async function writeArtifacts() {
const files = [];
const w = (name, content) => {
const p = path.join(S.outdir, name);
fs.writeFileSync(p, content);
files.push(p);
};
// verdicts.json
w('verdicts.json', JSON.stringify({
game: S.game, settings: S.settings, outdir: S.outdir,
segs: S.segs.map((s) => ({
idx: s.idx, voiceId: s.voiceId, t0: s.t0, t1: s.t1, dur: s.dur,
expEnd: s.expEnd, truncated: s.truncated, reason: s.reason,
verdict: s.result ? s.result.verdict : null,
heard: s.result ? s.result.heard : null,
similarity: s.result ? s.result.similarity : null,
})),
rows: S.rows,
}, null, 1));
// transcript.md:手势 + voice + 判定 + 震动 + console 错误的合并时间轴
const lines = [`# ${S.game} 实时会话 transcript`, ''];
const items = [];
for (const e of S.events) {
const t = e.ta;
if (t == null) continue;
if (e.ev === 'gesture') items.push([t, `**GESTURE** ${e.type} @(${e.x},${e.y})`]);
else if (e.ev === 'vstart') items.push([t, `VOICE ▶ ${voiceIdOf(e.url)} (dur ${e.dur}s)`]);
else if (e.ev === 'vend') items.push([t, `VOICE ■ ${voiceIdOf(e.url)} (${e.reason})`]);
else if (e.ev === 'start' && e.layer !== 'voice') items.push([t, `${(e.layer || 'sfx').toUpperCase()} ♪ ${e.url ? voiceIdOf(e.url) : '?'}${e.loop ? ' (loop)' : ''}`]);
else if (e.ev === 'vibrate') items.push([t, `VIBRATE ${e.pattern}`]);
else if (e.ev === 'asset404') items.push([t, `404 ${e.url}`]);
else if (e.ev === 'console.error') items.push([t, `CONSOLE.ERROR ${e.msg}`]);
else if (e.ev === 'pageerror' || e.ev === 'unhandledrejection') items.push([t, `PAGEERROR ${e.msg}`]);
else if (e.ev === 'recstart') items.push([t, `REC 开始 (${e.mime})`]);
}
for (const r of S.rows) {
const head = r.voiceId ? `VERDICT ${r.voiceId}: **${r.verdict}**` : `VERDICT (无语音区间): **${r.verdict}**`;
const detail = r.voiceId
? ` sim=${r.similarity}\n - 期望: ${r.expected}\n - 听到: ${r.heard}`
: '';
items.push([r.t1, `${head}${detail}`]);
}
items.sort((a, b) => a[0] - b[0]);
for (const [t, text] of items) lines.push(`- \`[${t.toFixed(2).padStart(8)}s]\` ${text}`);
w('transcript.md', lines.join('\n') + '\n');
if (S.mixPath && fs.existsSync(S.mixPath)) files.push(S.mixPath);
if (S.eventsPath && fs.existsSync(S.eventsPath)) files.push(S.eventsPath);
return files;
}
async function route(method, pathName, query, body) {
if (method === 'GET' && pathName === '/') {
return { ok: true, game: S.game, pageOpen: !!S.page, events: S.events.length, rows: S.rows.length };
}
if (method === 'POST' && pathName === '/session') {
return locked(async () => {
const game = body && body.game;
if (!game || !VOICE_TEXT[game]) throw httpError(400, `未知游戏: ${game}(可选 ${Object.keys(VOICE_TEXT).join('|')})`);
await endSession({ finalize: false });
await ensureBrowser();
S.game = game;
S.settings = mapSettings(body.settings || {});
S.baseSpeed = S.settings.speed ?? 1.0;
S.viewport = { width: body.viewport?.width || 390, height: body.viewport?.height || 844 };
const ts = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14);
S.outdir = path.join(OUT_BASE, `${game}-${ts}`);
fs.mkdirSync(S.outdir, { recursive: true });
S.mixPath = path.join(S.outdir, 'mix.webm');
S.eventsPath = path.join(S.outdir, 'events.jsonl');
fs.writeFileSync(S.mixPath, '');
fs.writeFileSync(S.eventsPath, '');
S.recStartT = null;
S.events = []; S.segs = []; S.boosts = []; S.rows = [];
S.listenCursor = 0; S.gapProbes = []; S.lastAudioDur = 0;
S.page = await S.browser.newPage(S.viewport);
const seed = `try{localStorage.setItem('soundgame.settings', ${JSON.stringify(JSON.stringify(S.settings))});}catch(e){}\n`;
await S.page.addInitScript(seed + INJECT_SRC);
await S.page.navigate(`${GAME_BASE}/${game}/`);
S.drainTimer = setInterval(() => { drain().catch(() => {}); }, DRAIN_MS);
return { ok: true, game, settings: S.settings, viewport: S.viewport, outdir: S.outdir };
});
}
if (method === 'POST' && pathName === '/gesture') {
return locked(async () => {
if (!S.page) throw httpError(409, '没有打开的会话(先 POST /session)');
await gesture(body || {});
return { ok: true };
});
}
if (method === 'GET' && pathName === '/events') {
const since = Math.max(0, Number(query.since) || 0);
await drain();
return { events: S.events.slice(since), next: S.events.length };
}
if (method === 'GET' && pathName === '/listen') {
return locked(async () => {
if (!S.page) throw httpError(409, '没有打开的会话(先 POST /session)');
await drain();
const newRows = await buildListenRows();
S.rows.push(...newRows);
const out = S.rows.slice(S.listenCursor);
S.listenCursor = S.rows.length;
return { audioDur: S.lastAudioDur, rows: out };
});
}
if (method === 'GET' && pathName === '/transcript') {
await drain();
return {
game: S.game, outdir: S.outdir, audioDur: S.lastAudioDur,
events: S.events,
segs: S.segs.map((s) => ({ ...s, sent: undefined })),
rows: S.rows,
};
}
if (method === 'POST' && pathName === '/stop') {
return locked(async () => {
if (!S.page) throw httpError(409, '没有打开的会话');
await endSession({ finalize: true });
const files = await writeArtifacts();
return { ok: true, outdir: S.outdir, files, segs: S.segs.length, rows: S.rows.length };
});
}
throw httpError(404, `${method} ${pathName} 不存在`);
}
function httpError(status, message) {
const e = new Error(message);
e.status = status;
return e;
}
const server = http.createServer(async (req, res) => {
const u = new URL(req.url, 'http://x');
let body = null;
if (req.method === 'POST') {
const chunks = [];
for await (const c of req) chunks.push(c);
const raw = Buffer.concat(chunks).toString('utf8');
if (raw.trim()) {
try { body = JSON.parse(raw); } catch (_) {
res.writeHead(400, { 'content-type': 'application/json' });
res.end(JSON.stringify({ ok: false, error: '请求体不是合法 JSON' }));
return;
}
}
}
try {
const out = await route(req.method, u.pathname, Object.fromEntries(u.searchParams), body);
res.writeHead(200, { 'content-type': 'application/json; charset=utf-8' });
res.end(JSON.stringify(out));
} catch (e) {
res.writeHead(e.status || 500, { 'content-type': 'application/json; charset=utf-8' });
res.end(JSON.stringify({ ok: false, error: e.message }));
}
});
server.listen(PORT, () => {
console.log(`[control] 全拟真玩家 agent 控制服务器: http://localhost:${PORT}/`);
console.log(`[control] 游戏服务器: ${GAME_BASE} 产物目录: ${OUT_BASE}`);
ensureListen().then(() => console.log('[control] listen_server (SenseVoice) 就绪')).catch((e) => console.error('[control] listen_server 启动失败:', e));
});
process.on('SIGINT', async () => {
await endSession({ finalize: false }).catch(() => {});
if (S.browser) await S.browser.dispose().catch(() => {});
if (S.listenProc) S.listenProc.proc.kill();
process.exit(0);
});