初始提交：3 款纯音频游戏 + 共享引擎 + 完整文档与资产管线
- 共享引擎（app/src/main）：GestureEngine 6 手势识别、AudioEngine 三层音频 （ExoPlayer BGM 交叉淡入 / 对白 ducking / SoundPool 低延迟）、Game/GameEngine 状态机接口、MainActivity 全屏封面 + TalkBack - 《声命线》storyline：末世互动广播剧，3 幕 8 抉择点 4 结局 - 《声东击西》reflex：听声辨位反应，12 波战斗 + BGM 三档 BPM 变速 - 《暗室》escape：3x3 网格密室逃脱，钥匙→铁柜→保险丝→电箱→大门谜题链 - 文档：setup（从零搭建）/ architecture（架构）/ engine-api（引擎 API）/ asset-pipeline（资产流水线）/ games/*（三款游戏设计文档） - 资产管线：Qwen3-TTS 对白 + 程序合成音效/BGM + Mureka BGM 的 gen 脚本与台账， 全部 ogg 资产入库可复现 - env.d 环境脚本：android/tts 入库；sfx/mureka 含密钥不入库，仅提交 .example.sh 模板
This commit is contained in:
SoundGame Dev
@@ -0,0 +1,173 @@
|
||||
"""Mel-spectrogram renderer ported from Underfit's dashboard.
|
||||
|
||||
Source: /weka2/cj/clod/underfit/dashboard/server.py — the 3-band tinted stereo
|
||||
mel spectrogram used in the Underfit run dashboards. Same algorithm,
|
||||
same constants, same Slaney mel scale + librosa-compatible mel filterbank.
|
||||
|
||||
Differences vs the Underfit original:
|
||||
- Input is an int16 (T, 2) numpy array (in-memory PCM) rather than a file
|
||||
path; no soundfile dependency. SAMPLE_RATE assumed 44.1 kHz, resampled
|
||||
to 32 kHz internally for the spec (matches Underfit's target_sr).
|
||||
- Output is PNG bytes returned in-memory (no file write).
|
||||
- Exposes render_spectrogram_png(pcm_int16, sample_rate, width, height)
|
||||
so callers can choose a render resolution.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import io
|
||||
import math
|
||||
from functools import lru_cache
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
# Underfit's 3-band tint
|
||||
SPEC_BANDS = [
|
||||
(0, 200, (1.0, 0.0, 0.0)), # Bass -> Red
|
||||
(200, 1500, (0.0, 1.0, 0.0)), # Mid -> Green
|
||||
(1500, 16000, (0.0, 0.0, 1.0)), # High -> Blue
|
||||
]
|
||||
_BAND_COLORS = np.array([c for _, _, c in SPEC_BANDS], dtype=np.float32)
|
||||
|
||||
# Slaney mel-scale constants (linear below 1 kHz, log above), matches librosa default.
|
||||
_F_SP = 200.0 / 3
|
||||
_MIN_LOG_HZ = 1000.0
|
||||
_MIN_LOG_MEL = _MIN_LOG_HZ / _F_SP
|
||||
_LOGSTEP = math.log(6.4) / 27.0
|
||||
|
||||
|
||||
def _hz_to_mel(hz):
|
||||
hz = np.asarray(hz, dtype=np.float64)
|
||||
log_term = np.log(np.maximum(hz, _MIN_LOG_HZ) / _MIN_LOG_HZ) / _LOGSTEP
|
||||
return np.where(hz >= _MIN_LOG_HZ, _MIN_LOG_MEL + log_term, hz / _F_SP)
|
||||
|
||||
|
||||
def _mel_to_hz(mels):
|
||||
mels = np.asarray(mels, dtype=np.float64)
|
||||
return np.where(mels >= _MIN_LOG_MEL,
|
||||
_MIN_LOG_HZ * np.exp(_LOGSTEP * (mels - _MIN_LOG_MEL)),
|
||||
_F_SP * mels)
|
||||
|
||||
|
||||
def _mel_frequencies(n_mels, fmax, fmin=0.0):
|
||||
return _mel_to_hz(np.linspace(_hz_to_mel(fmin), _hz_to_mel(fmax), n_mels))
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _mel_filterbank(n_mels, n_fft, sr, fmax):
|
||||
pts = _mel_to_hz(np.linspace(_hz_to_mel(0.0), _hz_to_mel(fmax), n_mels + 2))
|
||||
fft_f = np.linspace(0, sr / 2, n_fft // 2 + 1)
|
||||
filt = np.zeros((n_mels, n_fft // 2 + 1), dtype=np.float32)
|
||||
for i in range(n_mels):
|
||||
lo, ce, hi = pts[i], pts[i + 1], pts[i + 2]
|
||||
left = (fft_f - lo) / max(ce - lo, 1e-10)
|
||||
right = (hi - fft_f) / max(hi - ce, 1e-10)
|
||||
filt[i] = np.maximum(0, np.minimum(left, right))
|
||||
enorm = (2.0 / (pts[2:n_mels + 2] - pts[0:n_mels])).astype(np.float32)
|
||||
filt *= enorm[:, None]
|
||||
return torch.from_numpy(filt)
|
||||
|
||||
|
||||
def _melspectrogram(y_ch, sr, n_mels=30, fmax=16000, hop_length=2048, n_fft=2048):
|
||||
y_t = torch.from_numpy(np.ascontiguousarray(y_ch)).float()
|
||||
win = torch.hann_window(n_fft)
|
||||
spec = torch.stft(y_t, n_fft=n_fft, hop_length=hop_length, window=win,
|
||||
center=True, return_complex=True, pad_mode='reflect')
|
||||
return (_mel_filterbank(n_mels, n_fft, sr, fmax) @ spec.abs().square()).numpy()
|
||||
|
||||
|
||||
def _power_to_db(S, top_db=80.0):
|
||||
log_spec = 10.0 * np.log10(np.maximum(S, 1e-10))
|
||||
return np.maximum(log_spec - log_spec.max(), -top_db)
|
||||
|
||||
|
||||
def _mel_channel(y_ch, sr, n_mels=30):
|
||||
"""dB-scaled mel + band-tinted RGB for one mono channel."""
|
||||
S = _melspectrogram(y_ch, sr, n_mels=n_mels, fmax=16000, hop_length=2048)
|
||||
|
||||
S_db = _power_to_db(S)
|
||||
np.clip(S_db, -60, 0, out=S_db)
|
||||
S_db += 60.0
|
||||
S_db /= 60.0
|
||||
np.power(S_db, 0.6, out=S_db)
|
||||
|
||||
mel_f = _mel_frequencies(n_mels, fmax=16000)
|
||||
n_frames = S.shape[1]
|
||||
band_norms = np.empty((3, n_frames), dtype=np.float32)
|
||||
for i, (flo, fhi, _) in enumerate(SPEC_BANDS):
|
||||
mask = (mel_f >= flo) & (mel_f < fhi)
|
||||
if mask.any():
|
||||
power = np.sum(S[mask], axis=0)
|
||||
db = 10.0 * np.log10(power + 1e-10)
|
||||
np.clip(db, -20, None, out=db)
|
||||
db -= -20
|
||||
mx = db.max()
|
||||
if mx > 0:
|
||||
db /= mx
|
||||
band_norms[i] = db
|
||||
else:
|
||||
band_norms[i] = 0.0
|
||||
|
||||
rgb = band_norms.T @ _BAND_COLORS
|
||||
for c in range(3):
|
||||
mx = rgb[:, c].max()
|
||||
if mx > 0:
|
||||
rgb[:, c] /= mx
|
||||
|
||||
return S_db, rgb
|
||||
|
||||
|
||||
def _resample_to_32k(pcm_int16, sr_in):
|
||||
"""Resample (T, 2) int16 PCM to (2, T') float32 at 32 kHz."""
|
||||
target_sr = 32000
|
||||
if sr_in == target_sr:
|
||||
y = pcm_int16.astype(np.float32) / 32768.0
|
||||
return y.T, target_sr
|
||||
# Convert int16 → float32 → (2, T), then torch interp.
|
||||
y = pcm_int16.astype(np.float32).T / 32768.0 # (2, T)
|
||||
y_t = torch.from_numpy(np.ascontiguousarray(y))
|
||||
new_len = int(round(y_t.shape[-1] * target_sr / sr_in))
|
||||
y_t = torch.nn.functional.interpolate(
|
||||
y_t.unsqueeze(0), size=new_len, mode='linear', align_corners=False
|
||||
).squeeze(0)
|
||||
return y_t.numpy(), target_sr
|
||||
|
||||
|
||||
def render_spectrogram_png(pcm_int16, sample_rate=44100, width=1200, height=240,
|
||||
n_mels=30) -> bytes:
|
||||
"""Render Underfit-style 3-band tinted stereo mel spectrogram to PNG bytes.
|
||||
|
||||
Args:
|
||||
pcm_int16: (T, 2) int16 numpy array — stereo PCM.
|
||||
sample_rate: input PCM sample rate (default 44100).
|
||||
width, height: final image size (default 1200x240; Underfit ships 300x60).
|
||||
n_mels: mel filterbank size (30 matches Underfit).
|
||||
|
||||
Returns: PNG bytes ready to base64 / save / etc.
|
||||
"""
|
||||
if pcm_int16.ndim != 2 or pcm_int16.shape[1] != 2:
|
||||
raise ValueError(f"expected (T, 2) PCM, got {pcm_int16.shape}")
|
||||
y, sr = _resample_to_32k(pcm_int16, sample_rate)
|
||||
# y is (2, T') float32
|
||||
S_L, rgb_L = _mel_channel(y[0], sr, n_mels=n_mels)
|
||||
S_R, rgb_R = _mel_channel(y[1], sr, n_mels=n_mels)
|
||||
|
||||
nf = min(S_L.shape[1], S_R.shape[1])
|
||||
S_L, S_R = S_L[:, :nf], S_R[:, :nf]
|
||||
rgb_L, rgb_R = rgb_L[:nf], rgb_R[:nf]
|
||||
nm = S_L.shape[0]
|
||||
|
||||
# L channel: flip vertically so bass-band is at bottom.
|
||||
S_L = S_L[::-1]
|
||||
|
||||
img = np.empty((nm * 2, nf, 3), dtype=np.float32)
|
||||
img[:nm] = S_L[:, :, np.newaxis] * rgb_L[np.newaxis, :, :]
|
||||
img[nm:] = S_R[:, :, np.newaxis] * rgb_R[np.newaxis, :, :]
|
||||
|
||||
np.clip(img, 0, 1, out=img)
|
||||
img *= 255
|
||||
pil = Image.fromarray(img.astype(np.uint8)).resize((width, height), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
pil.save(buf, format="PNG", optimize=True)
|
||||
return buf.getvalue()
|
||||
Reference in New Issue
Block a user