初始提交：3 款纯音频游戏 + 共享引擎 + 完整文档与资产管线
- 共享引擎（app/src/main）：GestureEngine 6 手势识别、AudioEngine 三层音频 （ExoPlayer BGM 交叉淡入 / 对白 ducking / SoundPool 低延迟）、Game/GameEngine 状态机接口、MainActivity 全屏封面 + TalkBack - 《声命线》storyline：末世互动广播剧，3 幕 8 抉择点 4 结局 - 《声东击西》reflex：听声辨位反应，12 波战斗 + BGM 三档 BPM 变速 - 《暗室》escape：3x3 网格密室逃脱，钥匙→铁柜→保险丝→电箱→大门谜题链 - 文档：setup（从零搭建）/ architecture（架构）/ engine-api（引擎 API）/ asset-pipeline（资产流水线）/ games/*（三款游戏设计文档） - 资产管线：Qwen3-TTS 对白 + 程序合成音效/BGM + Mureka BGM 的 gen 脚本与台账， 全部 ogg 资产入库可复现 - env.d 环境脚本：android/tts 入库；sfx/mureka 含密钥不入库，仅提交 .example.sh 模板
This commit is contained in:
SoundGame Dev
@@ -0,0 +1,165 @@
|
||||
"""
|
||||
Pre-encode a dataset of audio clips into latents using Stable Audio 3, saving the latents and metadata to disk.
|
||||
|
||||
Dataset layout:
|
||||
data_dir/
|
||||
clip1.wav (or .flac, .mp3, .ogg)
|
||||
clip1.txt ← text prompt for clip1
|
||||
clip2.wav
|
||||
clip2.txt
|
||||
...
|
||||
|
||||
Saves .npy files for latents and .json files for metadata, compatible with train_lora.py --encoded_dir.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/pre_encode_dataset.py --model same-s --data_dir ./my_data --output_path ./latents_out
|
||||
uv run python scripts/pre_encode_dataset.py --model same-l --data_dir ./my_data --output_path ./latents_out --batch_size 4
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import functional as F
|
||||
|
||||
from stable_audio_3 import AutoencoderModel
|
||||
from stable_audio_3.model_configs import ae_models
|
||||
from stable_audio_3.data.dataset import (
|
||||
LocalDatasetConfig,
|
||||
SampleDataset,
|
||||
collation_fn,
|
||||
)
|
||||
|
||||
|
||||
def caption_metadata_fn(info, _audio):
|
||||
txt = Path(info["path"]).with_suffix(".txt")
|
||||
if not txt.exists():
|
||||
return {"__reject__": True}
|
||||
return {"prompt": txt.read_text().strip()}
|
||||
|
||||
|
||||
def main(args):
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
ae = AutoencoderModel.from_pretrained(args.model, device=str(device))
|
||||
if args.model_half:
|
||||
ae.autoencoder = ae.autoencoder.half()
|
||||
|
||||
dataset = SampleDataset(
|
||||
[
|
||||
LocalDatasetConfig(
|
||||
id="train", path=args.data_dir, custom_metadata_fn=caption_metadata_fn
|
||||
)
|
||||
],
|
||||
sample_size=args.sample_size,
|
||||
sample_rate=ae.sample_rate,
|
||||
force_channels="stereo",
|
||||
)
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset,
|
||||
batch_size=args.batch_size,
|
||||
shuffle=False,
|
||||
num_workers=min(4, os.cpu_count() or 1),
|
||||
drop_last=False,
|
||||
collate_fn=collation_fn,
|
||||
)
|
||||
|
||||
os.makedirs(args.output_path, exist_ok=True)
|
||||
|
||||
silence_path = os.path.join(args.output_path, "silence.npy")
|
||||
if not os.path.exists(silence_path):
|
||||
print("Saving silence latent")
|
||||
silence_audio = torch.zeros(
|
||||
1, ae.autoencoder.io_channels, args.sample_size, device=device
|
||||
)
|
||||
if args.model_half:
|
||||
silence_audio = silence_audio.half()
|
||||
with torch.no_grad():
|
||||
silence_latent = ae.encode(silence_audio, ae.sample_rate)
|
||||
np.save(silence_path, silence_latent.cpu().numpy())
|
||||
|
||||
for nb, (audio, metadata) in enumerate(loader):
|
||||
print(f"Processing batch {nb}")
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
|
||||
audio = audio.to(device)
|
||||
if args.model_half:
|
||||
audio = audio.half()
|
||||
|
||||
latents = ae.encode(audio, ae.sample_rate)
|
||||
|
||||
for i, latent in enumerate(latents):
|
||||
latent_np = latent.cpu().numpy()
|
||||
latent_id = f"{nb:06d}{i:04d}"
|
||||
|
||||
md = dict(metadata[i])
|
||||
padding_mask = (
|
||||
F.interpolate(
|
||||
md["padding_mask"][0].unsqueeze(0).unsqueeze(1).float(),
|
||||
size=latent_np.shape[-1],
|
||||
mode="nearest",
|
||||
)
|
||||
.squeeze(0)
|
||||
.squeeze(0)
|
||||
.int()
|
||||
)
|
||||
if not args.pad:
|
||||
padding_np = padding_mask.cpu().numpy()
|
||||
valid_indices = np.where(padding_np == 1)[0]
|
||||
if len(valid_indices) > 0:
|
||||
valid_length = valid_indices[-1] + 1
|
||||
latent_np = latent_np[:, :valid_length]
|
||||
padding_mask = padding_mask[:valid_length]
|
||||
|
||||
np.save(os.path.join(args.output_path, f"{latent_id}.npy"), latent_np)
|
||||
|
||||
md["padding_mask"] = padding_mask.cpu().numpy().tolist()
|
||||
for k, v in md.items():
|
||||
if isinstance(v, torch.Tensor):
|
||||
md[k] = v.cpu().numpy().tolist()
|
||||
|
||||
with open(os.path.join(args.output_path, f"{latent_id}.json"), "w") as f:
|
||||
json.dump(md, f)
|
||||
|
||||
print("Done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Pre-encode audio dataset to latents")
|
||||
parser.add_argument("--model", choices=list(ae_models), default="same-l")
|
||||
parser.add_argument(
|
||||
"--data_dir",
|
||||
required=True,
|
||||
help="Folder with audio files and matching .txt captions",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_path", required=True, help="Folder to write .npy/.json latent pairs"
|
||||
)
|
||||
parser.add_argument("--batch_size", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--sample_size",
|
||||
type=int,
|
||||
default=12582912, # 380s at 44.1kHz, 2 channels
|
||||
help="Audio samples to pad/crop to (default ~380s at 44.1kHz)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model_half", action="store_true", help="Run autoencoder in fp16"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pad", action="store_true", help="Pad audio samples to --sample_size"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.pad and args.batch_size > 1:
|
||||
parser.error(
|
||||
"padding is required for batch_size > 1; pass --pad or use --batch_size 1"
|
||||
)
|
||||
|
||||
main(args)
|
||||
Reference in New Issue
Block a user