"""SA3 TFLite — gradio web UI (portable CPU / XNNPACK). The TFLite sibling of optimized/mlx/scripts/sa3_gradio.py, driving the baked-I/O varlen TFLite models (scripts/sa3_tflite.py) instead of MLX. Every generation mode is wired, plus a precision picker the MLX/TRT UIs don't need: - Model picker: sm-music / sm-sfx / medium (hot-swap; the large DiT interpreter is cached — first use of a model/precision/length loads + XNNPACK-packs the weights, subsequent runs reuse it and only re-bind the conditioning). - Precision picker: fp32 / w16a32 / w8a32 / w8a8-dyn (one knob for DiT + codec + encoder, exactly like the CLI's --precision). fp32 is the CPU fast-and-accurate default; w16a32 is ≈lossless half-size; w8a32/w8a8-dyn are GPTQ int8 (¼ size). - CFG 0-10 next to seconds/steps (0 = negative prompt takes over, 0.5 = halfway between prompts, 1 = off, >1 = extrapolate) + negative prompt/APG under Advanced. - Audio-to-audio: guide audio + init_noise_level (whole clip starts from its latents); global sigma_max under Advanced for prompt-only generations. - Inpainting: separate reference audio + start/end range sliders (kept bit-exact outside the range). Combinable with a2a. - LoRA: file upload + local-folder pick (loras//) + strength, per-model memory, add/remove. Merged once into the DiT weights at load (the frozen graph has no per-step gating — so no Min/Max-step sliders, unlike the MLX UI). Only fp32 / w16a32 can be LoRA-merged; the panel disables itself under int8. - Spectrogram-as-player (numpy port, white playhead, click-to-seek), history panel, Output options (Auto-play / Auto-download / Infinite Radio / Loop / Hotswap), MP3 (via ffmpeg) or WAV saving. Launch: ./sa3-gradio # share=True by default, sm-music + same-s, fp32 ./sa3-gradio --dit medium --precision w8a32 ./sa3-gradio --no-share # local-only """ from __future__ import annotations import argparse import base64 import gc import html as html_lib import os import re import shutil import subprocess import sys import time import urllib.parse import wave from pathlib import Path import numpy as np SCRIPTS_DIR = Path(__file__).resolve().parent REPO = SCRIPTS_DIR.parent sys.path.insert(0, str(REPO)) # so `from models.defs.* import` resolves sys.path.insert(0, str(SCRIPTS_DIR)) # so `from weights / lora_* / spec import` resolves from sa3_tflite import ( # noqa: E402 BakedDiT, BakedEncoder, BakedDecoder, read_wav, valid_T_lat, DEFAULT_DECODER, DIT_REL, DEC_REL, T5_REL, COND_TOKENS, COND_DIM, SAMPLE_RATE, SAMPLES_PER_LATENT, SAMEL_CHUNK, SAMEL_OVERLAP, MIN_SIGMA, ) from models.defs import tflite_pipeline as P # noqa: E402 from lora_core import parse_lora_spec, LoraError # noqa: E402 from lora_patch import get_patched_dit # noqa: E402 from weights import ensure_local, PRECISIONS, dit_rel, dec_rel, enc_rel # noqa: E402 from spec import render_spectrogram_png # noqa: E402 OUTPUT_DIR = REPO / "output" / "gradio" OUTPUT_DIR.mkdir(parents=True, exist_ok=True) # Local LoRA library: drop .safetensors under loras// and they appear in # the per-slot dropdown (scanned per selected DiT; hidden when empty). LORAS_DIR = REPO / "loras" LORA_DIR_NAMES = {"medium": "sa3-medium", "sm-sfx": "sa3-sm-sfx", "sm-music": "sa3-sm-music"} for _d in LORA_DIR_NAMES.values(): (LORAS_DIR / _d).mkdir(parents=True, exist_ok=True) _DD_NONE = "" # dropdown placeholder value ("--- Choose a LoRA ---") # Precisions whose weights can be LoRA-merged (fp32 consts + w16a32 fp16-behind- # DEQUANTIZE). Quantized-int8 graphs (w8a32 / w8a8-dyn) can't — see lora_patch. LORA_PRECISIONS = ("fp32", "w16a32") # Trained max clip length per model (repo README model table). MAX_SECONDS = {"sm-music": 120, "sm-sfx": 120, "medium": 380} DEFAULT_DECODERS = dict(DEFAULT_DECODER) # XNNPACK CPU threads (set from --threads in main()). _THREADS = 8 def _lora_dd_choices(dit_name: str) -> list: """Dropdown choices for ./loras//: placeholder + (name, abspath).""" d = LORAS_DIR / LORA_DIR_NAMES.get(dit_name, dit_name) hits = sorted(d.glob("*.safetensors")) if d.is_dir() else [] return [("--- Choose a LoRA ---", _DD_NONE)] + [(p.name, str(p)) for p in hits] # MP3 (V0) saving needs ffmpeg; without it we save WAV and hide the choice. FFMPEG = shutil.which("ffmpeg") is not None FORMAT_MP3, FORMAT_WAV = "Save to MP3 (V0)", "Save to WAV" # The baked TFLite models are frozen FlatBuffer graphs, so unlike MLX there is no # ThreadLocalStream hazard — but a single XNNPACK Interpreter must NOT be invoked # from two threads at once, and gradio runs each handler on a rotating anyio worker # thread. The categorical fix (same structure as the MLX UI): ALL model work # (pre-warm + every generation) runs on one dedicated owner thread; handlers submit # to it and wait. This also serializes generations. import concurrent.futures as _cf # noqa: E402 _EXECUTOR = _cf.ThreadPoolExecutor(max_workers=1, thread_name_prefix="tfl") def run_serial(fn, *args, **kwargs): """Run fn on the owner thread and return its result (re-raises errors).""" return _EXECUTOR.submit(fn, *args, **kwargs).result() def read_audio_any(path: str) -> np.ndarray: """(2, T) float32 @ 44.1 kHz from any common upload format. Layered: read_wav handles 16-bit/44.1k natively and shells out to ffmpeg for the rest when installed; if that fails (no ffmpeg), soundfile's bundled libsndfile decodes mp3/flac/ogg/24-bit/48 kHz directly, followed by a linear resample to 44.1 kHz.""" try: return read_wav(path) except Exception: import soundfile as sf data, sr = sf.read(path, dtype="float32", always_2d=True) # (T, ch) y = data.T y = np.stack([y[0], y[0]]) if y.shape[0] == 1 else y[:2] if sr != SAMPLE_RATE: in_len = y.shape[-1] new_len = int(round(in_len * SAMPLE_RATE / sr)) scale = in_len / new_len pos = np.clip((np.arange(new_len) + 0.5) * scale - 0.5, 0, in_len - 1) y = np.stack([np.interp(pos, np.arange(in_len), ch) for ch in y]) return np.ascontiguousarray(y, dtype=np.float32) def condense_prompt(prompt: str) -> str: """Prompt → filename fragment (the main repo's verbose-naming rule): filesystem-special characters become hyphens, capped at 150 chars.""" prompt = re.sub(r'[\\/:*?"<>|]', '-', prompt)[:150] return prompt or "_" def verbose_basename(prompt, negative_prompt, cfg, sigma_max, seed, precision) -> str: """prompt[.neg-…][.cfg{scale}][.smx{σ}][.{precision}].{seed} — the main repo's gradio 'verbose' file naming, plus a precision tag for non-fp32 runs so A/B runs across --precision don't overwrite each other.""" base = condense_prompt(prompt) if negative_prompt and negative_prompt.strip(): base += ".neg-" + condense_prompt(negative_prompt.strip()) if cfg != 1.0: base += f".cfg{cfg:g}" if sigma_max != 1.0: base += f".smx{sigma_max:g}" if precision and precision != "fp32": base += f".{precision}" return f"{base}.{seed}" def _save_wav(pcm_int16, out_path): """pcm_int16: (T, 2) int16 interleaved.""" with wave.open(str(out_path), "wb") as w: w.setnchannels(2) w.setsampwidth(2) w.setframerate(SAMPLE_RATE) w.writeframes(pcm_int16.tobytes()) # ── Model caches ───────────────────────────────────────────────────────────── # The T5Gemma front-end is loaded once (fixed 256 tokens). The DiT interpreter is # large (medium fp32 ≈ 5.8 GB) and holds its conditioning as RESIDENT tensors, so # it's cached by FILE PATH (dit family + precision, or the LoRA-patched clone) and # re-bound per generation via set_conditioning — resize+allocate only when the # batch (cfg) or length changes. Codec enc/dec are cached by (name, precision). _t5 = None _tok = None _dit_cache: dict[str, BakedDiT] = {} _dit_lru: list[str] = [] _DIT_CACHE_MAX = 1 # one big DiT resident at a time (medium fp32 = 5.8 GB) _dec_cache: dict[tuple, BakedDecoder] = {} _dec_lru: list[tuple] = [] _enc_cache: dict[tuple, BakedEncoder] = {} _enc_lru: list[tuple] = [] _CODEC_CACHE_MAX = 2 # Encoded-latent cache: rerunning a2a/inpainting with the same input audio skips # the encoder. Keyed on the file's CONTENT hash (gradio re-uploads get fresh temp # paths) + codec + precision + latent length. _latent_cache: dict[tuple, np.ndarray] = {} _LATENT_CACHE_MAX = 32 def get_t5(): global _t5, _tok if _t5 is None: _t5 = P.T5GemmaTFLite(ensure_local(T5_REL), _THREADS) _tok = P.Tokenizer() return _t5, _tok def get_dit(dit_path, T_lat, t5_hidden, t5_mask, seconds, cfg, apg, null_h, null_m, local_add_cond, batched): """Cache the (large) baked DiT interpreter by its file path; re-bind the per-generation conditioning via set_conditioning. Returns (model, load_ms).""" key = str(dit_path) if key in _dit_cache: if key in _dit_lru: _dit_lru.remove(key) _dit_lru.append(key) model = _dit_cache[key] model.set_conditioning(T_lat, t5_hidden, t5_mask, seconds, cfg=cfg, apg=apg, null_hidden=null_h, null_mask=null_m, local_add_cond=local_add_cond, batched=batched) return model, 0.0 while len(_dit_cache) >= _DIT_CACHE_MAX: old = _dit_lru.pop(0) print(f" ← LRU-evicting DiT {Path(old).name}") _dit_cache.pop(old, None) gc.collect() t0 = time.time() model = BakedDiT(dit_path, T_lat, t5_hidden, t5_mask, seconds, _THREADS, cfg=cfg, apg=apg, null_hidden=null_h, null_mask=null_m, local_add_cond=local_add_cond, batched=batched) load_ms = (time.time() - t0) * 1000 _dit_cache[key] = model _dit_lru.append(key) return model, load_ms def get_decoder(name: str, precision: str) -> BakedDecoder: key = (name, precision) if key in _dec_cache: _dec_lru.remove(key) _dec_lru.append(key) return _dec_cache[key] while len(_dec_cache) >= _CODEC_CACHE_MAX: old = _dec_lru.pop(0) _dec_cache.pop(old, None) gc.collect() dec = BakedDecoder(ensure_local(dec_rel(name, precision)), _THREADS, needs_even=(name == "same-s")) _dec_cache[key] = dec _dec_lru.append(key) return dec def get_encoder(name: str, precision: str) -> BakedEncoder: key = (name, precision) if key in _enc_cache: _enc_lru.remove(key) _enc_lru.append(key) return _enc_cache[key] while len(_enc_cache) >= _CODEC_CACHE_MAX: old = _enc_lru.pop(0) _enc_cache.pop(old, None) gc.collect() enc = BakedEncoder(ensure_local(enc_rel(name, precision)), _THREADS, needs_even=(name == "same-s")) _enc_cache[key] = enc _enc_lru.append(key) return enc # ── One full generation (mirrors sa3_tflite.py main(), all modes) ───────────── def run_generation(dit_name: str, decoder_name: str, precision: str, prompt: str, negative_prompt: str, seconds: float, steps: int, seed: int, cfg: float, apg: float, sigma_max: float, a2a_audio_path: str | None = None, inpaint_audio_path: str | None = None, inpaint_range_sec=None, lora_specs=None, cfg_batched: bool = True): """Returns (audio_np (2,T) float32, timings dict). a2a and inpainting are independent and combinable (same as the MLX UI): - a2a_audio_path: the whole generation STARTS from this audio's latents (x0 = lat*(1-σmax) + noise*σmax) - inpaint_audio_path + inpaint_range_sec: kept bit-exact outside the range (local_add_cond context + per-step paste-back) - both: the inpainted span regenerates FROM the a2a guide instead of pure noise """ t = {} dec = decoder_name needs_even = (dec == "same-s") T_lat = valid_T_lat(seconds) # 1. T5Gemma (tokenize + encode; + negative prompt for CFG) t0 = time.time() t5, tok = get_t5() ids, mask = tok(prompt) t5_hidden = t5(ids, mask) null_h = null_m = None if cfg != 1.0: if negative_prompt and negative_prompt.strip(): n_ids, n_mask = tok(negative_prompt.strip()) null_h = t5(n_ids, n_mask) null_m = n_mask.astype(np.float32) else: # All-zero hidden+mask → in-graph conditioner emits learned padding # embeds for every position (the standard unconditional branch). null_h = np.zeros((1, COND_TOKENS, COND_DIM), np.float32) null_m = np.zeros((1, COND_TOKENS), np.float32) t["t5_ms"] = (time.time() - t0) * 1000 # 2. Encode the provided audio inputs → latents (each optional). Memoized by # (file content, codec, precision, T_lat) so re-running with the same audio # skips the encoder. def encode_audio(path): import hashlib ck = (hashlib.sha1(Path(path).read_bytes()).hexdigest(), dec, precision, T_lat) if ck in _latent_cache: t["enc_cached"] = t.get("enc_cached", 0) + 1 return _latent_cache[ck] enc = get_encoder(dec, precision) # SAME-S encoder needs even L; round the encode grid up, trim to T_lat. enc_L = T_lat + 1 if (needs_even and T_lat % 2 != 0) else T_lat target = enc_L * SAMPLES_PER_LATENT audio_np = read_audio_any(path) if audio_np.shape[-1] >= target: audio_np = audio_np[:, :target] else: audio_np = np.pad(audio_np, ((0, 0), (0, target - audio_np.shape[-1]))) lat = enc.encode(audio_np[None], T_lat) # (1,256,T_lat) while len(_latent_cache) >= _LATENT_CACHE_MAX: _latent_cache.pop(next(iter(_latent_cache))) _latent_cache[ck] = lat return lat a2a_latents = None # start-point guide (whole clip) ctx_latents = None # inpaint context (kept outside the range) t["enc_ms"] = 0.0 if a2a_audio_path: t0 = time.time() a2a_latents = encode_audio(a2a_audio_path) t["enc_ms"] += (time.time() - t0) * 1000 if inpaint_audio_path and inpaint_range_sec is not None: t0 = time.time() ctx_latents = encode_audio(inpaint_audio_path) t["enc_ms"] += (time.time() - t0) * 1000 # 3. inpaint local_add_cond + paste-back (TRT channel layout: [1,257,L]) local_add_cond = None paste_back = None if ctx_latents is not None: s0 = max(0, int(round(inpaint_range_sec[0] * SAMPLE_RATE / SAMPLES_PER_LATENT))) s1 = min(T_lat, int(round(inpaint_range_sec[1] * SAMPLE_RATE / SAMPLES_PER_LATENT))) keep = np.ones((1, 1, T_lat), np.float32) keep[:, :, s0:s1] = 0.0 # 1=keep init, 0=regenerate masked = ctx_latents.astype(np.float32) * keep local_add_cond = np.concatenate([keep, masked], axis=1) # (1,257,T_lat) paste_back = (ctx_latents.astype(np.float32), keep) # 4. initial noise + a2a start-point mix (independent of inpainting) x0, step_noise = P.make_noise(T_lat, steps, seed) if a2a_latents is not None: x0 = a2a_latents.astype(np.float32) * (1.0 - sigma_max) + x0 * sigma_max # 5. DiT load (+ optional LoRA patch) + pingpong sample dit_path = ensure_local(dit_rel(dit_name, precision)) if lora_specs: # Raises LoraError for quantized-int8 precisions (caught upstream → error box). dit_path = get_patched_dit(dit_path, lora_specs, family=dit_name, precision=precision, log=lambda m: print(f" {m}")) backend, t["dit_load_ms"] = get_dit(str(dit_path), T_lat, t5_hidden, mask.astype(np.float32), float(seconds), cfg, apg, null_h, null_m, local_add_cond, cfg_batched) sigmas = P.build_pingpong_schedule(steps, sigma_max) t0 = time.time() latents = P.sample(backend, x0, step_noise, sigmas, None, None, paste_back=paste_back) t["sample_ms"] = (time.time() - t0) * 1000 t["n_fwd"] = backend.n_fwd # 6. Decode (whole for SAME-S; chunked for SAME-L past the window) decoder = get_decoder(dec, precision) t0 = time.time() if dec == "same-l" and T_lat > SAMEL_CHUNK: audio = decoder.decode_chunked(latents, SAMEL_CHUNK, SAMEL_OVERLAP) else: audio = decoder.decode_whole(latents) t["decode_ms"] = (time.time() - t0) * 1000 audio_np = audio[0] # (2, L*4096) requested = int(round(seconds * SAMPLE_RATE)) if audio_np.shape[-1] > requested: audio_np = audio_np[..., :requested] t["T_lat"] = T_lat t["samples"] = audio_np.shape[-1] t["inference_ms"] = sum(t.get(k, 0) for k in ("t5_ms", "enc_ms", "dit_load_ms", "sample_ms", "decode_ms")) t["realtime"] = (audio_np.shape[-1] / SAMPLE_RATE) / max(t["inference_ms"] / 1000, 1e-9) return audio_np, t # ── HTML rendering (inline handlers only — gradio HTML runs attributes, not #