"""SA3 MLX — gradio web UI (Apple Silicon). The MLX sibling of optimized/tensorRT/scripts/sa3_gradio.py, with every generation mode wired (the TRT one only exposes text-to-audio): - Model picker: sm-music / sm-sfx / medium (hot-swap; models cache in unified memory, LRU-evicted — first switch loads weights, subsequent instant) - CFG 0-10 next to seconds/steps (0 = negative prompt takes over, 0.5 = halfway between prompts, 1 = off, >1 = extrapolate) + negative prompt/APG under Advanced - Audio-to-audio: guide audio + init_noise_level (whole clip starts from its latents); global sigma_max under Advanced for prompt-only generations - Inpainting: separate reference audio + start/end range sliders (kept bit-exact outside the range). Combinable with a2a: the regenerated span then starts from the guide audio instead of noise. - Spectrogram-as-player: 3-band tinted stereo mel spectrogram (numpy port — no torch) with a white playhead; click to seek. Only one audio plays at a time. - History: previous generations in a scroll panel (newest on top, scroll position anchored while new items arrive), each replayable. - Output options: Auto-play, Auto-download, Infinite Radio (pre-generates the next clip while the current one plays, swaps it in when playback ends), MP3 (V0, via ffmpeg) or WAV saving. Launch: ./sa3-gradio # share=True by default, sm-music + same-s ./sa3-gradio --dit medium ./sa3-gradio --no-share # local-only """ from __future__ import annotations import argparse import base64 import copy import html as html_lib import math import os import re import shutil import subprocess import sys import time import urllib.parse import wave from pathlib import Path import numpy as np SCRIPTS_DIR = Path(__file__).resolve().parent REPO = SCRIPTS_DIR.parent sys.path.insert(0, str(REPO)) sys.path.insert(0, str(SCRIPTS_DIR)) import mlx.core as mx # noqa: E402 from sa3_mlx import ( # noqa: E402 DIT_CHOICES, DECODER_CHOICES, ENCODER_CHOICES, T5GEMMA_NPZ_REL, SAMPLE_RATE, SAMPLES_PER_LATENT, load_dit, load_decoder, load_encoder, read_wav, patch_audio, _free_to_pool, ) from models.defs.sa3_pipeline import ( # noqa: E402 apply_prompt_padding, build_pingpong_schedule, sample_flow_pingpong, patched_decode, load_conditioner_from_npz, ) from models.defs.lora_merge import ( # noqa: E402 prepare_loras, apply_conditioner_lora, LoraError, ) from mlx.utils import tree_flatten # noqa: E402 from models.defs.t5gemma_mlx import T5Gemma # noqa: E402 from weights import ensure_local # noqa: E402 from spec import render_spectrogram_png # noqa: E402 OUTPUT_DIR = REPO / "output" / "gradio" OUTPUT_DIR.mkdir(parents=True, exist_ok=True) # Local LoRA library: drop .safetensors under loras// and they appear in # the per-slot dropdown (scanned per selected DiT; hidden when empty). LORAS_DIR = REPO / "loras" LORA_DIR_NAMES = {"medium": "sa3-medium", "sm-sfx": "sa3-sm-sfx", "sm-music": "sa3-sm-music"} for _d in LORA_DIR_NAMES.values(): (LORAS_DIR / _d).mkdir(parents=True, exist_ok=True) _DD_NONE = "" # dropdown placeholder value ("--- Choose a LoRA ---") def _lora_dd_choices(dit_name: str) -> list: """Dropdown choices for ./loras//: placeholder + (name, abspath).""" d = LORAS_DIR / LORA_DIR_NAMES.get(dit_name, dit_name) hits = sorted(d.glob("*.safetensors")) if d.is_dir() else [] return [("--- Choose a LoRA ---", _DD_NONE)] + [(p.name, str(p)) for p in hits] MIN_SIGMA = 0.01 # MP3 (V0) saving needs ffmpeg; without it we save WAV and hide the choice. FFMPEG = shutil.which("ffmpeg") is not None FORMAT_MP3, FORMAT_WAV = "Save to MP3 (V0)", "Save to WAV" DEFAULT_DECODERS = {name: cfg["default_decoder"] for name, cfg in DIT_CHOICES.items()} # Trained max clip length per model (repo README model table). MAX_SECONDS = {"sm-music": 120, "sm-sfx": 120, "medium": 380} # MLX ≥0.31 registers GPU streams PER THREAD (ThreadLocalStream) while gradio # runs each handler on a rotating anyio worker thread — cross-thread MLX use # then dies with "There is no Stream(gpu, 0) in current thread". The categorical # fix: ALL MLX work (pre-warm + every generation) runs on one dedicated owner # thread; handlers submit to it and wait. This also serializes generations. import concurrent.futures as _cf _MLX_EXECUTOR = _cf.ThreadPoolExecutor(max_workers=1, thread_name_prefix="mlx") def mlx_call(fn, *args, **kwargs): """Run fn on the MLX owner thread and return its result (re-raises errors).""" return _MLX_EXECUTOR.submit(fn, *args, **kwargs).result() def read_audio_any(path: str) -> np.ndarray: """(2, T) float32 @ 44.1 kHz from any common upload format. Layered: read_wav handles 16-bit/44.1k natively and shells out to ffmpeg for the rest when installed; if that fails (no ffmpeg), soundfile's bundled libsndfile decodes mp3/flac/ogg/24-bit/48 kHz directly, followed by a linear resample to 44.1 kHz.""" try: return read_wav(path) except Exception: import soundfile as sf data, sr = sf.read(path, dtype="float32", always_2d=True) # (T, ch) y = data.T y = np.stack([y[0], y[0]]) if y.shape[0] == 1 else y[:2] if sr != SAMPLE_RATE: in_len = y.shape[-1] new_len = int(round(in_len * SAMPLE_RATE / sr)) scale = in_len / new_len pos = np.clip((np.arange(new_len) + 0.5) * scale - 0.5, 0, in_len - 1) y = np.stack([np.interp(pos, np.arange(in_len), ch) for ch in y]) return np.ascontiguousarray(y, dtype=np.float32) def condense_prompt(prompt: str) -> str: """Prompt → filename fragment (the main repo's verbose-naming rule): filesystem-special characters become hyphens, capped at 150 chars.""" prompt = re.sub(r'[\\/:*?"<>|]', '-', prompt)[:150] return prompt or "_" def verbose_basename(prompt, negative_prompt, cfg, sigma_max, seed) -> str: """prompt[.neg-…].cfg{scale}[.smx{σ}].{seed} — matches the main repo's gradio 'verbose' file naming (cfg segment only when cfg != 1).""" base = condense_prompt(prompt) if negative_prompt and negative_prompt.strip(): base += ".neg-" + condense_prompt(negative_prompt.strip()) if cfg != 1.0: base += f".cfg{cfg:g}" if sigma_max != 1.0: base += f".smx{sigma_max:g}" return f"{base}.{seed}" def _save_wav(pcm_int16, out_path): """pcm_int16: (T, 2) int16 interleaved.""" with wave.open(str(out_path), "wb") as w: w.setnchannels(2) w.setsampwidth(2) w.setframerate(SAMPLE_RATE) w.writeframes(pcm_int16.tobytes()) # ── Model caches (unified memory; ~0.9-2.8 GB per DiT, 0.2-1.7 GB per codec) ── # The MLX DiT bakes RoPE/mask lengths at load, so the DiT cache key includes # T_lat. Everything else is length-independent and cached by name. _t5: T5Gemma | None = None _cond_cache: dict[str, tuple] = {} # dit -> (padding_emb, secs_embedder) _dit_cache: dict[tuple[str, int], object] = {} # (dit, T_lat) -> model _dit_lru: list[tuple[str, int]] = [] _DIT_CACHE_MAX = 2 _dec_cache: dict[str, tuple] = {} # decoder -> (model, chunk_fn, chunk_cfg) _enc_cache: dict[str, tuple] = {} # decoder -> (model, pad_modulo) # Encoded-latent cache: rerunning a2a/inpainting with the same input audio skips # the encoder entirely. Keyed on the file's CONTENT hash (gradio re-uploads get # fresh temp paths) + encoder + latent length; values are ~160 KB fp32 arrays. _latent_cache: dict[tuple, np.ndarray] = {} _LATENT_CACHE_MAX = 32 def get_t5() -> T5Gemma: global _t5 if _t5 is None: _t5 = T5Gemma.from_npz(str(ensure_local(T5GEMMA_NPZ_REL))) return _t5 def get_conditioner(dit_name: str): if dit_name not in _cond_cache: _cond_cache[dit_name] = load_conditioner_from_npz( str(ensure_local(DIT_CHOICES[dit_name]["ckpt"])), prefix="cond.") return _cond_cache[dit_name] def _lora_sig(specs, num_steps: int) -> tuple: """Identity of a LoRA config on a cached DiT: file (path+mtime+size — gradio temp uploads keep their path within a session), strength, raw step range, and the schedule length (the plan's step sets depend on it).""" if not specs: return () sig = [] for s in specs: st = os.stat(s["path"]) sig.append((s["path"], st.st_mtime_ns, st.st_size, float(s["strength"]), s["steps"], int(num_steps))) return tuple(sig) def _reconcile_lora(model, dit_name: str, specs, num_steps: int) -> None: """Apply / replace / remove the LoRA config on a cached base DiT, in place. A cached model always carries base weights + its current LoraStepPlan state. Changing config = exact-inverse clear back to base, then a fresh plan built from the live module weights (gate_all=True — a permanent merge couldn't be undone in place). One boundary transition each way (~26/80 ms), so tweaking strength or step ranges between generations never reloads the DiT. Same config → no-op (before_step(0) rewinds the plan every run).""" sig = _lora_sig(specs, num_steps) if sig == getattr(model, "_gr_lora_sig", ()): return old = getattr(model, "_lora_plan", None) if old is not None: old.clear() model._lora_plan, model._gr_lora_sig = None, () if not specs: return wd_view = dict(tree_flatten(model.parameters())) try: plan = prepare_loras(wd_view, specs, num_steps=num_steps, log=lambda m: print(f" {m}"), gate_all=True) except LoraError as e: raise LoraError(f"{e} — selected model: {dit_name}") from None if plan is None: return plan.attach(model) # prepare_loras applied the step-1 state to the DICT slots (fp32) — point # the live modules at the new arrays, in the model dtype. swaps = [] for lkey, mod in plan.mods.items(): if wd_view[lkey] is not mod.weight: mod.weight = wd_view[lkey].astype(mod.weight.dtype) swaps.append(mod.weight) if swaps: mx.eval(swaps) model._lora_plan, model._gr_lora_sig = plan, sig def get_dit(dit_name: str, T_lat: int, dtype, lora_specs=None, num_steps: int = 8): key = (dit_name, T_lat) if key in _dit_cache: if key in _dit_lru: _dit_lru.remove(key) _dit_lru.append(key) model, load_ms = _dit_cache[key], 0.0 else: while len(_dit_cache) >= _DIT_CACHE_MAX: oldest = _dit_lru.pop(0) print(f" ← LRU-evicting DiT {oldest}") _dit_cache.pop(oldest, None) _free_to_pool() t0 = time.time() model, _ = load_dit(dit_name, T_lat=T_lat, dtype=dtype) load_ms = (time.time() - t0) * 1000 _dit_cache[key] = model _dit_lru.append(key) _reconcile_lora(model, dit_name, lora_specs, num_steps) return model, load_ms def get_decoder(decoder_name: str): if decoder_name not in _dec_cache: _dec_cache[decoder_name] = load_decoder(decoder_name, mx.float32) return _dec_cache[decoder_name] def get_encoder(decoder_name: str): if decoder_name not in _enc_cache: _enc_cache[decoder_name] = load_encoder(decoder_name, mx.float32) return _enc_cache[decoder_name] # ── One full generation (mirrors sa3_mlx.py main(), all modes) ───────────── def run_generation(dit_name: str, decoder_name: str, prompt: str, negative_prompt: str, seconds: float, steps: int, seed: int, cfg: float, apg: float, sigma_max: float, a2a_audio_path: str | None = None, inpaint_audio_path: str | None = None, inpaint_range_sec=None, lora_specs=None): """Returns (audio_np (2,T) float32, timings dict). a2a and inpainting are independent and combinable: - a2a_audio_path: the whole generation STARTS from this audio's latents (noise = lat*(1-σmax) + noise*σmax) - inpaint_audio_path + inpaint_range_sec: kept bit-exact outside the range (local_add_cond context + per-step paste-back) - both: the inpainted span regenerates FROM the a2a guide instead of pure noise """ t = {} mx.set_default_device(mx.gpu) # belt-and-braces; normally on the MLX owner thread dtype = mx.float16 # MLX canonical DiT dtype T_lat = max(1, math.ceil(seconds * SAMPLE_RATE / SAMPLES_PER_LATENT)) # 1. T5Gemma t0 = time.time() enc = get_t5() embeds, mask = enc.encode([prompt], max_len=256) mx.eval(embeds, mask) t["t5_ms"] = (time.time() - t0) * 1000 # 2. Conditioning t0 = time.time() padding_emb, secs_embedder = get_conditioner(dit_name) if lora_specs: # Seconds-conditioner deltas (underfit adapters carry one) apply to the # whole generation — conditioning runs once, before sampling. Work on a # copy so the cached embedder stays pristine. W_lora, _n_cond = apply_conditioner_lora(secs_embedder.W, lora_specs) if _n_cond: secs_embedder = copy.copy(secs_embedder) secs_embedder.W = W_lora embeds = embeds.astype(dtype) embeds_padded = apply_prompt_padding(embeds, mask, padding_emb.astype(dtype)) seconds_embed = secs_embedder(seconds).astype(dtype) cross_attn = mx.concatenate([embeds_padded, seconds_embed], axis=1) global_cond = seconds_embed[:, 0, :] null_cross_attn = None if cfg != 1.0: if negative_prompt and negative_prompt.strip(): neg_embeds, neg_mask = enc.encode([negative_prompt.strip()], max_len=256) mx.eval(neg_embeds, neg_mask) neg_padded = apply_prompt_padding(neg_embeds.astype(dtype), neg_mask, padding_emb.astype(dtype)) null_cross_attn = mx.concatenate([neg_padded, seconds_embed], axis=1) else: null_cross_attn = mx.zeros_like(cross_attn) mx.eval(null_cross_attn) mx.eval(cross_attn, global_cond) t["cond_ms"] = (time.time() - t0) * 1000 # 3a. encode the provided audio inputs → latents (each is optional). # Latents are memoized by (file content, encoder, T_lat) so re-running # a2a/inpainting with the same input audio skips the encoder. def encode_audio(path): import hashlib ck = (hashlib.sha1(Path(path).read_bytes()).hexdigest(), decoder_name, T_lat) if ck in _latent_cache: t["enc_cached"] = t.get("enc_cached", 0) + 1 return mx.array(_latent_cache[ck]).astype(dtype) enc_model, pad_mod = get_encoder(decoder_name) enc_T_lat = T_lat if (T_lat * 16) % pad_mod != 0: enc_T_lat = math.ceil((T_lat * 16) / pad_mod) * pad_mod // 16 target_samples = enc_T_lat * SAMPLES_PER_LATENT audio_np = read_audio_any(path) if audio_np.shape[-1] >= target_samples: audio_np = audio_np[:, :target_samples] else: audio_np = np.pad(audio_np, ((0, 0), (0, target_samples - audio_np.shape[-1]))) patches_np = patch_audio(audio_np[None, ...], patch_size=256) lat = enc_model(mx.array(patches_np))[..., :T_lat] mx.eval(lat) while len(_latent_cache) >= _LATENT_CACHE_MAX: _latent_cache.pop(next(iter(_latent_cache))) _latent_cache[ck] = np.array(lat.astype(mx.float32)) return lat.astype(dtype) a2a_latents = None # start-point guide (whole clip) ctx_latents = None # inpaint context (kept outside the range) t["enc_ms"] = 0.0 if a2a_audio_path: t0 = time.time() a2a_latents = encode_audio(a2a_audio_path) t["enc_ms"] += (time.time() - t0) * 1000 if inpaint_audio_path and inpaint_range_sec is not None: t0 = time.time() ctx_latents = encode_audio(inpaint_audio_path) t["enc_ms"] += (time.time() - t0) * 1000 # 3b. DiT + pingpong sample dit_model, t["dit_load_ms"] = get_dit(dit_name, T_lat, dtype, lora_specs=lora_specs, num_steps=steps) sigmas = build_pingpong_schedule(steps, sigma_max=sigma_max, use_logsnr_shift=True) key = mx.random.key(seed) pure_noise = mx.random.normal((1, 256, T_lat), dtype=dtype, key=key) # a2a start-point mix is independent of inpainting: when both are set, the # inpainted span regenerates FROM the guide audio instead of pure noise # (the kept span is pasted back from ctx_latents every step regardless). if a2a_latents is not None: noise = a2a_latents * (1.0 - sigma_max) + pure_noise * sigma_max else: noise = pure_noise mx.eval(noise) local_add_cond = None paste_back = None if ctx_latents is not None: s0 = max(0, int(round(inpaint_range_sec[0] * SAMPLE_RATE / SAMPLES_PER_LATENT))) s1 = min(T_lat, int(round(inpaint_range_sec[1] * SAMPLE_RATE / SAMPLES_PER_LATENT))) mask_np = np.ones((1, 1, T_lat), dtype=np.float32) mask_np[:, :, s0:s1] = 0.0 keep = mx.array(mask_np) masked_input = ctx_latents.astype(mx.float32) * keep local_add_cond = mx.concatenate([keep, masked_input], axis=1).transpose(0, 2, 1).astype(dtype) paste_back = (ctx_latents, keep) def model_fn(x, tt): if cfg == 1.0: return dit_model(x, tt, cross_attn, global_cond, local_add_cond=local_add_cond) x2 = mx.concatenate([x, x], axis=0) t2 = mx.concatenate([tt, tt], axis=0) cross2 = mx.concatenate([cross_attn, null_cross_attn], axis=0) global2 = mx.concatenate([global_cond, global_cond], axis=0) lac2 = None if local_add_cond is None else mx.concatenate([local_add_cond, local_add_cond], axis=0) v_batched = dit_model(x2, t2, cross2, global2, local_add_cond=lac2) cond_v, uncond_v = mx.split(v_batched, 2, axis=0) sigma = tt.reshape(-1, 1, 1).astype(mx.float32) cond_d = x.astype(mx.float32) - cond_v.astype(mx.float32) * sigma uncond_d = x.astype(mx.float32) - uncond_v.astype(mx.float32) * sigma diff = cond_d - uncond_d # cfg < 1 is the INTERPOLATION regime: cfg_d = lerp(uncond_d, cond_d, cfg), # so 0 = pure negative/uncond branch, 0.5 = halfway between both prompts. # APG's orthogonal projection would bend that line — it only applies to the # extrapolation regime (cfg > 1) it was designed for. if apg <= 0.0 or cfg < 1.0: cfg_diff = diff else: norm = mx.sqrt((cond_d * cond_d).sum(axis=(-2, -1), keepdims=True)) unit = cond_d / mx.maximum(norm, 1e-8) parallel = (diff * unit).sum(axis=(-2, -1), keepdims=True) * unit diff_orth = diff - parallel cfg_diff = diff_orth if apg >= 1.0 else (apg * diff_orth + (1.0 - apg) * diff) cfg_d = cond_d + (cfg - 1.0) * cfg_diff cfg_v = (x.astype(mx.float32) - cfg_d) / sigma return cfg_v.astype(x.dtype) t0 = time.time() _plan = getattr(dit_model, "_lora_plan", None) latents = sample_flow_pingpong(model_fn, noise, sigmas, seed=seed + 1, paste_back=paste_back, before_step=_plan.sync if _plan else None) mx.eval(latents) t["sample_ms"] = (time.time() - t0) * 1000 # 4. Decode (fp32 codec; parity-aware chunk dispatch from sa3_mlx.py) t0 = time.time() decoder, chunk_fn, (chunk, ovl) = get_decoder(decoder_name) latents_fp32 = latents.astype(mx.float32) kernel = chunk + 2 * ovl if T_lat > kernel: patches = chunk_fn(decoder, latents_fp32, chunk, ovl) elif T_lat % 2 == 0: patches = decoder(latents_fp32) elif T_lat > 6: patches = chunk_fn(decoder, latents_fp32, 2, 2) else: latents_even = mx.concatenate([latents_fp32, latents_fp32[..., -1:]], axis=-1) patches = decoder(latents_even)[..., : T_lat * 16] mx.eval(patches) t["decode_ms"] = (time.time() - t0) * 1000 # 5. Unpatch + trim audio = patched_decode(patches, patch_size=256, channels=2) mx.eval(audio) audio_np = np.array(audio.astype(mx.float32))[0] requested = int(round(seconds * SAMPLE_RATE)) if audio_np.shape[-1] > requested: audio_np = audio_np[..., :requested] t["T_lat"] = T_lat t["samples"] = audio_np.shape[-1] t["inference_ms"] = sum(t.get(k, 0) for k in ("t5_ms", "cond_ms", "enc_ms", "dit_load_ms", "sample_ms", "decode_ms")) t["realtime"] = (audio_np.shape[-1] / SAMPLE_RATE) / max(t["inference_ms"] / 1000, 1e-9) return audio_np, t # ── HTML rendering (inline handlers only — gradio HTML runs attributes, not #