import typing as tp from stable_audio_3.models.diffusion import DiTWrapper, ConditionedDiffusionModelWrapper from stable_audio_3.models.autoencoders import ( AudioAutoencoder, SAMEEncoder, SAMEDecoder, ) from stable_audio_3.models.conditioners import ( MultiConditioner, NumberConditioner, T5GemmaConditioner, ) from stable_audio_3.models.bottleneck import SoftNormBottleneck from stable_audio_3.models.pretransforms import ( PatchedPretransform, AutoencoderPretransform, ) def create_diffusion_cond_from_config(config: tp.Dict[str, tp.Any]): model_config = config["model"] diffusion_config = model_config.get("diffusion", None) diffusion_model_config = diffusion_config.get("config", None) diffusion_objective = diffusion_config.get("diffusion_objective", "v") modular_local_cond_configs = diffusion_config.get("modular_local_cond_configs", []) diffusion_model = DiTWrapper( diffusion_objective=diffusion_objective, modular_local_cond_configs=modular_local_cond_configs, **diffusion_model_config, ) io_channels = model_config.get("io_channels", None) sample_rate = config.get("sample_rate", None) cross_attention_ids = diffusion_config.get("cross_attention_cond_ids", []) global_cond_ids = diffusion_config.get("global_cond_ids", []) input_concat_ids = diffusion_config.get("input_concat_ids", []) local_add_cond_ids = diffusion_config.get("local_add_cond_ids", []) modular_local_cond_ids = [c["id"] for c in modular_local_cond_configs] prepend_cond_ids = diffusion_config.get("prepend_cond_ids", []) distribution_shift_options = diffusion_config.get( "distribution_shift_options", None ) sampling_distribution_shift_options = diffusion_config.get( "sampling_distribution_shift_options", None ) mask_padding_attention = diffusion_config.get("mask_padding_attention", False) use_effective_length_for_schedule = diffusion_config.get( "use_effective_length_for_schedule", False ) pretransform = create_pretransform_from_config(model_config, sample_rate) min_input_length = pretransform.downsampling_ratio conditioning_config = model_config.get("conditioning", None) conditioner = create_multi_conditioner_from_conditioning_config(conditioning_config) min_input_length *= diffusion_model.model.patch_size return ConditionedDiffusionModelWrapper( diffusion_model, conditioner, min_input_length=min_input_length, sample_rate=sample_rate, cross_attn_cond_ids=cross_attention_ids, global_cond_ids=global_cond_ids, input_concat_ids=input_concat_ids, local_add_cond_ids=local_add_cond_ids, modular_local_cond_ids=modular_local_cond_ids, prepend_cond_ids=prepend_cond_ids, pretransform=pretransform, io_channels=io_channels, distribution_shift_options=distribution_shift_options, sampling_distribution_shift_options=sampling_distribution_shift_options, mask_padding_attention=mask_padding_attention, use_effective_length_for_schedule=use_effective_length_for_schedule, diffusion_objective=diffusion_objective, ) def create_autoencoder_from_config(config, sample_rate): # AE-only configs (SAME-S/SAME-L) have encoder/decoder at the top of config["model"]. # Full SA3 configs nest them inside config["model"]["pretransform"]["config"]. if "encoder" in config: autoencoder_config = config else: autoencoder_config = config["pretransform"]["config"] encoder = SAMEEncoder(**autoencoder_config["encoder"]["config"]) decoder = SAMEDecoder(**autoencoder_config["decoder"]["config"]) latent_dim = autoencoder_config.get("latent_dim", None) downsampling_ratio = autoencoder_config.get("downsampling_ratio", None) io_channels = autoencoder_config.get("io_channels", None) in_channels = autoencoder_config.get("in_channels", None) out_channels = autoencoder_config.get("out_channels", None) soft_clip = autoencoder_config["decoder"].get("soft_clip", False) pretransform = PatchedPretransform(**autoencoder_config["pretransform"]["config"]) bottleneck = SoftNormBottleneck(**autoencoder_config["bottleneck"]["config"]) return AudioAutoencoder( encoder, decoder, io_channels=io_channels, latent_dim=latent_dim, downsampling_ratio=downsampling_ratio, sample_rate=sample_rate, bottleneck=bottleneck, pretransform=pretransform, in_channels=in_channels, out_channels=out_channels, soft_clip=soft_clip, ) def create_pretransform_from_config(config, sample_rate): pretransform_block = config["pretransform"] autoencoder = create_autoencoder_from_config(config, sample_rate) chunked = pretransform_block.get("chunked", False) iterate_batch = pretransform_block.get("iterate_batch", False) return AutoencoderPretransform( autoencoder, chunked=chunked, iterate_batch=iterate_batch ) def create_multi_conditioner_from_conditioning_config( config: tp.Dict[str, tp.Any], ) -> MultiConditioner: """Create a MultiConditioner from a conditioning config dictionary.""" conditioners = {} cond_dim = config["cond_dim"] default_keys = config.get("default_keys", {}) pre_encoded_keys = config.get("pre_encoded_keys", []) for conditioner_info in config["configs"]: id = conditioner_info["id"] conditioner_type = conditioner_info["type"] conditioner_config = {"output_dim": cond_dim} conditioner_config.update(conditioner_info["config"]) if conditioner_type == "t5gemma": conditioners[id] = T5GemmaConditioner(**conditioner_config) elif conditioner_type == "number": conditioners[id] = NumberConditioner(**conditioner_config) else: raise ValueError(f"Unknown conditioner type: {conditioner_type}") return MultiConditioner( conditioners, default_keys=default_keys, pre_encoded_keys=pre_encoded_keys )