configuration_uas_audio.py

from typing import Optional, Union
from transformers import Qwen2Config
from transformers.configuration_utils import PretrainedConfig


class UASAudioEncoderConfig(PretrainedConfig):
    model_type = "uas_audio_encoder"
    def __init__(
        self,
        num_mel_bins: Optional[int] = 128,
        encoder_layers: Optional[int] = 32,
        encoder_attention_heads: Optional[int] = 20,
        encoder_ffn_dim: Optional[int] = 5120,
        d_model: Optional[int] = 1280,
        dropout: Optional[int] = 0,
        attention_dropout: Optional[int] = 0,
        activation_function: Optional[int] = "gelu",
        activation_dropout: Optional[int] = 0,
        initializer_range: Optional[int] = 0.02,
        max_source_positions: Optional[int] = 1500,
        n_window: Optional[int] = 50,
        n_window_infer: Optional[int] = 800,
        conv_chunksize: Optional[int] = 500,
        downsample_hidden_size: Optional[int] = 480,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.num_mel_bins = num_mel_bins
        self.d_model = d_model
        self.encoder_layers = encoder_layers
        self.encoder_attention_heads = encoder_attention_heads
        self.encoder_ffn_dim = encoder_ffn_dim
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_function = activation_function
        self.activation_dropout = activation_dropout
        self.num_hidden_layers = encoder_layers
        self.initializer_range = initializer_range
        self.max_source_positions = max_source_positions
        self.n_window = n_window
        self.n_window_infer = n_window_infer
        self.conv_chunksize = conv_chunksize
        self.downsample_hidden_size = downsample_hidden_size


class UASAudioTextConfig(PretrainedConfig):
    model_type = "uas_audio_text"

    def __init__(
        self,
        vocab_size=64012,
        hidden_size=4096,
        intermediate_size=11008,
        num_hidden_layers=48,
        num_attention_heads=32,
        num_attention_groups=4,
        num_key_value_heads=4,
        hidden_act="silu",
        max_position_embeddings=8192,
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        rope_theta=1000000.0,
        rope_scaling=None,
        eos_token_id=None,
        **kwargs
    ):

        super().__init__(
            **kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.num_attention_groups = num_attention_groups
        self.num_key_value_heads = num_key_value_heads
        assert self.num_attention_groups == self.num_key_value_heads, \
            "num_attention_groups must be equal to num_key_value_heads"
        self.hidden_act = hidden_act
        self.max_position_embeddings = max_position_embeddings
        self.initializer_range = initializer_range
        self.rms_norm_eps = rms_norm_eps
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self.eos_token_id = eos_token_id

        self.text_config = Qwen2Config(
            vocab_size=vocab_size,
            hidden_size=hidden_size,
            intermediate_size=intermediate_size,
            num_hidden_layers=num_hidden_layers,
            num_attention_heads=num_attention_heads,
            num_key_value_heads=num_key_value_heads,
            hidden_act=hidden_act,
            max_position_embeddings=max_position_embeddings,
            initializer_range=initializer_range,
            rms_norm_eps=rms_norm_eps,
            rope_theta=rope_theta,
            rope_scaling=rope_scaling,
            architectures=["Qwen2ForCausalLM"],
            dtype=getattr(self, "dtype", "bfloat16"),
        )


class UASAudioConfig(PretrainedConfig):
    model_type = "uas_audio"
    architectures = ["UASAudioForCausalLM"]

    def __init__(
        self,
        audio_encoder_config: Optional[Union[dict, UASAudioEncoderConfig]] = None,
        text_config: Optional[Union[dict, UASAudioTextConfig]] = None,
        **kwargs
    ):
        super().__init__(**kwargs)
        if text_config is None:
            text_config = UASAudioTextConfig().text_config
        elif isinstance(text_config, dict):
            text_config = UASAudioTextConfig(**text_config).text_config
        self.text_config = text_config

        if audio_encoder_config is None:
            self.audio_encoder_config = UASAudioEncoderConfig()
        elif isinstance(audio_encoder_config, dict):
            self.audio_encoder_config = UASAudioEncoderConfig(**audio_encoder_config)
        elif isinstance(audio_encoder_config, UASAudioEncoderConfig):
            self.audio_encoder_config = audio_encoder_config


class UASAudioEncoderOnlyConfig(PretrainedConfig):
    model_type = "uas_audio_encoder_only"
    architectures = ["UASAudioEncoderOnly"]

    def __init__(
        self,
        audio_encoder_config: Optional[Union[dict, UASAudioEncoderConfig]] = None,
        hidden_size: Optional[int] = 4096,  # LLM hidden size for adapter output
        **kwargs
    ):
        super().__init__(**kwargs)
        if audio_encoder_config is None:
            self.audio_encoder_config = UASAudioEncoderConfig()
        elif isinstance(audio_encoder_config, dict):
            self.audio_encoder_config = UASAudioEncoderConfig(**audio_encoder_config)
        elif isinstance(audio_encoder_config, UASAudioEncoderConfig):
            self.audio_encoder_config = audio_encoder_config
        self.hidden_size = hidden_size  # Output dimension of adapter