Upload ultravox_config.py with huggingface_hub

ultravox_config.py

@@ -9,7 +9,6 @@ import transformers
 class LoraConfigSimplified:
     """
     Low Rank Approximation (LoRA) configuration.
-
     Used for language and audio models separately.
     """
 
@@ -23,17 +22,6 @@ class LoraConfigSimplified:
     unfreeze_layers: Optional[List[str]] = None
 
 
-class LossMaskType(str, Enum):
-    """Type of loss mask to use."""
-
-    LAST_ASSISTANT = "last_assistant"
-    """This applies the loss mask up until the last assistant token"""
-    ALL = "all"  # This does not work with KL loss
-    """No loss mask, all inputs are used for loss"""
-    AFTER_AUDIO = "after_audio"
-    """Applies the loss mask up until the audio token"""
-
-
 class LossFunction(str, Enum):
     CrossEntropy = "ce"
     KL_Divergence = "kl"
@@ -57,10 +45,8 @@ class UltravoxConfig(transformers.PretrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an
     Ultravox model according to the specified arguments, defining the model architecture.
-
     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
     documentation from [`PretrainedConfig`] for more information.
-
     Args:
         audio_config (`WhisperConfig`,  *optional*):
             Custom audio config or dict
@@ -82,28 +68,19 @@ class UltravoxConfig(transformers.PretrainedConfig):
             The LoRA configuration for finetuning the audio model.
         audio_latency_block_size (`int`, *optional*, defaults to `None`):
             The latency block size for simulating audio streaming.
-
-
     Example:
-
     ```python
     >>> from transformers import UltravoxModel, WhisperConfig, UltravoxConfig, LlamaConfig
-
     >>> # Initializing an audio encoder config
     >>> audio_config = WhisperConfig()
-
     >>> # Initializing a Llama config
     >>> text_config = LlamaConfig()
-
     >>> # Initializing a default configuration
     >>> configuration = UltravoxConfig(audio_config, text_config)
-
     >>> # Initializing a completely untrained model from the configuration
     >>> model = UltravoxModel(configuration)
-
     >>> # Accessing the model configuration
     >>> configuration = model.config
-
     >>> # Initialize a model from pretrained checkpoints and random projector weights
     >>> config = UltravoxConfig(audio_model_id="openai/whisper-tiny", text_model_id="meta-llama/Llama-2-7b-chat-hf")
     ```"""
@@ -117,9 +94,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
         text_config: dict[str, Any] | transformers.PretrainedConfig | None = None,
         audio_model_id: str | None = None,
         text_model_id: str | None = None,
-        llm_only_training: bool = False,
         ignore_index: int = -100,
-        audio_token_index: int | None = None,
         hidden_size: int = 4096,
         stack_factor: int = 8,
         norm_init: float = 0.4,
@@ -135,8 +110,6 @@ class UltravoxConfig(transformers.PretrainedConfig):
         self.audio_model_id = audio_model_id
         self.text_model_id = text_model_id
 
-        self.audio_token_index = audio_token_index
-
         self.hidden_size = hidden_size
         self.stack_factor = stack_factor
         self.norm_init = norm_init
@@ -163,7 +136,6 @@ class UltravoxConfig(transformers.PretrainedConfig):
         self.text_config = text_config
         self.audio_config = audio_config
 
-        self.llm_only_training = llm_only_training
         self.text_model_lora_config = (
             text_model_lora_config
             if isinstance(text_model_lora_config, dict)
@@ -176,10 +148,6 @@ class UltravoxConfig(transformers.PretrainedConfig):
         )
         self.audio_latency_block_size = audio_latency_block_size
 
-        if hasattr(text_config, "text_config"):
-            text_config.vocab_size = text_config.text_config.vocab_size
-            text_config.hidden_size = text_config.text_config.hidden_size
-
         self.vocab_size = text_config.vocab_size
 
         self.initializer_range = text_config.initializer_range