Any-to-Any
Transformers
Safetensors
PyTorch
NemotronH_Nano_Omni_Reasoning_V3
feature-extraction
nvidia
multimodal
custom_code
Eval Results
Instructions to use nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Commit ·
9de8c68
1
Parent(s): 8e00a7f
Enable batched audio inference (#3)
Browse files- Enable batched audio inference (239ad827db3dd78892ad1c241db9bb5b7c047a51)
Co-authored-by: Piotr Żelasko <piotrzelasko@users.noreply.huggingface.co>
- modeling.py +26 -2
- processing.py +63 -39
- tokenizer_config.json +3 -1
modeling.py
CHANGED
|
@@ -572,11 +572,35 @@ class NemotronH_Nano_Omni_Reasoning_V3(PreTrainedModel):
|
|
| 572 |
assert video_mask.sum() != 0, "No video tokens found in input_ids"
|
| 573 |
inputs_embeds[video_mask] = video_vit_embeds.reshape(-1, C).to(inputs_embeds.device, inputs_embeds.dtype)
|
| 574 |
|
| 575 |
-
# Replace sound tokens with sound embeddings
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
if sound_embeds is not None and self.sound_context_token_id is not None:
|
| 577 |
sound_mask = (input_ids_copy == self.sound_context_token_id)
|
| 578 |
assert sound_mask.sum() != 0, "No sound tokens found in input_ids"
|
| 579 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
|
| 581 |
# Apply video pruning (EVS) if enabled
|
| 582 |
if video_vit_embeds is not None and self.video_pruning_rate > 0: # EVS
|
|
|
|
| 572 |
assert video_mask.sum() != 0, "No video tokens found in input_ids"
|
| 573 |
inputs_embeds[video_mask] = video_vit_embeds.reshape(-1, C).to(inputs_embeds.device, inputs_embeds.dtype)
|
| 574 |
|
| 575 |
+
# Replace sound tokens with sound embeddings.
|
| 576 |
+
# `sound_embeds` has shape `(B_sound, T_out_max, C)` where `T_out_max`
|
| 577 |
+
# is the encoder output length for the longest clip in the batch.
|
| 578 |
+
# When `B_sound > 1` the shorter clips have padding at the tail, so
|
| 579 |
+
# we must gather only the valid positions per row before scattering
|
| 580 |
+
# into `sound_mask`. The encoder's `_get_subsampling_output_length`
|
| 581 |
+
# converts each input mel-frame count (from the feature extractor's
|
| 582 |
+
# attention_mask) to its post-subsampling token count.
|
| 583 |
if sound_embeds is not None and self.sound_context_token_id is not None:
|
| 584 |
sound_mask = (input_ids_copy == self.sound_context_token_id)
|
| 585 |
assert sound_mask.sum() != 0, "No sound tokens found in input_ids"
|
| 586 |
+
if sound_embeds.dim() == 3 and sound_embeds.shape[0] > 1 and sound_attention_mask is not None:
|
| 587 |
+
# `attention_mask.sum() = L_i // hop` per row, but
|
| 588 |
+
# `ParakeetFeatureExtractor` pads each row to `1 + L_i // hop`
|
| 589 |
+
# mel frames in single-call mode (the trailing frame comes
|
| 590 |
+
# from STFT center padding) — and the existing batch=1 path
|
| 591 |
+
# consumes that frame's embed too. Add 1 here to match.
|
| 592 |
+
natural_input_lengths = sound_attention_mask.sum(-1) + 1
|
| 593 |
+
output_lengths = self.sound_encoder.encoder._get_subsampling_output_length(natural_input_lengths)
|
| 594 |
+
flat = torch.cat(
|
| 595 |
+
[sound_embeds[i, : int(n)] for i, n in enumerate(output_lengths.tolist())],
|
| 596 |
+
dim=0,
|
| 597 |
+
)
|
| 598 |
+
else:
|
| 599 |
+
flat = sound_embeds.reshape(-1, C)
|
| 600 |
+
assert sound_mask.sum().item() == flat.shape[0], (
|
| 601 |
+
f"sound token count ({sound_mask.sum().item()}) != encoder output count ({flat.shape[0]})"
|
| 602 |
+
)
|
| 603 |
+
inputs_embeds[sound_mask] = flat.to(inputs_embeds.device, inputs_embeds.dtype)
|
| 604 |
|
| 605 |
# Apply video pruning (EVS) if enabled
|
| 606 |
if video_vit_embeds is not None and self.video_pruning_rate > 0: # EVS
|
processing.py
CHANGED
|
@@ -53,8 +53,8 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 53 |
r"""
|
| 54 |
Constructs a Nemotron-3-Nano-Omni-30B-A3B-Reasoning processor which wraps an image processor, audio feature extractor,
|
| 55 |
and a tokenizer into a single processor.
|
| 56 |
-
[`NemotronH_Nano_Omni_Reasoning_V3Processor`] offers all the functionalities of the image processor, audio processor,
|
| 57 |
-
and tokenizer. See the [`~NemotronH_Nano_Omni_Reasoning_V3Processor.__call__`] and [`~NemotronH_Nano_Omni_Reasoning_V3Processor.decode`]
|
| 58 |
for more information.
|
| 59 |
Args:
|
| 60 |
image_processor ([`AutoImageProcessor`], *optional*):
|
|
@@ -110,12 +110,12 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 110 |
if getattr(tokenizer, "audio_token_id", None)
|
| 111 |
else tokenizer.convert_tokens_to_ids(self.audio_token)
|
| 112 |
)
|
| 113 |
-
|
| 114 |
# Audio processing parameters
|
| 115 |
self.audio_sampling_rate = audio_sampling_rate
|
| 116 |
self.audio_subsampling_factor = audio_subsampling_factor
|
| 117 |
self.audio_hop_length = audio_hop_length
|
| 118 |
-
|
| 119 |
super().__init__(image_processor, tokenizer, chat_template=chat_template)
|
| 120 |
|
| 121 |
def __call__(
|
|
@@ -127,13 +127,13 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 127 |
**kwargs: Unpack[NemotronH_Nano_Omni_Reasoning_V3ProcessorKwargs],
|
| 128 |
) -> BatchFeature:
|
| 129 |
"""
|
| 130 |
-
Main method to prepare multimodal inputs (text, images, videos, audio) for the model. This method processes
|
| 131 |
-
text by replacing image/video/audio tokens with appropriate placeholder sequences, processes images and videos
|
| 132 |
through the image processor, and tokenizes the final text.
|
| 133 |
|
| 134 |
The method performs the following key operations:
|
| 135 |
1. Processes images using the image processor to get pixel values and patch counts
|
| 136 |
-
2. Processes videos using the image processor with max_num_tiles=1 to get video pixel values
|
| 137 |
3. Processes audio to compute the number of audio tokens based on duration
|
| 138 |
4. Replaces `<image>` tokens in text with `<img>` + image tokens + `</img>` sequences
|
| 139 |
5. Replaces `<video>` tokens in text with frame-by-frame descriptions including timestamps (if metadata provided)
|
|
@@ -149,15 +149,15 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 149 |
special tokens `<image>`, `<video>`, and `<audio>` that will be replaced with appropriate token sequences.
|
| 150 |
videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
|
| 151 |
The video or batch of videos to be prepared. Each video should be a 4D NumPy array or PyTorch
|
| 152 |
-
tensor with shape (num_frames, channels, height, width). Both channels-first and channels-last formats
|
| 153 |
are supported. Note: Currently only supports batch size of 1 for videos.
|
| 154 |
audio (`str`, `np.ndarray`, `torch.Tensor`, `List[str]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
|
| 155 |
-
The audio or batch of audio clips to be prepared. Can be file paths, numpy arrays (waveforms),
|
| 156 |
or torch tensors. Waveforms should be 1D arrays at the expected sampling rate.
|
| 157 |
images_kwargs (`Dict`, *optional*):
|
| 158 |
Additional keyword arguments for image processing, including:
|
| 159 |
- `min_pixels` (`int`, *optional*): Minimum number of pixels for image processing
|
| 160 |
-
- `max_pixels` (`int`, *optional*): Maximum number of pixels for image processing
|
| 161 |
- `patch_size` (`int`, *optional*): Size of patches for image processing
|
| 162 |
- `temporal_patch_size` (`int`, *optional*): Size of temporal patches
|
| 163 |
- `merge_size` (`int`, *optional*): Size for merging patches
|
|
@@ -199,8 +199,8 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 199 |
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
| 200 |
**kwargs,
|
| 201 |
)
|
| 202 |
-
image_inputs
|
| 203 |
-
|
| 204 |
if images is not None:
|
| 205 |
image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
|
| 206 |
image_num_tokens = image_inputs["num_tokens"]
|
|
@@ -243,7 +243,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 243 |
)
|
| 244 |
index += 1
|
| 245 |
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 246 |
-
|
| 247 |
if videos is not None:
|
| 248 |
assert len(text) == 1, "Video is not supported for batch size > 1"
|
| 249 |
video_metadata = output_kwargs.get("videos_kwargs", {}).get("video_metadata", None)
|
|
@@ -299,9 +299,11 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 299 |
# model distinguishes image vs. video by which `pixel_values_*` arg was passed.
|
| 300 |
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 301 |
|
| 302 |
-
# Replace audio tokens with the correct number of placeholder tokens
|
|
|
|
|
|
|
|
|
|
| 303 |
if audio is not None:
|
| 304 |
-
assert len(text) == 1, "Audio is not supported for batch size > 1"
|
| 305 |
index = 0
|
| 306 |
for i in range(len(text)):
|
| 307 |
while self.audio_token in text[i]:
|
|
@@ -313,41 +315,41 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 313 |
|
| 314 |
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
| 315 |
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
| 316 |
-
|
| 317 |
# Build output - exclude audio from tensor conversion since it's raw waveforms
|
| 318 |
output_data = {**text_inputs, **image_inputs, **videos_inputs}
|
| 319 |
result = BatchFeature(data=output_data, tensor_type=return_tensors)
|
| 320 |
-
|
| 321 |
# Add audio clips separately (as list of numpy arrays, not tensors)
|
| 322 |
if audio_inputs:
|
| 323 |
result["sound_clips"] = audio_inputs["sound_clips"]
|
| 324 |
-
|
| 325 |
return result
|
| 326 |
-
|
| 327 |
def _process_audio(
|
| 328 |
-
self,
|
| 329 |
-
audio: AudioInput,
|
| 330 |
audio_kwargs: dict
|
| 331 |
) -> tuple:
|
| 332 |
"""Process audio inputs and compute the number of audio tokens.
|
| 333 |
-
|
| 334 |
Args:
|
| 335 |
audio: Audio input (file path, waveform array, or list thereof)
|
| 336 |
audio_kwargs: Additional audio processing arguments
|
| 337 |
-
|
| 338 |
Returns:
|
| 339 |
Tuple of (audio_clips, num_tokens_per_clip)
|
| 340 |
"""
|
| 341 |
# Get sampling rate from kwargs or use default
|
| 342 |
sampling_rate = audio_kwargs.get("sampling_rate", self.audio_sampling_rate)
|
| 343 |
-
|
| 344 |
# Normalize audio to list
|
| 345 |
if not isinstance(audio, list):
|
| 346 |
audio = [audio]
|
| 347 |
-
|
| 348 |
audio_clips = []
|
| 349 |
num_tokens = []
|
| 350 |
-
|
| 351 |
for audio_item in audio:
|
| 352 |
# Load audio if it's a file path
|
| 353 |
if isinstance(audio_item, str):
|
|
@@ -358,25 +360,47 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 358 |
waveform = audio_item.squeeze() if audio_item.ndim > 1 else audio_item
|
| 359 |
else:
|
| 360 |
raise ValueError(f"Unsupported audio type: {type(audio_item)}")
|
| 361 |
-
|
| 362 |
audio_clips.append(waveform)
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
# num_frames ≈ audio_samples / hop_length
|
| 366 |
-
# num_tokens = num_frames / subsampling_factor
|
| 367 |
-
num_frames = len(waveform) // self.audio_hop_length
|
| 368 |
-
n_tokens = math.ceil(num_frames / self.audio_subsampling_factor)
|
| 369 |
num_tokens.append(max(1, n_tokens)) # At least 1 token
|
| 370 |
-
|
| 371 |
return audio_clips, num_tokens
|
| 372 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
def _load_audio(self, audio_path: str, target_sr: int) -> np.ndarray:
|
| 374 |
"""Load audio from file and resample if necessary.
|
| 375 |
-
|
| 376 |
Args:
|
| 377 |
audio_path: Path to audio file
|
| 378 |
target_sr: Target sampling rate
|
| 379 |
-
|
| 380 |
Returns:
|
| 381 |
Audio waveform as numpy array
|
| 382 |
"""
|
|
@@ -386,7 +410,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 386 |
return waveform
|
| 387 |
except ImportError:
|
| 388 |
pass
|
| 389 |
-
|
| 390 |
try:
|
| 391 |
import soundfile as sf
|
| 392 |
waveform, sr = sf.read(audio_path)
|
|
@@ -400,7 +424,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
|
|
| 400 |
return waveform.astype(np.float32)
|
| 401 |
except ImportError:
|
| 402 |
pass
|
| 403 |
-
|
| 404 |
raise ImportError(
|
| 405 |
"Audio loading requires either librosa or soundfile. "
|
| 406 |
"Install with: pip install librosa soundfile"
|
|
|
|
| 53 |
r"""
|
| 54 |
Constructs a Nemotron-3-Nano-Omni-30B-A3B-Reasoning processor which wraps an image processor, audio feature extractor,
|
| 55 |
and a tokenizer into a single processor.
|
| 56 |
+
[`NemotronH_Nano_Omni_Reasoning_V3Processor`] offers all the functionalities of the image processor, audio processor,
|
| 57 |
+
and tokenizer. See the [`~NemotronH_Nano_Omni_Reasoning_V3Processor.__call__`] and [`~NemotronH_Nano_Omni_Reasoning_V3Processor.decode`]
|
| 58 |
for more information.
|
| 59 |
Args:
|
| 60 |
image_processor ([`AutoImageProcessor`], *optional*):
|
|
|
|
| 110 |
if getattr(tokenizer, "audio_token_id", None)
|
| 111 |
else tokenizer.convert_tokens_to_ids(self.audio_token)
|
| 112 |
)
|
| 113 |
+
|
| 114 |
# Audio processing parameters
|
| 115 |
self.audio_sampling_rate = audio_sampling_rate
|
| 116 |
self.audio_subsampling_factor = audio_subsampling_factor
|
| 117 |
self.audio_hop_length = audio_hop_length
|
| 118 |
+
|
| 119 |
super().__init__(image_processor, tokenizer, chat_template=chat_template)
|
| 120 |
|
| 121 |
def __call__(
|
|
|
|
| 127 |
**kwargs: Unpack[NemotronH_Nano_Omni_Reasoning_V3ProcessorKwargs],
|
| 128 |
) -> BatchFeature:
|
| 129 |
"""
|
| 130 |
+
Main method to prepare multimodal inputs (text, images, videos, audio) for the model. This method processes
|
| 131 |
+
text by replacing image/video/audio tokens with appropriate placeholder sequences, processes images and videos
|
| 132 |
through the image processor, and tokenizes the final text.
|
| 133 |
|
| 134 |
The method performs the following key operations:
|
| 135 |
1. Processes images using the image processor to get pixel values and patch counts
|
| 136 |
+
2. Processes videos using the image processor with max_num_tiles=1 to get video pixel values
|
| 137 |
3. Processes audio to compute the number of audio tokens based on duration
|
| 138 |
4. Replaces `<image>` tokens in text with `<img>` + image tokens + `</img>` sequences
|
| 139 |
5. Replaces `<video>` tokens in text with frame-by-frame descriptions including timestamps (if metadata provided)
|
|
|
|
| 149 |
special tokens `<image>`, `<video>`, and `<audio>` that will be replaced with appropriate token sequences.
|
| 150 |
videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
|
| 151 |
The video or batch of videos to be prepared. Each video should be a 4D NumPy array or PyTorch
|
| 152 |
+
tensor with shape (num_frames, channels, height, width). Both channels-first and channels-last formats
|
| 153 |
are supported. Note: Currently only supports batch size of 1 for videos.
|
| 154 |
audio (`str`, `np.ndarray`, `torch.Tensor`, `List[str]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
|
| 155 |
+
The audio or batch of audio clips to be prepared. Can be file paths, numpy arrays (waveforms),
|
| 156 |
or torch tensors. Waveforms should be 1D arrays at the expected sampling rate.
|
| 157 |
images_kwargs (`Dict`, *optional*):
|
| 158 |
Additional keyword arguments for image processing, including:
|
| 159 |
- `min_pixels` (`int`, *optional*): Minimum number of pixels for image processing
|
| 160 |
+
- `max_pixels` (`int`, *optional*): Maximum number of pixels for image processing
|
| 161 |
- `patch_size` (`int`, *optional*): Size of patches for image processing
|
| 162 |
- `temporal_patch_size` (`int`, *optional*): Size of temporal patches
|
| 163 |
- `merge_size` (`int`, *optional*): Size for merging patches
|
|
|
|
| 199 |
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
| 200 |
**kwargs,
|
| 201 |
)
|
| 202 |
+
image_inputs, videos_inputs, audio_inputs = {}, {}, {}
|
| 203 |
+
|
| 204 |
if images is not None:
|
| 205 |
image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
|
| 206 |
image_num_tokens = image_inputs["num_tokens"]
|
|
|
|
| 243 |
)
|
| 244 |
index += 1
|
| 245 |
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 246 |
+
|
| 247 |
if videos is not None:
|
| 248 |
assert len(text) == 1, "Video is not supported for batch size > 1"
|
| 249 |
video_metadata = output_kwargs.get("videos_kwargs", {}).get("video_metadata", None)
|
|
|
|
| 299 |
# model distinguishes image vs. video by which `pixel_values_*` arg was passed.
|
| 300 |
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 301 |
|
| 302 |
+
# Replace audio tokens with the correct number of placeholder tokens.
|
| 303 |
+
# The expansion loop is per-row, so batch size > 1 is supported as long
|
| 304 |
+
# as `audio_num_tokens` has one entry per `<so_embedding>` placeholder
|
| 305 |
+
# across the batch (in row-major order).
|
| 306 |
if audio is not None:
|
|
|
|
| 307 |
index = 0
|
| 308 |
for i in range(len(text)):
|
| 309 |
while self.audio_token in text[i]:
|
|
|
|
| 315 |
|
| 316 |
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
| 317 |
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
| 318 |
+
|
| 319 |
# Build output - exclude audio from tensor conversion since it's raw waveforms
|
| 320 |
output_data = {**text_inputs, **image_inputs, **videos_inputs}
|
| 321 |
result = BatchFeature(data=output_data, tensor_type=return_tensors)
|
| 322 |
+
|
| 323 |
# Add audio clips separately (as list of numpy arrays, not tensors)
|
| 324 |
if audio_inputs:
|
| 325 |
result["sound_clips"] = audio_inputs["sound_clips"]
|
| 326 |
+
|
| 327 |
return result
|
| 328 |
+
|
| 329 |
def _process_audio(
|
| 330 |
+
self,
|
| 331 |
+
audio: AudioInput,
|
| 332 |
audio_kwargs: dict
|
| 333 |
) -> tuple:
|
| 334 |
"""Process audio inputs and compute the number of audio tokens.
|
| 335 |
+
|
| 336 |
Args:
|
| 337 |
audio: Audio input (file path, waveform array, or list thereof)
|
| 338 |
audio_kwargs: Additional audio processing arguments
|
| 339 |
+
|
| 340 |
Returns:
|
| 341 |
Tuple of (audio_clips, num_tokens_per_clip)
|
| 342 |
"""
|
| 343 |
# Get sampling rate from kwargs or use default
|
| 344 |
sampling_rate = audio_kwargs.get("sampling_rate", self.audio_sampling_rate)
|
| 345 |
+
|
| 346 |
# Normalize audio to list
|
| 347 |
if not isinstance(audio, list):
|
| 348 |
audio = [audio]
|
| 349 |
+
|
| 350 |
audio_clips = []
|
| 351 |
num_tokens = []
|
| 352 |
+
|
| 353 |
for audio_item in audio:
|
| 354 |
# Load audio if it's a file path
|
| 355 |
if isinstance(audio_item, str):
|
|
|
|
| 360 |
waveform = audio_item.squeeze() if audio_item.ndim > 1 else audio_item
|
| 361 |
else:
|
| 362 |
raise ValueError(f"Unsupported audio type: {type(audio_item)}")
|
| 363 |
+
|
| 364 |
audio_clips.append(waveform)
|
| 365 |
+
|
| 366 |
+
n_tokens = self._estimate_audio_num_embeddings(len(waveform))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 367 |
num_tokens.append(max(1, n_tokens)) # At least 1 token
|
| 368 |
+
|
| 369 |
return audio_clips, num_tokens
|
| 370 |
+
|
| 371 |
+
def _estimate_audio_num_embeddings(self, audio_length_samples: int) -> int:
|
| 372 |
+
"""Predict the exact number of `<so_embedding>` tokens that the sound
|
| 373 |
+
encoder will produce for an audio clip of `audio_length_samples` raw
|
| 374 |
+
samples. Replaces the previous heuristic, which under-counted by 1 for
|
| 375 |
+
certain lengths and tripped a shape mismatch in `modeling.py::generate`.
|
| 376 |
+
|
| 377 |
+
Mirrors `ParakeetFeatureExtractor` (center-padded STFT → ``1 + L // hop``
|
| 378 |
+
mel frames) followed by `ParakeetEncoder._get_subsampling_output_length`
|
| 379 |
+
(``log2(subsampling_factor)`` stages of stride-2 conv with kernel-size
|
| 380 |
+
``subsampling_conv_kernel_size``, symmetric padding ``(kernel-1)//2`` on
|
| 381 |
+
each side).
|
| 382 |
+
"""
|
| 383 |
+
# Mel frame count (center=True is HF Parakeet's default).
|
| 384 |
+
n_frames = 1 + audio_length_samples // self.audio_hop_length
|
| 385 |
+
# Conv-subsampling: ``log2(subsampling_factor)`` stages, stride 2,
|
| 386 |
+
# kernel ``subsampling_conv_kernel_size``, symmetric padding.
|
| 387 |
+
kernel_size = getattr(self, "audio_subsampling_conv_kernel_size", 3)
|
| 388 |
+
stride = getattr(self, "audio_subsampling_conv_stride", 2)
|
| 389 |
+
num_layers = int(math.log2(self.audio_subsampling_factor))
|
| 390 |
+
all_paddings = (kernel_size - 1) // 2 * 2
|
| 391 |
+
add_pad = all_paddings - kernel_size # kernel=3, sym pad=1 → -1
|
| 392 |
+
L = n_frames
|
| 393 |
+
for _ in range(num_layers):
|
| 394 |
+
L = (L + add_pad) // stride + 1
|
| 395 |
+
return L
|
| 396 |
+
|
| 397 |
def _load_audio(self, audio_path: str, target_sr: int) -> np.ndarray:
|
| 398 |
"""Load audio from file and resample if necessary.
|
| 399 |
+
|
| 400 |
Args:
|
| 401 |
audio_path: Path to audio file
|
| 402 |
target_sr: Target sampling rate
|
| 403 |
+
|
| 404 |
Returns:
|
| 405 |
Audio waveform as numpy array
|
| 406 |
"""
|
|
|
|
| 410 |
return waveform
|
| 411 |
except ImportError:
|
| 412 |
pass
|
| 413 |
+
|
| 414 |
try:
|
| 415 |
import soundfile as sf
|
| 416 |
waveform, sr = sf.read(audio_path)
|
|
|
|
| 424 |
return waveform.astype(np.float32)
|
| 425 |
except ImportError:
|
| 426 |
pass
|
| 427 |
+
|
| 428 |
raise ImportError(
|
| 429 |
"Audio loading requires either librosa or soundfile. "
|
| 430 |
"Install with: pip install librosa soundfile"
|
tokenizer_config.json
CHANGED
|
@@ -8015,5 +8015,7 @@
|
|
| 8015 |
"model_max_length": 262144,
|
| 8016 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 8017 |
"unk_token": "<unk>",
|
| 8018 |
-
"chat_template": "{% macro render_extra_keys(json_dict, handled_keys) %}\n {%- if json_dict is mapping %}\n {%- for json_key in json_dict if json_key not in handled_keys %}\n {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}\n {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}\n {%- else %}\n {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n{% endmacro %}\n{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}\n{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \"<|im_start|>system\\n\" }}\n {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n {%- if system_message is defined and system_message | length > 0 %}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {{- \"<tools>\" }}\n {%- for tool in tools %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {{- \"\\n<function>\\n<name>\" ~ tool.name ~ \"</name>\" }}\n {%- if tool.description is defined %}\n {{- '\\n<description>' ~ (tool.description | trim) ~ '</description>' }}\n {%- endif %}\n {{- '\\n<parameters>' }}\n {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}\n {%- for param_name, param_fields in tool.parameters.properties|items %}\n {{- '\\n<parameter>' }}\n {{- '\\n<name>' ~ param_name ~ '</name>' }}\n {%- if param_fields.type is defined %}\n {{- '\\n<type>' ~ (param_fields.type | string) ~ '</type>' }}\n {%- endif %}\n {%- if param_fields.description is defined %}\n {{- '\\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}\n {%- endif %}\n {%- if param_fields.enum is defined %}\n {{- '\\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}\n {%- endif %}\n {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}\n {{- render_extra_keys(param_fields, handled_keys) }}\n {{- '\\n</parameter>' }}\n {%- endfor %}\n {%- endif %}\n {% set handled_keys = ['type', 'properties', 'required'] %}\n {{- render_extra_keys(tool.parameters, handled_keys) }}\n {%- if tool.parameters is defined and tool.parameters.required is defined %}\n {{- '\\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}\n {%- endif %}\n {{- '\\n</parameters>' }}\n {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}\n {{- render_extra_keys(tool, handled_keys) }}\n {{- '\\n</function>' }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == \"assistant\" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n {%- else %}\n {%- set content = message.content | default('', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if '<think>' not in content and '</think>' not in content -%}\n {%- set content = \"<think></think>\" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- '<|im_start|>assistant\\n' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ '\\n' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if '</think>' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split('</think>')[-1] %}\n {%- elif '<think>' in c %}\n {# If <think> was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split('<think>')[0] %}\n {%- endif %}\n {%- set c = \"<think></think>\" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ '\\n' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- \"<think></think>\" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ '\\n</parameter>\\n' -}}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>\\n' -}}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- else %}\n {# Assistant message doesn't have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }}\n {%- else %}\n {%- set c = (content | default('', true) | string) %}\n {%- if '<think>' in c and '</think>' in c %}\n {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == \"user\" or message.role == \"system\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- set content = message.content | string %}\n {{- content }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user\\n' }}\n {%- endif %}\n {{- '<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think></think>' }}\n {%- endif %}\n{%- endif %}\n"
|
|
|
|
|
|
|
| 8019 |
}
|
|
|
|
| 8015 |
"model_max_length": 262144,
|
| 8016 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 8017 |
"unk_token": "<unk>",
|
| 8018 |
+
"chat_template": "{% macro render_extra_keys(json_dict, handled_keys) %}\n {%- if json_dict is mapping %}\n {%- for json_key in json_dict if json_key not in handled_keys %}\n {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}\n {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}\n {%- else %}\n {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n{% endmacro %}\n{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}\n{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \"<|im_start|>system\\n\" }}\n {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n {%- if system_message is defined and system_message | length > 0 %}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {{- \"<tools>\" }}\n {%- for tool in tools %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {{- \"\\n<function>\\n<name>\" ~ tool.name ~ \"</name>\" }}\n {%- if tool.description is defined %}\n {{- '\\n<description>' ~ (tool.description | trim) ~ '</description>' }}\n {%- endif %}\n {{- '\\n<parameters>' }}\n {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}\n {%- for param_name, param_fields in tool.parameters.properties|items %}\n {{- '\\n<parameter>' }}\n {{- '\\n<name>' ~ param_name ~ '</name>' }}\n {%- if param_fields.type is defined %}\n {{- '\\n<type>' ~ (param_fields.type | string) ~ '</type>' }}\n {%- endif %}\n {%- if param_fields.description is defined %}\n {{- '\\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}\n {%- endif %}\n {%- if param_fields.enum is defined %}\n {{- '\\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}\n {%- endif %}\n {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}\n {{- render_extra_keys(param_fields, handled_keys) }}\n {{- '\\n</parameter>' }}\n {%- endfor %}\n {%- endif %}\n {% set handled_keys = ['type', 'properties', 'required'] %}\n {{- render_extra_keys(tool.parameters, handled_keys) }}\n {%- if tool.parameters is defined and tool.parameters.required is defined %}\n {{- '\\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}\n {%- endif %}\n {{- '\\n</parameters>' }}\n {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}\n {{- render_extra_keys(tool, handled_keys) }}\n {{- '\\n</function>' }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == \"assistant\" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n {%- else %}\n {%- set content = message.content | default('', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if '<think>' not in content and '</think>' not in content -%}\n {%- set content = \"<think></think>\" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- '<|im_start|>assistant\\n' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ '\\n' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if '</think>' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split('</think>')[-1] %}\n {%- elif '<think>' in c %}\n {# If <think> was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split('<think>')[0] %}\n {%- endif %}\n {%- set c = \"<think></think>\" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ '\\n' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- \"<think></think>\" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ '\\n</parameter>\\n' -}}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>\\n' -}}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- else %}\n {# Assistant message doesn't have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }}\n {%- else %}\n {%- set c = (content | default('', true) | string) %}\n {%- if '<think>' in c and '</think>' in c %}\n {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == \"user\" or message.role == \"system\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- set content = message.content | string %}\n {{- content }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user\\n' }}\n {%- endif %}\n {{- '<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think></think>' }}\n {%- endif %}\n{%- endif %}\n",
|
| 8019 |
+
"pad_token": "<|im_end|>",
|
| 8020 |
+
"padding_side": "left"
|
| 8021 |
}
|