DanialMT piotrzelasko commited on
Commit
9de8c68
·
1 Parent(s): 8e00a7f

Enable batched audio inference (#3)

Browse files

- Enable batched audio inference (239ad827db3dd78892ad1c241db9bb5b7c047a51)


Co-authored-by: Piotr Żelasko <piotrzelasko@users.noreply.huggingface.co>

Files changed (3) hide show
  1. modeling.py +26 -2
  2. processing.py +63 -39
  3. tokenizer_config.json +3 -1
modeling.py CHANGED
@@ -572,11 +572,35 @@ class NemotronH_Nano_Omni_Reasoning_V3(PreTrainedModel):
572
  assert video_mask.sum() != 0, "No video tokens found in input_ids"
573
  inputs_embeds[video_mask] = video_vit_embeds.reshape(-1, C).to(inputs_embeds.device, inputs_embeds.dtype)
574
 
575
- # Replace sound tokens with sound embeddings
 
 
 
 
 
 
 
576
  if sound_embeds is not None and self.sound_context_token_id is not None:
577
  sound_mask = (input_ids_copy == self.sound_context_token_id)
578
  assert sound_mask.sum() != 0, "No sound tokens found in input_ids"
579
- inputs_embeds[sound_mask] = sound_embeds.reshape(-1, C).to(inputs_embeds.device, inputs_embeds.dtype)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
580
 
581
  # Apply video pruning (EVS) if enabled
582
  if video_vit_embeds is not None and self.video_pruning_rate > 0: # EVS
 
572
  assert video_mask.sum() != 0, "No video tokens found in input_ids"
573
  inputs_embeds[video_mask] = video_vit_embeds.reshape(-1, C).to(inputs_embeds.device, inputs_embeds.dtype)
574
 
575
+ # Replace sound tokens with sound embeddings.
576
+ # `sound_embeds` has shape `(B_sound, T_out_max, C)` where `T_out_max`
577
+ # is the encoder output length for the longest clip in the batch.
578
+ # When `B_sound > 1` the shorter clips have padding at the tail, so
579
+ # we must gather only the valid positions per row before scattering
580
+ # into `sound_mask`. The encoder's `_get_subsampling_output_length`
581
+ # converts each input mel-frame count (from the feature extractor's
582
+ # attention_mask) to its post-subsampling token count.
583
  if sound_embeds is not None and self.sound_context_token_id is not None:
584
  sound_mask = (input_ids_copy == self.sound_context_token_id)
585
  assert sound_mask.sum() != 0, "No sound tokens found in input_ids"
586
+ if sound_embeds.dim() == 3 and sound_embeds.shape[0] > 1 and sound_attention_mask is not None:
587
+ # `attention_mask.sum() = L_i // hop` per row, but
588
+ # `ParakeetFeatureExtractor` pads each row to `1 + L_i // hop`
589
+ # mel frames in single-call mode (the trailing frame comes
590
+ # from STFT center padding) — and the existing batch=1 path
591
+ # consumes that frame's embed too. Add 1 here to match.
592
+ natural_input_lengths = sound_attention_mask.sum(-1) + 1
593
+ output_lengths = self.sound_encoder.encoder._get_subsampling_output_length(natural_input_lengths)
594
+ flat = torch.cat(
595
+ [sound_embeds[i, : int(n)] for i, n in enumerate(output_lengths.tolist())],
596
+ dim=0,
597
+ )
598
+ else:
599
+ flat = sound_embeds.reshape(-1, C)
600
+ assert sound_mask.sum().item() == flat.shape[0], (
601
+ f"sound token count ({sound_mask.sum().item()}) != encoder output count ({flat.shape[0]})"
602
+ )
603
+ inputs_embeds[sound_mask] = flat.to(inputs_embeds.device, inputs_embeds.dtype)
604
 
605
  # Apply video pruning (EVS) if enabled
606
  if video_vit_embeds is not None and self.video_pruning_rate > 0: # EVS
processing.py CHANGED
@@ -53,8 +53,8 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
53
  r"""
54
  Constructs a Nemotron-3-Nano-Omni-30B-A3B-Reasoning processor which wraps an image processor, audio feature extractor,
55
  and a tokenizer into a single processor.
56
- [`NemotronH_Nano_Omni_Reasoning_V3Processor`] offers all the functionalities of the image processor, audio processor,
57
- and tokenizer. See the [`~NemotronH_Nano_Omni_Reasoning_V3Processor.__call__`] and [`~NemotronH_Nano_Omni_Reasoning_V3Processor.decode`]
58
  for more information.
59
  Args:
60
  image_processor ([`AutoImageProcessor`], *optional*):
@@ -110,12 +110,12 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
110
  if getattr(tokenizer, "audio_token_id", None)
111
  else tokenizer.convert_tokens_to_ids(self.audio_token)
112
  )
113
-
114
  # Audio processing parameters
115
  self.audio_sampling_rate = audio_sampling_rate
116
  self.audio_subsampling_factor = audio_subsampling_factor
117
  self.audio_hop_length = audio_hop_length
118
-
119
  super().__init__(image_processor, tokenizer, chat_template=chat_template)
120
 
121
  def __call__(
@@ -127,13 +127,13 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
127
  **kwargs: Unpack[NemotronH_Nano_Omni_Reasoning_V3ProcessorKwargs],
128
  ) -> BatchFeature:
129
  """
130
- Main method to prepare multimodal inputs (text, images, videos, audio) for the model. This method processes
131
- text by replacing image/video/audio tokens with appropriate placeholder sequences, processes images and videos
132
  through the image processor, and tokenizes the final text.
133
 
134
  The method performs the following key operations:
135
  1. Processes images using the image processor to get pixel values and patch counts
136
- 2. Processes videos using the image processor with max_num_tiles=1 to get video pixel values
137
  3. Processes audio to compute the number of audio tokens based on duration
138
  4. Replaces `<image>` tokens in text with `<img>` + image tokens + `</img>` sequences
139
  5. Replaces `<video>` tokens in text with frame-by-frame descriptions including timestamps (if metadata provided)
@@ -149,15 +149,15 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
149
  special tokens `<image>`, `<video>`, and `<audio>` that will be replaced with appropriate token sequences.
150
  videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
151
  The video or batch of videos to be prepared. Each video should be a 4D NumPy array or PyTorch
152
- tensor with shape (num_frames, channels, height, width). Both channels-first and channels-last formats
153
  are supported. Note: Currently only supports batch size of 1 for videos.
154
  audio (`str`, `np.ndarray`, `torch.Tensor`, `List[str]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
155
- The audio or batch of audio clips to be prepared. Can be file paths, numpy arrays (waveforms),
156
  or torch tensors. Waveforms should be 1D arrays at the expected sampling rate.
157
  images_kwargs (`Dict`, *optional*):
158
  Additional keyword arguments for image processing, including:
159
  - `min_pixels` (`int`, *optional*): Minimum number of pixels for image processing
160
- - `max_pixels` (`int`, *optional*): Maximum number of pixels for image processing
161
  - `patch_size` (`int`, *optional*): Size of patches for image processing
162
  - `temporal_patch_size` (`int`, *optional*): Size of temporal patches
163
  - `merge_size` (`int`, *optional*): Size for merging patches
@@ -199,8 +199,8 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
199
  tokenizer_init_kwargs=self.tokenizer.init_kwargs,
200
  **kwargs,
201
  )
202
- image_inputs = videos_inputs = audio_inputs = {}
203
-
204
  if images is not None:
205
  image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
206
  image_num_tokens = image_inputs["num_tokens"]
@@ -243,7 +243,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
243
  )
244
  index += 1
245
  text[i] = text[i].replace("<|placeholder|>", self.image_token)
246
-
247
  if videos is not None:
248
  assert len(text) == 1, "Video is not supported for batch size > 1"
249
  video_metadata = output_kwargs.get("videos_kwargs", {}).get("video_metadata", None)
@@ -299,9 +299,11 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
299
  # model distinguishes image vs. video by which `pixel_values_*` arg was passed.
300
  text[i] = text[i].replace("<|placeholder|>", self.image_token)
301
 
302
- # Replace audio tokens with the correct number of placeholder tokens
 
 
 
303
  if audio is not None:
304
- assert len(text) == 1, "Audio is not supported for batch size > 1"
305
  index = 0
306
  for i in range(len(text)):
307
  while self.audio_token in text[i]:
@@ -313,41 +315,41 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
313
 
314
  return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
315
  text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
316
-
317
  # Build output - exclude audio from tensor conversion since it's raw waveforms
318
  output_data = {**text_inputs, **image_inputs, **videos_inputs}
319
  result = BatchFeature(data=output_data, tensor_type=return_tensors)
320
-
321
  # Add audio clips separately (as list of numpy arrays, not tensors)
322
  if audio_inputs:
323
  result["sound_clips"] = audio_inputs["sound_clips"]
324
-
325
  return result
326
-
327
  def _process_audio(
328
- self,
329
- audio: AudioInput,
330
  audio_kwargs: dict
331
  ) -> tuple:
332
  """Process audio inputs and compute the number of audio tokens.
333
-
334
  Args:
335
  audio: Audio input (file path, waveform array, or list thereof)
336
  audio_kwargs: Additional audio processing arguments
337
-
338
  Returns:
339
  Tuple of (audio_clips, num_tokens_per_clip)
340
  """
341
  # Get sampling rate from kwargs or use default
342
  sampling_rate = audio_kwargs.get("sampling_rate", self.audio_sampling_rate)
343
-
344
  # Normalize audio to list
345
  if not isinstance(audio, list):
346
  audio = [audio]
347
-
348
  audio_clips = []
349
  num_tokens = []
350
-
351
  for audio_item in audio:
352
  # Load audio if it's a file path
353
  if isinstance(audio_item, str):
@@ -358,25 +360,47 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
358
  waveform = audio_item.squeeze() if audio_item.ndim > 1 else audio_item
359
  else:
360
  raise ValueError(f"Unsupported audio type: {type(audio_item)}")
361
-
362
  audio_clips.append(waveform)
363
-
364
- # Estimate number of audio tokens based on waveform length
365
- # num_frames ≈ audio_samples / hop_length
366
- # num_tokens = num_frames / subsampling_factor
367
- num_frames = len(waveform) // self.audio_hop_length
368
- n_tokens = math.ceil(num_frames / self.audio_subsampling_factor)
369
  num_tokens.append(max(1, n_tokens)) # At least 1 token
370
-
371
  return audio_clips, num_tokens
372
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
373
  def _load_audio(self, audio_path: str, target_sr: int) -> np.ndarray:
374
  """Load audio from file and resample if necessary.
375
-
376
  Args:
377
  audio_path: Path to audio file
378
  target_sr: Target sampling rate
379
-
380
  Returns:
381
  Audio waveform as numpy array
382
  """
@@ -386,7 +410,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
386
  return waveform
387
  except ImportError:
388
  pass
389
-
390
  try:
391
  import soundfile as sf
392
  waveform, sr = sf.read(audio_path)
@@ -400,7 +424,7 @@ class NemotronH_Nano_Omni_Reasoning_V3Processor(ProcessorMixin):
400
  return waveform.astype(np.float32)
401
  except ImportError:
402
  pass
403
-
404
  raise ImportError(
405
  "Audio loading requires either librosa or soundfile. "
406
  "Install with: pip install librosa soundfile"
 
53
  r"""
54
  Constructs a Nemotron-3-Nano-Omni-30B-A3B-Reasoning processor which wraps an image processor, audio feature extractor,
55
  and a tokenizer into a single processor.
56
+ [`NemotronH_Nano_Omni_Reasoning_V3Processor`] offers all the functionalities of the image processor, audio processor,
57
+ and tokenizer. See the [`~NemotronH_Nano_Omni_Reasoning_V3Processor.__call__`] and [`~NemotronH_Nano_Omni_Reasoning_V3Processor.decode`]
58
  for more information.
59
  Args:
60
  image_processor ([`AutoImageProcessor`], *optional*):
 
110
  if getattr(tokenizer, "audio_token_id", None)
111
  else tokenizer.convert_tokens_to_ids(self.audio_token)
112
  )
113
+
114
  # Audio processing parameters
115
  self.audio_sampling_rate = audio_sampling_rate
116
  self.audio_subsampling_factor = audio_subsampling_factor
117
  self.audio_hop_length = audio_hop_length
118
+
119
  super().__init__(image_processor, tokenizer, chat_template=chat_template)
120
 
121
  def __call__(
 
127
  **kwargs: Unpack[NemotronH_Nano_Omni_Reasoning_V3ProcessorKwargs],
128
  ) -> BatchFeature:
129
  """
130
+ Main method to prepare multimodal inputs (text, images, videos, audio) for the model. This method processes
131
+ text by replacing image/video/audio tokens with appropriate placeholder sequences, processes images and videos
132
  through the image processor, and tokenizes the final text.
133
 
134
  The method performs the following key operations:
135
  1. Processes images using the image processor to get pixel values and patch counts
136
+ 2. Processes videos using the image processor with max_num_tiles=1 to get video pixel values
137
  3. Processes audio to compute the number of audio tokens based on duration
138
  4. Replaces `<image>` tokens in text with `<img>` + image tokens + `</img>` sequences
139
  5. Replaces `<video>` tokens in text with frame-by-frame descriptions including timestamps (if metadata provided)
 
149
  special tokens `<image>`, `<video>`, and `<audio>` that will be replaced with appropriate token sequences.
150
  videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
151
  The video or batch of videos to be prepared. Each video should be a 4D NumPy array or PyTorch
152
+ tensor with shape (num_frames, channels, height, width). Both channels-first and channels-last formats
153
  are supported. Note: Currently only supports batch size of 1 for videos.
154
  audio (`str`, `np.ndarray`, `torch.Tensor`, `List[str]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
155
+ The audio or batch of audio clips to be prepared. Can be file paths, numpy arrays (waveforms),
156
  or torch tensors. Waveforms should be 1D arrays at the expected sampling rate.
157
  images_kwargs (`Dict`, *optional*):
158
  Additional keyword arguments for image processing, including:
159
  - `min_pixels` (`int`, *optional*): Minimum number of pixels for image processing
160
+ - `max_pixels` (`int`, *optional*): Maximum number of pixels for image processing
161
  - `patch_size` (`int`, *optional*): Size of patches for image processing
162
  - `temporal_patch_size` (`int`, *optional*): Size of temporal patches
163
  - `merge_size` (`int`, *optional*): Size for merging patches
 
199
  tokenizer_init_kwargs=self.tokenizer.init_kwargs,
200
  **kwargs,
201
  )
202
+ image_inputs, videos_inputs, audio_inputs = {}, {}, {}
203
+
204
  if images is not None:
205
  image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
206
  image_num_tokens = image_inputs["num_tokens"]
 
243
  )
244
  index += 1
245
  text[i] = text[i].replace("<|placeholder|>", self.image_token)
246
+
247
  if videos is not None:
248
  assert len(text) == 1, "Video is not supported for batch size > 1"
249
  video_metadata = output_kwargs.get("videos_kwargs", {}).get("video_metadata", None)
 
299
  # model distinguishes image vs. video by which `pixel_values_*` arg was passed.
300
  text[i] = text[i].replace("<|placeholder|>", self.image_token)
301
 
302
+ # Replace audio tokens with the correct number of placeholder tokens.
303
+ # The expansion loop is per-row, so batch size > 1 is supported as long
304
+ # as `audio_num_tokens` has one entry per `<so_embedding>` placeholder
305
+ # across the batch (in row-major order).
306
  if audio is not None:
 
307
  index = 0
308
  for i in range(len(text)):
309
  while self.audio_token in text[i]:
 
315
 
316
  return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
317
  text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
318
+
319
  # Build output - exclude audio from tensor conversion since it's raw waveforms
320
  output_data = {**text_inputs, **image_inputs, **videos_inputs}
321
  result = BatchFeature(data=output_data, tensor_type=return_tensors)
322
+
323
  # Add audio clips separately (as list of numpy arrays, not tensors)
324
  if audio_inputs:
325
  result["sound_clips"] = audio_inputs["sound_clips"]
326
+
327
  return result
328
+
329
  def _process_audio(
330
+ self,
331
+ audio: AudioInput,
332
  audio_kwargs: dict
333
  ) -> tuple:
334
  """Process audio inputs and compute the number of audio tokens.
335
+
336
  Args:
337
  audio: Audio input (file path, waveform array, or list thereof)
338
  audio_kwargs: Additional audio processing arguments
339
+
340
  Returns:
341
  Tuple of (audio_clips, num_tokens_per_clip)
342
  """
343
  # Get sampling rate from kwargs or use default
344
  sampling_rate = audio_kwargs.get("sampling_rate", self.audio_sampling_rate)
345
+
346
  # Normalize audio to list
347
  if not isinstance(audio, list):
348
  audio = [audio]
349
+
350
  audio_clips = []
351
  num_tokens = []
352
+
353
  for audio_item in audio:
354
  # Load audio if it's a file path
355
  if isinstance(audio_item, str):
 
360
  waveform = audio_item.squeeze() if audio_item.ndim > 1 else audio_item
361
  else:
362
  raise ValueError(f"Unsupported audio type: {type(audio_item)}")
363
+
364
  audio_clips.append(waveform)
365
+
366
+ n_tokens = self._estimate_audio_num_embeddings(len(waveform))
 
 
 
 
367
  num_tokens.append(max(1, n_tokens)) # At least 1 token
368
+
369
  return audio_clips, num_tokens
370
+
371
+ def _estimate_audio_num_embeddings(self, audio_length_samples: int) -> int:
372
+ """Predict the exact number of `<so_embedding>` tokens that the sound
373
+ encoder will produce for an audio clip of `audio_length_samples` raw
374
+ samples. Replaces the previous heuristic, which under-counted by 1 for
375
+ certain lengths and tripped a shape mismatch in `modeling.py::generate`.
376
+
377
+ Mirrors `ParakeetFeatureExtractor` (center-padded STFT → ``1 + L // hop``
378
+ mel frames) followed by `ParakeetEncoder._get_subsampling_output_length`
379
+ (``log2(subsampling_factor)`` stages of stride-2 conv with kernel-size
380
+ ``subsampling_conv_kernel_size``, symmetric padding ``(kernel-1)//2`` on
381
+ each side).
382
+ """
383
+ # Mel frame count (center=True is HF Parakeet's default).
384
+ n_frames = 1 + audio_length_samples // self.audio_hop_length
385
+ # Conv-subsampling: ``log2(subsampling_factor)`` stages, stride 2,
386
+ # kernel ``subsampling_conv_kernel_size``, symmetric padding.
387
+ kernel_size = getattr(self, "audio_subsampling_conv_kernel_size", 3)
388
+ stride = getattr(self, "audio_subsampling_conv_stride", 2)
389
+ num_layers = int(math.log2(self.audio_subsampling_factor))
390
+ all_paddings = (kernel_size - 1) // 2 * 2
391
+ add_pad = all_paddings - kernel_size # kernel=3, sym pad=1 → -1
392
+ L = n_frames
393
+ for _ in range(num_layers):
394
+ L = (L + add_pad) // stride + 1
395
+ return L
396
+
397
  def _load_audio(self, audio_path: str, target_sr: int) -> np.ndarray:
398
  """Load audio from file and resample if necessary.
399
+
400
  Args:
401
  audio_path: Path to audio file
402
  target_sr: Target sampling rate
403
+
404
  Returns:
405
  Audio waveform as numpy array
406
  """
 
410
  return waveform
411
  except ImportError:
412
  pass
413
+
414
  try:
415
  import soundfile as sf
416
  waveform, sr = sf.read(audio_path)
 
424
  return waveform.astype(np.float32)
425
  except ImportError:
426
  pass
427
+
428
  raise ImportError(
429
  "Audio loading requires either librosa or soundfile. "
430
  "Install with: pip install librosa soundfile"
tokenizer_config.json CHANGED
@@ -8015,5 +8015,7 @@
8015
  "model_max_length": 262144,
8016
  "tokenizer_class": "PreTrainedTokenizerFast",
8017
  "unk_token": "<unk>",
8018
- "chat_template": "{% macro render_extra_keys(json_dict, handled_keys) %}\n {%- if json_dict is mapping %}\n {%- for json_key in json_dict if json_key not in handled_keys %}\n {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}\n {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}\n {%- else %}\n {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n{% endmacro %}\n{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}\n{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \"<|im_start|>system\\n\" }}\n {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n {%- if system_message is defined and system_message | length > 0 %}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {{- \"<tools>\" }}\n {%- for tool in tools %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {{- \"\\n<function>\\n<name>\" ~ tool.name ~ \"</name>\" }}\n {%- if tool.description is defined %}\n {{- '\\n<description>' ~ (tool.description | trim) ~ '</description>' }}\n {%- endif %}\n {{- '\\n<parameters>' }}\n {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}\n {%- for param_name, param_fields in tool.parameters.properties|items %}\n {{- '\\n<parameter>' }}\n {{- '\\n<name>' ~ param_name ~ '</name>' }}\n {%- if param_fields.type is defined %}\n {{- '\\n<type>' ~ (param_fields.type | string) ~ '</type>' }}\n {%- endif %}\n {%- if param_fields.description is defined %}\n {{- '\\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}\n {%- endif %}\n {%- if param_fields.enum is defined %}\n {{- '\\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}\n {%- endif %}\n {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}\n {{- render_extra_keys(param_fields, handled_keys) }}\n {{- '\\n</parameter>' }}\n {%- endfor %}\n {%- endif %}\n {% set handled_keys = ['type', 'properties', 'required'] %}\n {{- render_extra_keys(tool.parameters, handled_keys) }}\n {%- if tool.parameters is defined and tool.parameters.required is defined %}\n {{- '\\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}\n {%- endif %}\n {{- '\\n</parameters>' }}\n {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}\n {{- render_extra_keys(tool, handled_keys) }}\n {{- '\\n</function>' }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == \"assistant\" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n {%- else %}\n {%- set content = message.content | default('', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if '<think>' not in content and '</think>' not in content -%}\n {%- set content = \"<think></think>\" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- '<|im_start|>assistant\\n' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ '\\n' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if '</think>' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split('</think>')[-1] %}\n {%- elif '<think>' in c %}\n {# If <think> was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split('<think>')[0] %}\n {%- endif %}\n {%- set c = \"<think></think>\" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ '\\n' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- \"<think></think>\" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ '\\n</parameter>\\n' -}}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>\\n' -}}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- else %}\n {# Assistant message doesn't have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }}\n {%- else %}\n {%- set c = (content | default('', true) | string) %}\n {%- if '<think>' in c and '</think>' in c %}\n {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == \"user\" or message.role == \"system\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- set content = message.content | string %}\n {{- content }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user\\n' }}\n {%- endif %}\n {{- '<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think></think>' }}\n {%- endif %}\n{%- endif %}\n"
 
 
8019
  }
 
8015
  "model_max_length": 262144,
8016
  "tokenizer_class": "PreTrainedTokenizerFast",
8017
  "unk_token": "<unk>",
8018
+ "chat_template": "{% macro render_extra_keys(json_dict, handled_keys) %}\n {%- if json_dict is mapping %}\n {%- for json_key in json_dict if json_key not in handled_keys %}\n {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}\n {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}\n {%- else %}\n {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n{% endmacro %}\n{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}\n{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n {%- if m[\"role\"] == \"user\" %}\n {%- set ns.last_user_idx = loop.index0 %}\n {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- \"<|im_start|>system\\n\" }}\n {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n {%- if system_message is defined and system_message | length > 0 %}\n {{- \"\\n\\n\" }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {{- \"<tools>\" }}\n {%- for tool in tools %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {{- \"\\n<function>\\n<name>\" ~ tool.name ~ \"</name>\" }}\n {%- if tool.description is defined %}\n {{- '\\n<description>' ~ (tool.description | trim) ~ '</description>' }}\n {%- endif %}\n {{- '\\n<parameters>' }}\n {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}\n {%- for param_name, param_fields in tool.parameters.properties|items %}\n {{- '\\n<parameter>' }}\n {{- '\\n<name>' ~ param_name ~ '</name>' }}\n {%- if param_fields.type is defined %}\n {{- '\\n<type>' ~ (param_fields.type | string) ~ '</type>' }}\n {%- endif %}\n {%- if param_fields.description is defined %}\n {{- '\\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}\n {%- endif %}\n {%- if param_fields.enum is defined %}\n {{- '\\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}\n {%- endif %}\n {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}\n {{- render_extra_keys(param_fields, handled_keys) }}\n {{- '\\n</parameter>' }}\n {%- endfor %}\n {%- endif %}\n {% set handled_keys = ['type', 'properties', 'required'] %}\n {{- render_extra_keys(tool.parameters, handled_keys) }}\n {%- if tool.parameters is defined and tool.parameters.required is defined %}\n {{- '\\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}\n {%- endif %}\n {{- '\\n</parameters>' }}\n {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}\n {{- render_extra_keys(tool, handled_keys) }}\n {{- '\\n</function>' }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if tools is iterable and tools | length > 0 %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n {%- if message.role == \"assistant\" %}\n {# Add reasoning content in to content field for unified processing below. #}\n {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n {%- else %}\n {%- set content = message.content | default('', true) %}\n {%- if content is string -%}\n {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n {%- if '<think>' not in content and '</think>' not in content -%}\n {%- set content = \"<think></think>\" ~ content -%}\n {%- endif -%}\n {%- else -%}\n {%- set content = content -%}\n {%- endif -%}\n {%- endif %}\n {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n {# Assistant message has tool calls. #}\n {{- '<|im_start|>assistant\\n' }}\n {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {%- if content is string and content | trim | length > 0 %}\n {%- if include_content %}\n {{- (content | trim) ~ '\\n' -}}\n {%- else %}\n {%- set c = (content | string) %}\n {%- if '</think>' in c %}\n {# Keep only content after the last closing think. Also generation prompt causes this. #}\n {%- set c = c.split('</think>')[-1] %}\n {%- elif '<think>' in c %}\n {# If <think> was opened but never closed, drop the trailing think segment #}\n {%- set c = c.split('<think>')[0] %}\n {%- endif %}\n {%- set c = \"<think></think>\" ~ c | trim %}\n {%- if c | length > 0 %}\n {{- c ~ '\\n' -}}\n {%- endif %}\n {%- endif %}\n {%- else %}\n {{- \"<think></think>\" -}}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value ~ '\\n</parameter>\\n' -}}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>\\n' -}}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- else %}\n {# Assistant message doesn't have tool calls. #}\n {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n {{- '<|im_start|>assistant\\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\\n' }}\n {%- else %}\n {%- set c = (content | default('', true) | string) %}\n {%- if '<think>' in c and '</think>' in c %}\n {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n {%- endif %}\n {%- set c = c | trim %}\n {%- if c | length > 0 %}\n {{- '<|im_start|>assistant\\n' ~ c ~ '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n {%- endif %}\n {%- elif message.role == \"user\" or message.role == \"system\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- set content = message.content | string %}\n {{- content }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user\\n' }}\n {%- endif %}\n {{- '<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {%- if enable_thinking %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n {%- else %}\n {{- '<|im_start|>assistant\\n<think></think>' }}\n {%- endif %}\n{%- endif %}\n",
8019
+ "pad_token": "<|im_end|>",
8020
+ "padding_side": "left"
8021
  }