Add Transformers weights (#2)

- Add Transformers weights + update model card (47b9e36ec942f6ff9900555727478a1c96f63225)


Co-authored-by: Yoni Gozlan <[email protected]>

README.md

@@ -1,7 +1,7 @@
 ---
 license: apache-2.0
 pipeline_tag: mask-generation
-library_name: sam2
+library_name: transformers
 ---
 
 Repository for SAM 2: Segment Anything in Images and Videos, a foundation model towards solving promptable visual segmentation in images and videos from FAIR. See the [SAM 2 paper](https://arxiv.org/abs/2408.00714) for more information.
@@ -16,7 +16,7 @@ For image prediction:
 import torch
 from sam2.sam2_image_predictor import SAM2ImagePredictor
 
-predictor = SAM2ImagePredictor.from_pretrained("facebook/sam2-hiera-base-plus")
+predictor = SAM2ImagePredictor.from_pretrained("facebook/sam2.1-hiera-base-plus")
 
 with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
     predictor.set_image(<your_image>)
@@ -29,7 +29,7 @@ For video prediction:
 import torch
 from sam2.sam2_video_predictor import SAM2VideoPredictor
 
-predictor = SAM2VideoPredictor.from_pretrained("facebook/sam2-hiera-base-plus")
+predictor = SAM2VideoPredictor.from_pretrained("facebook/sam2.1-hiera-base-plus")
 
 with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
     state = predictor.init_state(<your_video>)
@@ -44,6 +44,490 @@ with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
 
 Refer to the [demo notebooks](https://github.com/facebookresearch/segment-anything-2/tree/main/notebooks) for details.
 
+## Usage with 🤗 Transformers
+
+
+### Automatic Mask Generation with Pipeline
+
+SAM2 can be used for automatic mask generation to segment all objects in an image using the `mask-generation` pipeline:
+
+```python
+>>> from transformers import pipeline
+
+>>> generator = pipeline("mask-generation", model="facebook/sam2.1-hiera-base-plus", device=0)
+>>> image_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg"
+>>> outputs = generator(image_url, points_per_batch=64)
+
+>>> len(outputs["masks"])  # Number of masks generated
+39
+```
+
+### Basic Image Segmentation
+
+#### Single Point Click
+
+You can segment objects by providing a single point click on the object you want to segment:
+
+```python
+>>> from transformers import Sam2Processor, Sam2Model
+>>> import torch
+>>> from PIL import Image
+>>> import requests
+
+>>> device = "cuda" if torch.cuda.is_available() else "cpu"
+
+>>> model = Sam2Model.from_pretrained("facebook/sam2.1-hiera-base-plus").to(device)
+>>> processor = Sam2Processor.from_pretrained("facebook/sam2.1-hiera-base-plus")
+
+>>> image_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg"
+>>> raw_image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+
+>>> input_points = [[[[500, 375]]]]  # Single point click, 4 dimensions (image_dim, object_dim, point_per_object_dim, coordinates)
+>>> input_labels = [[[1]]]  # 1 for positive click, 0 for negative click, 3 dimensions (image_dim, object_dim, point_label)
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+
+>>> # The model outputs multiple mask predictions ranked by quality score
+>>> print(f"Generated {masks.shape[1]} masks with shape {masks.shape}")
+Generated 3 masks with shape torch.Size(1, 3, 1500, 2250)
+```
+
+#### Multiple Points for Refinement
+
+You can provide multiple points to refine the segmentation:
+
+```python
+>>> # Add both positive and negative points to refine the mask
+>>> input_points = [[[[500, 375], [1125, 625]]]]  # Multiple points for refinement
+>>> input_labels = [[[1, 1]]]  # Both positive clicks
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+```
+
+#### Bounding Box Input
+
+SAM2 also supports bounding box inputs for segmentation:
+
+```python
+>>> # Define bounding box as [x_min, y_min, x_max, y_max]
+>>> input_boxes = [[[75, 275, 1725, 850]]]
+
+>>> inputs = processor(images=raw_image, input_boxes=input_boxes, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+```
+
+#### Multiple Objects Segmentation
+
+You can segment multiple objects simultaneously:
+
+```python
+>>> # Define points for two different objects
+>>> input_points = [[[[500, 375]], [[650, 750]]]]  # Points for two objects in same image
+>>> input_labels = [[[1], [1]]]  # Positive clicks for both objects
+
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> # Each object gets its own mask
+>>> masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])[0]
+>>> print(f"Generated masks for {masks.shape[0]} objects")
+Generated masks for 2 objects
+```
+
+### Batch Inference
+
+#### Batched Images
+
+Process multiple images simultaneously for improved efficiency:
+
+```python
+>>> from transformers import Sam2Processor, Sam2Model
+>>> import torch
+>>> from PIL import Image
+>>> import requests
+
+>>> device = "cuda" if torch.cuda.is_available() else "cpu"
+
+>>> model = Sam2Model.from_pretrained("facebook/sam2.1-hiera-base-plus").to(device)
+>>> processor = Sam2Processor.from_pretrained("facebook/sam2.1-hiera-base-plus")
+
+>>> # Load multiple images
+>>> image_urls = [
+...     "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg",
+...     "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/dog-sam.png"
+... ]
+>>> raw_images = [Image.open(requests.get(url, stream=True).raw).convert("RGB") for url in image_urls]
+
+>>> # Single point per image
+>>> input_points = [[[[500, 375]]], [[[770, 200]]]]  # One point for each image
+>>> input_labels = [[[1]], [[1]]]  # Positive clicks for both images
+
+>>> inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> # Post-process masks for each image
+>>> all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])
+>>> print(f"Processed {len(all_masks)} images, each with {all_masks[0].shape[0]} objects")
+Processed 2 images, each with 1 objects
+```
+
+#### Batched Objects per Image
+
+Segment multiple objects within each image using batch inference:
+
+```python
+>>> # Multiple objects per image - different numbers of objects per image
+>>> input_points = [
+...     [[[500, 375]], [[650, 750]]],  # Truck image: 2 objects
+...     [[[770, 200]]]  # Dog image: 1 object
+... ]
+>>> input_labels = [
+...     [[1], [1]],  # Truck image: positive clicks for both objects
+...     [[1]]  # Dog image: positive click for the object
+... ]
+
+>>> inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])
+```
+
+#### Batched Images with Batched Objects and Multiple Points
+
+Handle complex batch scenarios with multiple points per object:
+
+```python
+>>> # Add groceries image for more complex example
+>>> groceries_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/groceries.jpg"
+>>> groceries_image = Image.open(requests.get(groceries_url, stream=True).raw).convert("RGB")
+>>> raw_images = [raw_images[0], groceries_image]  # Use truck and groceries images
+
+>>> # Complex batching: multiple images, multiple objects, multiple points per object
+>>> input_points = [
+...     [[[500, 375]], [[650, 750]]],  # Truck image: 2 objects with 1 point each
+...     [[[400, 300]], [[630, 300], [550, 300]]]  # Groceries image: obj1 has 1 point, obj2 has 2 points
+... ]
+>>> input_labels = [
+...     [[1], [1]],  # Truck image: positive clicks
+...     [[1], [1, 1]]  # Groceries image: positive clicks for refinement
+... ]
+
+>>> inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])
+```
+
+#### Batched Bounding Boxes
+
+Process multiple images with bounding box inputs:
+
+```python
+>>> # Multiple bounding boxes per image (using truck and groceries images)
+>>> input_boxes = [
+...     [[75, 275, 1725, 850], [425, 600, 700, 875], [1375, 550, 1650, 800], [1240, 675, 1400, 750]],  # Truck image: 4 boxes
+...     [[450, 170, 520, 350], [350, 190, 450, 350], [500, 170, 580, 350], [580, 170, 640, 350]]  # Groceries image: 4 boxes
+... ]
+
+>>> # Update images for this example
+>>> raw_images = [raw_images[0], groceries_image]  # truck and groceries
+
+>>> inputs = processor(images=raw_images, input_boxes=input_boxes, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs, multimask_output=False)
+
+>>> all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"])
+>>> print(f"Processed {len(input_boxes)} images with {len(input_boxes[0])} and {len(input_boxes[1])} boxes respectively")
+Processed 2 images with 4 and 4 boxes respectively
+```
+
+### Using Previous Masks as Input
+
+SAM2 can use masks from previous predictions as input to refine segmentation:
+
+```python
+>>> # Get initial segmentation
+>>> input_points = [[[[500, 375]]]]
+>>> input_labels = [[[1]]]
+>>> inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors="pt").to(device)
+
+>>> with torch.no_grad():
+...     outputs = model(**inputs)
+
+>>> # Use the best mask as input for refinement
+>>> mask_input = outputs.pred_masks[:, :, torch.argmax(outputs.iou_scores.squeeze())]
+
+>>> # Add additional points with the mask input
+>>> new_input_points = [[[[500, 375], [450, 300]]]]
+>>> new_input_labels = [[[1, 1]]]
+>>> inputs = processor(
+...     input_points=new_input_points,
+...     input_labels=new_input_labels,
+...     original_sizes=inputs["original_sizes"],
+...     return_tensors="pt",
+... ).to(device)
+
+>>> with torch.no_grad():
+...     refined_outputs = model(
+...         **inputs,
+...         input_masks=mask_input,
+...         image_embeddings=outputs.image_embeddings,
+...         multimask_output=False,
+...     )
+```
+
+
+### Video Segmentation and Tracking
+
+SAM2's key strength is its ability to track objects across video frames. Here's how to use it for video segmentation:
+
+#### Basic Video Tracking
+
+```python
+>>> from transformers import Sam2VideoModel, Sam2VideoProcessor
+>>> import torch
+
+>>> device = "cuda" if torch.cuda.is_available() else "cpu"
+>>> model = Sam2VideoModel.from_pretrained("facebook/sam2.1-hiera-base-plus").to(device, dtype=torch.bfloat16)
+>>> processor = Sam2VideoProcessor.from_pretrained("facebook/sam2.1-hiera-base-plus")
+
+>>> # Load video frames (example assumes you have a list of PIL Images)
+>>> # video_frames = [Image.open(f"frame_{i:05d}.jpg") for i in range(num_frames)]
+
+>>> # For this example, we'll use the video loading utility
+>>> from transformers.video_utils import load_video
+>>> video_url = "https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/bedroom.mp4"
+>>> video_frames, _ = load_video(video_url)
+
+>>> # Initialize video inference session
+>>> inference_session = processor.init_video_session(
+...     video=video_frames,
+...     inference_device=device,
+...     torch_dtype=torch.bfloat16,
+... )
+
+>>> # Add click on first frame to select object
+>>> ann_frame_idx = 0
+>>> ann_obj_id = 1
+>>> points = [[[[210, 350]]]]
+>>> labels = [[[1]]]
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+...     obj_ids=ann_obj_id,
+...     input_points=points,
+...     input_labels=labels,
+... )
+
+>>> # Segment the object on the first frame
+>>> outputs = model(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+... )
+>>> video_res_masks = processor.post_process_masks(
+...     [outputs.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+... )[0]
+>>> print(f"Segmentation shape: {video_res_masks.shape}")
+Segmentation shape: torch.Size([1, 1, 480, 854])
+
+>>> # Propagate through the entire video
+>>> video_segments = {}
+>>> for sam2_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam2_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam2_video_output.frame_idx] = video_res_masks
+
+>>> print(f"Tracked object through {len(video_segments)} frames")
+Tracked object through 180 frames
+```
+
+#### Multi-Object Video Tracking
+
+Track multiple objects simultaneously across video frames:
+
+```python
+>>> # Reset for new tracking session
+>>> inference_session.reset_inference_session()
+
+>>> # Add multiple objects on the first frame
+>>> ann_frame_idx = 0
+>>> obj_ids = [2, 3]
+>>> input_points = [[[[200, 300]], [[400, 150]]]]  # Points for two objects (batched)
+>>> input_labels = [[[1], [1]]]
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+...     obj_ids=obj_ids,
+...     input_points=input_points,
+...     input_labels=input_labels,
+... )
+
+>>> # Get masks for both objects on first frame
+>>> outputs = model(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+... )
+
+>>> # Propagate both objects through video
+>>> video_segments = {}
+>>> for sam2_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam2_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam2_video_output.frame_idx] = {
+...         obj_id: video_res_masks[i]
+...         for i, obj_id in enumerate(inference_session.obj_ids)
+...     }
+
+>>> print(f"Tracked {len(inference_session.obj_ids)} objects through {len(video_segments)} frames")
+Tracked 2 objects through 180 frames
+```
+
+#### Refining Video Segmentation
+
+You can add additional clicks on any frame to refine the tracking:
+
+```python
+>>> # Add refinement click on a later frame
+>>> refine_frame_idx = 50
+>>> ann_obj_id = 2  # Refining first object
+>>> points = [[[[220, 280]]]]  # Additional point
+>>> labels = [[[1]]]  # Positive click
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=refine_frame_idx,
+...     obj_ids=ann_obj_id,
+...     input_points=points,
+...     input_labels=labels,
+... )
+
+>>> # Re-propagate with the additional information
+>>> video_segments = {}
+>>> for sam2_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam2_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam2_video_output.frame_idx] = video_res_masks
+```
+
+### Streaming Video Inference
+
+For real-time applications, SAM2 supports processing video frames as they arrive:
+
+```python
+>>> # Initialize session for streaming
+>>> inference_session = processor.init_video_session(
+...     inference_device=device,
+...     torch_dtype=torch.bfloat16,
+... )
+
+>>> # Process frames one by one
+>>> for frame_idx, frame in enumerate(video_frames[:10]):  # Process first 10 frames
+...     inputs = processor(images=frame, device=device, return_tensors="pt")
+...
+...     if frame_idx == 0:
+...         # Add point input on first frame
+...         processor.add_inputs_to_inference_session(
+...             inference_session=inference_session,
+...             frame_idx=0,
+...             obj_ids=1,
+...             input_points=[[[[210, 350], [250, 220]]]],
+...             input_labels=[[[1, 1]]],
+...             original_size=inputs.original_sizes[0], # need to be provided when using streaming video inference
+...         )
+...
+...     # Process current frame
+...     sam2_video_output = model(inference_session=inference_session, frame=inputs.pixel_values[0])
+...
+...     video_res_masks = processor.post_process_masks(
+...         [sam2_video_output.pred_masks], original_sizes=inputs.original_sizes, binarize=False
+...     )[0]
+...     print(f"Frame {frame_idx}: mask shape {video_res_masks.shape}")
+```
+
+#### Video Batch Processing for Multiple Objects
+
+Track multiple objects simultaneously in video by adding them all at once:
+
+```python
+>>> # Initialize video session
+>>> inference_session = processor.init_video_session(
+...     video=video_frames,
+...     inference_device=device,
+...     torch_dtype=torch.bfloat16,
+... )
+
+>>> # Add multiple objects on the first frame using batch processing
+>>> ann_frame_idx = 0
+>>> obj_ids = [2, 3]  # Track two different objects
+>>> input_points = [
+...     [[[200, 300], [230, 250], [275, 175]], [[400, 150]]]
+... ]  # Object 2: 3 points (2 positive, 1 negative); Object 3: 1 point
+>>> input_labels = [
+...     [[1, 1, 0], [1]]
+... ]  # Object 2: positive, positive, negative; Object 3: positive
+
+>>> processor.add_inputs_to_inference_session(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+...     obj_ids=obj_ids,
+...     input_points=input_points,
+...     input_labels=input_labels,
+... )
+
+>>> # Get masks for all objects on the first frame
+>>> outputs = model(
+...     inference_session=inference_session,
+...     frame_idx=ann_frame_idx,
+... )
+>>> video_res_masks = processor.post_process_masks(
+...     [outputs.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+... )[0]
+>>> print(f"Generated masks for {video_res_masks.shape[0]} objects")
+Generated masks for 2 objects
+
+>>> # Propagate all objects through the video
+>>> video_segments = {}
+>>> for sam2_video_output in model.propagate_in_video_iterator(inference_session):
+...     video_res_masks = processor.post_process_masks(
+...         [sam2_video_output.pred_masks], original_sizes=[[inference_session.video_height, inference_session.video_width]], binarize=False
+...     )[0]
+...     video_segments[sam2_video_output.frame_idx] = {
+...         obj_id: video_res_masks[i]
+...         for i, obj_id in enumerate(inference_session.obj_ids)
+...     }
+
+>>> print(f"Tracked {len(inference_session.obj_ids)} objects through {len(video_segments)} frames")
+Tracked 2 objects through 180 frames
+```
+
 ### Citation
 
 To cite the paper, model, or software, please use the below:
@@ -55,4 +539,4 @@ To cite the paper, model, or software, please use the below:
   url={https://arxiv.org/abs/2408.00714},
   year={2024}
 }
-```
+```

config.json

@@ -0,0 +1,233 @@
+{
+  "architectures": [
+    "Sam2VideoModel"
+  ],
+  "enable_occlusion_spatial_embedding": true,
+  "enable_temporal_pos_encoding_for_object_pointers": true,
+  "image_size": 1024,
+  "initializer_range": 0.02,
+  "mask_decoder_config": {
+    "attention_downsample_rate": 2,
+    "dynamic_multimask_stability_delta": 0.05,
+    "dynamic_multimask_stability_thresh": 0.98,
+    "dynamic_multimask_via_stability": true,
+    "hidden_act": "gelu",
+    "hidden_size": 256,
+    "iou_head_depth": 3,
+    "iou_head_hidden_dim": 256,
+    "mlp_dim": 2048,
+    "model_type": "",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 2,
+    "num_multimask_outputs": 3
+  },
+  "mask_downsampler_embed_dim": 256,
+  "mask_downsampler_hidden_act": "gelu",
+  "mask_downsampler_kernel_size": 3,
+  "mask_downsampler_padding": 1,
+  "mask_downsampler_stride": 2,
+  "mask_downsampler_total_stride": 16,
+  "max_object_pointers_in_encoder": 16,
+  "memory_attention_downsample_rate": 1,
+  "memory_attention_dropout": 0.1,
+  "memory_attention_feed_forward_hidden_act": "relu",
+  "memory_attention_feed_forward_hidden_size": 2048,
+  "memory_attention_hidden_size": 256,
+  "memory_attention_num_attention_heads": 1,
+  "memory_attention_num_layers": 4,
+  "memory_attention_rope_dropout": 0.1,
+  "memory_attention_rope_feat_sizes": [
+    64,
+    64
+  ],
+  "memory_attention_rope_theta": 10000,
+  "memory_encoder_hidden_size": 256,
+  "memory_encoder_output_channels": 64,
+  "memory_fuser_embed_dim": 256,
+  "memory_fuser_hidden_act": "gelu",
+  "memory_fuser_intermediate_dim": 1024,
+  "memory_fuser_kernel_size": 7,
+  "memory_fuser_layer_scale_init_value": 1e-06,
+  "memory_fuser_num_layers": 2,
+  "memory_fuser_padding": 3,
+  "model_type": "sam2_video",
+  "multimask_max_pt_num": 1,
+  "multimask_min_pt_num": 0,
+  "multimask_output_for_tracking": true,
+  "multimask_output_in_sam": true,
+  "num_maskmem": 7,
+  "prompt_encoder_config": {
+    "hidden_act": "gelu",
+    "hidden_size": 256,
+    "image_size": 1024,
+    "layer_norm_eps": 1e-06,
+    "mask_input_channels": 16,
+    "model_type": "",
+    "num_point_embeddings": 4,
+    "patch_size": 16,
+    "scale": 1
+  },
+  "sigmoid_bias_for_mem_enc": -10.0,
+  "sigmoid_scale_for_mem_enc": 20.0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.56.0.dev0",
+  "vision_config": {
+    "backbone_channel_list": [
+      896,
+      448,
+      224,
+      112
+    ],
+    "backbone_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "blocks_per_stage": [
+        2,
+        3,
+        16,
+        3
+      ],
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "early_stopping": false,
+      "embed_dim_per_stage": [
+        112,
+        224,
+        448,
+        896
+      ],
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "global_attention_blocks": [
+        12,
+        16,
+        20
+      ],
+      "hidden_act": "gelu",
+      "hidden_size": 112,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_size": [
+        1024,
+        1024
+      ],
+      "initializer_range": 0.02,
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "layer_norm_eps": 1e-06,
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "mlp_ratio": 4.0,
+      "model_type": "sam2_hiera_det_model",
+      "no_repeat_ngram_size": 0,
+      "num_attention_heads": 1,
+      "num_attention_heads_per_stage": [
+        2,
+        4,
+        8,
+        16
+      ],
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_channels": 3,
+      "num_query_pool_stages": 3,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "patch_kernel_size": [
+        7,
+        7
+      ],
+      "patch_padding": [
+        3,
+        3
+      ],
+      "patch_stride": [
+        4,
+        4
+      ],
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "query_stride": [
+        2,
+        2
+      ],
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "typical_p": 1.0,
+      "use_bfloat16": false,
+      "window_positional_embedding_background_size": [
+        14,
+        14
+      ],
+      "window_size_per_stage": [
+        8,
+        4,
+        14,
+        7
+      ]
+    },
+    "backbone_feature_sizes": [
+      [
+        256,
+        256
+      ],
+      [
+        128,
+        128
+      ],
+      [
+        64,
+        64
+      ]
+    ],
+    "fpn_hidden_size": 256,
+    "fpn_kernel_size": 1,
+    "fpn_padding": 0,
+    "fpn_stride": 1,
+    "fpn_top_down_levels": [
+      2,
+      3
+    ],
+    "hidden_act": "gelu",
+    "initializer_range": 0.02,
+    "layer_norm_eps": 1e-06,
+    "model_type": "sam2_vision_model",
+    "num_feature_levels": 3
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780
+size 323476296

preprocessor_config.json

@@ -0,0 +1,36 @@
+{
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "disable_grouping": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "Sam2ImageProcessorFast",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "input_data_format": null,
+  "mask_size": {
+    "height": 256,
+    "width": 256
+  },
+  "processor_class": "Sam2VideoProcessor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "return_tensors": null,
+  "size": {
+    "height": 1024,
+    "width": 1024
+  }
+}

processor_config.json

@@ -0,0 +1,5 @@
+{
+  "point_pad_value": -10,
+  "processor_class": "Sam2VideoProcessor",
+  "target_size": 1024
+}

video_preprocessor_config.json

@@ -0,0 +1,36 @@
+{
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "do_sample_frames": null,
+  "fps": null,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "input_data_format": null,
+  "num_frames": null,
+  "processor_class": "Sam2VideoProcessor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 1024,
+    "width": 1024
+  },
+  "size_divisor": null,
+  "video_metadata": null,
+  "video_processor_type": "Sam2VideoVideoProcessor"
+}