Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,79 @@
+---
+license: apache-2.0
+pipeline_tag: video-classification
+tags:
+- video
+---
+
+# V-JEPA 2
+
+A frontier video understanding model developed by FAIR, Meta, which extends the pretraining objectives of [VJEPA](https://ai.meta.com/blog/v-jepa-yann-lecun-ai-model-video-joint-embedding-predictive-architecture/), resulting in state-of-the-art video understanding capabilities, leveraging data and model sizes at scale.
+The code is released [in this repository](https://github.com/facebookresearch/vjepa2).
+
+<img src="https://dl.fbaipublicfiles.com/vjepa2/vjepa2-pretrain.gif">&nbsp;
+
+## Intended Uses
+
+V-JEPA 2 is intended to represent any video (and image) to perform video classification, retrieval, or as a video encoder for VLMs.
+
+```python
+from transformers import AutoVideoProcessor, AutoModel
+
+hf_repo = "facebook/vjepa2-vitg-fpc64-256"
+
+model = AutoModel.from_pretrained(hf_repo)
+processor = AutoVideoProcessor.from_pretrained(hf_repo)
+```
+
+
+
+To load a video, sample the number of frames according to the model. For this model, we use 64.
+
+```python
+import torch
+from torchcodec.decoders import VideoDecoder
+import numpy as np
+
+video_url = "https://huggingface.co/datasets/nateraw/kinetics-mini/resolve/main/val/archery/-Qz25rXdMjE_000014_000024.mp4"
+vr = VideoDecoder(video_url)
+frame_idx = np.arange(0, 64) # choosing some frames. here, you can define more complex sampling strategy
+video = vr.get_frames_at(indices=frame_idx).data  # T x C x H x W
+video = processor(video, return_tensors="pt").to(model.device)
+with torch.no_grad():
+    video_embeddings = model.get_vision_features(**video)
+
+print(video_embeddings.shape)
+```
+
+To load an image, simply copy the image to the desired number of frames.
+
+```python
+from transformers.image_utils import load_image
+
+image = load_image("https://huggingface.co/datasets/merve/coco/resolve/main/val2017/000000000285.jpg")
+pixel_values = processor(image, return_tensors="pt").to(model.device)["pixel_values_videos"]
+pixel_values = pixel_values.repeat(1, 16, 1, 1, 1) # repeating image 16 times
+
+with torch.no_grad():
+    image_embeddings = model.get_vision_features(pixel_values)    
+
+print(image_embeddings.shape)
+```
+
+For more code examples, please refer to the V-JEPA 2 documentation.
+
+
+### Citation
+
+```
+@techreport{assran2025vjepa2,
+  title={V-JEPA~2: Self-Supervised Video Models Enable Understanding, Prediction and Planning},
+  author={Assran, Mahmoud and Bardes, Adrien and Fan, David and Garrido, Quentin and Howes, Russell and
+Komeili, Mojtaba and Muckley, Matthew and Rizvi, Ammar and Roberts, Claire and Sinha, Koustuv and Zholus, Artem and
+Arnaud, Sergio and Gejji, Abha and Martin, Ada and Robert Hogan, Francois and Dugas, Daniel and
+Bojanowski, Piotr and Khalidov, Vasil and Labatut, Patrick and Massa, Francisco and Szafraniec, Marc and
+Krishnakumar, Kapil and Li, Yong and Ma, Xiaodong and Chandar, Sarath and Meier, Franziska and LeCun, Yann and
+Rabbat, Michael and Ballas, Nicolas},
+  institution={FAIR at Meta},
+  year={2025}
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "VJEPA2Model"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "crop_size": 256,
+  "drop_path_rate": 0.0,
+  "frames_per_clip": 64,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 1408,
+  "image_size": 256,
+  "in_chans": 3,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "mlp_ratio": 4.363636363636363,
+  "model_type": "vjepa2",
+  "num_attention_heads": 22,
+  "num_hidden_layers": 40,
+  "patch_size": 16,
+  "pred_hidden_size": 384,
+  "pred_mlp_ratio": 4.0,
+  "pred_num_attention_heads": 12,
+  "pred_num_hidden_layers": 12,
+  "pred_num_mask_tokens": 10,
+  "pred_zero_init_mask_tokens": true,
+  "qkv_bias": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.0.dev0",
+  "tubelet_size": 2,
+  "use_SiLU": false,
+  "wide_SiLU": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f205e77aa2ade168db6b09d4bc420d156141f64ab964278a9c181a2bdf2a232b
+size 4138311608

original/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67129f011434e605d894e69f2c8e13d9db118deabe59d54bf6e0fa62c2c5cb8e
+size 16464208210

video_preprocessor_config.json

@@ -0,0 +1,71 @@
+{
+  "_valid_kwargs_names": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device"
+  ],
+  "crop_size": {
+    "height": 256,
+    "width": 256
+  },
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": true,
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "input_data_format": null,
+  "model_valid_processing_keys": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device"
+  ],
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 292
+  },
+  "size_divisor": null,
+  "video_processor_type": "VJEPA2VideoProcessor"
+}