Update README.md

README.md

@@ -1,3 +1,109 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+language:
+- en
+tags:
+- video
+- audio
+- language
+- multimodal
+---
+
+# LongVU
+
+This repository contains the model weight of TDC-Qwen2-7B as presented in [Multimodal Long Video Modeling Based on Temporal Dynamic Context](https://arxiv.org/abs/2504.10443).
+
+
+# Use
+
+We provide the simple inference code for using our model. For more details, you could refer to [Github](https://github.com/Hoar012/TDC-Video)
+
+```python
+import numpy as np
+import torch
+from tdc.builder import load_pretrained_model
+from tdc.constants import (
+    DEFAULT_IMAGE_TOKEN,
+    IMAGE_TOKEN_INDEX,
+)
+from tdc.conversation import conv_templates, SeparatorStyle
+from tdc.mm_datautils import (
+    KeywordsStoppingCriteria,
+    process_images,
+    tokenizer_image_token,
+)
+from decord import cpu, VideoReader
+from utils.processor import Processor
+
+tokenizer, model, image_processor, context_len = load_pretrained_model(
+    "checkpoints/TDC-Qwen2-7B", None, "cambrian_qwen",
+)
+audio_processor = Processor("checkpoints/audio_encoder/whisper-large-v3")
+
+model.eval()
+model.cuda()
+video_path = "./examples/video1.mp4"
+audio_path = "./examples/audio1.wav"
+instruction = qs = "Describe this video in detail, what can you see and hear?"
+
+vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+fps = float(vr.get_avg_fps())
+frame_indices = np.array([i for i in range(0, len(vr), round(fps),)])
+video = []
+for frame_index in frame_indices:
+    img = vr[frame_index].asnumpy()
+    video.append(img)
+video = np.stack(video)
+image_sizes = [video[0].shape[:2]]
+video = process_images(video, image_processor, model.config)
+video = [item.unsqueeze(0) for item in video]
+
+if audio_path is not None:
+    audio_data = {
+        "audio": [{'audio_file': audio_path, 'start_time': None, 'end_time': None}]
+                }
+    audio = audio_processor(audio_data)
+else:
+    audio = None
+    
+qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
+conv = conv_templates["qwen"].copy()
+conv.append_message(conv.roles[0], qs)
+conv.append_message(conv.roles[1], None)
+prompt = conv.get_prompt()
+
+input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(model.device)
+stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+keywords = [stop_str]
+stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+with torch.inference_mode():
+    output_ids = model.generate(
+        input_ids,
+        images=video,
+        image_sizes=image_sizes,
+        do_sample=True,
+        temperature=0.2,
+        max_new_tokens=128,
+        use_cache=True,
+        stopping_criteria=[stopping_criteria],
+        prompt=instruction,
+        audio=audio
+    )
+pred = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+print(pred)
+```
+
+# Citation
+
+```
+@misc{hao2025multimodallongvideomodeling,
+        title={Multimodal Long Video Modeling Based on Temporal Dynamic Context}, 
+        author={Haoran Hao and Jiaming Han and Yiyuan Zhang and Xiangyu Yue},
+        year={2025},
+        eprint={2504.10443},
+        archivePrefix={arXiv},
+        primaryClass={cs.CV},
+        url={https://arxiv.org/abs/2504.10443}, 
+  }
+```