Update modeling file

modeling_llava.py

@@ -6,8 +6,6 @@ from typing import List, Optional, Tuple, Union
 import torch
 import torch.nn.functional as F
 import torch.utils.checkpoint
-from configuration_llava import LlavaConfig
-from configuration_phi import PhiConfig
 from torch import nn
 from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
 from transformers import PreTrainedModel, SiglipVisionModel
@@ -34,9 +32,138 @@ except Exception as exp:
     print(exp)
 
 
+from transformers.configuration_utils import PretrainedConfig
+from transformers import SiglipVisionConfig
+
+
 logger = logging.get_logger(__name__)
 
 
+class PhiConfig(PretrainedConfig):
+    model_type = "phi"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=51200,
+        hidden_size=2048,
+        intermediate_size=8192,
+        num_hidden_layers=24,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        resid_pdrop=0.0,
+        embd_pdrop=0.0,
+        attention_dropout=0.0,
+        hidden_act="gelu_new",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        layer_norm_eps=1e-5,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        partial_rotary_factor=0.5,
+        qk_layernorm=False,
+        bos_token_id=1,
+        eos_token_id=2,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attention_dropout = attention_dropout
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.partial_rotary_factor = partial_rotary_factor
+        self.qk_layernorm = qk_layernorm
+        self._rope_scaling_validation()
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_factor = self.rope_scaling.get("factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if (
+            rope_scaling_factor is None
+            or not isinstance(rope_scaling_factor, float)
+            or rope_scaling_factor <= 1.0
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}"
+            )
+
+
+class LlavaConfig(PretrainedConfig):
+    model_type = "mc-llava"
+    is_composition = False
+
+    def __init__(
+        self,
+        text_config=None,
+        vision_config=None,
+        ignore_index=-100,
+        image_token_index=50297,
+        projector_hidden_act="gelu",
+        projector_tokens_num=1,
+        vocab_size=51200,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+        self.image_token_index = image_token_index
+        self.projector_hidden_act = projector_hidden_act
+        self.projector_tokens_num = projector_tokens_num
+        self.vocab_size = vocab_size
+
+        self.text_config = text_config
+        if isinstance(self.text_config, dict):
+            text_config["model_type"] = (
+                text_config["model_type"] if "model_type" in text_config else "phi"
+            )
+            self.text_config = PhiConfig(**text_config)
+            self.vocab_size = self.text_config.vocab_size
+
+        self.vision_config = vision_config
+        if isinstance(self.vision_config, dict):
+            self.vision_config = SiglipVisionConfig(**vision_config)
+            self.vision_embed_dim = self.vision_config.hidden_size
+
+        super().__init__(**kwargs)
+
+
 # Copied from transformers.models.llama.modeling_llama._get_unpad_data
 def _get_unpad_data(attention_mask):
     seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
@@ -1324,7 +1451,7 @@ class SiglipVisionEncoder(nn.Module):
 
         self.num_tokens = 728
 
-    def feature_select(self, image_forward_outs, coord_feature, num_tokens = None):
+    def feature_select(self, image_forward_outs, coord_feature, num_tokens=None):
         image_features = image_forward_outs
         image_features = image_features[:, 1:]
         if num_tokens is None:
@@ -1344,24 +1471,30 @@ class SiglipVisionEncoder(nn.Module):
         image_features = torch.cat(output_list)
         return image_features
 
-    def process_image_chunks(self, image_tensor, coord_tensor, num_tokens = None):
+    def process_image_chunks(self, image_tensor, coord_tensor, num_tokens=None):
         if image_tensor.shape[0] > 50:
             image_forward_out = []
-            for i in range(0,image_tensor.shape[0],50):
-                part_forward_out = self.vision_tower(image_tensor[i:i+50], output_hidden_states=True).hidden_states[-1]
+            for i in range(0, image_tensor.shape[0], 50):
+                part_forward_out = self.vision_tower(
+                    image_tensor[i : i + 50], output_hidden_states=True
+                ).hidden_states[-1]
                 image_forward_out.append(part_forward_out)
             image_forward_out = torch.cat(image_forward_out, dim=0)
         else:
-            image_forward_out = self.vision_tower(image_tensor, output_hidden_states=True).hidden_states[-1]
+            image_forward_out = self.vision_tower(
+                image_tensor, output_hidden_states=True
+            ).hidden_states[-1]
         coord_feature = self.coord_embed(coord_tensor)
         if len(coord_feature.shape) == 1:
             coord_feature = coord_feature.unsqueeze(0)
-        image_feature = self.feature_select(image_forward_out, coord_feature, num_tokens).to(
-            image_tensor.dtype
-        )
+        image_feature = self.feature_select(
+            image_forward_out, coord_feature, num_tokens
+        ).to(image_tensor.dtype)
         return image_feature
 
-    def forward(self, images: List[torch.Tensor], coords: List[torch.Tensor], num_tokens = None):
+    def forward(
+        self, images: List[torch.Tensor], coords: List[torch.Tensor], num_tokens=None
+    ):
         image_features = []
         for i, image in enumerate(images):
             image_feature = self.process_image_chunks(image, coords[i], num_tokens)