Updated model card

README.md

@@ -20,45 +20,53 @@ Make sure you have the following dependencies installed:
 You can install the required packages using pip:
 
 ```bash
-pip install torch transformers
+!git clone https://github.com/huggingface/transformers.git
+%cd transformers
+!git checkout <commit_id_for_4.47.0.dev0>
+!pip install .
+!pip install -q accelerate==0.34.2 bitsandbytes==0.44.1 peft==0.13.1
+
 ```
 
 ```py
-# loading tokenizer & model
-model_id = "Ahanaas/Hermes-3-Llama-3.1-8B_finetune_prashu"
+# quantization of model
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type='nf4'
+)
+```
 
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
+```py
+# Load model & tokenizer
+model_id = "Ahanaas/Hermes-3-Llama-3.1-8B_finetune_prashu"
 
-# Load base model
-model = AutoModelForCausalLM.from_pretrained(
+from transformers import AutoTokenizer, LlamaTokenizer, PreTrainedTokenizerFast
+base_model = AutoModelForCausalLM.from_pretrained(
     model_id,
+    low_cpu_mem_usage=True,
+    return_dict=True,
+    torch_dtype=torch.float16,
     quantization_config=bnb_config,
-    device_map='auto'
+    device_map=0,
 )
-model.config.use_cache = False
-model.config.pretraining_tp = 1
-
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+# Tokenizer 
+tokenizer = AutoTokenizer.from_pretrained(model_id, padding_side="right", use_fast=False)
 tokenizer.pad_token = tokenizer.eos_token
-tokenizer.padding_side = "right"  # Fix weird overflow issue with fp16 training
 ```
 
 ```py
-# Ignore warnings
-logging.set_verbosity(logging.CRITICAL)
-
 # Run text generation pipeline with our next model
-system_prompt = """"""
-prompt = ""
+system_prompt = ''''''
+prompt = ''''''
 
 pipe = pipeline(
     task="text-generation", 
-    model=model, 
+    model=base_model, 
     tokenizer=tokenizer, 
     max_new_tokens=128,  # Increase this to allow for longer outputs
-    temperature=0.5,  # Encourages more varied outputs
+    temperature=0.4,  # Encourages more varied outputs
     top_k=50,  # Limits to the top 50 tokens
     do_sample=True,  # Enables sampling
     return_full_text=True
@@ -67,11 +75,5 @@ pipe = pipeline(
 result = pipe(f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>")
 # print(result[0]['generated_text'])
 generated_text = result[0]['generated_text']
-
-# Remove the leading system prompt and special tokens
-# start_idx = generated_text.find("[/INST]") + len("[/INST]")
-# response_text = generated_text[start_idx:].strip()  # Get text after [/INST]
-
-# Print the extracted response text
 print(generated_text)
 ```