Update backend/query_llm.py

backend/query_llm.py

@@ -24,10 +24,10 @@ HF_TOKEN = getenv("HUGGING_FACE_HUB_TOKEN")
 #        )
 
 
-# hf_client = InferenceClient(
-#         "mistralai/Mixtral-8x7B-Instruct-v0.1",
-#         token=HF_TOKEN
-#         )
+hf_client = InferenceClient(
+        "mistralai/Mixtral-8x7B-Instruct-v0.1",
+        token=HF_TOKEN
+        )
 def format_prompt(message: str, api_kind: str):
     """
     Formats the given message using a chat template.
@@ -83,24 +83,24 @@ def generate_hf(prompt: str, history: str, temperature: float = 0.5, max_new_tok
     formatted_prompt = format_prompt(prompt, "hf")
 
     try:
-        # stream = hf_client.text_generation(formatted_prompt, **generate_kwargs,
-        #                                     stream=True, details=True, return_full_text=False)
-        client_Qwen = Client("Qwen/Qwen1.5-110B-Chat-demo")
-        response = client_Qwen.predict(
-                    query=prompt,
-                    history=[],
-                    system="You are a helpful assistant.",
-                    api_name="/model_chat"
-                )
-                 # Extract the API output text
-        api_output = response[1] if response and len(response) > 1 else "No output received from the API."
+        stream = hf_client.text_generation(formatted_prompt, **generate_kwargs,
+                                            stream=True, details=True, return_full_text=False)
+        # client_Qwen = Client("Qwen/Qwen1.5-110B-Chat-demo")
+        # response = client_Qwen.predict(
+        #             query=prompt,
+        #             history=[],
+        #             system="You are a helpful assistant.",
+        #             api_name="/model_chat"
+        #         )
+        #          # Extract the API output text
+        # api_output = response[1] if response and len(response) > 1 else "No output received from the API."
         
-                #chatbot_responses.append((txt, api_output[0][1]))
-        print(response)
+        #         #chatbot_responses.append((txt, api_output[0][1]))
+        # print(response)
         
-        # Print the generated code
-        print(api_output[0][1])
-        stream=api_output[0][1]
+        # # Print the generated code
+        # print(api_output[0][1])
+        # stream=api_output[0][1]
         output = ""
         for response in stream:
             output += response.token.text