feat(data): add other pdfs

chroma_db/13934663-2db5-404d-be0f-51734d442e08/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a13e72541800c513c73dccea69f79e39cf4baef4fa23f7e117c0d6b0f5f99670
+size 3212000

chroma_db/13934663-2db5-404d-be0f-51734d442e08/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec6df10978b056a10062ed99efeef2702fa4a1301fad702b53dd2517103c746
+size 100

chroma_db/13934663-2db5-404d-be0f-51734d442e08/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
+size 4000

chroma_db/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db081ece29301d223a01bac97e8b2905fada2e7c376cec96bf44fee0f5c95069
-size 1843200
+oid sha256:4dc2c64a9de7507097ab452fdce23fc6348f38e0d34484d791a8c43366b78001
+size 2564096

climateqa/build_index.py

@@ -1,11 +1,12 @@
-# import
+import os
+
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.document_loaders import PyPDFLoader
 
 from .embeddings import EMBEDDING_MODEL_NAME
-from .vectorstore import get_vectorstore
+from .vectorstore import PERSIST_DIRECTORY, get_vectorstore
 
 
 def load_data():
@@ -15,24 +16,33 @@ def load_data():
 
     assert isinstance(vectorstore, Chroma)
     vectorstore.from_documents(
-        docs, embedding_function, persist_directory="./chroma_db"
+        docs, embedding_function, persist_directory=PERSIST_DIRECTORY
     )
     return vectorstore
 
 
 def parse_data():
-    loader = PyPDFLoader("data/daoism/tao-te-ching.pdf")
-    pages = loader.load_and_split()
+    docs = []
+    for root, dirs, files in os.walk("data"):
+        for file in files:
+            if file.endswith(".pdf"):
+                file_path = os.path.join(root, file)
+                loader = PyPDFLoader(file_path)
+                pages = loader.load_and_split()
+
+                # split it into chunks
+                text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=1000, chunk_overlap=0
+                )
+                doc_chunks = text_splitter.split_documents(pages)
+
+                for chunk in doc_chunks:
+                    chunk.metadata["name"] = parse_name(chunk.metadata["source"])
+                    chunk.metadata["domain"] = parse_domain(chunk.metadata["source"])
+                    chunk.metadata["page_number"] = chunk.metadata["page"]
+                    chunk.metadata["short_name"] = chunk.metadata["name"]
+                    docs.append(chunk)
 
-    # split it into chunks
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=250, chunk_overlap=0)
-    docs = text_splitter.split_documents(pages)
-    print(docs)
-    for doc in docs:
-        doc.metadata["name"] = parse_name(doc.metadata["source"])
-        doc.metadata["domain"] = parse_domain(doc.metadata["source"])
-        doc.metadata["page_number"] = doc.metadata["page"]
-        doc.metadata["short_name"] = doc.metadata["name"]
     return docs
 
 
@@ -41,10 +51,22 @@ def parse_name(source: str) -> str:
 
 
 def parse_domain(source: str) -> str:
-    return source.split("/")[2]
+    return source.split("/")[1]
+
+
+def clear_index():
+    folder = PERSIST_DIRECTORY
+    for filename in os.listdir(folder):
+        file_path = os.path.join(folder, filename)
+        try:
+            if os.path.isfile(file_path) or os.path.islink(file_path):
+                os.unlink(file_path)
+        except Exception as e:
+            print("Failed to delete %s. Reason: %s" % (file_path, e))
 
 
 if __name__ == "__main__":
+    clear_index()
     db = load_data()
     # query it
     query = (

climateqa/qa_logging.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import os
 
+
 def log(question, history, docs, user_id):
     if has_blob_config():
         log_in_azure(question, history, docs, user_id)
@@ -48,6 +49,7 @@ def get_azure_blob_client():
     file_share_name = "climategpt"
     # I don't know why this is necessary, but it cause an error otherwise when running build_index.py
     from azure.storage.fileshare import ShareServiceClient
+
     service = ShareServiceClient(account_url=account_url, credential=credential)
     share_client = service.get_share_client(file_share_name)
     return share_client

climateqa/vectorstore.py

@@ -5,6 +5,8 @@ import os
 import pinecone
 from langchain.vectorstores import Chroma, Pinecone
 
+PERSIST_DIRECTORY = "./chroma_db"
+
 try:
     from dotenv import load_dotenv
 
@@ -21,7 +23,7 @@ def get_vectorstore(embeddings_function):
 
 def get_chroma_vectore_store(embedding_function):
     return Chroma(
-        persist_directory="./chroma_db", embedding_function=embedding_function
+        persist_directory=PERSIST_DIRECTORY, embedding_function=embedding_function
     )