Update auditqa/doc_process.py

auditqa/doc_process.py

@@ -13,8 +13,8 @@ path_to_data = "./data/"
 
 
 def process_pdf():
-    files = {'ABC':'./data/MWTS2021.pdf',
-            'XYZ':'./data/MWTS2022.pdf'}
+    files = {'MWTS2021':'./data/MWTS2021.pdf',
+            'MWTS2022':'./data/MWTS2022.pdf'}
     docs = {}
     for file,value in files.items():
         try:
@@ -26,7 +26,7 @@ def process_pdf():
     # text splitter based on the tokenizer of a model of your choosing
     # to make texts fit exactly a transformer's context window size
     # langchain text splitters: https://python.langchain.com/docs/modules/data_connection/document_transformers/
-    chunk_size = 256
+    chunk_size = 512
     text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
             AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5"),
             chunk_size=chunk_size,