Update app.py

updated code to store files inside of hugging face dataset folder

app.py

@@ -24,17 +24,23 @@ embedding_model = SentenceTransformer(
     "togethercomputer/m2-bert-80M-8k-retrieval", 
     trust_remote_code=True  # Allow remote code execution
 )
+
+# Define dataset storage folder
+DATASET_DIR = "/home/user/.cache/huggingface/datasets/my_documents"
+os.makedirs(DATASET_DIR, exist_ok=True)  # Ensure directory exists
+
+# Define file paths inside dataset folder
+INDEX_FILE = os.path.join(DATASET_DIR, "faiss_index.bin")  # FAISS index file
+METADATA_FILE = os.path.join(DATASET_DIR, "metadata.json")  # Metadata file
+
 embedding_dim = 768  # Adjust according to model
 
 # Initialize FAISS index
 index = faiss.IndexFlatL2(embedding_dim)
 
-# Initialize paths
-INDEX_FILE = "faiss_index.bin"  # FAISS index file (binary format)
-METADATA_FILE = "metadata.json"  # Document metadata
-
-print(os.getcwd())  # This will print the current working directory
-print(os.listdir("."))  # This will show files in the current director
+# Debugging: Check working directory and available files
+print("Current working directory:", os.getcwd())
+print("Files in dataset directory:", os.listdir(DATASET_DIR))
 
 # Load FAISS index if it exists
 if os.path.exists(INDEX_FILE):
@@ -46,7 +52,7 @@ else:
 
 # Load metadata
 if os.path.exists(METADATA_FILE):
-    print("metadata exists")
+    print(" Metadata file exists")
     with open(METADATA_FILE, "r") as f:
         metadata = json.load(f)
 else:
@@ -55,9 +61,9 @@ else:
 def store_document(text):
     print(" Storing document...")
 
-    # Generate a unique filename
+    # Generate a unique filename inside the dataset folder
     doc_id = len(metadata) + 1
-    filename = f"doc_{doc_id}.txt"
+    filename = os.path.join(DATASET_DIR, f"doc_{doc_id}.txt")
     print(f"Saving document at: {filename}")
 
     # Save document to file
@@ -76,33 +82,14 @@ def store_document(text):
     # Update metadata with FAISS index
     metadata[str(doc_index)] = filename
     with open(METADATA_FILE, "w") as f:
-        print(metadata)
         json.dump(metadata, f)
-    print("saved Metadata")
+    print(" Saved Metadata")
 
-    # Save FAISS index properly
+    # Save FAISS index
     faiss.write_index(index, INDEX_FILE)
+    print(" FAISS index saved")
 
-    print(f" Document stored successfully at: {filename}")
-    return "Document stored!"
-
-def retrieve_document(query):
-    print(f"retrieving doc based on: \n{query}")
-
-    query_embedding = embedding_model.encode([query]).astype(np.float32)
-    _, closest_idx = index.search(query_embedding, 1)
-
-    if not closest_idx or closest_idx[0][0] not in metadata:
-        print("No relevant Document found")
-        return None
-
-    
-    if closest_idx[0][0] in metadata:  # Ensure a valid match
-        filename = metadata[str(closest_idx[0][0])]
-        with open(filename, "r") as f:
-            return f.read()
-    else:
-        return None
+    return f"Document stored at: {filename}"
 
 
 def clean_text(text):