add: alias to document loader artifacts and datasets + enable mps fallback for marker

medrag_multi_modal/document_loader/image_loader/base_img_loader.py

@@ -100,7 +100,11 @@ class BaseImageLoader(BaseTextLoader):
             await task
 
         if wandb_artifact_name:
-            artifact = wandb.Artifact(name=wandb_artifact_name, type="dataset")
+            artifact = wandb.Artifact(
+                name=wandb_artifact_name,
+                type="dataset",
+                metadata={"loader_name": self.__class__.__name__},
+            )
             artifact.add_dir(local_path=image_save_dir)
             artifact.save()
             rich.print("Artifact saved and uploaded to wandb!")

medrag_multi_modal/document_loader/image_loader/marker_img_loader.py

@@ -6,6 +6,8 @@ from marker.models import load_all_models
 
 from .base_img_loader import BaseImageLoader
 
+os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+
 
 class MarkerImageLoader(BaseImageLoader):
     """

medrag_multi_modal/document_loader/text_loader/base_text_loader.py

@@ -131,6 +131,7 @@ class BaseTextLoader(ABC):
         async def process_page(page_idx):
             nonlocal processed_pages_counter
             page_data = await self.extract_page_data(page_idx, **kwargs)
+            page_data["loader_name"] = self.__class__.__name__
             pages.append(page_data)
             rich.print(
                 f"Processed page idx: {page_idx}, progress: {processed_pages_counter}/{total_pages}"

medrag_multi_modal/document_loader/text_loader/marker_text_loader.py

@@ -1,3 +1,4 @@
+import os
 from typing import Dict
 
 from marker.convert import convert_single_pdf
@@ -5,6 +6,8 @@ from marker.models import load_all_models
 
 from .base_text_loader import BaseTextLoader
 
+os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+
 
 class MarkerTextLoader(BaseTextLoader):
     """