Spaces:
Sleeping
Sleeping
EMMA
commited on
Commit
·
781dd97
1
Parent(s):
eb23805
Final deployment: Model loaded from Hub
Browse files- Emma/app.py +20 -2
Emma/app.py
CHANGED
|
@@ -40,8 +40,26 @@ from utils.memory_utils import (
|
|
| 40 |
extract_semantic_memory,
|
| 41 |
)
|
| 42 |
|
| 43 |
-
# Ensure NLTK data path
|
| 44 |
-
nltk.data.path = [os.path.join(os.path.dirname(__file__), "nltk_data")] + nltk.data.path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
tokenizer = tiktoken.get_encoding("cl100k_base")
|
| 47 |
|
|
|
|
| 40 |
extract_semantic_memory,
|
| 41 |
)
|
| 42 |
|
| 43 |
+
# Ensure NLTK data path(for local)
|
| 44 |
+
#nltk.data.path = [os.path.join(os.path.dirname(__file__), "nltk_data")] + nltk.data.path
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
#for Hugginface
|
| 48 |
+
# دانلود هوشمند دیتای مورد نیاز NLTK برای سرور
|
| 49 |
+
try:
|
| 50 |
+
nltk.data.find('tokenizers/punkt')
|
| 51 |
+
except LookupError:
|
| 52 |
+
nltk.download('punkt')
|
| 53 |
+
|
| 54 |
+
try:
|
| 55 |
+
nltk.data.find('tokenizers/punkt_tab')
|
| 56 |
+
except LookupError:
|
| 57 |
+
nltk.download('punkt_tab')
|
| 58 |
+
|
| 59 |
+
# مسیر لوکال (اختیاری - اگر وجود داشته باشد استفاده میکند)
|
| 60 |
+
local_nltk_path = os.path.join(os.path.dirname(__file__), "nltk_data")
|
| 61 |
+
if os.path.exists(local_nltk_path):
|
| 62 |
+
nltk.data.path.append(local_nltk_path)
|
| 63 |
|
| 64 |
tokenizer = tiktoken.get_encoding("cl100k_base")
|
| 65 |
|