Update app.py

app.py

@@ -14,27 +14,27 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 # Define a writable directory for NLTK data in Hugging Face Spaces
 NLTK_DATA_DIR = "/tmp/nltk_data"
 
-# Add this directory to NLTK's data path
+# Ensure the NLTK data directory exists
+os.makedirs(NLTK_DATA_DIR, exist_ok=True)
+logging.info(f"Ensured NLTK data directory exists: {NLTK_DATA_DIR}")
+
+# Add this directory to NLTK's data path and prioritize it
 if NLTK_DATA_DIR not in nltk.data.path:
-    nltk.data.path.append(NLTK_DATA_DIR)
-    logging.info(f"Added {NLTK_DATA_DIR} to NLTK data path.")
+    nltk.data.path.insert(0, NLTK_DATA_DIR) # Use insert(0) to prioritize this path
+    logging.info(f"Prioritized {NLTK_DATA_DIR} in NLTK data path.")
 
-# Ensure 'punkt' tokenizer is downloaded.
-# This must happen before any `nltk.sent_tokenize` or `nltk.word_tokenize` calls.
+# Attempt to download 'punkt' if it's not fully accessible or missing sub-components like 'punkt_tab'.
+# This step is crucial and will run every time the app starts to ensure the resource is available.
 try:
-    # Try to find 'punkt' in the specified download directory first
-    nltk.data.find('tokenizers/punkt', paths=[NLTK_DATA_DIR])
-    logging.info("NLTK 'punkt' tokenizer already downloaded to /tmp/nltk_data.")
-except LookupError:
-    logging.info("NLTK 'punkt' tokenizer not found, attempting to download to /tmp/nltk_data...")
-    try:
-        # Download 'punkt' explicitly to the writable directory
-        nltk.download('punkt', download_dir=NLTK_DATA_DIR)
-        logging.info("NLTK 'punkt' tokenizer downloaded successfully to /tmp/nltk_data.")
-    except Exception as e:
-        logging.error(f"Failed to download NLTK 'punkt' tokenizer: {e}")
-        # Re-raise the error as it's a critical dependency for the application to function.
-        raise
+    logging.info("Attempting to verify/download NLTK 'punkt' tokenizer to ensure all components are available...")
+    # This will download 'punkt' if not found, or verify if already there.
+    # quiet=False provides more verbose output during download in logs.
+    nltk.download('punkt', download_dir=NLTK_DATA_DIR, quiet=False)
+    logging.info("NLTK 'punkt' tokenizer confirmed (or downloaded) successfully.")
+except Exception as e:
+    logging.error(f"Critical error: Failed to download NLTK 'punkt' tokenizer (this might affect 'punkt_tab'): {e}")
+    # Re-raise the error as the application cannot proceed without this resource.
+    raise
 
 # --- Groq API Configuration ---
 # IMPORTANT: It's highly recommended to set your GROQ_API_KEY as an environment variable