Fix Firebase rate limiting infinite loop - progressive batch reduction and max retries

cve_factchecker/app.py

@@ -159,8 +159,8 @@ def _background_ingest() -> None:
         
         for attempt in range(max_retries):
             try:
-                # Use smaller limit to avoid rate limiting, prioritize English articles collection
-                limit = min(1000, 5000)  # Start with smaller batch
+                # Use conservative limit to avoid rate limiting
+                limit = min(500, 5000)  # Further reduced initial batch
                 
                 # Use dedicated English articles collection for better results
                 if LANGUAGE_FILTER.lower() in ["english", "en"]:

cve_factchecker/firebase_loader.py

@@ -67,13 +67,20 @@ class FirebaseNewsLoader:
             
             articles: List[NewsArticle] = []
             page_token: Optional[str] = None
-            batch_size = min(300, limit or 300)  # Firestore max pageSize
+            batch_size = min(100, limit or 100)  # Start conservative
             remaining = limit
+            consecutive_rate_limits = 0
+            max_rate_limit_retries = 5  # Prevent infinite loops
             
             while True:
                 if remaining is not None and remaining <= 0:
                     break
                 
+                # Progressive batch size reduction on rate limits
+                if consecutive_rate_limits > 2:
+                    batch_size = max(10, batch_size // 2)
+                    print(f"🔽 Reducing batch size to {batch_size} due to rate limits")
+                
                 page_size = batch_size if remaining is None else min(batch_size, remaining)
                 params = {
                     "key": self.config.api_key,
@@ -87,11 +94,18 @@ class FirebaseNewsLoader:
                 resp = requests.get(base_url, params=params, timeout=30)
                 
                 if resp.status_code == 429:  # Rate limit
-                    retry_after = int(resp.headers.get('Retry-After', 30))
-                    print(f"⏳ Rate limited, waiting {retry_after}s...")
+                    consecutive_rate_limits += 1
+                    if consecutive_rate_limits > max_rate_limit_retries:
+                        print(f"❌ Too many consecutive rate limits ({consecutive_rate_limits}), stopping fetch")
+                        break
+                    
+                    retry_after = int(resp.headers.get('Retry-After', min(60, 10 * consecutive_rate_limits)))
+                    print(f"⏳ Rate limited #{consecutive_rate_limits}, waiting {retry_after}s...")
                     time.sleep(retry_after)
                     continue
-                elif resp.status_code != 200:
+                else:
+                    consecutive_rate_limits = 0  # Reset on success
+                if resp.status_code != 200:
                     print(f"❌ Failed to fetch English articles: {resp.status_code}")
                     if resp.status_code == 404:
                         print(f"💡 Collection '{collection_name}' not found. Falling back to language filtering...")
@@ -541,7 +555,7 @@ class FirebaseNewsLoader:
                     print(f"❌ Firebase API rate limited: waiting {retry_after}s")
                     time.sleep(retry_after)
                     continue
-                elif resp.status_code != 200:
+                if resp.status_code != 200:
                     print(f"❌ Firebase API failed: {resp.status_code}")
                     if resp.status_code >= 500:  # Server error, might be temporary
                         time.sleep(5)