updates to the threading service and crawling from Rasmus PR's

python/src/server/services/threading_service.py

@@ -93,18 +93,19 @@ class RateLimiter:
             self._clean_old_entries(now)
 
             # Check if we can make the request
-            while not self._can_make_request(estimated_tokens):
+            if not self._can_make_request(estimated_tokens):
                 wait_time = self._calculate_wait_time(estimated_tokens)
                 if wait_time > 0:
                     logfire_logger.info(
-                        f"Rate limiting: waiting {wait_time:.1f}s (tokens={estimated_tokens}, current_usage={self._get_current_usage()})"
+                        f"Rate limiting: waiting {wait_time:.1f}s",
+                        extra={
+                            "tokens": estimated_tokens,
+                            "current_usage": self._get_current_usage(),
+                        }
                     )
                     await asyncio.sleep(wait_time)
-                    # Clean old entries after waiting
-                    now = time.time()
-                    self._clean_old_entries(now)
-                else:
-                    return False
+                    return await self.acquire(estimated_tokens)
+                return False
 
             # Record the request
             self.request_times.append(now)
@@ -199,13 +200,21 @@ class MemoryAdaptiveDispatcher:
             # Reduce workers when memory is high
             workers = max(1, base // 2)
             logfire_logger.warning(
-                f"High memory usage detected, reducing workers (memory_percent={metrics.memory_percent}, workers={workers})"
+                "High memory usage detected, reducing workers",
+                extra={
+                    "memory_percent": metrics.memory_percent,
+                    "workers": workers,
+                }
             )
         elif metrics.cpu_percent > self.config.cpu_threshold * 100:
             # Reduce workers when CPU is high
             workers = max(1, base // 2)
             logfire_logger.warning(
-                f"High CPU usage detected, reducing workers (cpu_percent={metrics.cpu_percent}, workers={workers})"
+                "High CPU usage detected, reducing workers",
+                extra={
+                    "cpu_percent": metrics.cpu_percent,
+                    "workers": workers,
+                }
             )
         elif metrics.memory_percent < 50 and metrics.cpu_percent < 50:
             # Increase workers when resources are available
@@ -235,7 +244,14 @@ class MemoryAdaptiveDispatcher:
         semaphore = asyncio.Semaphore(optimal_workers)
 
         logfire_logger.info(
-            f"Starting adaptive processing (items_count={len(items)}, workers={optimal_workers}, mode={mode}, memory_percent={self.last_metrics.memory_percent}, cpu_percent={self.last_metrics.cpu_percent})"
+            "Starting adaptive processing",
+            extra={
+                "items_count": len(items),
+                "workers": optimal_workers,
+                "mode": mode,
+                "memory_percent": self.last_metrics.memory_percent,
+                "cpu_percent": self.last_metrics.cpu_percent,
+            }
         )
 
         # Track active workers
@@ -310,7 +326,8 @@ class MemoryAdaptiveDispatcher:
                             del active_workers[worker_id]
 
                     logfire_logger.error(
-                        f"Processing failed for item {index} (error={str(e)}, item_index={index})"
+                        f"Processing failed for item {index}",
+                        extra={"error": str(e), "item_index": index}
                     )
                     return None
 
@@ -325,7 +342,13 @@ class MemoryAdaptiveDispatcher:
 
         success_rate = len(successful_results) / len(items) * 100
         logfire_logger.info(
-            f"Adaptive processing completed (total_items={len(items)}, successful={len(successful_results)}, success_rate={success_rate:.1f}%, workers_used={optimal_workers})"
+            "Adaptive processing completed",
+            extra={
+                "total_items": len(items),
+                "successful": len(successful_results),
+                "success_rate": f"{success_rate:.1f}%",
+                "workers_used": optimal_workers,
+            }
         )
 
         return successful_results
@@ -343,7 +366,8 @@ class WebSocketSafeProcessor:
         await websocket.accept()
         self.active_connections.append(websocket)
         logfire_logger.info(
-            f"WebSocket client connected (total_connections={len(self.active_connections)})"
+            "WebSocket client connected",
+            extra={"total_connections": len(self.active_connections)}
         )
 
     def disconnect(self, websocket: WebSocket):
@@ -351,7 +375,8 @@ class WebSocketSafeProcessor:
         if websocket in self.active_connections:
             self.active_connections.remove(websocket)
             logfire_logger.info(
-                f"WebSocket client disconnected (remaining_connections={len(self.active_connections)})"
+                "WebSocket client disconnected",
+                extra={"remaining_connections": len(self.active_connections)}
             )
 
     async def broadcast_progress(self, message: dict[str, Any]):
@@ -462,7 +487,7 @@ class ThreadingService:
 
         self._running = True
         self._health_check_task = asyncio.create_task(self._health_check_loop())
-        logfire_logger.info(f"Threading service started (config={self.config.__dict__})")
+        logfire_logger.info("Threading service started", extra={"config": self.config.__dict__})
 
     async def stop(self):
         """Stop the threading service"""
@@ -498,7 +523,8 @@ class ThreadingService:
             finally:
                 duration = time.time() - start_time
                 logfire_logger.debug(
-                    f"Rate limited operation completed (duration={duration}, tokens={estimated_tokens})"
+                    "Rate limited operation completed",
+                    extra={"duration": duration, "tokens": estimated_tokens},
                 )
 
     async def run_cpu_intensive(self, func: Callable, *args, **kwargs) -> Any:
@@ -550,30 +576,44 @@ class ThreadingService:
 
                 # Log system metrics
                 logfire_logger.info(
-                    f"System health check (memory_percent={metrics.memory_percent}, cpu_percent={metrics.cpu_percent}, available_memory_gb={metrics.available_memory_gb}, active_threads={metrics.active_threads}, active_websockets={len(self.websocket_processor.active_connections)})"
+                    "System health check",
+                    extra={
+                        "memory_percent": metrics.memory_percent,
+                        "cpu_percent": metrics.cpu_percent,
+                        "available_memory_gb": metrics.available_memory_gb,
+                        "active_threads": metrics.active_threads,
+                        "active_websockets": len(self.websocket_processor.active_connections),
+                    }
                 )
 
                 # Alert on critical thresholds
                 if metrics.memory_percent > 90:
                     logfire_logger.warning(
-                        f"Critical memory usage (memory_percent={metrics.memory_percent})"
+                        "Critical memory usage",
+                        extra={"memory_percent": metrics.memory_percent}
                     )
                     # Force garbage collection
                     gc.collect()
 
                 if metrics.cpu_percent > 95:
-                    logfire_logger.warning(f"Critical CPU usage (cpu_percent={metrics.cpu_percent})")
+                    logfire_logger.warning(
+                        "Critical CPU usage", extra={"cpu_percent": metrics.cpu_percent}
+                    )
 
                 # Check for memory leaks (too many threads)
                 if metrics.active_threads > self.config.max_workers * 3:
                     logfire_logger.warning(
-                        f"High thread count detected (active_threads={metrics.active_threads}, max_expected={self.config.max_workers * 3})"
+                        "High thread count detected",
+                        extra={
+                            "active_threads": metrics.active_threads,
+                            "max_expected": self.config.max_workers * 3,
+                        }
                     )
 
                 await asyncio.sleep(self.config.health_check_interval)
 
             except Exception as e:
-                logfire_logger.error(f"Health check failed (error={str(e)})")
+                logfire_logger.error("Health check failed", extra={"error": str(e)})
                 await asyncio.sleep(self.config.health_check_interval)
 
 
@@ -601,4 +641,4 @@ async def stop_threading_service():
     global _threading_service
     if _threading_service:
         await _threading_service.stop()
-        _threading_service = None
+        _threading_service = None