From 9f22659f4c56cf7b6a3b544d7292746213c258ca Mon Sep 17 00:00:00 2001 From: Cole Medin Date: Sat, 30 Aug 2025 10:33:11 -0600 Subject: [PATCH] Moving Dockerfiles to uv for package installation (#533) * Moving Dockerfiles to uv for package installation * Updating uv installation for CI --- .github/workflows/ci.yml | 2 +- README.md | 10 +- python/Dockerfile.agents | 20 ++-- python/Dockerfile.mcp | 17 ++-- python/Dockerfile.server | 30 +++--- python/pyproject.toml | 167 +++++++++++++++++++++++---------- python/requirements.agents.txt | 26 ----- python/requirements.mcp.txt | 8 -- python/requirements.server.txt | 47 ---------- 9 files changed, 163 insertions(+), 164 deletions(-) delete mode 100644 python/requirements.agents.txt delete mode 100644 python/requirements.mcp.txt delete mode 100644 python/requirements.server.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91383ae7..f02d8224 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: - name: Install dependencies run: | - uv sync --dev + uv sync --group all --group dev uv add pytest-cov - name: Run linting with ruff (if available) diff --git a/README.md b/README.md index 9542bb7c..2dac63df 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,11 @@ This new vision for Archon replaces the old one (the agenteer). Archon used to b # SUPABASE_SERVICE_KEY=your-service-key-here ``` - NOTE: Supabase introduced a new type of service key but use the legacy one (the longer one). + IMPORTANT NOTES: + - For cloud Supabase: they recently introduced a new type of service role key but use the legacy one (the longer one). + - For local Supabase: set SUPABASE_URL to http://host.docker.internal:8000 (unless you have an IP address set up). - OPTIONAL: If you want to enable the reranking RAG strategy, uncomment lines 20-22 in `python\requirements.server.txt`. This will significantly increase the size of the Archon Server container which is why it's off by default. + OPTIONAL: If you want to enable the reranking RAG strategy, add " --group server-reranking" to the end of the uv install on line 18 of `python/server/Dockerfile.server`. This will significantly increase the size of the Archon Server container which is why it's off by default. 3. **Database Setup**: In your [Supabase project](https://supabase.com/dashboard) SQL Editor, copy, paste, and execute the contents of `migration/complete_setup.sql` @@ -205,7 +207,7 @@ Once everything is running: ### 🤖 AI Integration - **Model Context Protocol (MCP)**: Connect any MCP-compatible client (Claude Code, Cursor, even non-AI coding assistants like Claude Desktop) -- **10 MCP Tools**: Comprehensive yet simple set of tools for RAG queries, task management, and project operations +- **MCP Tools**: Comprehensive yet simple set of tools for RAG queries, task management, and project operations - **Multi-LLM Support**: Works with OpenAI, Ollama, and Google Gemini models - **RAG Strategies**: Hybrid search, contextual embeddings, and result reranking for optimal AI responses - **Real-time Streaming**: Live responses from AI agents with progress tracking @@ -256,7 +258,7 @@ Archon uses true microservices architecture with clear separation of concerns: | -------------- | -------------------- | ---------------------------- | ------------------------------------------------------------------ | | **Frontend** | `archon-ui-main/` | Web interface and dashboard | React, TypeScript, TailwindCSS, Socket.IO client | | **Server** | `python/src/server/` | Core business logic and APIs | FastAPI, service layer, Socket.IO broadcasts, all ML/AI operations | -| **MCP Server** | `python/src/mcp/` | MCP protocol interface | Lightweight HTTP wrapper, 10 MCP tools, session management | +| **MCP Server** | `python/src/mcp/` | MCP protocol interface | Lightweight HTTP wrapper, MCP tools, session management | | **Agents** | `python/src/agents/` | PydanticAI agent hosting | Document and RAG agents, streaming responses | ### Communication Patterns diff --git a/python/Dockerfile.agents b/python/Dockerfile.agents index b15d60fc..2013bc3c 100644 --- a/python/Dockerfile.agents +++ b/python/Dockerfile.agents @@ -1,13 +1,18 @@ -# Agents Service - Lightweight PydanticAI agents ONLY (no ML models) -FROM python:3.11-slim +# Agents Service - Lightweight Pydantic AI agents +FROM python:3.12-slim WORKDIR /app -# Install dependencies -COPY requirements.agents.txt . -RUN pip install --no-cache-dir -r requirements.agents.txt +# Install uv +RUN pip install --no-cache-dir uv -# Copy ONLY agents code - no dependencies on server code +# Copy pyproject.toml for dependency installation +COPY pyproject.toml . + +# Install only agents dependencies using uv +RUN uv pip install --system --group agents + +# Copy agents code - no dependencies on server code # Agents use MCP tools for all operations COPY src/agents/ src/agents/ COPY src/__init__.py src/ @@ -16,9 +21,6 @@ COPY src/__init__.py src/ ENV PYTHONPATH="/app:$PYTHONPATH" ENV PYTHONUNBUFFERED=1 -# NO ML models in agents container! -# Agents use MCP tools for all ML operations - # Expose Agents port ARG ARCHON_AGENTS_PORT=8052 ENV ARCHON_AGENTS_PORT=${ARCHON_AGENTS_PORT} diff --git a/python/Dockerfile.mcp b/python/Dockerfile.mcp index 310d1154..045da712 100644 --- a/python/Dockerfile.mcp +++ b/python/Dockerfile.mcp @@ -1,20 +1,25 @@ # MCP Service - Lightweight HTTP-based microservice -FROM python:3.11-slim +FROM python:3.12-slim WORKDIR /app -# Install dependencies -COPY requirements.mcp.txt . -RUN pip install --no-cache-dir -r requirements.mcp.txt +# Install uv +RUN pip install --no-cache-dir uv + +# Copy pyproject.toml for dependency installation +COPY pyproject.toml . + +# Install only mcp dependencies using uv +RUN uv pip install --system --group mcp # Create minimal directory structure RUN mkdir -p src/mcp_server/features/projects src/mcp_server/features/tasks src/mcp_server/features/documents src/server/services src/server/config -# Copy only MCP-specific files (lightweight protocol wrapper) +# Copy only MCP-specific files COPY src/mcp_server/ src/mcp_server/ COPY src/__init__.py src/ -# Copy only the minimal server files MCP needs for HTTP communication +# Copy the server files MCP needs for HTTP communication COPY src/server/__init__.py src/server/ COPY src/server/services/__init__.py src/server/services/ COPY src/server/services/mcp_service_client.py src/server/services/ diff --git a/python/Dockerfile.server b/python/Dockerfile.server index 5aa752a4..fe4aaf58 100644 --- a/python/Dockerfile.server +++ b/python/Dockerfile.server @@ -1,21 +1,24 @@ # Server Service - Web crawling and document processing microservice - -# Build stage -FROM python:3.11 AS builder +FROM python:3.12 AS builder WORKDIR /build -# Install build dependencies +# Install build dependencies and uv RUN apt-get update && apt-get install -y \ build-essential \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && pip install --no-cache-dir uv -# Copy and install Python dependencies -COPY requirements.server.txt . -RUN pip install --user --no-cache-dir -r requirements.server.txt +# Copy pyproject.toml for dependency installation +COPY pyproject.toml . + +# Install server dependencies to a virtual environment using uv +RUN uv venv /venv && \ + . /venv/bin/activate && \ + uv pip install --group server # Runtime stage -FROM python:3.11-slim +FROM python:3.12-slim WORKDIR /app @@ -45,11 +48,11 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Copy Python packages from builder -COPY --from=builder /root/.local /root/.local +# Copy the virtual environment from builder +COPY --from=builder /venv /venv # Install Playwright browsers -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/venv/bin:$PATH RUN playwright install chromium # Copy server code and tests @@ -58,8 +61,9 @@ COPY src/__init__.py src/ COPY tests/ tests/ # Set environment variables -ENV PYTHONPATH="/app:/root/.local/lib/python3.11/site-packages:$PYTHONPATH" +ENV PYTHONPATH="/app:$PYTHONPATH" ENV PYTHONUNBUFFERED=1 +ENV PATH="/venv/bin:$PATH" # Expose Server port ARG ARCHON_SERVER_PORT=8181 diff --git a/python/pyproject.toml b/python/pyproject.toml index c1be5ab4..0d32fabb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,64 +1,131 @@ [project] name = "archon" version = "0.1.0" -description = "MCP server for integrating web crawling and RAG into AI agents and AI coding assistants" +description = "Archon - the command center for AI coding assistants." readme = "README.md" requires-python = ">=3.12" -dependencies = [ - "crawl4ai==0.6.2", - "mcp==1.7.1", - "supabase==2.15.1", - "openai==1.71.0", - "dotenv==0.9.9", - "python-dotenv>=1.0.0", - "sentence-transformers>=4.1.0", - "cryptography>=41.0.0", - "asyncpg>=0.29.0", - "pypdf2>=3.0.1", - "python-multipart>=0.0.20", - "pdfplumber>=0.11.6", - "python-docx>=1.1.2", - "markdown>=3.8", - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - "pydantic>=2.0.0", - "python-jose[cryptography]>=3.3.0", - "slowapi>=0.1.9", - "httpx>=0.24.0", - "pydantic-ai>=0.0.13", - "logfire>=0.30.0", - "python-socketio[asyncio]>=5.11.0", - "pytest-asyncio>=1.0.0", - "docker>=7.1.0", -] +# Base dependencies - empty since we're using dependency groups +dependencies = [] -[project.optional-dependencies] -test = [ +[dependency-groups] +# Development dependencies for linting and testing +dev = [ + "mypy>=1.17.0", "pytest>=8.0.0", "pytest-asyncio>=0.21.0", "pytest-mock>=3.12.0", "pytest-timeout>=2.3.0", - "httpx>=0.24.0", - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - "requests>=2.31.0", - "docker>=7.0.0", - "factory-boy>=3.3.0", -] -api = [ - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - "pydantic>=2.0.0", - "python-jose[cryptography]>=3.3.0", - "slowapi>=0.1.9", - "httpx>=0.24.0", -] - -[dependency-groups] -dev = [ - "mypy>=1.17.0", "pytest-cov>=6.2.1", "ruff>=0.12.5", + "requests>=2.31.0", + "factory-boy>=3.3.0", +] + +# Server container dependencies +server = [ + # Web framework + "fastapi>=0.104.0", + "uvicorn>=0.24.0", + "python-multipart>=0.0.20", + "watchfiles>=0.18", + # Web crawling + "crawl4ai==0.6.2", + # Real-time communication + "python-socketio[asyncio]>=5.11.0", + # Database and storage + "supabase==2.15.1", + "asyncpg>=0.29.0", + # AI/ML libraries + "openai==1.71.0", + # Document processing + "pypdf2>=3.0.1", + "pdfplumber>=0.11.6", + "python-docx>=1.1.2", + "markdown>=3.8", + # Security and utilities + "python-jose[cryptography]>=3.3.0", + "cryptography>=41.0.0", + "slowapi>=0.1.9", + # Core utilities + "httpx>=0.24.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", + "docker>=6.1.0", + # Logging + "logfire>=0.30.0", + # Testing (needed for UI-triggered tests) + "pytest>=8.0.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.12.0", +] + +# Optional reranking dependencies for server +server-reranking = [ + "sentence-transformers>=4.1.0", + "torch>=2.0.0", + "transformers>=4.30.0", +] + +# MCP container dependencies +mcp = [ + "mcp==1.12.2", + "httpx>=0.24.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", + "supabase==2.15.1", + "logfire>=0.30.0", + "fastapi>=0.104.0", +] + +# Agents container dependencies +agents = [ + "pydantic-ai>=0.0.13", + "pydantic>=2.0.0", + "fastapi>=0.104.0", + "uvicorn>=0.24.0", + "httpx>=0.24.0", + "python-dotenv>=1.0.0", + "structlog>=23.1.0", +] + +# All dependencies for running unit tests locally +# This combines all container dependencies plus test-specific ones +all = [ + # All server dependencies + "fastapi>=0.104.0", + "uvicorn>=0.24.0", + "python-multipart>=0.0.20", + "watchfiles>=0.18", + "crawl4ai==0.6.2", + "python-socketio[asyncio]>=5.11.0", + "supabase==2.15.1", + "asyncpg>=0.29.0", + "openai==1.71.0", + "pypdf2>=3.0.1", + "pdfplumber>=0.11.6", + "python-docx>=1.1.2", + "markdown>=3.8", + "python-jose[cryptography]>=3.3.0", + "cryptography>=41.0.0", + "slowapi>=0.1.9", + "docker>=6.1.0", + "logfire>=0.30.0", + # MCP specific (mcp version) + "mcp==1.12.2", + # Agents specific + "pydantic-ai>=0.0.13", + "structlog>=23.1.0", + # Shared utilities + "httpx>=0.24.0", + "pydantic>=2.0.0", + "python-dotenv>=1.0.0", + # Test dependencies + "pytest>=8.0.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.12.0", + "pytest-timeout>=2.3.0", + "requests>=2.31.0", + "factory-boy>=3.3.0", ] [tool.ruff] @@ -102,4 +169,4 @@ check_untyped_defs = true # Third-party libraries often don't have type stubs # We'll explicitly type our own code but not fail on external libs -ignore_missing_imports = true +ignore_missing_imports = true \ No newline at end of file diff --git a/python/requirements.agents.txt b/python/requirements.agents.txt deleted file mode 100644 index 371f6dfa..00000000 --- a/python/requirements.agents.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Agents Service Dependencies - ONLY PydanticAI agents -# This container should be lightweight (~200MB) - -# PydanticAI for agent framework -pydantic-ai>=0.0.13 -pydantic>=2.0.0 - -# Web framework for agent server -fastapi>=0.104.0 -uvicorn>=0.24.0 - -# HTTP client for MCP tool calls -httpx>=0.24.0 - -# Basic utilities -python-dotenv>=1.0.0 - -# Logging (lightweight) -structlog>=23.1.0 - -# NOTE: NO ML libraries here! -# - NO sentence-transformers -# - NO OpenAI SDK (agents use MCP tools) -# - NO database clients (agents use MCP tools) -# - NO embeddings libraries -# All ML and data operations happen in the Server service \ No newline at end of file diff --git a/python/requirements.mcp.txt b/python/requirements.mcp.txt deleted file mode 100644 index e98c4c45..00000000 --- a/python/requirements.mcp.txt +++ /dev/null @@ -1,8 +0,0 @@ -# MCP Service Dependencies - Minimal -mcp==1.12.2 -httpx>=0.24.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -supabase==2.15.1 -logfire>=0.30.0 -fastapi>=0.104.0 # Required for mcp tools \ No newline at end of file diff --git a/python/requirements.server.txt b/python/requirements.server.txt deleted file mode 100644 index 4a0956cb..00000000 --- a/python/requirements.server.txt +++ /dev/null @@ -1,47 +0,0 @@ -# Server Service Dependencies -# Web framework -fastapi>=0.104.0 -uvicorn>=0.24.0 -python-multipart>=0.0.20 -watchfiles>=0.18 # For better hot reload performance - -# Web crawling -crawl4ai==0.6.2 - -# Real-time communication -python-socketio[asyncio]>=5.11.0 - -# Database and storage -supabase==2.15.1 -asyncpg>=0.29.0 - -# AI/ML libraries (ALL ML models belong here) -openai==1.71.0 -# sentence-transformers>=4.1.0 # For reranking and advanced embeddings -# torch>=2.0.0 # Required by sentence-transformers -# transformers>=4.30.0 # Required by sentence-transformers - -# Document processing -pypdf2>=3.0.1 -pdfplumber>=0.11.6 -python-docx>=1.1.2 -markdown>=3.8 - -# Security and utilities -python-jose[cryptography]>=3.3.0 -cryptography>=41.0.0 -slowapi>=0.1.9 - -# Core utilities -httpx>=0.24.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -docker>=6.1.0 # For MCP container control - -# Logging -logfire>=0.30.0 - -# Testing (needed for UI-triggered tests) -pytest>=8.0.0 -pytest-asyncio>=0.21.0 -pytest-mock>=3.12.0 \ No newline at end of file