diff --git a/.env.example b/.env.example index 4077e9cd..9647c8fa 100644 --- a/.env.example +++ b/.env.example @@ -53,9 +53,6 @@ VITE_SHOW_DEVTOOLS=false # proxy where you want to expose the frontend on a single external domain. PROD=false -# Embedding Configuration -# Dimensions for embedding vectors (1536 for OpenAI text-embedding-3-small) -EMBEDDING_DIMENSIONS=1536 # NOTE: All other configuration has been moved to database management! # Run the credentials_setup.sql file in your Supabase SQL editor to set up the credentials table. diff --git a/.gitignore b/.gitignore index e9b1084a..96c7a645 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,11 @@ __pycache__ .claude/settings.local.json PRPs/local PRPs/completed/ +PRPs/stories/ /logs/ .zed tmp/ temp/ +UAT/ + +.DS_Store diff --git a/AGENTS.md b/AGENTS.md index 80f4261a..344b43d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,6 @@ -# AGENTS.md +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Beta Development Guidelines @@ -6,9 +8,13 @@ ### Core Principles -- **No backwards compatibility** - remove deprecated code immediately +- **No backwards compatibility; we follow a fix‑forward approach** — remove deprecated code immediately - **Detailed errors over graceful failures** - we want to identify and fix issues fast - **Break things to improve them** - beta is for rapid iteration +- **Continuous improvement** - embrace change and learn from mistakes +- **KISS** - keep it simple +- **DRY** when appropriate +- **YAGNI** — don't implement features that are not needed ### Error Handling @@ -38,51 +44,7 @@ These operations should continue but track and report failures clearly: #### Critical Nuance: Never Accept Corrupted Data -When a process should continue despite failures, it must **skip the failed item entirely** rather than storing corrupted data: - -**❌ WRONG - Silent Corruption:** - -```python -try: - embedding = create_embedding(text) -except Exception as e: - embedding = [0.0] * 1536 # NEVER DO THIS - corrupts database - store_document(doc, embedding) -``` - -**✅ CORRECT - Skip Failed Items:** - -```python -try: - embedding = create_embedding(text) - store_document(doc, embedding) # Only store on success -except Exception as e: - failed_items.append({'doc': doc, 'error': str(e)}) - logger.error(f"Skipping document {doc.id}: {e}") - # Continue with next document, don't store anything -``` - -**✅ CORRECT - Batch Processing with Failure Tracking:** - -```python -def process_batch(items): - results = {'succeeded': [], 'failed': []} - - for item in items: - try: - result = process_item(item) - results['succeeded'].append(result) - except Exception as e: - results['failed'].append({ - 'item': item, - 'error': str(e), - 'traceback': traceback.format_exc() - }) - logger.error(f"Failed to process {item.id}: {e}") - - # Always return both successes and failures - return results -``` +When a process should continue despite failures, it must **skip the failed item entirely** rather than storing corrupted data #### Error Message Guidelines @@ -96,22 +58,11 @@ def process_batch(items): ### Code Quality - Remove dead code immediately rather than maintaining it - no backward compatibility or legacy functions -- Prioritize functionality over production-ready patterns +- Avoid backward compatibility mappings or legacy function wrappers +- Fix forward - Focus on user experience and feature completeness -- When updating code, don't reference what is changing (avoid keywords like LEGACY, CHANGED, REMOVED), instead focus on comments that document just the functionality of the code - -## Architecture Overview - -Archon V2 Beta is a microservices-based knowledge management system with MCP (Model Context Protocol) integration: - -- **Frontend (port 3737)**: React + TypeScript + Vite + TailwindCSS - - **UI Strategy**: Radix UI primitives in `/features`, custom components in legacy `/components` - - **State Management**: TanStack Query for all data fetching in `/features` - - **Styling**: Tron-inspired glassmorphism with Tailwind CSS -- **Main Server (port 8181)**: FastAPI with HTTP polling for updates -- **MCP Server (port 8051)**: Lightweight HTTP-based MCP protocol server -- **Agents Service (port 8052)**: PydanticAI agents for AI/ML operations -- **Database**: Supabase (PostgreSQL + pgvector for embeddings) +- When updating code, don't reference what is changing (avoid keywords like SIMPLIFIED, ENHANCED, LEGACY, CHANGED, REMOVED), instead focus on comments that document just the functionality of the code +- When commenting on code in the codebase, only comment on the functionality and reasoning behind the code. Refrain from speaking to Archon being in "beta" or referencing anything else that comes from these global rules. ## Development Commands @@ -120,210 +71,134 @@ Archon V2 Beta is a microservices-based knowledge management system with MCP (Mo ```bash npm run dev # Start development server on port 3737 npm run build # Build for production -npm run lint # Run ESLint -npm run test # Run Vitest tests -npm run test:coverage # Run tests with coverage report -``` +npm run lint # Run ESLint on legacy code (excludes /features) +npm run lint:files path/to/file.tsx # Lint specific files -# Biome Linter Guide for AI Assistants +# Biome for /src/features directory only +npm run biome # Check features directory +npm run biome:fix # Auto-fix issues +npm run biome:format # Format code (120 char lines) +npm run biome:ai # Machine-readable JSON output for AI +npm run biome:ai-fix # Auto-fix with JSON output -## Overview +# Testing +npm run test # Run all tests in watch mode +npm run test:ui # Run with Vitest UI interface +npm run test:coverage:stream # Run once with streaming output +vitest run src/features/projects # Test specific directory -This project uses Biome for linting and formatting the `/src/features` directory. Biome provides fast, machine-readable feedback that AI assistants can use to improve code quality. - -## Configuration - -Biome is configured in `biome.json`: - -- **Scope**: Only checks `/src/features/**` directory -- **Formatting**: 2 spaces, 80 char line width -- **Linting**: Recommended rules enabled -- **Import Organization**: Automatically sorts and groups imports - -## AI Assistant Workflow in the new /features directory - -1. **Check Issues**: Run `npm run biome:ai` to get JSON output -2. **Parse Output**: Extract error locations and types -3. **Apply Fixes**: - - Run `npm run biome:ai-fix` for auto-fixable issues - - Manually fix remaining issues based on patterns above -4. **Verify**: Run `npm run biome:ai` again to confirm fixes - -## JSON Output Format - -When using `biome:ai`, the output is structured JSON: - -```json -{ - "diagnostics": [ - { - "file": "path/to/file.tsx", - "line": 10, - "column": 5, - "severity": "error", - "message": "Description of the issue", - "rule": "lint/a11y/useButtonType" - } - ] -} +# TypeScript +npx tsc --noEmit # Check all TypeScript errors +npx tsc --noEmit 2>&1 | grep "src/features" # Check features only ``` ### Backend (python/) ```bash -# Using uv package manager -uv sync # Install/update dependencies -uv run pytest # Run tests -uv run python -m src.server.main # Run server locally +# Using uv package manager (preferred) +uv sync --group all # Install all dependencies +uv run python -m src.server.main # Run server locally on 8181 +uv run pytest # Run all tests +uv run pytest tests/test_api_essentials.py -v # Run specific test +uv run ruff check # Run linter +uv run ruff check --fix # Auto-fix linting issues +uv run mypy src/ # Type check -# With Docker -docker-compose up --build -d # Start all services -docker-compose logs -f # View logs -docker-compose restart # Restart services +# Docker operations +docker compose up --build -d # Start all services +docker compose --profile backend up -d # Backend only (for hybrid dev) +docker compose logs -f archon-server # View server logs +docker compose logs -f archon-mcp # View MCP server logs +docker compose restart archon-server # Restart after code changes +docker compose down # Stop all services +docker compose down -v # Stop and remove volumes ``` -### Testing +### Quick Workflows ```bash -# Frontend tests (from archon-ui-main/) -npm run test:coverage:stream # Run with streaming output -npm run test:ui # Run with Vitest UI +# Hybrid development (recommended) - backend in Docker, frontend local +make dev # Or manually: docker compose --profile backend up -d && cd archon-ui-main && npm run dev -# Backend tests (from python/) -uv run pytest tests/test_api_essentials.py -v -uv run pytest tests/test_service_integration.py -v +# Full Docker mode +make dev-docker # Or: docker compose up --build -d + +# Run linters before committing +make lint # Runs both frontend and backend linters +make lint-fe # Frontend only (ESLint + Biome) +make lint-be # Backend only (Ruff + MyPy) + +# Testing +make test # Run all tests +make test-fe # Frontend tests only +make test-be # Backend tests only ``` -## Key API Endpoints +## Architecture Overview -### Knowledge Base +@PRPs/ai_docs/ARCHITECTURE.md -- `POST /api/knowledge/crawl` - Crawl a website -- `POST /api/knowledge/upload` - Upload documents (PDF, DOCX, MD) -- `GET /api/knowledge/items` - List knowledge items -- `POST /api/knowledge/search` - RAG search +#### TanStack Query Implementation -### MCP Integration +For architecture and file references: +@PRPs/ai_docs/DATA_FETCHING_ARCHITECTURE.md -- `GET /api/mcp/health` - MCP server status -- `POST /api/mcp/tools/{tool_name}` - Execute MCP tool -- `GET /api/mcp/tools` - List available tools +For code patterns and examples: +@PRPs/ai_docs/QUERY_PATTERNS.md -### Projects & Tasks (when enabled) +#### Service Layer Pattern -- `GET /api/projects` - List all projects -- `POST /api/projects` - Create project -- `GET /api/projects/{id}` - Get single project -- `PUT /api/projects/{id}` - Update project -- `DELETE /api/projects/{id}` - Delete project -- `GET /api/projects/{id}/tasks` - Get tasks for project (use this, not getTasks) -- `POST /api/tasks` - Create task -- `PUT /api/tasks/{id}` - Update task -- `DELETE /api/tasks/{id}` - Delete task +See implementation examples: -## Polling Architecture +- API routes: `python/src/server/api_routes/projects_api.py` +- Service layer: `python/src/server/services/project_service.py` +- Pattern: API Route → Service → Database -### HTTP Polling (replaced Socket.IO) +#### Error Handling Patterns -- **Polling intervals**: 1-2s for active operations, 5-10s for background data -- **ETag caching**: Reduces bandwidth by ~70% via 304 Not Modified responses -- **Smart pausing**: Stops polling when browser tab is inactive -- **Progress endpoints**: `/api/progress/crawl`, `/api/progress/project-creation` +See implementation examples: -### Key Polling Hooks +- Custom exceptions: `python/src/server/exceptions.py` +- Exception handlers: `python/src/server/main.py` (search for @app.exception_handler) +- Service error handling: `python/src/server/services/` (various services) -- `usePolling` - Generic polling with ETag support -- `useDatabaseMutation` - Optimistic updates with rollback -- `useProjectMutation` - Project-specific operations +## ETag Implementation -## Environment Variables - -Required in `.env`: - -```bash -SUPABASE_URL=https://your-project.supabase.co -SUPABASE_SERVICE_KEY=your-service-key-here -``` - -Optional: - -```bash -OPENAI_API_KEY=your-openai-key # Can be set via UI -LOGFIRE_TOKEN=your-logfire-token # For observability -LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR -``` - -## File Organization - -### Frontend Structure - -- `src/components/` - Legacy UI components (custom-built) -- `src/features/` - Modern vertical slice architecture with Radix UI - - `ui/primitives/` - Radix UI primitives with Tron glassmorphism - - `projects/` - Project management feature - - `tasks/` - Task management sub-feature -- `src/pages/` - Main application pages -- `src/services/` - API communication and business logic -- `src/hooks/` - Custom React hooks -- `src/contexts/` - React context providers - -### UI Libraries - -- **Radix UI** (@radix-ui/react-\*) - Unstyled, accessible primitives for `/features` -- **TanStack Query** - Data fetching and caching for `/features` -- **React DnD** - Drag and drop for Kanban boards -- **Tailwind CSS** - Utility-first styling with Tron-inspired glassmorphism -- **Framer Motion** - Animations (minimal usage) - -### Theme Management - -- **ThemeContext** - Manages light/dark theme state -- **Tailwind dark mode** - Uses `dark:` prefix with selector strategy -- **Automatic switching** - All components respect theme via Tailwind classes -- **Persistent** - Theme choice saved in localStorage -- **Tron aesthetic** - Stronger neon glows in dark mode, subtle in light mode - -We're migrating to a vertical slice architecture where each feature is self-contained. Features are organized by domain hierarchy - main features contain their sub-features. For example, tasks are a sub-feature of projects, so they live at `features/projects/tasks/` rather than as separate siblings. Each feature level has its own components, hooks, types, and services folders. This keeps related code together and makes the codebase more maintainable as it scales. - -### Backend Structure - -- `src/server/` - Main FastAPI application -- `src/server/api_routes/` - API route handlers -- `src/server/services/` - Business logic services -- `src/mcp/` - MCP server implementation -- `src/agents/` - PydanticAI agent implementations +@PRPs/ai_docs/ETAG_IMPLEMENTATION.md ## Database Schema Key tables in Supabase: - `sources` - Crawled websites and uploaded documents + - Stores metadata, crawl status, and configuration - `documents` - Processed document chunks with embeddings + - Text chunks with vector embeddings for semantic search - `projects` - Project management (optional feature) + - Contains features array, documents, and metadata - `tasks` - Task tracking linked to projects + - Status: todo, doing, review, done + - Assignee: User, Archon, AI IDE Agent - `code_examples` - Extracted code snippets + - Language, summary, and relevance metadata ## API Naming Conventions -### Task Status Values +@PRPs/ai_docs/API_NAMING_CONVENTIONS.md -Use database values directly (no UI mapping): +Use database values directly (no mapping in the FE typesafe from BE and up): -- `todo`, `doing`, `review`, `done` +## Environment Variables -### Service Method Patterns +Required in `.env`: -- `get[Resource]sByProject(projectId)` - Scoped queries -- `get[Resource](id)` - Single resource -- `create[Resource](data)` - Create operations -- `update[Resource](id, updates)` - Updates -- `delete[Resource](id)` - Soft deletes +```bash +SUPABASE_URL=https://your-project.supabase.co # Or http://host.docker.internal:8000 for local +SUPABASE_SERVICE_KEY=your-service-key-here # Use legacy key format for cloud Supabase +``` -### State Naming - -- `is[Action]ing` - Loading states (e.g., `isSwitchingProject`) -- `[resource]Error` - Error messages -- `selected[Resource]` - Current selection +Optional variables and full configuration: +See `python/.env.example` for complete list ## Common Development Tasks @@ -332,57 +207,96 @@ Use database values directly (no UI mapping): 1. Create route handler in `python/src/server/api_routes/` 2. Add service logic in `python/src/server/services/` 3. Include router in `python/src/server/main.py` -4. Update frontend service in `archon-ui-main/src/services/` +4. Update frontend service in `archon-ui-main/src/features/[feature]/services/` -### Add a new UI component - -For **features** directory (preferred for new components): +### Add a new UI component in features directory 1. Use Radix UI primitives from `src/features/ui/primitives/` -2. Create component in relevant feature folder under `src/features/` -3. Use TanStack Query for data fetching -4. Apply Tron-inspired glassmorphism styling with Tailwind +2. Create component in relevant feature folder under `src/features/[feature]/components/` +3. Define types in `src/features/[feature]/types/` +4. Use TanStack Query hook from `src/features/[feature]/hooks/` +5. Apply Tron-inspired glassmorphism styling with Tailwind -For **legacy** components: +### Add or modify MCP tools -1. Create component in `archon-ui-main/src/components/` -2. Add to page in `archon-ui-main/src/pages/` -3. Include any new API calls in services -4. Add tests in `archon-ui-main/test/` +1. MCP tools are in `python/src/mcp_server/features/[feature]/[feature]_tools.py` +2. Follow the pattern: + - `find_[resource]` - Handles list, search, and get single item operations + - `manage_[resource]` - Handles create, update, delete with an "action" parameter +3. Register tools in the feature's `__init__.py` file ### Debug MCP connection issues 1. Check MCP health: `curl http://localhost:8051/health` -2. View MCP logs: `docker-compose logs archon-mcp` +2. View MCP logs: `docker compose logs archon-mcp` 3. Test tool execution via UI MCP page 4. Verify Supabase connection and credentials +### Fix TypeScript/Linting Issues + +```bash +# TypeScript errors in features +npx tsc --noEmit 2>&1 | grep "src/features" + +# Biome auto-fix for features +npm run biome:fix + +# ESLint for legacy code +npm run lint:files src/components/SomeComponent.tsx +``` + ## Code Quality Standards -We enforce code quality through automated linting and type checking: +### Frontend + +- **TypeScript**: Strict mode enabled, no implicit any +- **Biome** for `/src/features/`: 120 char lines, double quotes, trailing commas +- **ESLint** for legacy code: Standard React rules +- **Testing**: Vitest with React Testing Library + +### Backend - **Python 3.12** with 120 character line length -- **Ruff** for linting - checks for errors, warnings, unused imports, and code style -- **Mypy** for type checking - ensures type safety across the codebase -- **Auto-formatting** on save in IDEs to maintain consistent style -- Run `uv run ruff check` and `uv run mypy src/` locally before committing +- **Ruff** for linting - checks for errors, warnings, unused imports +- **Mypy** for type checking - ensures type safety +- **Pytest** for testing with async support ## MCP Tools Available -When connected to Cursor/Windsurf: +When connected to Claude/Cursor/Windsurf, the following tools are available: -- `archon:perform_rag_query` - Search knowledge base -- `archon:search_code_examples` - Find code snippets -- `archon:manage_project` - Project operations -- `archon:manage_task` - Task management -- `archon:get_available_sources` - List knowledge sources +### Knowledge Base Tools + +- `archon:rag_search_knowledge_base` - Search knowledge base for relevant content +- `archon:rag_search_code_examples` - Find code snippets in the knowledge base +- `archon:rag_get_available_sources` - List available knowledge sources + +### Project Management + +- `archon:find_projects` - Find all projects, search, or get specific project (by project_id) +- `archon:manage_project` - Manage projects with actions: "create", "update", "delete" + +### Task Management + +- `archon:find_tasks` - Find tasks with search, filters, or get specific task (by task_id) +- `archon:manage_task` - Manage tasks with actions: "create", "update", "delete" + +### Document Management + +- `archon:find_documents` - Find documents, search, or get specific document (by document_id) +- `archon:manage_document` - Manage documents with actions: "create", "update", "delete" + +### Version Control + +- `archon:find_versions` - Find version history or get specific version +- `archon:manage_version` - Manage versions with actions: "create", "restore" ## Important Notes - Projects feature is optional - toggle in Settings UI -- All services communicate via HTTP, not gRPC -- HTTP polling handles all updates (Socket.IO removed) +- HTTP polling handles all updates - Frontend uses Vite proxy for API calls in development - Python backend uses `uv` for dependency management - Docker Compose handles service orchestration -- we use tanstack query NO PROP DRILLING! refacring in progress! +- TanStack Query for all data fetching - NO PROP DRILLING +- Vertical slice architecture in `/features` - features own their sub-features diff --git a/CLAUDE.md b/CLAUDE.md index f147d077..77673db7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,9 +8,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ### Core Principles -- **No backwards compatibility** - remove deprecated code immediately +- **No backwards compatibility; we follow a fix‑forward approach** — remove deprecated code immediately - **Detailed errors over graceful failures** - we want to identify and fix issues fast - **Break things to improve them** - beta is for rapid iteration +- **Continuous improvement** - embrace change and learn from mistakes +- **KISS** - keep it simple +- **DRY** when appropriate +- **YAGNI** — don't implement features that are not needed ### Error Handling @@ -40,51 +44,7 @@ These operations should continue but track and report failures clearly: #### Critical Nuance: Never Accept Corrupted Data -When a process should continue despite failures, it must **skip the failed item entirely** rather than storing corrupted data: - -**❌ WRONG - Silent Corruption:** - -```python -try: - embedding = create_embedding(text) -except Exception as e: - embedding = [0.0] * 1536 # NEVER DO THIS - corrupts database - store_document(doc, embedding) -``` - -**✅ CORRECT - Skip Failed Items:** - -```python -try: - embedding = create_embedding(text) - store_document(doc, embedding) # Only store on success -except Exception as e: - failed_items.append({'doc': doc, 'error': str(e)}) - logger.error(f"Skipping document {doc.id}: {e}") - # Continue with next document, don't store anything -``` - -**✅ CORRECT - Batch Processing with Failure Tracking:** - -```python -def process_batch(items): - results = {'succeeded': [], 'failed': []} - - for item in items: - try: - result = process_item(item) - results['succeeded'].append(result) - except Exception as e: - results['failed'].append({ - 'item': item, - 'error': str(e), - 'traceback': traceback.format_exc() - }) - logger.error(f"Failed to process {item.id}: {e}") - - # Always return both successes and failures - return results -``` +When a process should continue despite failures, it must **skip the failed item entirely** rather than storing corrupted data #### Error Message Guidelines @@ -98,9 +58,10 @@ def process_batch(items): ### Code Quality - Remove dead code immediately rather than maintaining it - no backward compatibility or legacy functions -- Prioritize functionality over production-ready patterns +- Avoid backward compatibility mappings or legacy function wrappers +- Fix forward - Focus on user experience and feature completeness -- When updating code, don't reference what is changing (avoid keywords like LEGACY, CHANGED, REMOVED), instead focus on comments that document just the functionality of the code +- When updating code, don't reference what is changing (avoid keywords like SIMPLIFIED, ENHANCED, LEGACY, CHANGED, REMOVED), instead focus on comments that document just the functionality of the code - When commenting on code in the codebase, only comment on the functionality and reasoning behind the code. Refrain from speaking to Archon being in "beta" or referencing anything else that comes from these global rules. ## Development Commands @@ -175,139 +136,33 @@ make test-be # Backend tests only ## Architecture Overview -Archon Beta is a microservices-based knowledge management system with MCP (Model Context Protocol) integration: +@PRPs/ai_docs/ARCHITECTURE.md -### Service Architecture +#### TanStack Query Implementation -- **Frontend (port 3737)**: React + TypeScript + Vite + TailwindCSS - - **Dual UI Strategy**: - - `/features` - Modern vertical slice with Radix UI primitives + TanStack Query - - `/components` - Legacy custom components (being migrated) - - **State Management**: TanStack Query for all data fetching (no prop drilling) - - **Styling**: Tron-inspired glassmorphism with Tailwind CSS - - **Linting**: Biome for `/features`, ESLint for legacy code +For architecture and file references: +@PRPs/ai_docs/DATA_FETCHING_ARCHITECTURE.md -- **Main Server (port 8181)**: FastAPI with HTTP polling for updates - - Handles all business logic, database operations, and external API calls - - WebSocket support removed in favor of HTTP polling with ETag caching - -- **MCP Server (port 8051)**: Lightweight HTTP-based MCP protocol server - - Provides tools for AI assistants (Claude, Cursor, Windsurf) - - Exposes knowledge search, task management, and project operations - -- **Agents Service (port 8052)**: PydanticAI agents for AI/ML operations - - Handles complex AI workflows and document processing - -- **Database**: Supabase (PostgreSQL + pgvector for embeddings) - - Cloud or local Supabase both supported - - pgvector for semantic search capabilities - -### Frontend Architecture Details - -#### Vertical Slice Architecture (/features) - -Features are organized by domain hierarchy with self-contained modules: - -``` -src/features/ -├── ui/ -│ ├── primitives/ # Radix UI base components -│ ├── hooks/ # Shared UI hooks (useSmartPolling, etc) -│ └── types/ # UI type definitions -├── projects/ -│ ├── components/ # Project UI components -│ ├── hooks/ # Project hooks (useProjectQueries, etc) -│ ├── services/ # Project API services -│ ├── types/ # Project type definitions -│ ├── tasks/ # Tasks sub-feature (nested under projects) -│ │ ├── components/ -│ │ ├── hooks/ # Task-specific hooks -│ │ ├── services/ # Task API services -│ │ └── types/ -│ └── documents/ # Documents sub-feature -│ ├── components/ -│ ├── services/ -│ └── types/ -``` - -#### TanStack Query Patterns - -All data fetching uses TanStack Query with consistent patterns: - -```typescript -// Query keys factory pattern -export const projectKeys = { - all: ["projects"] as const, - lists: () => [...projectKeys.all, "list"] as const, - detail: (id: string) => [...projectKeys.all, "detail", id] as const, -}; - -// Smart polling with visibility awareness -const { refetchInterval } = useSmartPolling(10000); // Pauses when tab inactive - -// Optimistic updates with rollback -useMutation({ - onMutate: async (data) => { - await queryClient.cancelQueries(key); - const previous = queryClient.getQueryData(key); - queryClient.setQueryData(key, optimisticData); - return { previous }; - }, - onError: (err, vars, context) => { - if (context?.previous) { - queryClient.setQueryData(key, context.previous); - } - }, -}); -``` - -### Backend Architecture Details +For code patterns and examples: +@PRPs/ai_docs/QUERY_PATTERNS.md #### Service Layer Pattern -```python -# API Route -> Service -> Database -# src/server/api_routes/projects.py -@router.get("/{project_id}") -async def get_project(project_id: str): - return await project_service.get_project(project_id) - -# src/server/services/project_service.py -async def get_project(project_id: str): - # Business logic here - return await db.fetch_project(project_id) -``` +See implementation examples: +- API routes: `python/src/server/api_routes/projects_api.py` +- Service layer: `python/src/server/services/project_service.py` +- Pattern: API Route → Service → Database #### Error Handling Patterns -```python -# Use specific exceptions -class ProjectNotFoundError(Exception): pass -class ValidationError(Exception): pass +See implementation examples: +- Custom exceptions: `python/src/server/exceptions.py` +- Exception handlers: `python/src/server/main.py` (search for @app.exception_handler) +- Service error handling: `python/src/server/services/` (various services) -# Rich error responses -@app.exception_handler(ProjectNotFoundError) -async def handle_not_found(request, exc): - return JSONResponse( - status_code=404, - content={"detail": str(exc), "type": "not_found"} - ) -``` +## ETag Implementation -## Polling Architecture - -### HTTP Polling (replaced Socket.IO) - -- **Polling intervals**: 1-2s for active operations, 5-10s for background data -- **ETag caching**: Reduces bandwidth by ~70% via 304 Not Modified responses -- **Smart pausing**: Stops polling when browser tab is inactive -- **Progress endpoints**: `/api/progress/{id}` for operation tracking - -### Key Polling Hooks - -- `useSmartPolling` - Adjusts interval based on page visibility/focus -- `useCrawlProgressPolling` - Specialized for crawl progress with auto-cleanup -- `useProjectTasks` - Smart polling for task lists +@PRPs/ai_docs/ETAG_IMPLEMENTATION.md ## Database Schema @@ -327,25 +182,9 @@ Key tables in Supabase: ## API Naming Conventions -### Task Status Values +@PRPs/ai_docs/API_NAMING_CONVENTIONS.md -Use database values directly (no UI mapping): - -- `todo`, `doing`, `review`, `done` - -### Service Method Patterns - -- `get[Resource]sByProject(projectId)` - Scoped queries -- `get[Resource](id)` - Single resource -- `create[Resource](data)` - Create operations -- `update[Resource](id, updates)` - Updates -- `delete[Resource](id)` - Soft deletes - -### State Naming - -- `is[Action]ing` - Loading states (e.g., `isSwitchingProject`) -- `[resource]Error` - Error messages -- `selected[Resource]` - Current selection +Use database values directly (no FE mapping; type‑safe end‑to‑end from BE upward): ## Environment Variables @@ -356,15 +195,8 @@ SUPABASE_URL=https://your-project.supabase.co # Or http://host.docker.internal: SUPABASE_SERVICE_KEY=your-service-key-here # Use legacy key format for cloud Supabase ``` -Optional: - -```bash -LOGFIRE_TOKEN=your-logfire-token # For observability -LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR -ARCHON_SERVER_PORT=8181 # Server port -ARCHON_MCP_PORT=8051 # MCP server port -ARCHON_UI_PORT=3737 # Frontend port -``` +Optional variables and full configuration: +See `python/.env.example` for complete list ## Common Development Tasks @@ -383,6 +215,14 @@ ARCHON_UI_PORT=3737 # Frontend port 4. Use TanStack Query hook from `src/features/[feature]/hooks/` 5. Apply Tron-inspired glassmorphism styling with Tailwind +### Add or modify MCP tools + +1. MCP tools are in `python/src/mcp_server/features/[feature]/[feature]_tools.py` +2. Follow the pattern: + - `find_[resource]` - Handles list, search, and get single item operations + - `manage_[resource]` - Handles create, update, delete with an "action" parameter +3. Register tools in the feature's `__init__.py` file + ### Debug MCP connection issues 1. Check MCP health: `curl http://localhost:8051/health` @@ -421,22 +261,38 @@ npm run lint:files src/components/SomeComponent.tsx ## MCP Tools Available -When connected to Client/Cursor/Windsurf: +When connected to Claude/Cursor/Windsurf, the following tools are available: -- `archon:perform_rag_query` - Search knowledge base -- `archon:search_code_examples` - Find code snippets -- `archon:create_project` - Create new project -- `archon:list_projects` - List all projects -- `archon:create_task` - Create task in project -- `archon:list_tasks` - List and filter tasks -- `archon:update_task` - Update task status/details -- `archon:get_available_sources` - List knowledge sources +### Knowledge Base Tools + +- `archon:rag_search_knowledge_base` - Search knowledge base for relevant content +- `archon:rag_search_code_examples` - Find code snippets in the knowledge base +- `archon:rag_get_available_sources` - List available knowledge sources + +### Project Management + +- `archon:find_projects` - Find all projects, search, or get specific project (by project_id) +- `archon:manage_project` - Manage projects with actions: "create", "update", "delete" + +### Task Management + +- `archon:find_tasks` - Find tasks with search, filters, or get specific task (by task_id) +- `archon:manage_task` - Manage tasks with actions: "create", "update", "delete" + +### Document Management + +- `archon:find_documents` - Find documents, search, or get specific document (by document_id) +- `archon:manage_document` - Manage documents with actions: "create", "update", "delete" + +### Version Control + +- `archon:find_versions` - Find version history or get specific version +- `archon:manage_version` - Manage versions with actions: "create", "restore" ## Important Notes - Projects feature is optional - toggle in Settings UI -- All services communicate via HTTP, not gRPC -- HTTP polling handles all updates +- TanStack Query handles all data fetching; smart HTTP polling is used where appropriate (no WebSockets) - Frontend uses Vite proxy for API calls in development - Python backend uses `uv` for dependency management - Docker Compose handles service orchestration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47be2e31..9c2f0c6d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,7 +149,7 @@ Test these things using both the UI and the MCP server. This process will be sim - This creates your own copy of the repository ```bash - # Clone your fork (replace 'your-username' with your GitHub username) + # Clone your fork from main branch for contributing (replace 'your-username' with your GitHub username) git clone https://github.com/your-username/archon.git cd archon @@ -157,6 +157,8 @@ Test these things using both the UI and the MCP server. This process will be sim git remote add upstream https://github.com/coleam00/archon.git ``` + **Note:** The `main` branch is used for contributions and contains the latest development work. The `stable` branch is for users who want a more tested, stable version of Archon. + 2. **🤖 AI Coding Assistant Setup** **IMPORTANT**: If you're using AI coding assistants to help contribute to Archon, set up our global rules for optimal results. @@ -169,7 +171,7 @@ Test these things using both the UI and the MCP server. This process will be sim 3. **Create Feature Branch** - **Best Practice**: Always create a feature branch rather than working directly on main. This keeps your main branch clean and makes it easier to sync with the upstream repository. + **Best Practice**: Always create a feature branch from main rather than working directly on it. This keeps your main branch clean and makes it easier to sync with the upstream repository. ```bash git checkout -b feature/your-feature-name diff --git a/PRPs/ai_docs/API_NAMING_CONVENTIONS.md b/PRPs/ai_docs/API_NAMING_CONVENTIONS.md index 82a97dfb..5688912b 100644 --- a/PRPs/ai_docs/API_NAMING_CONVENTIONS.md +++ b/PRPs/ai_docs/API_NAMING_CONVENTIONS.md @@ -1,163 +1,249 @@ # API Naming Conventions ## Overview -This document defines the naming conventions used throughout the Archon V2 codebase for consistency and clarity. -## Task Status Values -**Database values only - no UI mapping:** -- `todo` - Task is in backlog/todo state -- `doing` - Task is actively being worked on -- `review` - Task is pending review -- `done` - Task is completed +This document describes the actual naming conventions used throughout Archon's codebase based on current implementation patterns. All examples reference real files where these patterns are implemented. -## Service Method Naming +## Backend API Endpoints -### Project Service (`projectService.ts`) +### RESTful Route Patterns +**Reference**: `python/src/server/api_routes/projects_api.py` -#### Projects +Standard REST patterns used: +- `GET /api/{resource}` - List all resources +- `POST /api/{resource}` - Create new resource +- `GET /api/{resource}/{id}` - Get single resource +- `PUT /api/{resource}/{id}` - Update resource +- `DELETE /api/{resource}/{id}` - Delete resource + +Nested resource patterns: +- `GET /api/projects/{project_id}/tasks` - Tasks scoped to project +- `GET /api/projects/{project_id}/docs` - Documents scoped to project +- `POST /api/projects/{project_id}/versions` - Create version for project + +### Actual Endpoint Examples +From `python/src/server/api_routes/`: + +**Projects** (`projects_api.py`): +- `/api/projects` - Project CRUD +- `/api/projects/{project_id}/features` - Get project features +- `/api/projects/{project_id}/tasks` - Project-scoped tasks +- `/api/projects/{project_id}/docs` - Project documents +- `/api/projects/{project_id}/versions` - Version history + +**Knowledge** (`knowledge_api.py`): +- `/api/knowledge/sources` - Knowledge sources +- `/api/knowledge/crawl` - Start web crawl +- `/api/knowledge/upload` - Upload document +- `/api/knowledge/search` - RAG search +- `/api/knowledge/code-search` - Code-specific search + +**Progress** (`progress_api.py`): +- `/api/progress/active` - Active operations +- `/api/progress/{operation_id}` - Specific operation status + +**MCP** (`mcp_api.py`): +- `/api/mcp/status` - MCP server status +- `/api/mcp/execute` - Execute MCP tool + +## Frontend Service Methods + +### Service Object Pattern +**Reference**: `archon-ui-main/src/features/projects/services/projectService.ts` + +Services are exported as objects with async methods: +```typescript +export const serviceNameService = { + async methodName(): Promise { ... } +} +``` + +### Standard Service Method Names +Actual patterns from service files: + +**List Operations**: - `listProjects()` - Get all projects -- `getProject(projectId)` - Get single project by ID -- `createProject(projectData)` - Create new project -- `updateProject(projectId, updates)` - Update project -- `deleteProject(projectId)` - Delete project +- `getTasksByProject(projectId)` - Get filtered list +- `getTasksByStatus(status)` - Get by specific criteria -#### Tasks -- `getTasksByProject(projectId)` - Get all tasks for a specific project -- `getTask(taskId)` - Get single task by ID -- `createTask(taskData)` - Create new task -- `updateTask(taskId, updates)` - Update task with partial data -- `updateTaskStatus(taskId, status)` - Update only task status -- `updateTaskOrder(taskId, newOrder, newStatus?)` - Update task position/order -- `deleteTask(taskId)` - Delete task (soft delete/archive) -- `getTasksByStatus(status)` - Get all tasks with specific status +**Single Item Operations**: +- `getProject(projectId)` - Get single item +- `getTask(taskId)` - Direct ID access -#### Documents -- `getDocuments(projectId)` - Get all documents for project -- `getDocument(projectId, docId)` - Get single document -- `createDocument(projectId, documentData)` - Create document -- `updateDocument(projectId, docId, updates)` - Update document -- `deleteDocument(projectId, docId)` - Delete document +**Create Operations**: +- `createProject(data)` - Returns created entity +- `createTask(data)` - Includes server-generated fields -#### Versions -- `createVersion(projectId, field, content)` - Create version snapshot -- `listVersions(projectId, fieldName?)` - List version history -- `getVersion(projectId, fieldName, versionNumber)` - Get specific version -- `restoreVersion(projectId, fieldName, versionNumber)` - Restore version +**Update Operations**: +- `updateProject(id, updates)` - Partial updates +- `updateTaskStatus(id, status)` - Specific field update +- `updateTaskOrder(id, order, status?)` - Complex updates -## API Endpoint Patterns +**Delete Operations**: +- `deleteProject(id)` - Returns void +- `deleteTask(id)` - Soft delete pattern -### RESTful Endpoints -``` -GET /api/projects - List all projects -POST /api/projects - Create project -GET /api/projects/{project_id} - Get project -PUT /api/projects/{project_id} - Update project -DELETE /api/projects/{project_id} - Delete project +### Service File Locations +- **Projects**: `archon-ui-main/src/features/projects/services/projectService.ts` +- **Tasks**: `archon-ui-main/src/features/projects/tasks/services/taskService.ts` +- **Knowledge**: `archon-ui-main/src/features/knowledge/services/knowledgeService.ts` +- **Progress**: `archon-ui-main/src/features/progress/services/progressService.ts` -GET /api/projects/{project_id}/tasks - Get project tasks -POST /api/tasks - Create task (project_id in body) -GET /api/tasks/{task_id} - Get task -PUT /api/tasks/{task_id} - Update task -DELETE /api/tasks/{task_id} - Delete task +## React Hook Naming -GET /api/projects/{project_id}/docs - Get project documents -POST /api/projects/{project_id}/docs - Create document -GET /api/projects/{project_id}/docs/{doc_id} - Get document -PUT /api/projects/{project_id}/docs/{doc_id} - Update document -DELETE /api/projects/{project_id}/docs/{doc_id} - Delete document -``` +### Query Hooks +**Reference**: `archon-ui-main/src/features/projects/tasks/hooks/useTaskQueries.ts` -### Progress/Polling Endpoints -``` -GET /api/progress/{operation_id} - Generic operation progress -GET /api/knowledge/crawl-progress/{id} - Crawling progress -GET /api/agent-chat/sessions/{id}/messages - Chat messages -``` +Standard patterns: +- `use[Resource]()` - List query (e.g., `useProjects`) +- `use[Resource]Detail(id)` - Single item query +- `use[Parent][Resource](parentId)` - Scoped query (e.g., `useProjectTasks`) + +### Mutation Hooks +- `useCreate[Resource]()` - Creation mutation +- `useUpdate[Resource]()` - Update mutation +- `useDelete[Resource]()` - Deletion mutation + +### Utility Hooks +**Reference**: `archon-ui-main/src/features/ui/hooks/` +- `useSmartPolling()` - Visibility-aware polling +- `useToast()` - Toast notifications +- `useDebounce()` - Debounced values + +## Type Naming Conventions + +### Type Definition Patterns +**Reference**: `archon-ui-main/src/features/projects/types/` + +**Entity Types**: +- `Project` - Core entity type +- `Task` - Business object +- `Document` - Data model + +**Request/Response Types**: +- `Create[Entity]Request` - Creation payload +- `Update[Entity]Request` - Update payload +- `[Entity]Response` - API response wrapper + +**Database Types**: +- `DatabaseTaskStatus` - Exact database values +**Location**: `archon-ui-main/src/features/projects/tasks/types/task.ts` +Values: `"todo" | "doing" | "review" | "done"` + +### Type File Organization +Following vertical slice architecture: +- Core types in `{feature}/types/` +- Sub-feature types in `{feature}/{subfeature}/types/` +- Shared types in `shared/types/` + +## Query Key Factories + +**Reference**: Each feature's `hooks/use{Feature}Queries.ts` file + +Standard factory pattern: +- `{resource}Keys.all` - Base key for invalidation +- `{resource}Keys.lists()` - List queries +- `{resource}Keys.detail(id)` - Single item queries +- `{resource}Keys.byProject(projectId)` - Scoped queries + +Examples: +- `projectKeys` - Projects domain +- `taskKeys` - Tasks (dual nature: global and project-scoped) +- `knowledgeKeys` - Knowledge base +- `progressKeys` - Progress tracking +- `documentKeys` - Document management ## Component Naming -### Hooks -- `use[Feature]` - Custom hooks (e.g., `usePolling`, `useProjectMutation`) -- Returns object with: `{ data, isLoading, error, refetch }` +### Page Components +**Location**: `archon-ui-main/src/pages/` +- `[Feature]Page.tsx` - Top-level pages +- `[Feature]View.tsx` - Main view components -### Services -- `[feature]Service` - Service modules (e.g., `projectService`, `crawlProgressService`) -- Methods return Promises with typed responses +### Feature Components +**Location**: `archon-ui-main/src/features/{feature}/components/` +- `[Entity]Card.tsx` - Card displays +- `[Entity]List.tsx` - List containers +- `[Entity]Form.tsx` - Form components +- `New[Entity]Modal.tsx` - Creation modals +- `Edit[Entity]Modal.tsx` - Edit modals -### Components -- `[Feature][Type]` - UI components (e.g., `TaskBoardView`, `EditTaskModal`) -- Props interfaces: `[Component]Props` +### Shared Components +**Location**: `archon-ui-main/src/features/ui/primitives/` +- Radix UI-based primitives +- Generic, reusable components ## State Variable Naming ### Loading States -- `isLoading[Feature]` - Boolean loading indicators -- `isSwitchingProject` - Specific operation states -- `movingTaskIds` - Set/Array of items being processed +**Examples from**: `archon-ui-main/src/features/projects/views/ProjectsView.tsx` +- `isLoading` - Generic loading +- `is[Action]ing` - Specific operations (e.g., `isSwitchingProject`) +- `[action]ingIds` - Sets of items being processed ### Error States -- `[feature]Error` - Error message strings -- `taskOperationError` - Specific operation errors +- `error` - Query errors +- `[operation]Error` - Specific operation errors -### Data States -- `[feature]s` - Plural for collections (e.g., `tasks`, `projects`) -- `selected[Feature]` - Currently selected item -- `[feature]Data` - Raw data from API +### Selection States +- `selected[Entity]` - Currently selected item +- `active[Entity]Id` - Active item ID -## Type Definitions +## Constants and Enums -### Database Types (from backend) -```typescript -type DatabaseTaskStatus = 'todo' | 'doing' | 'review' | 'done'; -type Assignee = 'User' | 'Archon' | 'AI IDE Agent'; -``` +### Status Values +**Location**: `archon-ui-main/src/features/projects/tasks/types/task.ts` +Database values used directly - no mapping layers: +- Task statuses: `"todo"`, `"doing"`, `"review"`, `"done"` +- Operation statuses: `"pending"`, `"processing"`, `"completed"`, `"failed"` -### Request/Response Types -```typescript -Create[Feature]Request // e.g., CreateTaskRequest -Update[Feature]Request // e.g., UpdateTaskRequest -[Feature]Response // e.g., TaskResponse -``` +### Time Constants +**Location**: `archon-ui-main/src/features/shared/queryPatterns.ts` +- `STALE_TIMES.instant` - 0ms +- `STALE_TIMES.realtime` - 3 seconds +- `STALE_TIMES.frequent` - 5 seconds +- `STALE_TIMES.normal` - 30 seconds +- `STALE_TIMES.rare` - 5 minutes +- `STALE_TIMES.static` - Infinity -## Function Naming Patterns +## File Naming Patterns -### Event Handlers -- `handle[Event]` - Generic handlers (e.g., `handleProjectSelect`) -- `on[Event]` - Props callbacks (e.g., `onTaskMove`, `onRefresh`) +### Service Layer +- `{feature}Service.ts` - Service modules +- Use lower camelCase with "Service" suffix (e.g., `projectService.ts`) -### Operations -- `load[Feature]` - Fetch data (e.g., `loadTasksForProject`) -- `save[Feature]` - Persist changes (e.g., `saveTask`) -- `delete[Feature]` - Remove items (e.g., `deleteTask`) -- `refresh[Feature]` - Reload data (e.g., `refreshTasks`) +### Hook Files +- `use{Feature}Queries.ts` - Query hooks and keys +- `use{Feature}.ts` - Feature-specific hooks -### Formatting/Transformation -- `format[Feature]` - Format for display (e.g., `formatTask`) -- `validate[Feature]` - Validate data (e.g., `validateUpdateTask`) +### Type Files +- `index.ts` - Barrel exports +- `{entity}.ts` - Specific entity types + +### Test Files +- `{filename}.test.ts` - Unit tests +- Located in `tests/` subdirectories ## Best Practices -### ✅ Do Use -- `getTasksByProject(projectId)` - Clear scope with context -- `status` - Single source of truth from database -- Direct database values everywhere (no mapping) -- Polling with `usePolling` hook for data fetching -- Async/await with proper error handling -- ETag headers for efficient polling -- Loading indicators during operations +### Do Follow +- Use exact database values (no translation layers) +- Keep consistent patterns within features +- Use query key factories for all cache operations +- Follow vertical slice architecture +- Reference shared constants -## Current Architecture Patterns +### Don't Do +- Don't create mapping layers for database values +- Don't hardcode time values +- Don't mix query keys between features +- Don't use inconsistent naming within a feature +- Don't embed business logic in components -### Polling & Data Fetching -- HTTP polling with `usePolling` and `useCrawlProgressPolling` hooks -- ETag-based caching for bandwidth efficiency -- Loading state indicators (`isLoading`, `isSwitchingProject`) -- Error toast notifications for user feedback -- Manual refresh triggers via `refetch()` -- Immediate UI updates followed by API calls +## Common Patterns Reference -### Service Architecture -- Specialized services for different domains (`projectService`, `crawlProgressService`) -- Direct database value usage (no UI/DB mapping) -- Promise-based async operations -- Typed request/response interfaces \ No newline at end of file +For implementation examples, see: +- Query patterns: Any `use{Feature}Queries.ts` file +- Service patterns: Any `{feature}Service.ts` file +- Type patterns: Any `{feature}/types/` directory +- Component patterns: Any `{feature}/components/` directory \ No newline at end of file diff --git a/PRPs/ai_docs/ARCHITECTURE.md b/PRPs/ai_docs/ARCHITECTURE.md index 04494b39..a5c0ae7a 100644 --- a/PRPs/ai_docs/ARCHITECTURE.md +++ b/PRPs/ai_docs/ARCHITECTURE.md @@ -2,480 +2,194 @@ ## Overview -Archon follows a **Vertical Slice Architecture** pattern where features are organized by business capability rather than technical layers. Each module is self-contained with its own API, business logic, and data access, making the system modular, maintainable, and ready for future microservice extraction if needed. +Archon is a knowledge management system with AI capabilities, built as a monolithic application with vertical slice organization. The frontend uses React with TanStack Query, while the backend runs FastAPI with multiple service components. -## Core Principles +## Tech Stack -1. **Feature Cohesion**: All code for a feature lives together -2. **Module Independence**: Modules communicate through well-defined interfaces -3. **Vertical Slices**: Each feature contains its complete stack (API → Service → Repository) -4. **Shared Minimal**: Only truly cross-cutting concerns go in shared -5. **Migration Ready**: Structure supports easy extraction to microservices +**Frontend**: React 18, TypeScript 5, TanStack Query v5, Tailwind CSS, Vite +**Backend**: Python 3.12, FastAPI, Supabase, PydanticAI +**Infrastructure**: Docker, PostgreSQL + pgvector ## Directory Structure -``` -archon/ -├── python/ -│ ├── src/ -│ │ ├── knowledge/ # Knowledge Management Module -│ │ │ ├── __init__.py -│ │ │ ├── main.py # Knowledge module entry point -│ │ │ ├── shared/ # Shared within knowledge context -│ │ │ │ ├── models.py -│ │ │ │ ├── exceptions.py -│ │ │ │ └── utils.py -│ │ │ └── features/ # Knowledge feature slices -│ │ │ ├── crawling/ # Web crawling feature -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Crawl endpoints -│ │ │ │ ├── service.py # Crawling orchestration -│ │ │ │ ├── models.py # Crawl-specific models -│ │ │ │ ├── repository.py # Crawl data storage -│ │ │ │ └── tests/ -│ │ │ ├── document_processing/ # Document upload & processing -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Upload endpoints -│ │ │ │ ├── service.py # PDF/DOCX processing -│ │ │ │ ├── extractors.py # Text extraction -│ │ │ │ └── tests/ -│ │ │ ├── embeddings/ # Vector embeddings -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Embedding endpoints -│ │ │ │ ├── service.py # OpenAI/local embeddings -│ │ │ │ ├── models.py -│ │ │ │ └── repository.py # Vector storage -│ │ │ ├── search/ # RAG search -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Search endpoints -│ │ │ │ ├── service.py # Search algorithms -│ │ │ │ ├── reranker.py # Result reranking -│ │ │ │ └── tests/ -│ │ │ ├── code_extraction/ # Code snippet extraction -│ │ │ │ ├── __init__.py -│ │ │ │ ├── service.py # Code parsing -│ │ │ │ ├── analyzers.py # Language detection -│ │ │ │ └── repository.py -│ │ │ └── source_management/ # Knowledge source CRUD -│ │ │ ├── __init__.py -│ │ │ ├── api.py -│ │ │ ├── service.py -│ │ │ └── repository.py -│ │ │ -│ │ ├── projects/ # Project Management Module -│ │ │ ├── __init__.py -│ │ │ ├── main.py # Projects module entry point -│ │ │ ├── shared/ # Shared within projects context -│ │ │ │ ├── database.py # Project DB utilities -│ │ │ │ ├── models.py # Shared project models -│ │ │ │ └── exceptions.py # Project-specific exceptions -│ │ │ └── features/ # Project feature slices -│ │ │ ├── project_management/ # Project CRUD -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Project endpoints -│ │ │ │ ├── service.py # Project business logic -│ │ │ │ ├── models.py # Project models -│ │ │ │ ├── repository.py # Project DB operations -│ │ │ │ └── tests/ -│ │ │ ├── task_management/ # Task CRUD -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Task endpoints -│ │ │ │ ├── service.py # Task business logic -│ │ │ │ ├── models.py # Task models -│ │ │ │ ├── repository.py # Task DB operations -│ │ │ │ └── tests/ -│ │ │ ├── task_ordering/ # Drag-and-drop reordering -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Reorder endpoints -│ │ │ │ ├── service.py # Reordering algorithm -│ │ │ │ └── tests/ -│ │ │ ├── document_management/ # Project documents -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Document endpoints -│ │ │ │ ├── service.py # Document logic -│ │ │ │ ├── models.py -│ │ │ │ └── repository.py -│ │ │ ├── document_versioning/ # Version control -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Version endpoints -│ │ │ │ ├── service.py # Versioning logic -│ │ │ │ ├── models.py # Version models -│ │ │ │ └── repository.py # Version storage -│ │ │ ├── ai_generation/ # AI project creation -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Generate endpoints -│ │ │ │ ├── service.py # AI orchestration -│ │ │ │ ├── agents.py # Agent interactions -│ │ │ │ ├── progress.py # Progress tracking -│ │ │ │ └── prompts.py # Generation prompts -│ │ │ ├── source_linking/ # Link to knowledge base -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api.py # Link endpoints -│ │ │ │ ├── service.py # Linking logic -│ │ │ │ └── repository.py # Junction table ops -│ │ │ └── bulk_operations/ # Batch updates -│ │ │ ├── __init__.py -│ │ │ ├── api.py # Bulk endpoints -│ │ │ ├── service.py # Batch processing -│ │ │ └── tests/ -│ │ │ -│ │ ├── mcp_server/ # MCP Protocol Server (IDE Integration) -│ │ │ ├── __init__.py -│ │ │ ├── main.py # MCP server entry point -│ │ │ ├── server.py # FastMCP server setup -│ │ │ ├── features/ # MCP tool implementations -│ │ │ │ ├── projects/ # Project tools for IDEs -│ │ │ │ │ ├── __init__.py -│ │ │ │ │ ├── project_tools.py -│ │ │ │ │ └── tests/ -│ │ │ │ ├── tasks/ # Task tools for IDEs -│ │ │ │ │ ├── __init__.py -│ │ │ │ │ ├── task_tools.py -│ │ │ │ │ └── tests/ -│ │ │ │ ├── documents/ # Document tools for IDEs -│ │ │ │ │ ├── __init__.py -│ │ │ │ │ ├── document_tools.py -│ │ │ │ │ ├── version_tools.py -│ │ │ │ │ └── tests/ -│ │ │ │ └── feature_tools.py # Feature management -│ │ │ ├── modules/ # MCP modules -│ │ │ │ └── archon.py # Main Archon MCP module -│ │ │ └── utils/ # MCP utilities -│ │ │ └── tool_utils.py -│ │ │ -│ │ ├── agents/ # AI Agents Module -│ │ │ ├── __init__.py -│ │ │ ├── main.py # Agents module entry point -│ │ │ ├── config.py # Agent configurations -│ │ │ ├── features/ # Agent capabilities -│ │ │ │ ├── document_agent/ # Document processing agent -│ │ │ │ │ ├── __init__.py -│ │ │ │ │ ├── agent.py # PydanticAI agent -│ │ │ │ │ ├── prompts.py # Agent prompts -│ │ │ │ │ └── tools.py # Agent tools -│ │ │ │ ├── code_agent/ # Code analysis agent -│ │ │ │ │ ├── __init__.py -│ │ │ │ │ ├── agent.py -│ │ │ │ │ └── analyzers.py -│ │ │ │ └── project_agent/ # Project creation agent -│ │ │ │ ├── __init__.py -│ │ │ │ ├── agent.py -│ │ │ │ ├── prp_generator.py -│ │ │ │ └── task_generator.py -│ │ │ └── shared/ # Shared agent utilities -│ │ │ ├── base_agent.py -│ │ │ ├── llm_client.py -│ │ │ └── response_models.py -│ │ │ -│ │ ├── shared/ # Shared Across All Modules -│ │ │ ├── database/ # Database utilities -│ │ │ │ ├── __init__.py -│ │ │ │ ├── supabase.py # Supabase client -│ │ │ │ ├── migrations.py # DB migrations -│ │ │ │ └── connection_pool.py -│ │ │ ├── auth/ # Authentication -│ │ │ │ ├── __init__.py -│ │ │ │ ├── api_keys.py -│ │ │ │ └── permissions.py -│ │ │ ├── config/ # Configuration -│ │ │ │ ├── __init__.py -│ │ │ │ ├── settings.py # Environment settings -│ │ │ │ └── logfire_config.py # Logging config -│ │ │ ├── middleware/ # HTTP middleware -│ │ │ │ ├── __init__.py -│ │ │ │ ├── cors.py -│ │ │ │ └── error_handler.py -│ │ │ └── utils/ # General utilities -│ │ │ ├── __init__.py -│ │ │ ├── datetime_utils.py -│ │ │ └── json_utils.py -│ │ │ -│ │ └── main.py # Application orchestrator -│ │ -│ └── tests/ # Integration tests -│ ├── test_api_essentials.py -│ ├── test_service_integration.py -│ └── fixtures/ -│ -├── archon-ui-main/ # Frontend Application -│ ├── src/ -│ │ ├── pages/ # Page components -│ │ │ ├── KnowledgeBasePage.tsx -│ │ │ ├── ProjectPage.tsx -│ │ │ ├── SettingsPage.tsx -│ │ │ └── MCPPage.tsx -│ │ ├── components/ # Reusable components -│ │ │ ├── knowledge-base/ # Knowledge features -│ │ │ ├── project-tasks/ # Project features -│ │ │ └── ui/ # Shared UI components -│ │ ├── services/ # API services -│ │ │ ├── api.ts # Base API client -│ │ │ ├── knowledgeBaseService.ts -│ │ │ ├── projectService.ts -│ │ │ └── pollingService.ts # New polling utilities -│ │ ├── hooks/ # React hooks -│ │ │ ├── usePolling.ts # Polling hook -│ │ │ ├── useDatabaseMutation.ts # DB-first mutations -│ │ │ └── useAsyncAction.ts -│ │ └── contexts/ # React contexts -│ │ ├── ToastContext.tsx -│ │ └── ThemeContext.tsx -│ │ -│ └── tests/ # Frontend tests -│ -├── PRPs/ # Product Requirement Prompts -│ ├── templates/ # PRP templates -│ ├── ai_docs/ # AI context documentation -│ └── *.md # Feature PRPs -│ -├── docs/ # Documentation -│ └── architecture/ # Architecture decisions -│ -└── docker/ # Docker configurations - ├── Dockerfile - └── docker-compose.yml +### Backend (`python/src/`) +```text +server/ # Main FastAPI application +├── api_routes/ # HTTP endpoints +├── services/ # Business logic +├── models/ # Data models +├── config/ # Configuration +├── middleware/ # Request processing +└── utils/ # Shared utilities + +mcp_server/ # MCP server for IDE integration +└── features/ # MCP tool implementations + +agents/ # AI agents (PydanticAI) +└── features/ # Agent capabilities ``` -## Module Descriptions +### Frontend (`archon-ui-main/src/`) +```text +features/ # Vertical slice architecture +├── knowledge/ # Knowledge base feature +├── projects/ # Project management +│ ├── tasks/ # Task sub-feature +│ └── documents/ # Document sub-feature +├── progress/ # Operation tracking +├── mcp/ # MCP integration +├── shared/ # Cross-feature utilities +└── ui/ # UI components & hooks -### Knowledge Module (`src/knowledge/`) - -Core knowledge management functionality including web crawling, document processing, embeddings, and RAG search. This is the heart of Archon's knowledge engine. - -**Key Features:** - -- Web crawling with JavaScript rendering -- Document upload and text extraction -- Vector embeddings and similarity search -- Code snippet extraction and indexing -- Source management and organization - -### Projects Module (`src/projects/`) - -Project and task management system with AI-powered project generation. Currently optional via feature flag. - -**Key Features:** - -- Project CRUD operations -- Task management with drag-and-drop ordering -- Document management with versioning -- AI-powered project generation -- Integration with knowledge base sources - -### MCP Server Module (`src/mcp_server/`) - -Model Context Protocol server that exposes Archon functionality to IDEs like Cursor and Windsurf. - -**Key Features:** - -- Tool-based API for IDE integration -- Project and task management tools -- Document operations -- Async operation support - -### Agents Module (`src/agents/`) - -AI agents powered by PydanticAI for intelligent document processing and project generation. - -**Key Features:** - -- Document analysis and summarization -- Code understanding and extraction -- Project requirement generation -- Task breakdown and planning - -### Shared Module (`src/shared/`) - -Cross-cutting concerns shared across all modules. Kept minimal to maintain module independence. - -**Key Components:** - -- Database connections and utilities -- Authentication and authorization -- Configuration management -- Logging and observability -- Common middleware - -## Communication Patterns - -### Inter-Module Communication - -Modules communicate through: - -1. **Direct HTTP API Calls** (current) - - Projects module calls Knowledge module APIs - - Simple and straightforward - - Works well for current scale - -2. **Event Bus** (future consideration) - - ```python - # Example event-driven communication - await event_bus.publish("project.created", { - "project_id": "123", - "created_by": "user" - }) - ``` - -3. **Shared Database** (current reality) - - All modules use same Supabase instance - - Direct foreign keys between contexts - - Will need refactoring for true microservices - -## Feature Flags - -Features can be toggled via environment variables: - -```python -# settings.py -PROJECTS_ENABLED = env.bool("PROJECTS_ENABLED", default=False) -TASK_ORDERING_ENABLED = env.bool("TASK_ORDERING_ENABLED", default=True) -AI_GENERATION_ENABLED = env.bool("AI_GENERATION_ENABLED", default=True) +pages/ # Route components +components/ # Legacy components (migrating) ``` -## Database Architecture +## Core Modules -Currently using a shared Supabase (PostgreSQL) database: +### Knowledge Management +**Backend**: `python/src/server/services/knowledge_service.py` +**Frontend**: `archon-ui-main/src/features/knowledge/` +**Features**: Web crawling, document upload, embeddings, RAG search -```sql --- Knowledge context tables -sources -documents -code_examples +### Project Management +**Backend**: `python/src/server/services/project_*_service.py` +**Frontend**: `archon-ui-main/src/features/projects/` +**Features**: Projects, tasks, documents, version history --- Projects context tables -archon_projects -archon_tasks -archon_document_versions +### MCP Server +**Location**: `python/src/mcp_server/` +**Purpose**: Exposes tools to AI IDEs (Cursor, Windsurf) +**Port**: 8051 --- Cross-context junction tables -archon_project_sources -- Links projects to knowledge -``` +### AI Agents +**Location**: `python/src/agents/` +**Purpose**: Document processing, code analysis, project generation +**Port**: 8052 ## API Structure -Each feature exposes its own API routes: +### RESTful Endpoints +Pattern: `{METHOD} /api/{resource}/{id?}/{sub-resource?}` -``` -/api/knowledge/ - /crawl # Web crawling - /upload # Document upload - /search # RAG search - /sources # Source management +**Examples from** `python/src/server/api_routes/`: +- `/api/projects` - CRUD operations +- `/api/projects/{id}/tasks` - Nested resources +- `/api/knowledge/search` - RAG search +- `/api/progress/{id}` - Operation status -/api/projects/ - /projects # Project CRUD - /tasks # Task management - /tasks/reorder # Task ordering - /documents # Document management - /generate # AI generation +### Service Layer +**Pattern**: `python/src/server/services/{feature}_service.py` +- Handles business logic +- Database operations via Supabase client +- Returns typed responses + +## Frontend Architecture + +### Data Fetching +**Core**: TanStack Query v5 +**Configuration**: `archon-ui-main/src/features/shared/queryClient.ts` +**Patterns**: `archon-ui-main/src/features/shared/queryPatterns.ts` + +### State Management +- **Server State**: TanStack Query +- **UI State**: React hooks & context +- **No Redux/Zustand**: Query cache handles all data + +### Feature Organization +Each feature follows vertical slice pattern: +```text +features/{feature}/ +├── components/ # UI components +├── hooks/ # Query hooks & keys +├── services/ # API calls +└── types/ # TypeScript types ``` -## Deployment Architecture +### Smart Polling +**Implementation**: `archon-ui-main/src/features/ui/hooks/useSmartPolling.ts` +- Visibility-aware (pauses when tab hidden) +- Variable intervals based on focus state -### Current mixed +## Database -### Future (service modules) +**Provider**: Supabase (PostgreSQL + pgvector) +**Client**: `python/src/server/config/database.py` -Each module can become its own service: +### Main Tables +- `sources` - Knowledge sources +- `documents` - Document chunks with embeddings +- `code_examples` - Extracted code +- `archon_projects` - Projects +- `archon_tasks` - Tasks +- `archon_document_versions` - Version history -```yaml -# docker-compose.yml (future) -services: - knowledge: - image: archon-knowledge - ports: ["8001:8000"] +## Key Architectural Decisions - projects: - image: archon-projects - ports: ["8002:8000"] +### Vertical Slices +Features own their entire stack (UI → API → DB). See any `features/{feature}/` directory. - mcp-server: - image: archon-mcp - ports: ["8051:8051"] +### No WebSockets +HTTP polling with smart intervals. ETag caching reduces bandwidth by ~70%. - agents: - image: archon-agents - ports: ["8052:8052"] +### Query-First State +TanStack Query is the single source of truth. No separate state management needed. + +### Direct Database Values +No translation layers. Database values (e.g., `"todo"`, `"doing"`) used directly in UI. + +### Browser-Native Caching +ETags handled by browser, not JavaScript. See `archon-ui-main/src/features/shared/apiWithEtag.ts`. + +## Deployment + +### Development +```bash +# Backend +docker compose up -d +# or +cd python && uv run python -m src.server.main + +# Frontend +cd archon-ui-main && npm run dev ``` -## Migration Path +### Production +Single Docker Compose deployment with all services. -### Phase 1: Current State (Modules/service) +## Configuration -- All code in one repository -- Shared database -- Single deployment +### Environment Variables +**Required**: `SUPABASE_URL`, `SUPABASE_SERVICE_KEY` +**Optional**: See `.env.example` -### Phase 2: Vertical Slices +### Feature Flags +Controlled via Settings UI. Projects feature can be disabled. -- Reorganize by feature -- Clear module boundaries -- Feature flags for control +## Recent Refactors (Phases 1-5) -## Development Guidelines +1. **Removed ETag cache layer** - Browser handles HTTP caching +2. **Standardized query keys** - Each feature owns its keys +3. **Fixed optimistic updates** - UUID-based with nanoid +4. **Configured deduplication** - Centralized QueryClient +5. **Removed manual invalidations** - Trust backend consistency -### Adding a New Feature +## Performance Optimizations -1. **Identify the Module**: Which bounded context does it belong to? -2. **Create Feature Slice**: New folder under `module/features/` -3. **Implement Vertical Slice**: - - `api.py` - HTTP endpoints - - `service.py` - Business logic - - `models.py` - Data models - - `repository.py` - Data access - - `tests/` - Feature tests +- **Request Deduplication**: Same query key = one request +- **Smart Polling**: Adapts to tab visibility +- **ETag Caching**: 70% bandwidth reduction +- **Optimistic Updates**: Instant UI feedback -### Testing Strategy +## Testing -- **Unit Tests**: Each feature has its own tests -- **Integration Tests**: Test module boundaries -- **E2E Tests**: Test complete user flows - -### Code Organization Rules - -1. **Features are Self-Contained**: All code for a feature lives together -2. **No Cross-Feature Imports**: Use module's shared or API calls -3. **Shared is Minimal**: Only truly cross-cutting concerns -4. **Dependencies Point Inward**: Features → Module Shared → Global Shared - -## Technology Stack - -### Backend - -- **FastAPI**: Web framework -- **Supabase**: Database and auth -- **PydanticAI**: AI agents -- **OpenAI**: Embeddings and LLM -- **Crawl4AI**: Web crawling - -### Frontend - -- **React**: UI framework -- **TypeScript**: Type safety -- **TailwindCSS**: Styling -- **React Query**: Data fetching -- **Vite**: Build tool - -### Infrastructure - -- **Docker**: Containerization -- **PostgreSQL**: Database (via Supabase, desire to support any PostgreSQL) -- **pgvector**: Vector storage, Desire to support ChromaDB, Pinecone, Weaviate, etc. +**Frontend Tests**: `archon-ui-main/src/features/*/tests/` +**Backend Tests**: `python/tests/` +**Patterns**: Mock services and query patterns, not implementation ## Future Considerations -### Planned Improvements - -1. **Remove Socket.IO**: Replace with polling (in progress) -2. **API Gateway**: Central entry point for all services -3. **Separate Databases**: One per bounded context - -### Scalability Path - -1. **Vertical Scaling**: Current approach, works for single-user -2. **Horizontal Scaling**: Add load balancer and multiple instances - ---- - -This architecture provides a clear path from the current monolithic application to a more modular approach with vertical slicing, for easy potential to service separation if needed. +- Server-Sent Events for real-time updates +- GraphQL for selective field queries +- Separate databases per bounded context +- Multi-tenant support \ No newline at end of file diff --git a/PRPs/ai_docs/DATA_FETCHING_ARCHITECTURE.md b/PRPs/ai_docs/DATA_FETCHING_ARCHITECTURE.md new file mode 100644 index 00000000..d8a9822b --- /dev/null +++ b/PRPs/ai_docs/DATA_FETCHING_ARCHITECTURE.md @@ -0,0 +1,192 @@ +# Data Fetching Architecture + +## Overview + +Archon uses **TanStack Query v5** for all data fetching, caching, and synchronization. This replaces the former custom polling layer with a query‑centric design that handles caching, deduplication, and smart refetching (including visibility‑aware polling) automatically. + +## Core Components + +### 1. Query Client Configuration + +**Location**: `archon-ui-main/src/features/shared/queryClient.ts` + +Centralized QueryClient with: + +- 30-second default stale time +- 10-minute garbage collection +- Smart retry logic (skips 4xx errors) +- Request deduplication enabled +- Structural sharing for optimized re-renders + +### 2. Smart Polling Hook + +**Location**: `archon-ui-main/src/features/ui/hooks/useSmartPolling.ts` + +Visibility-aware polling that: + +- Pauses when browser tab is hidden +- Slows down (1.5x interval) when tab is unfocused +- Returns `refetchInterval` for use with TanStack Query + +### 3. Query Patterns + +**Location**: `archon-ui-main/src/features/shared/queryPatterns.ts` + +Shared constants: + +- `DISABLED_QUERY_KEY` - For disabled queries +- `STALE_TIMES` - Standardized cache durations (instant, realtime, frequent, normal, rare, static) + +## Feature Implementation Patterns + +### Query Key Factories + +Each feature maintains its own query keys: + +- **Projects**: `archon-ui-main/src/features/projects/hooks/useProjectQueries.ts` (projectKeys) +- **Tasks**: `archon-ui-main/src/features/projects/tasks/hooks/useTaskQueries.ts` (taskKeys) +- **Knowledge**: `archon-ui-main/src/features/knowledge/hooks/useKnowledgeQueries.ts` (knowledgeKeys) +- **Progress**: `archon-ui-main/src/features/progress/hooks/useProgressQueries.ts` (progressKeys) +- **MCP**: `archon-ui-main/src/features/mcp/hooks/useMcpQueries.ts` (mcpKeys) +- **Documents**: `archon-ui-main/src/features/projects/documents/hooks/useDocumentQueries.ts` (documentKeys) + +### Data Fetching Hooks + +Standard pattern across all features: + +- `use[Feature]()` - List queries +- `use[Feature]Detail(id)` - Single item queries +- `useCreate[Feature]()` - Creation mutations +- `useUpdate[Feature]()` - Update mutations +- `useDelete[Feature]()` - Deletion mutations + +## Backend Integration + +### ETag Support + +**Location**: `archon-ui-main/src/features/shared/apiWithEtag.ts` + +ETag implementation: + +- Browser handles ETag headers automatically +- 304 responses reduce bandwidth +- TanStack Query manages cache state + +### API Structure + +Backend endpoints follow RESTful patterns: + +- **Knowledge**: `python/src/server/api_routes/knowledge_api.py` +- **Projects**: `python/src/server/api_routes/projects_api.py` +- **Progress**: `python/src/server/api_routes/progress_api.py` +- **MCP**: `python/src/server/api_routes/mcp_api.py` + +## Optimistic Updates + +**Utilities**: `archon-ui-main/src/features/shared/optimistic.ts` + +All mutations use nanoid-based optimistic updates: + +- Creates temporary entities with `_optimistic` flag +- Replaces with server data on success +- Rollback on error +- Visual indicators for pending state + +## Refetch Strategies + +### Smart Polling Usage + +**Implementation**: `archon-ui-main/src/features/ui/hooks/useSmartPolling.ts` + +Polling intervals are defined in each feature's query hooks. See actual implementations: +- **Projects**: `archon-ui-main/src/features/projects/hooks/useProjectQueries.ts` +- **Tasks**: `archon-ui-main/src/features/projects/tasks/hooks/useTaskQueries.ts` +- **Knowledge**: `archon-ui-main/src/features/knowledge/hooks/useKnowledgeQueries.ts` +- **Progress**: `archon-ui-main/src/features/progress/hooks/useProgressQueries.ts` +- **MCP**: `archon-ui-main/src/features/mcp/hooks/useMcpQueries.ts` + +Standard intervals from `archon-ui-main/src/features/shared/queryPatterns.ts`: +- `STALE_TIMES.instant`: 0ms (always fresh) +- `STALE_TIMES.frequent`: 5 seconds (frequently changing data) +- `STALE_TIMES.normal`: 30 seconds (standard cache) + +### Manual Refetch + +All queries expose `refetch()` for manual updates. + +## Performance Optimizations + +### Request Deduplication + +Handled automatically by TanStack Query when same query key is used. + +### Stale Time Configuration + +Defined in `STALE_TIMES` and used consistently: + +- Auth/Settings: `Infinity` (never stale) +- Active operations: `0` (always fresh) +- Normal data: `30_000` (30 seconds) +- Rare updates: `300_000` (5 minutes) + +### Garbage Collection + +Unused data removed after 10 minutes (configurable in queryClient). + +## Migration from Polling + +### What Changed (Phases 1-5) + +1. **Phase 1**: Removed ETag cache layer +2. **Phase 2**: Standardized query keys +3. **Phase 3**: Fixed optimistic updates with UUIDs +4. **Phase 4**: Configured request deduplication +5. **Phase 5**: Removed manual invalidations + +### Deprecated Patterns + +- `usePolling` hook (removed) +- `useCrawlProgressPolling` (removed) +- Manual cache invalidation with setTimeout +- Socket.IO connections +- Double-layer caching + +## Testing Patterns + +### Hook Testing + +**Example**: `archon-ui-main/src/features/projects/hooks/tests/useProjectQueries.test.ts` + +Standard mocking approach for: + +- Service methods +- Query patterns (STALE_TIMES, DISABLED_QUERY_KEY) +- Smart polling behavior + +### Integration Testing + +Use React Testing Library with QueryClientProvider wrapper. + +## Developer Guidelines + +### Adding New Data Fetching + +1. Create query key factory in `{feature}/hooks/use{Feature}Queries.ts` +2. Use `useQuery` with appropriate stale time from `STALE_TIMES` +3. Add smart polling if real-time updates needed +4. Implement optimistic updates for mutations +5. Follow existing patterns in similar features + +### Common Patterns to Follow + +- Always use query key factories +- Never hardcode stale times +- Use `DISABLED_QUERY_KEY` for conditional queries +- Implement optimistic updates for better UX +- Add loading and error states + +## Future Considerations + +- Server-Sent Events for true real-time (post-Phase 5) +- WebSocket fallback for critical updates +- GraphQL migration for selective field updates diff --git a/PRPs/ai_docs/ETAG_IMPLEMENTATION.md b/PRPs/ai_docs/ETAG_IMPLEMENTATION.md index b8ebcedc..70e4ce63 100644 --- a/PRPs/ai_docs/ETAG_IMPLEMENTATION.md +++ b/PRPs/ai_docs/ETAG_IMPLEMENTATION.md @@ -1,39 +1,149 @@ # ETag Implementation -## Current Implementation +## Overview -Our ETag implementation provides efficient HTTP caching for polling endpoints to reduce bandwidth usage. +Archon implements HTTP ETag caching to optimize bandwidth usage by reducing redundant data transfers. The implementation leverages browser-native HTTP caching combined with backend ETag generation for efficient cache validation. -### What It Does -- **Generates ETags**: Creates MD5 hashes of JSON response data -- **Checks ETags**: Simple string equality comparison between client's `If-None-Match` header and current data's ETag -- **Returns 304**: When ETags match, returns `304 Not Modified` with no body (saves bandwidth) +## How It Works -### How It Works -1. Server generates ETag from response data using MD5 hash -2. Client sends previous ETag in `If-None-Match` header -3. Server compares ETags: - - **Match**: Returns 304 (no body) - - **No match**: Returns 200 with new data and new ETag +### Backend ETag Generation +**Location**: `python/src/server/utils/etag_utils.py` -### Example -```python -# Server generates: ETag: "a3c2f1e4b5d6789" -# Client sends: If-None-Match: "a3c2f1e4b5d6789" -# Server returns: 304 Not Modified (no body) -``` +The backend generates ETags for API responses: +- Creates MD5 hash of JSON-serialized response data +- Returns quoted ETag string (RFC 7232 format) +- Sets `Cache-Control: no-cache, must-revalidate` headers +- Compares client's `If-None-Match` header with current data's ETag +- Returns `304 Not Modified` when ETags match -## Limitations +### Frontend Handling +**Location**: `archon-ui-main/src/features/shared/apiWithEtag.ts` -Our implementation is simplified and doesn't support full RFC 7232 features: -- ❌ Wildcard (`*`) matching -- ❌ Multiple ETags (`"etag1", "etag2"`) -- ❌ Weak validators (`W/"etag"`) -- ✅ Single ETag comparison only +The frontend relies on browser-native HTTP caching: +- Browser automatically sends `If-None-Match` headers with cached ETags +- Browser handles 304 responses by returning cached data from HTTP cache +- No manual ETag tracking or cache management needed +- TanStack Query manages data freshness through `staleTime` configuration -This works perfectly for our browser-to-API polling use case but may need enhancement for CDN/proxy support. +#### Browser vs Non-Browser Behavior +- **Standard Browsers**: Per the Fetch spec, a 304 response freshens the HTTP cache and returns the cached body to JavaScript +- **Non-Browser Runtimes** (React Native, custom fetch): May surface 304 with empty body to JavaScript +- **Client Fallback**: The `apiWithEtag.ts` implementation handles both scenarios, ensuring consistent behavior across environments -## Files -- Implementation: `python/src/server/utils/etag_utils.py` -- Tests: `python/tests/server/utils/test_etag_utils.py` -- Used in: Progress API, Projects API polling endpoints \ No newline at end of file +## Implementation Details + +### Backend API Integration + +ETags are used in these API routes: +- **Projects**: `python/src/server/api_routes/projects_api.py` + - Project lists + - Task lists + - Task counts +- **Progress**: `python/src/server/api_routes/progress_api.py` + - Active operations tracking + +### ETag Generation Process + +1. **Data Serialization**: Response data is JSON-serialized with sorted keys for consistency +2. **Hash Creation**: MD5 hash generated from JSON string +3. **Format**: Returns quoted string per RFC 7232 (e.g., `"a3c2f1e4b5d6789"`) + +### Cache Validation Flow + +1. **Initial Request**: Server generates ETag and sends with response +2. **Subsequent Requests**: Browser sends `If-None-Match` header with cached ETag +3. **Server Validation**: + - ETags match → Returns `304 Not Modified` (no body) + - ETags differ → Returns `200 OK` with new data and new ETag +4. **Browser Behavior**: On 304, browser serves cached response to JavaScript + +## Key Design Decisions + +### Browser-Native Caching +The implementation leverages browser HTTP caching instead of manual cache management: +- Reduces code complexity +- Eliminates cache synchronization issues +- Works seamlessly with TanStack Query +- Maintains bandwidth optimization + +### No Manual ETag Tracking +Unlike previous implementations, the current approach: +- Does NOT maintain ETag maps in JavaScript +- Does NOT manually handle 304 responses +- Lets browser and TanStack Query handle caching layers + +## Integration with TanStack Query + +### Cache Coordination +- **Browser Cache**: Handles HTTP-level caching (ETags/304s) +- **TanStack Query Cache**: Manages application-level data freshness +- **Separation of Concerns**: HTTP caching for bandwidth, TanStack for state + +### Configuration +Cache behavior is controlled through TanStack Query's `staleTime`: +- See `archon-ui-main/src/features/shared/queryPatterns.ts` for standard times +- See `archon-ui-main/src/features/shared/queryClient.ts` for global configuration + +## Performance Benefits + +### Bandwidth Reduction +- ~70% reduction in data transfer for unchanged responses (based on internal measurements) +- Especially effective for polling patterns +- Significant improvement for mobile/slow connections + +### Server Load +- Reduced JSON serialization for 304 responses +- Lower network I/O +- Faster response times for cached data + +## Files and References + +### Core Implementation +- **Backend Utilities**: `python/src/server/utils/etag_utils.py` +- **Frontend Client**: `archon-ui-main/src/features/shared/apiWithEtag.ts` +- **Tests**: `python/tests/server/utils/test_etag_utils.py` + +### Usage Examples +- **Projects API**: `python/src/server/api_routes/projects_api.py` (lines with `generate_etag`, `check_etag`) +- **Progress API**: `python/src/server/api_routes/progress_api.py` (active operations tracking) + +## Testing + +### Backend Testing +Tests in `python/tests/server/utils/test_etag_utils.py` verify: +- Correct ETag generation format +- Consistent hashing for same data +- Different hashes for different data +- Proper quote formatting + +### Frontend Testing +Browser DevTools verification: +1. Network tab shows `If-None-Match` headers on requests +2. 304 responses have no body +3. Response served from cache on 304 +4. New ETag values when data changes + +## Monitoring + +### How to Verify ETags are Working +1. Open Chrome DevTools → Network tab +2. Make a request to a supported endpoint +3. Note the `ETag` response header +4. Refresh or re-request the same data +5. Observe: + - Request includes `If-None-Match` header + - Server returns `304 Not Modified` if unchanged + - Response body is empty on 304 + - Browser serves cached data + +### Metrics to Track +- Ratio of 304 vs 200 responses +- Bandwidth saved through 304 responses +- Cache hit rate in production + +## Future Considerations + +- Consider implementing strong vs weak ETags for more granular control +- Evaluate adding ETag support to more endpoints +- Monitor cache effectiveness in production +- Consider Last-Modified headers as supplementary validation \ No newline at end of file diff --git a/PRPs/ai_docs/POLLING_ARCHITECTURE.md b/PRPs/ai_docs/POLLING_ARCHITECTURE.md deleted file mode 100644 index 0c034b62..00000000 --- a/PRPs/ai_docs/POLLING_ARCHITECTURE.md +++ /dev/null @@ -1,194 +0,0 @@ -# Polling Architecture Documentation - -## Overview -Archon V2 uses HTTP polling instead of WebSockets for real-time updates. This simplifies the architecture, reduces complexity, and improves maintainability while providing adequate responsiveness for project management tasks. - -## Core Components - -### 1. usePolling Hook (`archon-ui-main/src/hooks/usePolling.ts`) -Generic polling hook that manages periodic data fetching with smart optimizations. - -**Key Features:** -- Configurable polling intervals (default: 3 seconds) -- Automatic pause during browser tab inactivity -- ETag-based caching to reduce bandwidth -- Manual refresh capability - -**Usage:** -```typescript -const { data, isLoading, error, refetch } = usePolling('/api/projects', { - interval: 5000, - enabled: true, - onSuccess: (data) => console.log('Projects updated:', data) -}); -``` - -### 2. Specialized Progress Services -Individual services handle specific progress tracking needs: - -**CrawlProgressService (`archon-ui-main/src/services/crawlProgressService.ts`)** -- Tracks website crawling operations -- Maps backend status to UI-friendly format -- Includes in-flight request guard to prevent overlapping fetches -- 1-second polling interval during active crawls - -**Polling Endpoints:** -- `/api/projects` - Project list updates -- `/api/projects/{project_id}/tasks` - Task list for active project -- `/api/crawl-progress/{progress_id}` - Website crawling progress -- `/api/agent-chat/sessions/{session_id}/messages` - Chat messages - -## Backend Support - -### ETag Implementation (`python/src/server/utils/etag_utils.py`) -Server-side optimization to reduce unnecessary data transfer. - -**How it works:** -1. Server generates ETag hash from response data -2. Client sends `If-None-Match` header with cached ETag -3. Server returns 304 Not Modified if data unchanged -4. Client uses cached data, reducing bandwidth by ~70% - -### Progress API (`python/src/server/api_routes/progress_api.py`) -Dedicated endpoints for progress tracking: -- `GET /api/crawl-progress/{progress_id}` - Returns crawling status with ETag support -- Includes completion percentage, current step, and error details - -## State Management - -### Loading States -Visual feedback during operations: -- `movingTaskIds: Set` - Tracks tasks being moved -- `isSwitchingProject: boolean` - Project transition state -- Loading overlays prevent concurrent operations - -## Error Handling - -### Retry Strategy -```typescript -retryCount: 3 -retryDelay: attempt => Math.min(1000 * 2 ** attempt, 30000) -``` -- Exponential backoff: 1s, 2s, 4s... -- Maximum retry delay: 30 seconds -- Automatic recovery after network issues - -### User Feedback -- Toast notifications for errors -- Loading spinners during operations -- Clear error messages with recovery actions - -## Performance Optimizations - -### 1. Request Deduplication -Prevents multiple components from making identical requests: -```typescript -const cacheKey = `${endpoint}-${JSON.stringify(params)}`; -if (pendingRequests.has(cacheKey)) { - return pendingRequests.get(cacheKey); -} -``` - -### 2. Smart Polling Intervals -- Active operations: 1-2 second intervals -- Background data: 5-10 second intervals -- Paused when tab inactive (visibility API) - -### 3. Selective Updates -Only polls active/relevant data: -- Tasks poll only for selected project -- Progress polls only during active operations -- Chat polls only for open sessions - -## Architecture Benefits - -### What We Have -- **Simple HTTP polling** - Standard request/response pattern -- **Automatic error recovery** - Built-in retry with exponential backoff -- **ETag caching** - 70% bandwidth reduction via 304 responses -- **Easy debugging** - Standard HTTP requests visible in DevTools -- **No connection limits** - Scales with standard HTTP infrastructure -- **Consolidated polling hooks** - Single pattern for all data fetching - -### Trade-offs -- **Latency:** 1-5 second delay vs instant updates -- **Bandwidth:** More requests, but mitigated by ETags -- **Battery:** Slightly higher mobile battery usage - -## Developer Guidelines - -### Adding New Polling Endpoint - -1. **Frontend - Use the usePolling hook:** -```typescript -// In your component or custom hook -const { data, isLoading, error, refetch } = usePolling('/api/new-endpoint', { - interval: 5000, - enabled: true, - staleTime: 2000 -}); -``` - -2. **Backend - Add ETag support:** -```python -from ..utils.etag_utils import generate_etag, check_etag - -@router.get("/api/new-endpoint") -async def get_data(request: Request): - data = fetch_data() - etag = generate_etag(data) - - if check_etag(request, etag): - return Response(status_code=304) - - return JSONResponse( - content=data, - headers={"ETag": etag} - ) -``` - -3. **For progress tracking, use useCrawlProgressPolling:** -```typescript -const { data, isLoading } = useCrawlProgressPolling(operationId, { - onSuccess: (data) => { - if (data.status === 'completed') { - // Handle completion - } - } -}); -``` - -### Best Practices - -1. **Always provide loading states** - Users should know when data is updating -2. **Handle errors gracefully** - Show toast notifications with clear messages -3. **Respect polling intervals** - Don't poll faster than necessary -4. **Clean up on unmount** - Cancel pending requests when components unmount -5. **Use ETag caching** - Reduce bandwidth with 304 responses - -## Testing Polling Behavior - -### Manual Testing -1. Open Network tab in DevTools -2. Look for requests with 304 status (cache hits) -3. Verify polling stops when switching tabs -4. Test error recovery by stopping backend - -### Debugging Tips -- Check `localStorage` for cached ETags -- Monitor `console.log` for polling lifecycle events -- Use React DevTools to inspect hook states -- Watch for memory leaks in long-running sessions - -## Future Improvements - -### Planned Enhancements -- WebSocket fallback for critical updates -- Configurable per-user polling rates -- Smart polling based on user activity patterns -- GraphQL subscriptions for selective field updates - -### Considered Alternatives -- Server-Sent Events (SSE) - One-way real-time updates -- Long polling - Reduced request frequency -- WebRTC data channels - P2P updates between clients \ No newline at end of file diff --git a/PRPs/ai_docs/QUERY_PATTERNS.md b/PRPs/ai_docs/QUERY_PATTERNS.md new file mode 100644 index 00000000..3c3204db --- /dev/null +++ b/PRPs/ai_docs/QUERY_PATTERNS.md @@ -0,0 +1,237 @@ +# TanStack Query Patterns Guide + +This guide documents the standardized patterns for using TanStack Query v5 in the Archon frontend. + +## Core Principles + +1. **Feature Ownership**: Each feature owns its query keys in `{feature}/hooks/use{Feature}Queries.ts` +2. **Consistent Patterns**: Always use shared patterns from `shared/queryPatterns.ts` +3. **No Hardcoded Values**: Never hardcode stale times or disabled keys +4. **Mirror Backend API**: Query keys should exactly match backend API structure + +## Query Key Factory Pattern + +Every feature MUST implement a query key factory following this pattern: + +```typescript +// features/{feature}/hooks/use{Feature}Queries.ts +export const featureKeys = { + all: ["feature"] as const, // Base key for the domain + lists: () => [...featureKeys.all, "list"] as const, // For list endpoints + detail: (id: string) => [...featureKeys.all, "detail", id] as const, // For single item + // Add more as needed following backend routes +}; +``` + +### Examples from Codebase + +```typescript +// Projects - Simple hierarchy +export const projectKeys = { + all: ["projects"] as const, + lists: () => [...projectKeys.all, "list"] as const, + detail: (id: string) => [...projectKeys.all, "detail", id] as const, + features: (id: string) => [...projectKeys.all, id, "features"] as const, +}; + +// Tasks - Dual nature (global and project-scoped) +export const taskKeys = { + all: ["tasks"] as const, + lists: () => [...taskKeys.all, "list"] as const, // /api/tasks + detail: (id: string) => [...taskKeys.all, "detail", id] as const, + byProject: (projectId: string) => ["projects", projectId, "tasks"] as const, // /api/projects/{id}/tasks + counts: () => [...taskKeys.all, "counts"] as const, +}; +``` + +## Shared Patterns Usage + +### Import Required Patterns + +```typescript +import { DISABLED_QUERY_KEY, STALE_TIMES } from "@/features/shared/queryPatterns"; +``` + +### Disabled Queries + +Always use `DISABLED_QUERY_KEY` when a query should not execute: + +```typescript +// ✅ CORRECT +queryKey: projectId ? projectKeys.detail(projectId) : DISABLED_QUERY_KEY, + +// ❌ WRONG - Don't create custom disabled keys +queryKey: projectId ? projectKeys.detail(projectId) : ["projects-undefined"], +``` + +### Stale Times + +Always use `STALE_TIMES` constants for cache configuration: + +```typescript +// ✅ CORRECT +staleTime: STALE_TIMES.normal, // 30 seconds +staleTime: STALE_TIMES.frequent, // 5 seconds +staleTime: STALE_TIMES.instant, // 0 - always fresh + +// ❌ WRONG - Don't hardcode times +staleTime: 30000, +staleTime: 0, +``` + +#### STALE_TIMES Reference + +- `instant: 0` - Always fresh (real-time data like active progress) +- `realtime: 3_000` - 3 seconds (near real-time updates) +- `frequent: 5_000` - 5 seconds (frequently changing data) +- `normal: 30_000` - 30 seconds (standard cache time) +- `rare: 300_000` - 5 minutes (rarely changing config) +- `static: Infinity` - Never stale (settings, auth) + +## Complete Hook Pattern + +```typescript +export function useFeatureDetail(id: string | undefined) { + return useQuery({ + queryKey: id ? featureKeys.detail(id) : DISABLED_QUERY_KEY, + queryFn: () => id + ? featureService.getFeatureById(id) + : Promise.reject("No ID provided"), + enabled: !!id, + staleTime: STALE_TIMES.normal, + }); +} +``` + +## Mutations with Optimistic Updates + +```typescript +import { createOptimisticEntity, replaceOptimisticEntity } from "@/features/shared/optimistic"; + +export function useCreateFeature() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: (data: CreateFeatureRequest) => featureService.create(data), + + onMutate: async (newData) => { + // Cancel in-flight queries + await queryClient.cancelQueries({ queryKey: featureKeys.lists() }); + + // Snapshot for rollback + const previous = queryClient.getQueryData(featureKeys.lists()); + + // Optimistic update with nanoid for stable IDs + const optimisticEntity = createOptimisticEntity(newData); + queryClient.setQueryData(featureKeys.lists(), (old: Feature[] = []) => + [...old, optimisticEntity] + ); + + return { previous, localId: optimisticEntity._localId }; + }, + + onError: (err, variables, context) => { + // Rollback on error + if (context?.previous) { + queryClient.setQueryData(featureKeys.lists(), context.previous); + } + }, + + onSuccess: (data, variables, context) => { + // Replace optimistic with real data + queryClient.setQueryData(featureKeys.lists(), (old: Feature[] = []) => + replaceOptimisticEntity(old, context?.localId, data) + ); + }, + }); +} +``` + +## Testing Query Hooks + +Always mock both services and shared patterns: + +```typescript +// Mock services +vi.mock("../../services", () => ({ + featureService: { + getList: vi.fn(), + getById: vi.fn(), + }, +})); + +// Mock shared patterns with ALL values +vi.mock("../../../shared/queryPatterns", () => ({ + DISABLED_QUERY_KEY: ["disabled"] as const, + STALE_TIMES: { + instant: 0, + realtime: 3_000, + frequent: 5_000, + normal: 30_000, + rare: 300_000, + static: Infinity, + }, +})); +``` + +## Vertical Slice Architecture + +Each feature is self-contained: + +```text +src/features/projects/ +├── components/ # UI components +├── hooks/ +│ └── useProjectQueries.ts # Query hooks & keys +├── services/ +│ └── projectService.ts # API calls +└── types/ + └── index.ts # TypeScript types +``` + +Sub-features (like tasks under projects) follow the same structure: + +```text +src/features/projects/tasks/ +├── components/ +├── hooks/ +│ └── useTaskQueries.ts # Own query keys! +├── services/ +└── types/ +``` + +## Migration Checklist + +When refactoring to these patterns: + +- [ ] Create query key factory in `hooks/use{Feature}Queries.ts` +- [ ] Import `DISABLED_QUERY_KEY` and `STALE_TIMES` from shared +- [ ] Replace all hardcoded disabled keys with `DISABLED_QUERY_KEY` +- [ ] Replace all hardcoded stale times with `STALE_TIMES` constants +- [ ] Update all `queryKey` references to use factory +- [ ] Update all `invalidateQueries` to use factory +- [ ] Update all `setQueryData` to use factory +- [ ] Add comprehensive tests for query keys +- [ ] Remove any backward compatibility code + +## Common Pitfalls to Avoid + +1. **Don't create centralized query keys** - Each feature owns its keys +2. **Don't hardcode values** - Use shared constants +3. **Don't mix concerns** - Tasks shouldn't import projectKeys +4. **Don't skip mocking in tests** - Mock both services and patterns +5. **Don't use inconsistent patterns** - Follow the established conventions + +## Completed Improvements (Phases 1-5) + +- ✅ Phase 1: Removed manual frontend ETag cache layer (backend ETags remain; browser-managed) +- ✅ Phase 2: Standardized query keys with factories +- ✅ Phase 3: Implemented UUID-based optimistic updates using nanoid +- ✅ Phase 4: Configured request deduplication +- ✅ Phase 5: Removed manual cache invalidations + +## Future Considerations + +- Add Server-Sent Events for real-time updates +- Consider WebSocket fallback for critical updates +- Evaluate Zustand for complex client state management \ No newline at end of file diff --git a/PRPs/ai_docs/optimistic_updates.md b/PRPs/ai_docs/optimistic_updates.md index 5884338b..7be11ea6 100644 --- a/PRPs/ai_docs/optimistic_updates.md +++ b/PRPs/ai_docs/optimistic_updates.md @@ -1,148 +1,135 @@ -# Optimistic Updates Pattern (Future State) +# Optimistic Updates Pattern Guide -**⚠️ STATUS:** This is not currently implemented. There is a proof‑of‑concept (POC) on the frontend Project page. This document describes the desired future state for handling optimistic updates in a simple, consistent way. +## Core Architecture -## Mental Model +### Shared Utilities Module +**Location**: `src/features/shared/optimistic.ts` -Think of optimistic updates as "assuming success" - update the UI immediately for instant feedback, then verify with the server. If something goes wrong, revert to the last known good state. - -## The Pattern +Provides type-safe utilities for managing optimistic state across all features: +- `createOptimisticId()` - Generates stable UUIDs using nanoid +- `createOptimisticEntity()` - Creates entities with `_optimistic` and `_localId` metadata +- `isOptimistic()` - Type guard for checking optimistic state +- `replaceOptimisticEntity()` - Replaces optimistic items by `_localId` (race-condition safe) +- `removeDuplicateEntities()` - Deduplicates after replacement +- `cleanOptimisticMetadata()` - Strips optimistic fields when needed +### TypeScript Interface ```typescript -// 1. Save current state (for rollback) — take an immutable snapshot -const previousState = structuredClone(currentState); - -// 2. Update UI immediately -setState(newState); - -// 3. Call API -try { - const serverState = await api.updateResource(newState); - // Success — use server as the source of truth - setState(serverState); -} catch (error) { - // 4. Rollback on failure - setState(previousState); - showToast("Failed to update. Reverted changes.", "error"); +interface OptimisticEntity { + _optimistic: boolean; + _localId: string; } ``` -## Implementation Approach +## Implementation Patterns -### Simple Hook Pattern +### Mutation Hooks Pattern +**Reference**: `src/features/projects/tasks/hooks/useTaskQueries.ts:44-108` -```typescript -function useOptimistic(initialValue: T, updateFn: (value: T) => Promise) { - const [value, setValue] = useState(initialValue); - const [isUpdating, setIsUpdating] = useState(false); - const previousValueRef = useRef(initialValue); - const opSeqRef = useRef(0); // monotonically increasing op id - const mountedRef = useRef(true); // avoid setState after unmount - useEffect(() => () => { mountedRef.current = false; }, []); +1. **onMutate**: Create optimistic entity with stable ID + - Use `createOptimisticEntity()` for type-safe creation + - Store `optimisticId` in context for later replacement - const optimisticUpdate = async (newValue: T) => { - const opId = ++opSeqRef.current; - // Save for rollback - previousValueRef.current = value; +2. **onSuccess**: Replace optimistic with server response + - Use `replaceOptimisticEntity()` matching by `_localId` + - Apply `removeDuplicateEntities()` to prevent duplicates - // Update immediately - if (mountedRef.current) setValue(newValue); - if (mountedRef.current) setIsUpdating(true); +3. **onError**: Rollback to previous state + - Restore snapshot from context - try { - const result = await updateFn(newValue); - // Apply only if latest op and still mounted - if (mountedRef.current && opId === opSeqRef.current) { - setValue(result); // Server is source of truth - } - } catch (error) { - // Rollback - if (mountedRef.current && opId === opSeqRef.current) { - setValue(previousValueRef.current); - } - throw error; - } finally { - if (mountedRef.current && opId === opSeqRef.current) { - setIsUpdating(false); - } - } - }; +### UI Component Pattern +**References**: +- `src/features/projects/tasks/components/TaskCard.tsx:39-40,160,186` +- `src/features/projects/components/ProjectCard.tsx:32-33,67,93` +- `src/features/knowledge/components/KnowledgeCard.tsx:49-50,176,244` - return { value, optimisticUpdate, isUpdating }; -} -``` +1. Check optimistic state: `const optimistic = isOptimistic(entity)` +2. Apply conditional styling: Add opacity and ring effect when optimistic +3. Display indicator: Use `` component for visual feedback -### Usage Example +### Visual Indicator Component +**Location**: `src/features/ui/primitives/OptimisticIndicator.tsx` -```typescript -// In a component -const { - value: task, - optimisticUpdate, - isUpdating, -} = useOptimistic(initialTask, (task) => - projectService.updateTask(task.id, task), -); +Reusable component showing: +- Spinning loader icon (Loader2 from lucide-react) +- "Saving..." text with pulse animation +- Configurable via props: `showSpinner`, `pulseAnimation` -// Handle user action -const handleStatusChange = (newStatus: string) => { - optimisticUpdate({ ...task, status: newStatus }).catch((error) => - showToast("Failed to update task", "error"), - ); -}; -``` +## Feature Integration -## Key Principles +### Tasks +- **Mutations**: `src/features/projects/tasks/hooks/useTaskQueries.ts` +- **UI**: `src/features/projects/tasks/components/TaskCard.tsx` +- Creates tasks with `priority: "medium"` default -1. **Keep it simple** — save, update, roll back. -2. **Server is the source of truth** — always use the server response as the final state. -3. **User feedback** — show loading states and clear error messages. -4. **Selective usage** — only where instant feedback matters: - - Drag‑and‑drop - - Status changes - - Toggle switches - - Quick edits +### Projects +- **Mutations**: `src/features/projects/hooks/useProjectQueries.ts` +- **UI**: `src/features/projects/components/ProjectCard.tsx` +- Handles `prd: null`, `data_schema: null` for new projects -## What NOT to Do +### Knowledge +- **Mutations**: `src/features/knowledge/hooks/useKnowledgeQueries.ts` +- **UI**: `src/features/knowledge/components/KnowledgeCard.tsx` +- Uses `createOptimisticId()` directly for progress tracking -- Don't track complex state histories -- Don't try to merge conflicts -- Use with caution for create/delete operations. If used, generate temporary client IDs, reconcile with server‑assigned IDs, ensure idempotency, and define clear rollback/error states. Prefer non‑optimistic flows when side effects are complex. -- Don't over-engineer with queues or reconciliation +### Toasts +- **Location**: `src/features/ui/hooks/useToast.ts:43` +- Uses `createOptimisticId()` for unique toast IDs -## When to Implement +## Testing -Implement optimistic updates when: +### Unit Tests +**Location**: `src/features/shared/optimistic.test.ts` -- Users complain about UI feeling "slow" -- Drag-and-drop or reordering feels laggy -- Quick actions (like checkbox toggles) feel unresponsive -- Network latency is noticeable (> 200ms) +Covers all utility functions with 8 test cases: +- ID uniqueness and format validation +- Entity creation with metadata +- Type guard functionality +- Replacement logic +- Deduplication +- Metadata cleanup -## Success Metrics +### Manual Testing Checklist +1. **Rapid Creation**: Create 5+ items quickly - verify no duplicates +2. **Visual Feedback**: Check optimistic indicators appear immediately +3. **ID Stability**: Confirm nanoid-based IDs after server response +4. **Error Handling**: Stop backend, attempt creation - verify rollback +5. **Race Conditions**: Use browser console script for concurrent creates -When implemented correctly: +## Performance Characteristics -- UI feels instant (< 100ms response) -- Rollbacks are rare (< 1% of updates) -- Error messages are clear -- Users understand what happened when things fail +- **Bundle Impact**: ~130 bytes ([nanoid v5, minified+gzipped](https://bundlephobia.com/package/nanoid@5.0.9)) - build/environment dependent +- **Update Speed**: Typically snappy on modern devices; actual latency varies by device and workload +- **ID Generation**: Per [nanoid benchmarks](https://github.com/ai/nanoid#benchmark): secure sync ≈5M ops/s, non-secure ≈2.7M ops/s, async crypto ≈135k ops/s +- **Memory**: Minimal - only `_optimistic` and `_localId` metadata added per optimistic entity -## Production Considerations +## Migration Notes -The examples above are simplified for clarity. Production implementations should consider: +### From Timestamp-based IDs +**Before**: `const tempId = \`temp-\${Date.now()}\`` +**After**: `const optimisticId = createOptimisticId()` -1. **Deep cloning**: Use `structuredClone()` or a deep clone utility for complex state +### Key Differences +- No timestamp collisions during rapid creation +- Stable IDs survive re-renders +- Type-safe with full TypeScript inference +- ~60% code reduction through shared utilities - ```typescript - const previousState = structuredClone(currentState); // Proper deep clone - ``` +## Best Practices -2. **Race conditions**: Handle out-of-order responses with operation IDs -3. **Unmount safety**: Avoid setState after component unmount -4. **Debouncing**: For rapid updates (e.g., sliders), debounce API calls -5. **Conflict resolution**: For collaborative editing, consider operational transforms -6. **Polling/ETag interplay**: When polling, ignore stale responses (e.g., compare opId or Last-Modified) and rely on ETag/304 to prevent flicker overriding optimistic state. -7. **Idempotency & retries**: Use idempotency keys on write APIs so client retries (or duplicate submits) don't create duplicate effects. +1. **Always use shared utilities** - Don't implement custom optimistic logic +2. **Match by _localId** - Never match by the entity's `id` field +3. **Include deduplication** - Always call `removeDuplicateEntities()` after replacement +4. **Show visual feedback** - Users should see pending state clearly +5. **Handle errors gracefully** - Always implement rollback in `onError` -These complexities are why we recommend starting simple and only adding optimistic updates where the UX benefit is clear. +## Dependencies + +- **nanoid**: v5.0.9 - UUID generation +- **@tanstack/react-query**: v5.x - Mutation state management +- **React**: v18.x - UI components +- **TypeScript**: v5.x - Type safety + +--- + +*Last updated: Phase 3 implementation (PR #695)* \ No newline at end of file diff --git a/README.md b/README.md index de2a9ed3..d0440f1c 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,13 @@ This new vision for Archon replaces the old one (the agenteer). Archon used to b 1. **Clone Repository**: ```bash - git clone https://github.com/coleam00/archon.git + git clone -b stable https://github.com/coleam00/archon.git ``` ```bash cd archon ``` + + **Note:** The `stable` branch is recommended for using Archon. If you want to contribute or try the latest features, use the `main` branch with `git clone https://github.com/coleam00/archon.git` 2. **Environment Configuration**: ```bash diff --git a/archon-ui-main/package-lock.json b/archon-ui-main/package-lock.json index 7d367133..a6653753 100644 --- a/archon-ui-main/package-lock.json +++ b/archon-ui-main/package-lock.json @@ -24,6 +24,7 @@ "fractional-indexing": "^3.2.0", "framer-motion": "^11.5.4", "lucide-react": "^0.441.0", + "nanoid": "^5.0.9", "prismjs": "^1.30.0", "react": "^18.3.1", "react-dnd": "^16.0.1", @@ -9030,10 +9031,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "dev": true, + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz", + "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==", "funding": [ { "type": "github", @@ -9042,10 +9042,10 @@ ], "license": "MIT", "bin": { - "nanoid": "bin/nanoid.cjs" + "nanoid": "bin/nanoid.js" }, "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + "node": "^18 || >=20" } }, "node_modules/natural-compare": { @@ -9651,6 +9651,25 @@ "dev": true, "license": "MIT" }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/archon-ui-main/package.json b/archon-ui-main/package.json index 336132fe..31c07574 100644 --- a/archon-ui-main/package.json +++ b/archon-ui-main/package.json @@ -17,7 +17,9 @@ "biome:ci": "biome ci", "preview": "npx vite preview", "test": "vitest", + "test:run": "vitest run", "test:ui": "vitest --ui", + "test:integration": "vitest run --config vitest.integration.config.ts", "test:coverage": "npm run test:coverage:run && npm run test:coverage:summary", "test:coverage:run": "vitest run --coverage --reporter=dot --reporter=json", "test:coverage:stream": "vitest run --coverage --reporter=default --reporter=json --bail=false || true", @@ -42,6 +44,7 @@ "fractional-indexing": "^3.2.0", "framer-motion": "^11.5.4", "lucide-react": "^0.441.0", + "nanoid": "^5.0.9", "prismjs": "^1.30.0", "react": "^18.3.1", "react-dnd": "^16.0.1", diff --git a/archon-ui-main/public/img/Grok.png b/archon-ui-main/public/img/Grok.png new file mode 100644 index 00000000..44677e7d Binary files /dev/null and b/archon-ui-main/public/img/Grok.png differ diff --git a/archon-ui-main/public/img/Ollama.png b/archon-ui-main/public/img/Ollama.png new file mode 100644 index 00000000..c4869b0e Binary files /dev/null and b/archon-ui-main/public/img/Ollama.png differ diff --git a/archon-ui-main/public/img/OpenAI.png b/archon-ui-main/public/img/OpenAI.png new file mode 100644 index 00000000..b1fd308e Binary files /dev/null and b/archon-ui-main/public/img/OpenAI.png differ diff --git a/archon-ui-main/public/img/OpenRouter.png b/archon-ui-main/public/img/OpenRouter.png new file mode 100644 index 00000000..7619de5f Binary files /dev/null and b/archon-ui-main/public/img/OpenRouter.png differ diff --git a/archon-ui-main/public/img/anthropic-logo.svg b/archon-ui-main/public/img/anthropic-logo.svg new file mode 100644 index 00000000..7f7ae2bb --- /dev/null +++ b/archon-ui-main/public/img/anthropic-logo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/archon-ui-main/public/img/google-logo.svg b/archon-ui-main/public/img/google-logo.svg new file mode 100644 index 00000000..25e68c76 --- /dev/null +++ b/archon-ui-main/public/img/google-logo.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/archon-ui-main/src/App.tsx b/archon-ui-main/src/App.tsx index 2a0cdc22..1d4e22d3 100644 --- a/archon-ui-main/src/App.tsx +++ b/archon-ui-main/src/App.tsx @@ -1,15 +1,15 @@ import { useState, useEffect } from 'react'; import { BrowserRouter as Router, Routes, Route, Navigate } from 'react-router-dom'; -import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { QueryClientProvider } from '@tanstack/react-query'; import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; +import { queryClient } from './features/shared/queryClient'; import { KnowledgeBasePage } from './pages/KnowledgeBasePage'; import { SettingsPage } from './pages/SettingsPage'; import { MCPPage } from './pages/MCPPage'; import { OnboardingPage } from './pages/OnboardingPage'; import { MainLayout } from './components/layout/MainLayout'; import { ThemeProvider } from './contexts/ThemeContext'; -import { ToastProvider } from './contexts/ToastContext'; -import { ToastProvider as FeaturesToastProvider } from './features/ui/components/ToastProvider'; +import { ToastProvider } from './features/ui/components/ToastProvider'; import { SettingsProvider, useSettings } from './contexts/SettingsContext'; import { TooltipProvider } from './features/ui/primitives/tooltip'; import { ProjectPage } from './pages/ProjectPage'; @@ -19,27 +19,6 @@ import { MigrationBanner } from './components/ui/MigrationBanner'; import { serverHealthService } from './services/serverHealthService'; import { useMigrationStatus } from './hooks/useMigrationStatus'; -// Create a client with optimized settings for our polling use case -const queryClient = new QueryClient({ - defaultOptions: { - queries: { - // Keep data fresh for 2 seconds by default - staleTime: 2000, - // Cache data for 5 minutes - gcTime: 5 * 60 * 1000, - // Retry failed requests 3 times - retry: 3, - // Refetch on window focus - refetchOnWindowFocus: true, - // Don't refetch on reconnect by default (we handle this manually) - refetchOnReconnect: false, - }, - mutations: { - // Retry mutations once on failure - retry: 1, - }, - }, -}); const AppRoutes = () => { const { projectsEnabled } = useSettings(); @@ -134,13 +113,11 @@ export function App() { - - - - - - - + + + + + {import.meta.env.VITE_SHOW_DEVTOOLS === 'true' && ( diff --git a/archon-ui-main/src/components/bug-report/BugReportModal.tsx b/archon-ui-main/src/components/bug-report/BugReportModal.tsx index bd321383..69b40262 100644 --- a/archon-ui-main/src/components/bug-report/BugReportModal.tsx +++ b/archon-ui-main/src/components/bug-report/BugReportModal.tsx @@ -5,12 +5,13 @@ import { Button } from "../ui/Button"; import { Input } from "../ui/Input"; import { Card } from "../ui/Card"; import { Select } from "../ui/Select"; -import { useToast } from "../../contexts/ToastContext"; +import { useToast } from "../../features/ui/hooks/useToast"; import { bugReportService, BugContext, BugReportData, } from "../../services/bugReportService"; +import { copyToClipboard } from "../../features/shared/utils/clipboard"; interface BugReportModalProps { isOpen: boolean; @@ -99,13 +100,21 @@ export const BugReportModal: React.FC = ({ // Fallback: copy to clipboard const formattedReport = bugReportService.formatReportForClipboard(bugReportData); - await navigator.clipboard.writeText(formattedReport); + const clipboardResult = await copyToClipboard(formattedReport); - showToast( - "Failed to create GitHub issue, but bug report was copied to clipboard. Please paste it in a new GitHub issue.", - "warning", - 10000, - ); + if (clipboardResult.success) { + showToast( + "Failed to create GitHub issue, but bug report was copied to clipboard. Please paste it in a new GitHub issue.", + "warning", + 10000, + ); + } else { + showToast( + "Failed to create GitHub issue and could not copy to clipboard. Please report manually.", + "error", + 10000, + ); + } } } catch (error) { console.error("Bug report submission failed:", error); @@ -118,15 +127,15 @@ export const BugReportModal: React.FC = ({ } }; - const copyToClipboard = async () => { + const handleCopyToClipboard = async () => { const bugReportData: BugReportData = { ...report, context }; const formattedReport = bugReportService.formatReportForClipboard(bugReportData); - try { - await navigator.clipboard.writeText(formattedReport); + const result = await copyToClipboard(formattedReport); + if (result.success) { showToast("Bug report copied to clipboard", "success"); - } catch { + } else { showToast("Failed to copy to clipboard", "error"); } }; @@ -372,7 +381,7 @@ export const BugReportModal: React.FC = ({ - - - - {/* URL Input */} - {method === 'url' && ( -
- setUrl(e.target.value)} - placeholder="https://example.com or example.com" - accentColor="blue" - /> - {url && !url.startsWith('http://') && !url.startsWith('https://') && ( -

- ℹ️ Will automatically add https:// prefix -

- )} -
- )} - - {/* File Upload */} - {method === 'file' && ( -
- -
- setSelectedFile(e.target.files?.[0] || null)} - className="sr-only" - /> - -
-

- Supports PDF, MD, DOC up to 10MB -

-
- )} - - {/* Crawl Depth - Only for URLs */} - {method === 'url' && ( -
- - - -
- )} - - {/* Tags */} -
- -
- {tags.map((tag) => ( - - {tag} - - - ))} -
- setNewTag(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter' && newTag.trim()) { - setTags([...tags, newTag.trim()]); - setNewTag(''); - } - }} - placeholder="Add tags..." - accentColor="purple" - /> -
- - {/* Action Buttons */} -
- - -
- - - ); -}; \ No newline at end of file diff --git a/archon-ui-main/src/components/knowledge-base/CrawlingProgressCard.tsx b/archon-ui-main/src/components/knowledge-base/CrawlingProgressCard.tsx deleted file mode 100644 index f5eeb5aa..00000000 --- a/archon-ui-main/src/components/knowledge-base/CrawlingProgressCard.tsx +++ /dev/null @@ -1,760 +0,0 @@ -import React, { useState, useEffect, useRef } from 'react'; -import { motion, AnimatePresence } from 'framer-motion'; -import { - Activity, - AlertTriangle, - CheckCircle, - ChevronDown, - ChevronUp, - Clock, - Globe, - FileText, - RotateCcw, - X, - FileCode, - Upload, - Search, - Cpu, - Database, - Code, - Zap, - Square, - Layers, - Download -} from 'lucide-react'; -import { Card } from '../ui/Card'; -import { Button } from '../ui/Button'; -import { Badge } from '../ui/Badge'; -import { CrawlProgressData } from '../../types/crawl'; -import { useCrawlProgressPolling } from '../../hooks/useCrawlQueries'; -import { useTerminalScroll } from '../../hooks/useTerminalScroll'; - -interface CrawlingProgressCardProps { - progressId: string; - initialData?: Partial; - onComplete?: (data: CrawlProgressData) => void; - onError?: (error: string) => void; - onRetry?: () => void; - onDismiss?: () => void; - onStop?: () => void; -} - -// Simple mapping of backend status to UI display -const STATUS_CONFIG = { - // Common statuses - 'starting': { label: 'Starting', icon: , color: 'blue' }, - 'initializing': { label: 'Initializing', icon: , color: 'blue' }, - - // Crawl statuses - 'analyzing': { label: 'Analyzing URL', icon: , color: 'purple' }, - 'crawling': { label: 'Crawling Pages', icon: , color: 'blue' }, - 'processing': { label: 'Processing Content', icon: , color: 'cyan' }, - 'source_creation': { label: 'Creating Source', icon: , color: 'indigo' }, - 'document_storage': { label: 'Storing Documents', icon: , color: 'green' }, - 'code_extraction': { label: 'Extracting Code', icon: , color: 'yellow' }, - 'finalization': { label: 'Finalizing', icon: , color: 'orange' }, - - // Upload statuses - 'reading': { label: 'Reading File', icon: , color: 'blue' }, - 'extracting': { label: 'Extracting Text', icon: , color: 'blue' }, - 'chunking': { label: 'Chunking Content', icon: , color: 'blue' }, - 'creating_source': { label: 'Creating Source', icon: , color: 'blue' }, - 'summarizing': { label: 'Generating Summary', icon: , color: 'purple' }, - 'storing': { label: 'Storing Chunks', icon: , color: 'green' }, - - // End states - 'completed': { label: 'Completed', icon: , color: 'green' }, - 'error': { label: 'Error', icon: , color: 'red' }, - 'failed': { label: 'Failed', icon: , color: 'red' }, - 'cancelled': { label: 'Cancelled', icon: , color: 'gray' }, - 'stopping': { label: 'Stopping', icon: , color: 'orange' }, -} as const; - -export const CrawlingProgressCard: React.FC = ({ - progressId, - initialData, - onComplete, - onError, - onRetry, - onDismiss, - onStop -}) => { - const [showDetailedProgress, setShowDetailedProgress] = useState(true); - const [showLogs, setShowLogs] = useState(false); - const [isStopping, setIsStopping] = useState(false); - - // Track completion/error handling - const [hasHandledCompletion, setHasHandledCompletion] = useState(false); - const [hasHandledError, setHasHandledError] = useState(false); - - // Poll for progress updates - const { data: progressData } = useCrawlProgressPolling(progressId, { - onError: (error: Error) => { - if (error.message === 'Resource no longer exists') { - if (onDismiss) { - onDismiss(); - } - } - } - }); - - // Merge polled data with initial data - preserve important fields - const displayData = progressData ? { - ...initialData, - ...progressData, - // Ensure we don't lose these fields during polling - currentUrl: progressData.currentUrl || progressData.current_url || initialData?.currentUrl, - crawlType: progressData.crawlType || progressData.crawl_type || initialData?.crawlType, - } : { - progressId, - status: 'starting', - progress: 0, - message: 'Initializing...', - ...initialData - } as CrawlProgressData; - - // Use terminal scroll hook for logs - const logsContainerRef = useTerminalScroll( - displayData?.logs || [], - showLogs - ); - - // Handle status changes - useEffect(() => { - if (!progressData) return; - - if (progressData.status === 'completed' && !hasHandledCompletion && onComplete) { - setHasHandledCompletion(true); - onComplete(progressData); - } else if ((progressData.status === 'error' || progressData.status === 'failed') && !hasHandledError && onError) { - setHasHandledError(true); - onError(progressData.error || 'Unknown error'); - } - }, [progressData?.status, hasHandledCompletion, hasHandledError, onComplete, onError]); - - // Get current status config with better fallback - const statusConfig = (() => { - const config = STATUS_CONFIG[displayData.status as keyof typeof STATUS_CONFIG]; - if (config) { - return config; - } - - // Better fallbacks based on progress - if (displayData.progress >= 100) { - return STATUS_CONFIG.completed; - } - if (displayData.progress > 90) { - return STATUS_CONFIG.finalization; - } - - // Log unknown statuses for debugging - console.warn(`Unknown status: ${displayData.status}, progress: ${displayData.progress}%, message: ${displayData.message}`); - - return STATUS_CONFIG.processing; - })(); - - // Debug log for status transitions - useEffect(() => { - if (displayData.status === 'finalization' || - (displayData.status === 'starting' && displayData.progress > 90)) { - console.log('Status transition debug:', { - status: displayData.status, - progress: displayData.progress, - message: displayData.message, - hasStatusConfig: !!STATUS_CONFIG[displayData.status as keyof typeof STATUS_CONFIG] - }); - } - }, [displayData.status, displayData.progress]); - - // Determine crawl type display - const getCrawlTypeDisplay = () => { - const crawlType = displayData.crawlType || - (displayData.uploadType === 'document' ? 'upload' : 'normal'); - - switch (crawlType) { - case 'sitemap': - return { icon: , label: 'Sitemap Crawl' }; - case 'llms-txt': - case 'text_file': - return { icon: , label: 'LLMs.txt Import' }; - case 'upload': - return { icon: , label: 'Document Upload' }; - default: - return { icon: , label: 'Web Crawl' }; - } - }; - - const crawlType = getCrawlTypeDisplay(); - - // Handle stop - const handleStop = async () => { - if (isStopping || !onStop) return; - setIsStopping(true); - try { - onStop(); - } finally { - setIsStopping(false); - } - }; - - // Get progress steps based on type - const getProgressSteps = () => { - const isUpload = displayData.uploadType === 'document'; - - const steps = isUpload ? [ - 'reading', 'extracting', 'chunking', 'creating_source', 'summarizing', 'storing' - ] : [ - 'analyzing', 'crawling', 'processing', 'source_creation', 'document_storage', 'code_extraction', 'finalization' - ]; - - return steps.map(stepId => { - const config = STATUS_CONFIG[stepId as keyof typeof STATUS_CONFIG]; - const currentIndex = steps.indexOf(displayData.status || ''); - const stepIndex = steps.indexOf(stepId); - - let status: 'pending' | 'active' | 'completed' | 'error' = 'pending'; - - if (displayData.status === 'completed') { - status = 'completed'; - } else if (displayData.status === 'error' || displayData.status === 'failed') { - status = stepIndex <= currentIndex ? 'error' : 'pending'; - } else if (stepIndex < currentIndex) { - status = 'completed'; - } else if (stepIndex === currentIndex) { - status = 'active'; - } - - return { - id: stepId, - label: config.label, - icon: config.icon, - status - }; - }); - }; - - const progressSteps = getProgressSteps(); - const isActive = !['completed', 'error', 'failed', 'cancelled'].includes(displayData.status || ''); - - return ( - - {/* Header */} -
- - {crawlType.icon} - {crawlType.label} - - -
-
- - {statusConfig.label} - - {isActive && ( - - {statusConfig.icon} - - )} -
- {displayData.currentUrl && ( -

- {displayData.currentUrl} -

- )} -
- - {/* Stop button */} - {isActive && onStop && ( - - )} -
- - {/* Main Progress Bar */} - {isActive && ( -
-
- - Overall Progress - - - {Math.round(displayData.progress || 0)}% - -
-
- -
- - {/* Current message with numeric progress */} - {displayData.message && ( -

- {displayData.message} - {displayData.status === 'crawling' && displayData.totalPages !== undefined && displayData.totalPages > 0 && ( - - ({displayData.processedPages || 0}/{displayData.totalPages} pages) - - )} -

- )} -
- )} - - {/* Finalization Progress */} - {isActive && displayData.status === 'finalization' && ( -
-
- - - Finalizing Results - -
-

- Completing crawl and saving final metadata... -

-
- )} - - {/* Crawling Statistics - Show detailed crawl progress */} - {isActive && displayData.status === 'crawling' && (displayData.totalPages > 0 || displayData.processedPages > 0) && ( -
-
- - - Crawling Progress - -
-
-
-
Pages Discovered
-
- {displayData.totalPages || 0} -
-
-
-
Pages Processed
-
- {displayData.processedPages || 0} -
-
-
- {displayData.currentUrl && ( -
-
Currently crawling:
-
- {displayData.currentUrl} -
-
- )} -
- )} - - {/* Code Extraction Progress - Special handling for long-running step */} - {isActive && displayData.status === 'code_extraction' && ( -
-
- - - Extracting Code Examples - -
- - {/* Show document scanning progress if available */} - {(displayData.completedDocuments !== undefined || displayData.totalDocuments !== undefined) && - displayData.completedDocuments < displayData.totalDocuments && ( -
-
- Scanning documents: {displayData.completedDocuments || 0} / {displayData.totalDocuments || 0} -
-
-
-
-
- )} - - {/* Show summary generation progress */} - {(displayData.completedSummaries !== undefined || displayData.totalSummaries !== undefined) && displayData.totalSummaries > 0 && ( -
-
- Generating summaries: {displayData.completedSummaries || 0} / {displayData.totalSummaries || 0} -
-
-
-
-
- )} - - {/* Show code blocks found and stored */} -
- {displayData.codeBlocksFound !== undefined && ( -
-
Code Blocks Found
-
- {displayData.codeBlocksFound} -
-
- )} - {displayData.codeExamplesStored !== undefined && ( -
-
Examples Stored
-
- {displayData.codeExamplesStored} -
-
- )} -
- - {/* Fallback to details if main fields not available */} - {!displayData.codeBlocksFound && displayData.details?.codeBlocksFound !== undefined && ( -
-
- - {displayData.details.codeBlocksFound} - - - code blocks found - -
- {displayData.details?.totalChunks && ( -
- Scanning chunk {displayData.details.currentChunk || 0} of {displayData.details.totalChunks} -
- )} -
- )} - -

- {displayData.completedSummaries !== undefined && displayData.totalSummaries > 0 - ? `Generating AI summaries for ${displayData.totalSummaries} code examples...` - : displayData.completedDocuments !== undefined && displayData.totalDocuments > 0 - ? `Scanning ${displayData.totalDocuments} document(s) for code blocks...` - : 'Analyzing content for code examples...'} -

-
- )} - - {/* Real-time Processing Stats */} - {isActive && displayData.status === 'document_storage' && ( -
- {displayData.details?.currentChunk !== undefined && displayData.details?.totalChunks && ( -
-
Chunks Processing
-
- {displayData.details.currentChunk} / {displayData.details.totalChunks} -
-
- {Math.round((displayData.details.currentChunk / displayData.details.totalChunks) * 100)}% complete -
-
- )} - - {displayData.details?.embeddingsCreated !== undefined && ( -
-
Embeddings
-
- {displayData.details.embeddingsCreated} -
-
created
-
- )} - - {displayData.details?.codeBlocksFound !== undefined && displayData.status === 'code_extraction' && ( -
-
Code Blocks
-
- {displayData.details.codeBlocksFound} -
-
extracted
-
- )} - - {displayData.details?.chunksPerSecond && ( -
-
Processing Speed
-
- {displayData.details.chunksPerSecond.toFixed(1)} -
-
chunks/sec
-
- )} - - {displayData.details?.estimatedTimeRemaining && ( -
-
Time Remaining
-
- {Math.ceil(displayData.details.estimatedTimeRemaining / 60)}m -
-
estimated
-
- )} -
- )} - - {/* Batch Processing Info - Enhanced */} - {(() => { - const shouldShowBatch = displayData.totalBatches && displayData.totalBatches > 0 && isActive && displayData.status === 'document_storage'; - return shouldShowBatch; - })() && ( -
-
-
- - - Batch Processing - -
- - {displayData.completedBatches || 0}/{displayData.totalBatches} batches - -
- - {/* Batch progress bar */} -
- -
- -
- {displayData.activeWorkers !== undefined && ( -
- {displayData.activeWorkers} parallel {displayData.activeWorkers === 1 ? 'worker' : 'workers'} -
- )} - - {displayData.currentBatch && displayData.totalChunksInBatch && ( -
- Current: {displayData.chunksInBatch || 0}/{displayData.totalChunksInBatch} chunks -
- )} - - {displayData.details?.totalChunks && ( -
- Total progress: {displayData.details.currentChunk || 0}/{displayData.details.totalChunks} chunks processed -
- )} -
-
- )} - - {/* Detailed Progress Steps */} - {isActive && ( -
- -
- )} - - - {showDetailedProgress && isActive && ( - -
- {progressSteps.map((step) => ( -
-
- {step.status === 'active' ? ( - - {step.icon} - - ) : ( - step.icon - )} -
-
- - {step.label} - - - {/* Show detailed progress for active step */} - {step.status === 'active' && ( -
- {step.id === 'document_storage' && displayData.completedBatches !== undefined && displayData.totalBatches ? ( - Batch {displayData.completedBatches + 1} of {displayData.totalBatches} - ) : step.id === 'code_extraction' && displayData.details?.codeBlocksFound !== undefined ? ( - {displayData.details.codeBlocksFound} code blocks found - ) : step.id === 'crawling' && (displayData.processedPages !== undefined || displayData.totalPages !== undefined) ? ( - - {displayData.processedPages !== undefined ? displayData.processedPages : '?'} of {displayData.totalPages !== undefined ? displayData.totalPages : '?'} pages - - ) : displayData.message ? ( - {displayData.message} - ) : null} -
- )} -
-
- ))} -
- - )} - - - {/* Statistics */} - {(displayData.status === 'completed' || !isActive) && ( -
- {displayData.totalPages && ( -
- Pages: - - {displayData.processedPages || 0} / {displayData.totalPages} - -
- )} - {displayData.chunksStored && ( -
- Chunks: - - {displayData.chunksStored} - -
- )} - {displayData.details?.embeddingsCreated && ( -
- Embeddings: - - {displayData.details.embeddingsCreated} - -
- )} - {displayData.details?.codeBlocksFound && ( -
- Code Blocks: - - {displayData.details.codeBlocksFound} - -
- )} -
- )} - - {/* Error Message */} - {displayData.error && ( -
-

- {displayData.error} -

-
- )} - - {/* Console Logs */} - {displayData.logs && displayData.logs.length > 0 && ( -
- - - - {showLogs && ( - -
-
- {displayData.logs.map((log, index) => ( -
- {log} -
- ))} -
-
- - )} - -
- )} - - {/* Action Buttons */} - {(displayData.status === 'error' || displayData.status === 'failed' || displayData.status === 'cancelled') && ( -
- {onDismiss && ( - - )} - {onRetry && ( - - )} -
- )} - - ); -}; \ No newline at end of file diff --git a/archon-ui-main/src/components/knowledge-base/CrawlingTab.tsx b/archon-ui-main/src/components/knowledge-base/CrawlingTab.tsx deleted file mode 100644 index 4bd498d3..00000000 --- a/archon-ui-main/src/components/knowledge-base/CrawlingTab.tsx +++ /dev/null @@ -1,112 +0,0 @@ -import { useState, useEffect } from 'react'; -import { motion, AnimatePresence } from 'framer-motion'; -import { CrawlingProgressCard } from './CrawlingProgressCard'; -import { CrawlProgressData } from '../../types/crawl'; -import { AlertCircle } from 'lucide-react'; - -interface CrawlingTabProps { - progressItems: CrawlProgressData[]; - onProgressComplete: (data: CrawlProgressData) => void; - onProgressError: (error: string, progressId?: string) => void; - onRetryProgress: (progressId: string) => void; - onStopProgress: (progressId: string) => void; - onDismissProgress: (progressId: string) => void; -} - -export const CrawlingTab = ({ - progressItems, - onProgressComplete, - onProgressError, - onRetryProgress, - onStopProgress, - onDismissProgress -}: CrawlingTabProps) => { - // Group progress items by type for better organization - const groupedItems = progressItems.reduce((acc, item) => { - const type = item.crawlType || (item.uploadType === 'document' ? 'upload' : 'normal'); - if (!acc[type]) acc[type] = []; - acc[type].push(item); - return acc; - }, {} as Record); - - const getSectionTitle = (type: string) => { - switch (type) { - case 'sitemap': return 'Sitemap Crawls'; - case 'llms-txt': return 'LLMs.txt Crawls'; - case 'upload': return 'Document Uploads'; - case 'refresh': return 'Refreshing Sources'; - default: return 'Web Crawls'; - } - }; - - const getSectionDescription = (type: string) => { - switch (type) { - case 'sitemap': - return 'Processing sitemap.xml files to discover and crawl all listed pages'; - case 'llms-txt': - return 'Extracting content from llms.txt files for AI model training'; - case 'upload': - return 'Processing uploaded documents and extracting content'; - case 'refresh': - return 'Re-crawling existing sources to update content'; - default: - return 'Recursively crawling websites to extract knowledge'; - } - }; - - if (progressItems.length === 0) { - return ( -
- -

- No Active Crawls -

-

- Start crawling a website or uploading a document to see progress here -

-
- ); - } - - return ( -
- - {Object.entries(groupedItems).map(([type, items]) => ( - - {/* Section Header */} -
-

- {getSectionTitle(type)} -

-

- {getSectionDescription(type)} -

-
- - {/* Progress Cards */} -
- {items.map((progressData) => ( - onProgressError(error, progressData.progressId)} - onRetry={() => onRetryProgress(progressData.progressId)} - onDismiss={() => onDismissProgress(progressData.progressId)} - onStop={() => onStopProgress(progressData.progressId)} - /> - ))} -
- - ))} - -
- ); -}; \ No newline at end of file diff --git a/archon-ui-main/src/components/knowledge-base/DocumentBrowser.tsx b/archon-ui-main/src/components/knowledge-base/DocumentBrowser.tsx deleted file mode 100644 index 4373cc0b..00000000 --- a/archon-ui-main/src/components/knowledge-base/DocumentBrowser.tsx +++ /dev/null @@ -1,319 +0,0 @@ -import React, { useState, useEffect, useMemo } from 'react'; -import { createPortal } from 'react-dom'; -import { Search, Filter, FileText, Globe, X } from 'lucide-react'; -import { motion, AnimatePresence } from 'framer-motion'; -import { Badge } from '../ui/Badge'; -import { Button } from '../ui/Button'; -import { knowledgeBaseService } from '../../services/knowledgeBaseService'; - -interface DocumentChunk { - id: string; - source_id: string; - content: string; - metadata?: any; - url?: string; -} - -interface DocumentBrowserProps { - sourceId: string; - isOpen: boolean; - onClose: () => void; -} - -const extractDomain = (url: string): string => { - try { - const urlObj = new URL(url); - const hostname = urlObj.hostname; - - // Remove 'www.' prefix if present - const withoutWww = hostname.startsWith('www.') ? hostname.slice(4) : hostname; - - // Keep full hostname (minus 'www.') to preserve subdomain-level filtering - return withoutWww; - } catch { - return url; // Return original if URL parsing fails - } -}; - -export const DocumentBrowser: React.FC = ({ - sourceId, - isOpen, - onClose, -}) => { - const [chunks, setChunks] = useState([]); - const [loading, setLoading] = useState(true); - const [searchQuery, setSearchQuery] = useState(''); - const [selectedDomain, setSelectedDomain] = useState('all'); - const [selectedChunkId, setSelectedChunkId] = useState(null); - const [error, setError] = useState(null); - - // Extract unique domains from chunks - const domains = useMemo(() => { - const domainSet = new Set(); - chunks.forEach(chunk => { - if (chunk.url) { - domainSet.add(extractDomain(chunk.url)); - } - }); - return Array.from(domainSet).sort(); - }, [chunks]); - - // Filter chunks based on search and domain - const filteredChunks = useMemo(() => { - return chunks.filter(chunk => { - // Search filter - const searchLower = searchQuery.toLowerCase(); - const searchMatch = !searchQuery || - chunk.content.toLowerCase().includes(searchLower) || - chunk.url?.toLowerCase().includes(searchLower); - - // Domain filter - const domainMatch = selectedDomain === 'all' || - (chunk.url && extractDomain(chunk.url) === selectedDomain); - - return searchMatch && domainMatch; - }); - }, [chunks, searchQuery, selectedDomain]); - - // Get selected chunk - const selectedChunk = useMemo(() => { - return filteredChunks.find(chunk => chunk.id === selectedChunkId) || filteredChunks[0]; - }, [filteredChunks, selectedChunkId]); - - // Load chunks when component opens - useEffect(() => { - if (isOpen && sourceId) { - loadChunks(); - } - }, [isOpen, sourceId]); - - const loadChunks = async () => { - try { - setLoading(true); - setError(null); - - const response = await knowledgeBaseService.getKnowledgeItemChunks(sourceId); - - if (response.success) { - setChunks(response.chunks); - // Auto-select first chunk if none selected - if (response.chunks.length > 0 && !selectedChunkId) { - setSelectedChunkId(response.chunks[0].id); - } - } else { - setError('Failed to load document chunks'); - } - } catch (error) { - console.error('Failed to load chunks:', error); - setError(error instanceof Error ? error.message : 'Failed to load document chunks'); - } finally { - setLoading(false); - } - }; - - const loadChunksWithDomainFilter = async (domain: string) => { - try { - setLoading(true); - setError(null); - - const domainFilter = domain === 'all' ? undefined : domain; - const response = await knowledgeBaseService.getKnowledgeItemChunks(sourceId, domainFilter); - - if (response.success) { - setChunks(response.chunks); - } else { - setError('Failed to load document chunks'); - } - } catch (error) { - console.error('Failed to load chunks with domain filter:', error); - setError(error instanceof Error ? error.message : 'Failed to load document chunks'); - } finally { - setLoading(false); - } - }; - - const handleDomainChange = (domain: string) => { - setSelectedDomain(domain); - // Note: We could reload with server-side filtering, but for now we'll do client-side filtering - // loadChunksWithDomainFilter(domain); - }; - - if (!isOpen) return null; - - return createPortal( - - e.stopPropagation()} - > - {/* Blue accent line at the top */} -
- - {/* Sidebar */} -
- {/* Sidebar Header */} -
-
-

- Document Chunks ({(filteredChunks || []).length}) -

-
- - {/* Search */} -
- - setSearchQuery(e.target.value)} - className="w-full pl-10 pr-3 py-2 bg-gray-900/70 border border-gray-800 rounded-lg text-sm text-gray-300 placeholder-gray-600 focus:outline-none focus:border-blue-500/50 focus:ring-1 focus:ring-blue-500/20 transition-all" - /> -
- - {/* Domain Filter */} -
- - -
-
- - {/* Document List */} -
- {filteredChunks.length === 0 ? ( -
- No documents found -
- ) : ( - filteredChunks.map((chunk, index) => ( - - )) - )} -
-
- - {/* Main Content Area */} -
- {/* Header */} -
-
-

- {selectedChunk ? `Document Chunk` : 'Document Browser'} -

- {selectedChunk?.url && ( - - - {extractDomain(selectedChunk.url)} - - )} -
- -
- - {/* Content */} -
- {loading ? ( -
-
-
-

Loading document chunks...

-
-
- ) : !selectedChunk || filteredChunks.length === 0 ? ( -
-
- -

Select a document chunk to view content

-
-
- ) : ( -
-
-
- {selectedChunk.url && ( -
- {selectedChunk.url} -
- )} - -
-
- {selectedChunk.content || 'No content available'} -
-
- - {selectedChunk.metadata && ( -
-
- - View Metadata - - 
-                            {JSON.stringify(selectedChunk.metadata, null, 2)}
-
-
-
- )} -
-
-
- )} -
-
- - , - document.body - ); -}; \ No newline at end of file diff --git a/archon-ui-main/src/components/knowledge-base/EditKnowledgeItemModal.tsx b/archon-ui-main/src/components/knowledge-base/EditKnowledgeItemModal.tsx deleted file mode 100644 index 242cce04..00000000 --- a/archon-ui-main/src/components/knowledge-base/EditKnowledgeItemModal.tsx +++ /dev/null @@ -1,277 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { createPortal } from 'react-dom'; -import { motion } from 'framer-motion'; -import { X, Save, RefreshCw, Users, UserX } from 'lucide-react'; -import { Input } from '../ui/Input'; -import { Button } from '../ui/Button'; -import { Card } from '../ui/Card'; -import { KnowledgeItem } from '../../services/knowledgeBaseService'; -import { knowledgeBaseService } from '../../services/knowledgeBaseService'; -import { useToast } from '../../contexts/ToastContext'; - -interface EditKnowledgeItemModalProps { - item: KnowledgeItem; - onClose: () => void; - onUpdate: () => void; -} - -export const EditKnowledgeItemModal: React.FC = ({ - item, - onClose, - onUpdate, -}) => { - const { showToast } = useToast(); - const [isLoading, setIsLoading] = useState(false); - const [isRemovingFromGroup, setIsRemovingFromGroup] = useState(false); - const [formData, setFormData] = useState({ - title: item.title, - description: item.metadata?.description || '', - }); - - const isInGroup = Boolean(item.metadata?.group_name); - - // Handle escape key to close modal - useEffect(() => { - const handleKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Escape') onClose(); - }; - window.addEventListener('keydown', handleKeyDown); - return () => window.removeEventListener('keydown', handleKeyDown); - }, [onClose]); - - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - - if (!formData.title.trim()) { - showToast('Title is required', 'error'); - return; - } - - setIsLoading(true); - - try { - // Update the knowledge item - const updates: any = {}; - - // Only include title if it has changed - if (formData.title !== item.title) { - updates.title = formData.title; - } - - // Only include description if it has changed - if (formData.description !== (item.metadata?.description || '')) { - updates.description = formData.description; - } - - await knowledgeBaseService.updateKnowledgeItem(item.source_id, updates); - - showToast('Knowledge item updated successfully', 'success'); - onUpdate(); - onClose(); - } catch (error) { - console.error('Failed to update knowledge item:', error); - showToast(`Failed to update: ${(error as any)?.message || 'Unknown error'}`, 'error'); - } finally { - setIsLoading(false); - } - }; - - const handleRemoveFromGroup = async () => { - if (!isInGroup) return; - - setIsRemovingFromGroup(true); - - try { - const currentGroupName = item.metadata?.group_name; - if (!currentGroupName) { - throw new Error('No group name found'); - } - - // Get all knowledge items to find other items in the same group - const allItemsResponse = await knowledgeBaseService.getKnowledgeItems({ per_page: 1000 }); - const itemsInGroup = allItemsResponse.items.filter( - knowledgeItem => knowledgeItem.metadata?.group_name === currentGroupName - ); - - console.log(`Found ${itemsInGroup.length} items in group "${currentGroupName}"`); - - if (itemsInGroup.length <= 2) { - // If there are only 2 items in the group, remove group_name from both - // This dissolves the group entirely - showToast('Dissolving group with 2 or fewer items...', 'info'); - - for (const groupItem of itemsInGroup) { - await knowledgeBaseService.updateKnowledgeItem(groupItem.source_id, { - group_name: "" - }); - } - - showToast('Group dissolved - all items are now individual', 'success'); - } else { - // If there are 3+ items, only remove this item from the group - await knowledgeBaseService.updateKnowledgeItem(item.source_id, { - group_name: "" - }); - - showToast('Item removed from group successfully', 'success'); - } - - onUpdate(); - onClose(); - } catch (error) { - console.error('Failed to remove from group:', error); - showToast(`Failed to remove from group: ${(error as any)?.message || 'Unknown error'}`, 'error'); - } finally { - setIsRemovingFromGroup(false); - } - }; - - // Using React Portal to render the modal at the root level - return createPortal( - - e.stopPropagation()} - > - {/* Pink accent line at the top */} -
- - - {/* Header */} -
-

- Edit Knowledge Item -

- -
- - {/* Form */} -
- setFormData({ ...formData, title: e.target.value })} - placeholder="Enter title" - accentColor="pink" - disabled={isLoading} - /> - - {/* Description field */} -
- -
-