From e5aa4f0aef05c74a064c3e0878c4663e2e50428f Mon Sep 17 00:00:00 2001 From: Spotizerr Date: Sat, 23 Aug 2025 12:53:37 -0600 Subject: [PATCH 1/6] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8ff2833..7231a57 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ A self-hosted music download manager with a lossless twist. Download everything If you self-host a music server with other users than yourself, you almost certainly have realized that the process of adding requested items to the library is not without its friction. No matter how automated your flow is, unless your users are tech-savvy enough to do it themselves, chances are the process always needs some type of manual intervention from you, be it to rip the CDs yourself, tag some random files from youtube, etc. No more! Spotizerr allows for your users to access a nice little frontend where they can add whatever they want to the library without bothering you. What's that? You want some screenshots? Sure, why not: +## How do I start? + +Docs are available at: https://spotizerr.rtfd.io +
Main page image From 09a623f98ba98d86957fc44bc05c29e8cb2c96be Mon Sep 17 00:00:00 2001 From: Spotizerr Date: Sat, 23 Aug 2025 12:55:27 -0600 Subject: [PATCH 2/6] Update .env.example --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 227381e..8d4b3a5 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ ### can leave the defaults as they are. ### ### If you plan on using for a server, -### see [insert docs url] +### see https://spotizerr.rtfd.io ### # Interface to bind to. Unless you know what you're doing, don't change this @@ -56,4 +56,4 @@ GOOGLE_CLIENT_SECRET= # GitHub SSO (get from GitHub Developer Settings) GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= \ No newline at end of file +GITHUB_CLIENT_SECRET= From 8b90c7b75b03c50db8cd6087f4695212e6b131bb Mon Sep 17 00:00:00 2001 From: Spotizerr Date: Sat, 23 Aug 2025 13:07:36 -0600 Subject: [PATCH 3/6] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7231a57..6a980a6 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,6 @@ A self-hosted music download manager with a lossless twist. Download everything If you self-host a music server with other users than yourself, you almost certainly have realized that the process of adding requested items to the library is not without its friction. No matter how automated your flow is, unless your users are tech-savvy enough to do it themselves, chances are the process always needs some type of manual intervention from you, be it to rip the CDs yourself, tag some random files from youtube, etc. No more! Spotizerr allows for your users to access a nice little frontend where they can add whatever they want to the library without bothering you. What's that? You want some screenshots? Sure, why not: -## How do I start? - -Docs are available at: https://spotizerr.rtfd.io -
Main page image @@ -31,6 +27,10 @@ Docs are available at: https://spotizerr.rtfd.io image
+## How do I start? + +Docs are available at: https://spotizerr.rtfd.io + ### Common Issues **Downloads not starting?** From f9cf953de142dc4de52fad48ffb49b2ca9ca1688 Mon Sep 17 00:00:00 2001 From: che-pj Date: Sat, 30 Aug 2025 09:32:44 +0200 Subject: [PATCH 4/6] feat(api): add per-task sse throttling and batching for robust updates --- routes/system/progress.py | 99 +++++++++++++++++++++++++++++------ routes/utils/celery_config.py | 3 +- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/routes/system/progress.py b/routes/system/progress.py index f5d5d78..e2a6cc0 100755 --- a/routes/system/progress.py +++ b/routes/system/progress.py @@ -8,7 +8,7 @@ from typing import Set, Optional import redis import threading -from routes.utils.celery_config import REDIS_URL +from routes.utils.celery_config import REDIS_URL, get_config_params from routes.utils.celery_tasks import ( get_task_info, @@ -37,6 +37,11 @@ router = APIRouter() class SSEBroadcaster: def __init__(self): self.clients: Set[asyncio.Queue] = set() + # Per-task throttling/batching/deduplication state + self._task_state = {} # task_id -> dict with last_sent, last_event, last_send_time, scheduled_handle + # Load configurable interval + config = get_config_params() + self.sse_update_interval = float(config.get("sseUpdateIntervalSeconds", 1)) async def add_client(self, queue: asyncio.Queue): """Add a new SSE client""" @@ -49,43 +54,105 @@ class SSEBroadcaster: logger.debug(f"SSE: Client disconnected (total: {len(self.clients)})") async def broadcast_event(self, event_data: dict): - """Broadcast an event to all connected clients""" - logger.debug( - f"SSE Broadcaster: Attempting to broadcast to {len(self.clients)} clients" - ) - + """ + Throttle, batch, and deduplicate SSE events per task. + Only emit at most 1 update/sec per task, aggregate within window, suppress redundant updates. + """ if not self.clients: logger.debug("SSE Broadcaster: No clients connected, skipping broadcast") return - # Add global task counts right before broadcasting - this is the single source of truth + # Defensive: always work with a list of tasks + tasks = event_data.get("tasks", []) + if not isinstance(tasks, list): + tasks = [tasks] + + # For each task, throttle/batch/dedupe + for task in tasks: + task_id = task.get("task_id") + if not task_id: + continue + + now = time.time() + state = self._task_state.setdefault(task_id, { + "last_sent": None, + "last_event": None, + "last_send_time": 0, + "scheduled_handle": None, + }) + + # Deduplication: if event is identical to last sent, skip + if state["last_sent"] is not None and self._events_equal(state["last_sent"], task): + logger.debug(f"SSE: Deduped event for task {task_id}") + continue + + # Throttling: if within interval, batch (store as last_event, schedule send) + elapsed = now - state["last_send_time"] + if elapsed < self.sse_update_interval: + state["last_event"] = task + if state["scheduled_handle"] is None: + delay = self.sse_update_interval - elapsed + loop = asyncio.get_event_loop() + state["scheduled_handle"] = loop.call_later( + delay, lambda: asyncio.create_task(self._send_batched_event(task_id)) + ) + continue + + # Otherwise, send immediately + await self._send_event(task_id, task) + state["last_send_time"] = now + state["last_sent"] = task + state["last_event"] = None + if state["scheduled_handle"]: + state["scheduled_handle"].cancel() + state["scheduled_handle"] = None + + async def _send_batched_event(self, task_id): + state = self._task_state.get(task_id) + if not state or not state["last_event"]: + return + await self._send_event(task_id, state["last_event"]) + state["last_send_time"] = time.time() + state["last_sent"] = state["last_event"] + state["last_event"] = None + state["scheduled_handle"] = None + + async def _send_event(self, task_id, task): + # Compose event_data for this task + event_data = { + "tasks": [task], + "current_timestamp": time.time(), + "change_type": "update", + } enhanced_event_data = add_global_task_counts_to_event(event_data.copy()) event_json = json.dumps(enhanced_event_data) sse_data = f"data: {event_json}\n\n" - logger.debug( - f"SSE Broadcaster: Broadcasting event: {enhanced_event_data.get('change_type', 'unknown')} with {enhanced_event_data.get('active_tasks', 0)} active tasks" - ) - - # Send to all clients, remove disconnected ones disconnected = set() sent_count = 0 for client_queue in self.clients.copy(): try: await client_queue.put(sse_data) sent_count += 1 - logger.debug("SSE: Successfully sent to client queue") except Exception as e: logger.error(f"SSE: Failed to send to client: {e}") disconnected.add(client_queue) - - # Clean up disconnected clients for client in disconnected: self.clients.discard(client) logger.debug( - f"SSE Broadcaster: Successfully sent to {sent_count} clients, removed {len(disconnected)} disconnected clients" + f"SSE Broadcaster: Sent throttled/batched event for task {task_id} to {sent_count} clients" ) + def _events_equal(self, a, b): + # Compare two task dicts for deduplication (ignore timestamps) + if not isinstance(a, dict) or not isinstance(b, dict): + return False + a_copy = dict(a) + b_copy = dict(b) + a_copy.pop("timestamp", None) + b_copy.pop("timestamp", None) + return a_copy == b_copy + # Global broadcaster instance sse_broadcaster = SSEBroadcaster() diff --git a/routes/utils/celery_config.py b/routes/utils/celery_config.py index 83814fd..e97c24f 100644 --- a/routes/utils/celery_config.py +++ b/routes/utils/celery_config.py @@ -52,6 +52,7 @@ DEFAULT_MAIN_CONFIG = { "watch": {}, "realTimeMultiplier": 0, "padNumberWidth": 3, + "sseUpdateIntervalSeconds": 1, # Configurable SSE update interval (default: 1s) } @@ -188,7 +189,7 @@ task_annotations = { "rate_limit": f"{MAX_CONCURRENT_DL}/m", }, "routes.utils.celery_tasks.trigger_sse_update_task": { - "rate_limit": "500/m", # Allow high rate for real-time SSE updates + "rate_limit": "60/m", # Throttle to 1 update/sec per task (matches SSE throttle) "default_retry_delay": 1, # Quick retry for SSE updates "max_retries": 1, # Limited retries for best-effort delivery "ignore_result": True, # Don't store results for SSE tasks From 1016d333ccba513f7570e0ea2f1f07526463b9a1 Mon Sep 17 00:00:00 2001 From: che-pj Date: Sat, 30 Aug 2025 12:24:37 +0200 Subject: [PATCH 5/6] ci(workflows): add pr-build workflow for dev/test container images - Introduces a new GitHub Actions workflow that automatically builds and pushes multi-arch Docker images for pull requests - Images are tagged with the PR number (e.g., dev-pr-123) for easy identification - Uses GHCR as the container registry with proper authentication via GITHUB_TOKEN - Implements BuildKit cache optimization for faster builds - Supports both linux/amd64 and linux/arm64 platforms --- .github/workflows/pr-build.yml | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/pr-build.yml diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 0000000..f572e76 --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,50 @@ +name: PR Dev/Test Container + +on: + pull_request: + types: [opened, synchronize, reopened] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # ✅ needed for GHCR push + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata for PR builds + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=dev-pr-${{ github.event.pull_request.number }} + + # Build and push multi-arch dev image + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From 6922b4a5da82c02d0ad37555d4929c1c82f34be9 Mon Sep 17 00:00:00 2001 From: che-pj Date: Sat, 30 Aug 2025 12:59:30 +0200 Subject: [PATCH 6/6] updates --- .github/workflows/pr-build.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index f572e76..f4cea3a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -3,6 +3,14 @@ name: PR Dev/Test Container on: pull_request: types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + pr_number: + description: 'Pull request number (optional, for manual runs)' + required: false + branch: + description: 'Branch to build (optional, defaults to PR head or main)' + required: false env: REGISTRY: ghcr.io @@ -13,10 +21,12 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - packages: write # ✅ needed for GHCR push + packages: write steps: - name: Checkout code uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch || github.head_ref || github.ref }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -28,14 +38,14 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Extract metadata for PR builds + # Extract Docker metadata - name: Extract Docker metadata id: meta uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=raw,value=dev-pr-${{ github.event.pull_request.number }} + type=raw,value=dev-pr-${{ github.event.inputs.pr_number || github.event.pull_request.number }} # Build and push multi-arch dev image - name: Build and push