Merge remote-tracking branch 'overseerr/develop' into develop

server/entity/Media.ts

@@ -205,8 +205,8 @@ class Media {
           ? externalHostname
           : hostname;
 
-      jellyfinHost = jellyfinHost!.endsWith('/')
-        ? jellyfinHost!.slice(0, -1)
+      jellyfinHost = jellyfinHost.endsWith('/')
+        ? jellyfinHost.slice(0, -1)
         : jellyfinHost;
 
       if (this.jellyfinMediaId) {

server/entity/User.ts

@@ -39,7 +39,7 @@ export class User {
     return users.map((u) => u.filter(showFiltered));
   }
 
-  static readonly filteredFields: string[] = ['email'];
+  static readonly filteredFields: string[] = ['email', 'plexId'];
 
   public displayName: string;
 
@@ -76,7 +76,7 @@ export class User {
   @Column({ type: 'integer', default: UserType.PLEX })
   public userType: UserType;
 
-  @Column({ nullable: true })
+  @Column({ nullable: true, select: true })
   public plexId?: number;
 
   @Column({ nullable: true })

server/index.ts

@@ -17,6 +17,7 @@ import WebPushAgent from '@server/lib/notifications/agents/webpush';
 import { getSettings } from '@server/lib/settings';
 import logger from '@server/logger';
 import routes from '@server/routes';
+import imageproxy from '@server/routes/imageproxy';
 import { getAppVersion } from '@server/utils/appVersion';
 import restartFlag from '@server/utils/restartFlag';
 import { getClientIp } from '@supercharge/request-ip';
@@ -186,6 +187,9 @@ app
       next();
     });
     server.use('/api/v1', routes);
+
+    server.use('/imageproxy', imageproxy);
+
     server.get('*', (req, res) => handle(req, res));
     server.use(
       (

server/interfaces/api/settingsInterfaces.ts

@@ -54,6 +54,11 @@ export interface CacheItem {
   };
 }
 
+export interface CacheResponse {
+  apiCaches: CacheItem[];
+  imageCache: Record<'tmdb', { size: number; imageCount: number }>;
+}
+
 export interface StatusResponse {
   version: string;
   commitTag: string;

server/job/schedule.ts

@@ -1,5 +1,6 @@
 import { MediaServerType } from '@server/constants/server';
 import downloadTracker from '@server/lib/downloadtracker';
+import ImageProxy from '@server/lib/imageproxy';
 import { plexFullScanner, plexRecentScanner } from '@server/lib/scanners/plex';
 import { radarrScanner } from '@server/lib/scanners/radarr';
 import { sonarrScanner } from '@server/lib/scanners/sonarr';
@@ -181,5 +182,21 @@ export const startJobs = (): void => {
     }),
   });
 
+  // Run image cache cleanup every 5 minutes
+  scheduledJobs.push({
+    id: 'image-cache-cleanup',
+    name: 'Image Cache Cleanup',
+    type: 'process',
+    interval: 'long',
+    cronSchedule: jobs['image-cache-cleanup'].schedule,
+    job: schedule.scheduleJob(jobs['image-cache-cleanup'].schedule, () => {
+      logger.info('Starting scheduled job: Image Cache Cleanup', {
+        label: 'Jobs',
+      });
+      // Clean TMDB image cache
+      ImageProxy.clearCache('tmdb');
+    }),
+  });
+
   logger.info('Scheduled jobs loaded', { label: 'Jobs' });
 };

server/lib/imageproxy.ts

@@ -0,0 +1,268 @@
+import logger from '@server/logger';
+import axios from 'axios';
+import rateLimit, { type rateLimitOptions } from 'axios-rate-limit';
+import { createHash } from 'crypto';
+import { promises } from 'fs';
+import path, { join } from 'path';
+
+type ImageResponse = {
+  meta: {
+    revalidateAfter: number;
+    curRevalidate: number;
+    isStale: boolean;
+    etag: string;
+    extension: string;
+    cacheKey: string;
+    cacheMiss: boolean;
+  };
+  imageBuffer: Buffer;
+};
+
+class ImageProxy {
+  public static async clearCache(key: string) {
+    let deletedImages = 0;
+    const cacheDirectory = path.join(
+      __dirname,
+      '../../config/cache/images/',
+      key
+    );
+
+    const files = await promises.readdir(cacheDirectory);
+
+    for (const file of files) {
+      const filePath = path.join(cacheDirectory, file);
+      const stat = await promises.lstat(filePath);
+
+      if (stat.isDirectory()) {
+        const imageFiles = await promises.readdir(filePath);
+
+        for (const imageFile of imageFiles) {
+          const [, expireAtSt] = imageFile.split('.');
+          const expireAt = Number(expireAtSt);
+          const now = Date.now();
+
+          if (now > expireAt) {
+            await promises.rm(path.join(filePath, imageFile));
+            deletedImages += 1;
+          }
+        }
+      }
+    }
+
+    logger.info(`Cleared ${deletedImages} stale image(s) from cache`, {
+      label: 'Image Cache',
+    });
+  }
+
+  public static async getImageStats(
+    key: string
+  ): Promise<{ size: number; imageCount: number }> {
+    const cacheDirectory = path.join(
+      __dirname,
+      '../../config/cache/images/',
+      key
+    );
+
+    const imageTotalSize = await ImageProxy.getDirectorySize(cacheDirectory);
+    const imageCount = await ImageProxy.getImageCount(cacheDirectory);
+
+    return {
+      size: imageTotalSize,
+      imageCount,
+    };
+  }
+
+  private static async getDirectorySize(dir: string): Promise<number> {
+    const files = await promises.readdir(dir, {
+      withFileTypes: true,
+    });
+
+    const paths = files.map(async (file) => {
+      const path = join(dir, file.name);
+
+      if (file.isDirectory()) return await ImageProxy.getDirectorySize(path);
+
+      if (file.isFile()) {
+        const { size } = await promises.stat(path);
+
+        return size;
+      }
+
+      return 0;
+    });
+
+    return (await Promise.all(paths))
+      .flat(Infinity)
+      .reduce((i, size) => i + size, 0);
+  }
+
+  private static async getImageCount(dir: string) {
+    const files = await promises.readdir(dir);
+
+    return files.length;
+  }
+
+  private axios;
+  private cacheVersion;
+  private key;
+
+  constructor(
+    key: string,
+    baseUrl: string,
+    options: {
+      cacheVersion?: number;
+      rateLimitOptions?: rateLimitOptions;
+    } = {}
+  ) {
+    this.cacheVersion = options.cacheVersion ?? 1;
+    this.key = key;
+    this.axios = axios.create({
+      baseURL: baseUrl,
+    });
+
+    if (options.rateLimitOptions) {
+      this.axios = rateLimit(this.axios, options.rateLimitOptions);
+    }
+  }
+
+  public async getImage(path: string): Promise<ImageResponse> {
+    const cacheKey = this.getCacheKey(path);
+
+    const imageResponse = await this.get(cacheKey);
+
+    if (!imageResponse) {
+      const newImage = await this.set(path, cacheKey);
+
+      if (!newImage) {
+        throw new Error('Failed to load image');
+      }
+
+      return newImage;
+    }
+
+    // If the image is stale, we will revalidate it in the background.
+    if (imageResponse.meta.isStale) {
+      this.set(path, cacheKey);
+    }
+
+    return imageResponse;
+  }
+
+  private async get(cacheKey: string): Promise<ImageResponse | null> {
+    try {
+      const directory = join(this.getCacheDirectory(), cacheKey);
+      const files = await promises.readdir(directory);
+      const now = Date.now();
+
+      for (const file of files) {
+        const [maxAgeSt, expireAtSt, etag, extension] = file.split('.');
+        const buffer = await promises.readFile(join(directory, file));
+        const expireAt = Number(expireAtSt);
+        const maxAge = Number(maxAgeSt);
+
+        return {
+          meta: {
+            curRevalidate: maxAge,
+            revalidateAfter: maxAge * 1000 + now,
+            isStale: now > expireAt,
+            etag,
+            extension,
+            cacheKey,
+            cacheMiss: false,
+          },
+          imageBuffer: buffer,
+        };
+      }
+    } catch (e) {
+      // No files. Treat as empty cache.
+    }
+
+    return null;
+  }
+
+  private async set(
+    path: string,
+    cacheKey: string
+  ): Promise<ImageResponse | null> {
+    try {
+      const directory = join(this.getCacheDirectory(), cacheKey);
+      const response = await this.axios.get(path, {
+        responseType: 'arraybuffer',
+      });
+
+      const buffer = Buffer.from(response.data, 'binary');
+      const extension = path.split('.').pop() ?? '';
+      const maxAge = Number(response.headers['cache-control'].split('=')[1]);
+      const expireAt = Date.now() + maxAge * 1000;
+      const etag = response.headers.etag.replace(/"/g, '');
+
+      await this.writeToCacheDir(
+        directory,
+        extension,
+        maxAge,
+        expireAt,
+        buffer,
+        etag
+      );
+
+      return {
+        meta: {
+          curRevalidate: maxAge,
+          revalidateAfter: expireAt,
+          isStale: false,
+          etag,
+          extension,
+          cacheKey,
+          cacheMiss: true,
+        },
+        imageBuffer: buffer,
+      };
+    } catch (e) {
+      logger.debug('Something went wrong caching image.', {
+        label: 'Image Cache',
+        errorMessage: e.message,
+      });
+      return null;
+    }
+  }
+
+  private async writeToCacheDir(
+    dir: string,
+    extension: string,
+    maxAge: number,
+    expireAt: number,
+    buffer: Buffer,
+    etag: string
+  ) {
+    const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`);
+
+    await promises.rm(dir, { force: true, recursive: true }).catch(() => {
+      // do nothing
+    });
+
+    await promises.mkdir(dir, { recursive: true });
+    await promises.writeFile(filename, buffer);
+  }
+
+  private getCacheKey(path: string) {
+    return this.getHash([this.key, this.cacheVersion, path]);
+  }
+
+  private getHash(items: (string | number | Buffer)[]) {
+    const hash = createHash('sha256');
+    for (const item of items) {
+      if (typeof item === 'number') hash.update(String(item));
+      else {
+        hash.update(item);
+      }
+    }
+    // See https://en.wikipedia.org/wiki/Base64#Filenames
+    return hash.digest('base64').replace(/\//g, '-');
+  }
+
+  private getCacheDirectory() {
+    return path.join(__dirname, '../../config/cache/images/', this.key);
+  }
+}
+
+export default ImageProxy;

server/lib/settings.ts

@@ -38,7 +38,7 @@ export interface PlexSettings {
 
 export interface JellyfinSettings {
   name: string;
-  hostname?: string;
+  hostname: string;
   externalHostname?: string;
   libraries: Library[];
   serverId: string;
@@ -263,7 +263,8 @@ export type JobId =
   | 'download-sync'
   | 'download-sync-reset'
   | 'jellyfin-recently-added-sync'
-  | 'jellyfin-full-sync';
+  | 'jellyfin-full-sync'
+  | 'image-cache-cleanup';
 
 interface AllSettings {
   clientId: string;
@@ -446,6 +447,9 @@ class Settings {
         'jellyfin-full-sync': {
           schedule: '0 0 3 * * *',
         },
+        'image-cache-cleanup': {
+          schedule: '0 0 5 * * *',
+        },
       },
     };
     if (initialSettings) {

server/routes/auth.ts

@@ -89,13 +89,28 @@ authRoutes.post('/plex', async (req, res, next) => {
       await userRepository.save(user);
     } else {
       const mainUser = await userRepository.findOneOrFail({
-        select: { id: true, plexToken: true, plexId: true },
+        select: { id: true, plexToken: true, plexId: true, email: true },
         where: { id: 1 },
       });
       const mainPlexTv = new PlexTvAPI(mainUser.plexToken ?? '');
 
+      if (!account.id) {
+        logger.error('Plex ID was missing from Plex.tv response', {
+          label: 'API',
+          ip: req.ip,
+          email: account.email,
+          plexUsername: account.username,
+        });
+
+        return next({
+          status: 500,
+          message: 'Something went wrong. Try again.',
+        });
+      }
+
       if (
         account.id === mainUser.plexId ||
+        (account.email === mainUser.email && !mainUser.plexId) ||
         (await mainPlexTv.checkUserAccess(account.id))
       ) {
         if (user) {
@@ -226,7 +241,7 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
     const hostname =
       settings.jellyfin.hostname !== ''
         ? settings.jellyfin.hostname
-        : body.hostname;
+        : body.hostname ?? '';
     const { externalHostname } = getSettings().jellyfin;
 
     // Try to find deviceId that corresponds to jellyfin user, else generate a new one
@@ -249,8 +264,8 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
         ? externalHostname
         : hostname;
 
-    jellyfinHost = jellyfinHost!.endsWith('/')
-      ? jellyfinHost!.slice(0, -1)
+    jellyfinHost = jellyfinHost.endsWith('/')
+      ? jellyfinHost.slice(0, -1)
       : jellyfinHost;
 
     const account = await jellyfinserver.login(body.username, body.password);

server/routes/imageproxy.ts

@@ -0,0 +1,39 @@
+import ImageProxy from '@server/lib/imageproxy';
+import logger from '@server/logger';
+import { Router } from 'express';
+
+const router = Router();
+const tmdbImageProxy = new ImageProxy('tmdb', 'https://image.tmdb.org', {
+  rateLimitOptions: {
+    maxRequests: 20,
+    maxRPS: 50,
+  },
+});
+
+/**
+ * Image Proxy
+ */
+router.get('/*', async (req, res) => {
+  const imagePath = req.path.replace('/image', '');
+  try {
+    const imageData = await tmdbImageProxy.getImage(imagePath);
+
+    res.writeHead(200, {
+      'Content-Type': `image/${imageData.meta.extension}`,
+      'Content-Length': imageData.imageBuffer.length,
+      'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`,
+      'OS-Cache-Key': imageData.meta.cacheKey,
+      'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT',
+    });
+
+    res.end(imageData.imageBuffer);
+  } catch (e) {
+    logger.error('Failed to proxy image', {
+      imagePath,
+      errorMessage: e.message,
+    });
+    res.status(500).send();
+  }
+});
+
+export default router;

server/routes/settings/index.ts

@@ -16,9 +16,10 @@ import { jobJellyfinFullSync } from '@server/job/jellyfinsync';
 import { scheduledJobs } from '@server/job/schedule';
 import type { AvailableCacheIds } from '@server/lib/cache';
 import cacheManager from '@server/lib/cache';
+import ImageProxy from '@server/lib/imageproxy';
 import { Permission } from '@server/lib/permissions';
 import { plexFullScanner } from '@server/lib/scanners/plex';
-import type { Library, MainSettings } from '@server/lib/settings';
+import type { JobId, Library, MainSettings } from '@server/lib/settings';
 import { getSettings } from '@server/lib/settings';
 import logger from '@server/logger';
 import { isAuthenticated } from '@server/middleware/auth';
@@ -312,8 +313,8 @@ settingsRoutes.get('/jellyfin/users', async (req, res) => {
       ? externalHostname
       : hostname;
 
-  jellyfinHost = jellyfinHost!.endsWith('/')
-    ? jellyfinHost!.slice(0, -1)
+  jellyfinHost = jellyfinHost.endsWith('/')
+    ? jellyfinHost.slice(0, -1)
     : jellyfinHost;
   const userRepository = getRepository(User);
   const admin = await userRepository.findOneOrFail({
@@ -604,7 +605,7 @@ settingsRoutes.post<{ jobId: string }>('/jobs/:jobId/run', (req, res, next) => {
   });
 });
 
-settingsRoutes.post<{ jobId: string }>(
+settingsRoutes.post<{ jobId: JobId }>(
   '/jobs/:jobId/cancel',
   (req, res, next) => {
     const scheduledJob = scheduledJobs.find(
@@ -631,7 +632,7 @@ settingsRoutes.post<{ jobId: string }>(
   }
 );
 
-settingsRoutes.post<{ jobId: string }>(
+settingsRoutes.post<{ jobId: JobId }>(
   '/jobs/:jobId/schedule',
   (req, res, next) => {
     const scheduledJob = scheduledJobs.find(
@@ -666,16 +667,23 @@ settingsRoutes.post<{ jobId: string }>(
   }
 );
 
-settingsRoutes.get('/cache', (req, res) => {
-  const caches = cacheManager.getAllCaches();
+settingsRoutes.get('/cache', async (_req, res) => {
+  const cacheManagerCaches = cacheManager.getAllCaches();
 
-  return res.status(200).json(
-    Object.values(caches).map((cache) => ({
-      id: cache.id,
-      name: cache.name,
-      stats: cache.getStats(),
-    }))
-  );
+  const apiCaches = Object.values(cacheManagerCaches).map((cache) => ({
+    id: cache.id,
+    name: cache.name,
+    stats: cache.getStats(),
+  }));
+
+  const tmdbImageCache = await ImageProxy.getImageStats('tmdb');
+
+  return res.status(200).json({
+    apiCaches,
+    imageCache: {
+      tmdb: tmdbImageCache,
+    },
+  });
 });
 
 settingsRoutes.post<{ cacheId: AvailableCacheIds }>(

server/routes/user/index.ts

@@ -502,8 +502,8 @@ router.post(
           ? externalHostname
           : hostname;
 
-      jellyfinHost = jellyfinHost!.endsWith('/')
-        ? jellyfinHost!.slice(0, -1)
+      jellyfinHost = jellyfinHost.endsWith('/')
+        ? jellyfinHost.slice(0, -1)
         : jellyfinHost;
       jellyfinClient.setUserId(admin.jellyfinUserId ?? '');
       const jellyfinUsers = await jellyfinClient.getUsers();