feat: thumbnails

package.json

@@ -40,6 +40,7 @@
     "dayjs": "^1.11.8",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
+    "ffmpeg-static": "^5.2.0",
     "highlight.js": "^11.8.0",
     "isomorphic-dompurify": "^1.8.0",
     "katex": "^0.16.8",

pnpm-lock.yaml

@@ -74,6 +74,9 @@ dependencies:
   express:
     specifier: ^4.18.2
     version: 4.18.2
+  ffmpeg-static:
+    specifier: ^5.2.0
+    version: 5.2.0
   highlight.js:
     specifier: ^11.8.0
     version: 11.8.0
@@ -1166,6 +1169,16 @@ packages:
       to-fast-properties: 2.0.0
     dev: false
 
+  /@derhuerst/http-basic@8.2.4:
+    resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==}
+    engines: {node: '>=6.0.0'}
+    dependencies:
+      caseless: 0.12.0
+      concat-stream: 2.0.0
+      http-response-object: 3.0.2
+      parse-cache-control: 1.0.1
+    dev: false
+
   /@emotion/babel-plugin@11.11.0:
     resolution: {integrity: sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==}
     dependencies:
@@ -2800,6 +2813,10 @@ packages:
       '@types/express': 4.17.17
     dev: true
 
+  /@types/node@10.17.60:
+    resolution: {integrity: sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==}
+    dev: false
+
   /@types/node@17.0.45:
     resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==}
     dev: false
@@ -3544,6 +3561,10 @@ packages:
     resolution: {integrity: sha512-Jj917pJtYg9HSJBF95HVX3Cdr89JUyLT4IZ8SvM5aDRni95swKgYi3TgYLH5hnGfPE/U1dg6IfZ50UsIlLkwSA==}
     dev: false
 
+  /caseless@0.12.0:
+    resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
+    dev: false
+
   /ccount@2.0.1:
     resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
     dev: false
@@ -3753,6 +3774,16 @@ packages:
       typedarray: 0.0.6
     dev: false
 
+  /concat-stream@2.0.0:
+    resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==}
+    engines: {'0': node >= 6.0}
+    dependencies:
+      buffer-from: 1.1.2
+      inherits: 2.0.4
+      readable-stream: 3.6.2
+      typedarray: 0.0.6
+    dev: false
+
   /console-control-strings@1.1.0:
     resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==}
     dev: false
@@ -4781,6 +4812,19 @@ packages:
     dependencies:
       reusify: 1.0.4
 
+  /ffmpeg-static@5.2.0:
+    resolution: {integrity: sha512-WrM7kLW+do9HLr+H6tk7LzQ7kPqbAgLjdzNE32+u3Ff11gXt9Kkkd2nusGFrlWMIe+XaA97t+I8JS7sZIrvRgA==}
+    engines: {node: '>=16'}
+    requiresBuild: true
+    dependencies:
+      '@derhuerst/http-basic': 8.2.4
+      env-paths: 2.2.1
+      https-proxy-agent: 5.0.1
+      progress: 2.0.3
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /file-entry-cache@6.0.1:
     resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==}
     engines: {node: ^10.12.0 || >=12.0.0}
@@ -5267,6 +5311,12 @@ packages:
       - supports-color
     dev: false
 
+  /http-response-object@3.0.2:
+    resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
+    dependencies:
+      '@types/node': 10.17.60
+    dev: false
+
   /https-proxy-agent@5.0.1:
     resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
     engines: {node: '>= 6'}
@@ -7044,6 +7094,10 @@ packages:
     dependencies:
       callsites: 3.1.0
 
+  /parse-cache-control@1.0.1:
+    resolution: {integrity: sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==}
+    dev: false
+
   /parse-json@5.2.0:
     resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==}
     engines: {node: '>=8'}

prisma/schema.prisma

@@ -135,6 +135,22 @@ model File {
 
   Folder   Folder? @relation(fields: [folderId], references: [id], onDelete: SetNull, onUpdate: Cascade)
   folderId String?
+
+  thumbnail Thumbnail?
+
+}
+
+model Thumbnail { 
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  path String
+
+  file File @relation(fields: [fileId], references: [id], onDelete: Cascade, onUpdate: Cascade) 
+  fileId String
+
+  @@unique([fileId])
 }
 
 model Folder {

src/components/file/DashboardFileType.tsx

@@ -1,5 +1,5 @@
 import type { File as DbFile } from '@/lib/db/models/file';
-import { Center, Image, Paper, Stack, Text } from '@mantine/core';
+import { Box, Center, Image, Paper, Stack, Text } from '@mantine/core';
 import {
   Icon,
   IconFileText,
@@ -113,6 +113,34 @@ export default function DashboardFileType({
           controls
           src={dbFile ? `/raw/${file.name}${password ? `?pw=${password}` : ''}` : URL.createObjectURL(file)}
         />
+      ) : (file as DbFile).thumbnail && dbFile ? (
+        <Box>
+          <Image
+            styles={{
+              imageWrapper: {
+                position: 'inherit',
+              },
+              image: {
+                maxHeight: dbFile ? '100vh' : 100,
+              },
+            }}
+            src={`/raw/${(file as DbFile).thumbnail.path}`}
+            alt={file.name}
+          />
+
+          <Center
+            sx={{
+              position: 'absolute',
+              top: 0,
+              left: 0,
+              height: '100%',
+              width: '100%',
+              cursor: 'pointer',
+            }}
+          >
+            <IconPlayerPlay size='4rem' stroke={3} />
+          </Center>
+        </Box>
       ) : (
         <Placeholder text={`Click to play video ${file.name}`} Icon={IconPlayerPlay} />
       );

src/lib/config/read.ts

@@ -18,6 +18,7 @@ export const rawConfig: any = {
     deleteInterval: undefined,
     clearInvitesInterval: undefined,
     maxViewsInterval: undefined,
+    thumbnailsInterval: undefined,
   },
   files: {
     route: undefined,
@@ -38,7 +39,6 @@ export const rawConfig: any = {
     type: undefined,
   },
   features: {
-    thumbnail: undefined,
     imageCompression: undefined,
     robotsTxt: undefined,
     healthcheck: undefined,
@@ -46,6 +46,10 @@ export const rawConfig: any = {
     userRegistration: undefined,
     oauthRegistration: undefined,
     deleteOnMaxViews: undefined,
+    thumbnails: {
+      enabled: undefined,
+      num_threads: undefined,
+    },
   },
   invites: {
     enabled: undefined,
@@ -105,6 +109,7 @@ export const PROP_TO_ENV: Record<string, string> = {
   'scheduler.deleteInterval': 'SCHEDULER_DELETE_INTERVAL',
   'scheduler.clearInvitesInterval': 'SCHEDULER_CLEAR_INVITES_INTERVAL',
   'scheduler.maxViewsInterval': 'SCHEDULER_MAX_VIEWS_INTERVAL',
+  'scheduler.thumbnailsInterval': 'SCHEDULER_THUMBNAILS_INTERVAL',
 
   'files.route': 'FILES_ROUTE',
   'files.length': 'FILES_LENGTH',
@@ -132,13 +137,14 @@ export const PROP_TO_ENV: Record<string, string> = {
 
   'datasource.local.directory': 'DATASOURCE_LOCAL_DIRECTORY',
 
-  'features.thumbnail': 'FEATURES_THUMBNAIL',
   'features.imageCompression': 'FEATURES_IMAGE_COMPRESSION',
   'features.robotsTxt': 'FEATURES_ROBOTS_TXT',
   'features.healthcheck': 'FEATURES_HEALTHCHECK',
   'features.userRegistration': 'FEATURES_USER_REGISTRATION',
   'features.oauthRegistration': 'FEATURES_OAUTH_REGISTRATION',
   'features.deleteOnMaxViews': 'FEATURES_DELETE_ON_MAX_VIEWS',
+  'features.thumbails.enabled': 'FEATURES_THUMBNAILS_ENABLED',
+  'features.thumbnails.num_threads': 'FEATURES_THUMBNAILS_NUM_THREADS',
 
   'invites.enabled': 'INVITES_ENABLED',
   'invites.length': 'INVITES_LENGTH',
@@ -184,6 +190,7 @@ export function readEnv() {
     env(PROP_TO_ENV['scheduler.deleteInterval'], 'scheduler.deleteInterval', 'ms'),
     env(PROP_TO_ENV['scheduler.clearInvitesInterval'], 'scheduler.clearInvitesInterval', 'ms'),
     env(PROP_TO_ENV['scheduler.maxViewsInterval'], 'scheduler.maxViewsInterval', 'ms'),
+    env(PROP_TO_ENV['scheduler.thumbnailsInterval'], 'scheduler.thumbnailsInterval', 'ms'),
 
     env(PROP_TO_ENV['files.route'], 'files.route', 'string'),
     env(PROP_TO_ENV['files.length'], 'files.length', 'number'),
@@ -206,7 +213,6 @@ export function readEnv() {
 
     env(PROP_TO_ENV['datasource.local.directory'], 'datasource.local.directory', 'string'),
 
-    env(PROP_TO_ENV['features.thumbnail'], 'features.thumbnail', 'boolean'),
     env(PROP_TO_ENV['features.imageCompression'], 'features.imageCompression', 'boolean'),
     env(PROP_TO_ENV['features.robotsTxt'], 'features.robotsTxt', 'boolean'),
     env(PROP_TO_ENV['features.healthcheck'], 'features.healthcheck', 'boolean'),
@@ -214,6 +220,8 @@ export function readEnv() {
     env(PROP_TO_ENV['features.userRegistration'], 'features.userRegistration', 'boolean'),
     env(PROP_TO_ENV['features.oauthRegistration'], 'features.oauthRegistration', 'boolean'),
     env(PROP_TO_ENV['features.deleteOnMaxViews'], 'features.deleteOnMaxViews', 'boolean'),
+    env(PROP_TO_ENV['features.thumbnails.enabled'], 'features.thumbnails.enabled', 'boolean'),
+    env(PROP_TO_ENV['features.thumbnails.num_threads'], 'features.thumbnails.num_threads', 'number'),
 
     env(PROP_TO_ENV['invites.enabled'], 'invites.enabled', 'boolean'),
     env(PROP_TO_ENV['invites.length'], 'invites.length', 'number'),

src/lib/config/validate.ts

@@ -44,6 +44,7 @@ export const schema = z.object({
     deleteInterval: z.number().default(ms('30min')),
     clearInvitesInterval: z.number().default(ms('30min')),
     maxViewsInterval: z.number().default(ms('30min')),
+    thumbnailsInterval: z.number().default(ms('15s')),
   }),
   files: z.object({
     route: z.string().startsWith('/').nonempty().trim().toLowerCase().default('/u'),
@@ -100,13 +101,16 @@ export const schema = z.object({
       }
     }),
   features: z.object({
-    thumbnails: z.boolean().default(true),
     imageCompression: z.boolean().default(true),
     robotsTxt: z.boolean().default(false),
     healthcheck: z.boolean().default(true),
     userRegistration: z.boolean().default(false),
     oauthRegistration: z.boolean().default(false),
     deleteOnMaxViews: z.boolean().default(true),
+    thumbnails: z.object({
+      enabled: z.boolean().default(true),
+      num_threads: z.number().default(4),
+    }),
   }),
   invites: z.object({
     enabled: z.boolean().default(true),

src/lib/db/models/file.ts

@@ -1,5 +1,5 @@
-import { formatRootUrl } from '@/lib/url';
 import { config } from '@/lib/config';
+import { formatRootUrl } from '@/lib/url';
 
 export type File = {
   createdAt: Date;
@@ -15,6 +15,10 @@ export type File = {
   password?: string | boolean | null;
   folderId: string | null;
 
+  thumbnail: {
+    path: string;
+  };
+
   url?: string;
   similarity?: number;
 };
@@ -31,6 +35,11 @@ export const fileSelect = {
   type: true,
   views: true,
   folderId: true,
+  thumbnail: {
+    select: {
+      path: true,
+    },
+  },
 };
 
 export function cleanFile(file: File) {

src/lib/scheduler/index.ts

@@ -6,6 +6,7 @@ export interface Job {
 
   started: boolean;
   logger: Logger;
+  scheduler: Scheduler;
 }
 
 export interface WorkerJob<Data = any> extends Job {
@@ -36,6 +37,7 @@ export class Scheduler {
       if (job.started) continue;
 
       job.logger = this.logger.c('jobs').c(job.id);
+      job.scheduler = this;
 
       if ('interval' in job) {
         this.logger.debug('running first run', {
@@ -78,6 +80,18 @@ export class Scheduler {
       workerData: job.data,
     });
 
+    worker.once('exit', (code) => {
+      this.logger.debug('worker exited', {
+        id: job.id,
+        code,
+      });
+
+      const index = this.jobs.findIndex((x) => x.id === job.id);
+      if (index === -1) return;
+
+      this.jobs.splice(index, 1);
+    });
+
     job.worker = worker;
 
     this.logger.debug('started worker job', {

src/lib/scheduler/jobs/thumbnails.ts

@@ -0,0 +1,47 @@
+import { IntervalJob, WorkerJob } from '..';
+
+export default function thumbnails(prisma: typeof globalThis.__db__) {
+  return async function (this: IntervalJob) {
+    const thumbnailWorkers = this.scheduler.jobs.filter(
+      (x) => 'worker' in x && x.id.startsWith('thumbnail'),
+    ) as unknown as WorkerJob[];
+    if (!thumbnailWorkers.length) return;
+
+    const thumbnailNeeded = await prisma.file.findMany({
+      where: {
+        thumbnail: {
+          is: null,
+        },
+        type: {
+          startsWith: 'video/',
+        },
+      },
+    });
+    if (!thumbnailNeeded.length) return;
+
+    this.logger.debug(`found ${thumbnailNeeded.length} files that need thumbnails`);
+
+    const thumbToWorker: { id: string; worker: number }[] = [];
+
+    let workerIndex = 0;
+    for (const file of thumbnailNeeded) {
+      thumbToWorker.push({
+        id: file.id,
+        worker: workerIndex,
+      });
+
+      workerIndex = (workerIndex + 1) % thumbnailWorkers.length;
+    }
+
+    const ids = thumbnailWorkers.map((_, i) => thumbToWorker.filter((x) => x.worker === i).map((x) => x.id));
+
+    for (let i = 0; i !== thumbnailWorkers.length; ++i) {
+      if (!ids[i].length) continue;
+
+      thumbnailWorkers[i].worker!.postMessage({
+        type: 0,
+        data: ids[i],
+      });
+    }
+  };
+}

src/offload/thumbnails.ts

@@ -0,0 +1,143 @@
+import { bytes } from '@/lib/bytes';
+import { datasource } from '@/lib/datasource';
+import { prisma } from '@/lib/db';
+import { log } from '@/lib/logger';
+import { spawn } from 'child_process';
+import ffmpegPath from 'ffmpeg-static';
+import { createWriteStream } from 'fs';
+import { join } from 'path';
+import { isMainThread, parentPort, workerData } from 'worker_threads';
+
+export type ThumbnailWorkerData = {
+  id: string;
+  enabled: boolean;
+};
+
+const { id, enabled } = workerData as ThumbnailWorkerData;
+
+const logger = log('scheduler').c('jobs').c(id);
+
+if (isMainThread) {
+  logger.error("thumbnail worker can't run on the main thread");
+  process.exit(1);
+}
+
+if (!enabled) {
+  logger.debug('thumbnail generation is disabled');
+  process.exit(0);
+}
+
+logger.debug('started thumbnail worker');
+
+async function ffmpeg(file: string): Promise<Buffer | undefined> {
+  const args = ['-i', file, '-frames:v', '1', '-f', 'mjpeg', 'pipe:1'];
+
+  const proc = spawn(ffmpegPath!, args, {
+    stdio: ['ignore', 'pipe', 'ignore'],
+  });
+
+  try {
+    const buffer = await new Promise<Buffer>((resolve, reject) => {
+      const data: Buffer[] = [];
+
+      proc.stdout!.on('data', (d) => {
+        data.push(d);
+      });
+
+      proc.once('error', reject);
+
+      proc.once('close', (code) => {
+        if (code !== 0) {
+          const stringed = Buffer.concat([...data]).toString();
+
+          logger.error('ffmpeg exited with non-zero code');
+
+          reject(stringed);
+        } else {
+          resolve(Buffer.concat([...data]));
+        }
+      });
+    });
+
+    return buffer;
+  } catch (e) {
+    logger.error('failed to generate thumbnail', {
+      file,
+      error: e,
+    });
+  }
+}
+
+async function generate(ids: string[]) {
+  for (const id of ids) {
+    const file = await prisma.file.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        thumbnail: true,
+      },
+    });
+
+    if (!file) return;
+    if (!file.type.startsWith('video/')) {
+      logger.debug('received file that is not a video', { id: file.id, type: file.type });
+      continue;
+    }
+
+    if (file.thumbnail) {
+      logger.debug('thumbnail already exists', { id: file.id });
+      continue;
+    }
+
+    const stream = await datasource.get(file.name);
+    if (!stream) return;
+
+    const tmpFile = join('/tmp', `zthumbnail_${file.id}.tmp`);
+    const writeStream = createWriteStream(tmpFile);
+    await new Promise((resolve, reject) => {
+      stream.pipe(writeStream);
+      stream.on('error', reject);
+      writeStream.on('error', reject);
+      writeStream.on('finish', resolve);
+    });
+
+    const thumbnail = await ffmpeg(tmpFile);
+    if (!thumbnail) return;
+
+    await datasource.put(`.thumbnail.${file.id}.jpg`, thumbnail);
+
+    const t = await prisma.thumbnail.create({
+      data: {
+        fileId: file.id,
+        path: `.thumbnail.${file.id}.jpg`,
+      },
+    });
+
+    logger.info('generated thumbnail', { id: t.id, fileId: file.id, size: bytes(thumbnail.length) });
+  }
+}
+
+async function main() {
+  parentPort!.on('message', async (d) => {
+    const { type, data } = d as {
+      type: 0 | 1;
+      data?: string[];
+    };
+
+    switch (type) {
+      case 0:
+        logger.debug('received thumbnail generation request', { ids: data });
+        await generate(data!);
+        break;
+      case 1:
+        logger.debug('received kill request');
+        process.exit(0);
+      default:
+        logger.error('received unknown message type', { type });
+        break;
+    }
+  });
+}
+
+main();

src/server/index.ts

@@ -17,6 +17,7 @@ import { Scheduler } from '@/lib/scheduler';
 import deleteFiles from '@/lib/scheduler/jobs/deleteFiles';
 import clearInvites from '@/lib/scheduler/jobs/clearInvites';
 import maxViews from '@/lib/scheduler/jobs/maxViews';
+import thumbnails from '@/lib/scheduler/jobs/thumbnails';
 
 const MODE = process.env.NODE_ENV || 'production';
 
@@ -94,20 +95,20 @@ async function main() {
       },
     });
 
-    if (!file) return app.render404(req, res, parsedUrl);
-
-    const stream = await datasource.get(file.name);
+    const stream = await datasource.get(file?.name ?? id);
     if (!stream) return app.render404(req, res, parsedUrl);
-    if (file.password) {
+    if (file?.password) {
       if (!pw) return res.status(403).json({ code: 403, message: 'Password protected.' });
       const verified = await verifyPassword(pw as string, file.password!);
 
       if (!verified) return res.status(403).json({ code: 403, message: 'Incorrect password.' });
     }
 
+    const size = await datasource.size(file?.name ?? id);
+
     res.setHeader('Content-Type', 'text/plain');
-    res.setHeader('Content-Length', file.size);
-    file.originalName &&
+    res.setHeader('Content-Length', size);
+    file?.originalName &&
       res.setHeader(
         'Content-Disposition',
         `${req.query.download ? 'attachment; ' : ''}filename="${file.originalName}"`,
@@ -129,7 +130,18 @@ async function main() {
 
     scheduler.addInterval('deletefiles', config.scheduler.deleteInterval, deleteFiles(prisma));
     scheduler.addInterval('maxviews', config.scheduler.maxViewsInterval, maxViews(prisma));
-    scheduler.addInterval('clearinvites', config.scheduler.clearInvitesInterval, clearInvites(prisma));
+    scheduler.addInterval('thumbnails', config.scheduler.thumbnailsInterval, thumbnails(prisma));
+
+    if (config.features.thumbnails.enabled) {
+      for (let i = 0; i !== config.features.thumbnails.num_threads; ++i) {
+        scheduler.addWorker(`thumbnail-${i}`, './build/offload/thumbnails.js', {
+          id: `thumbnail-${i}`,
+          enabled: config.features.thumbnails.enabled,
+        });
+      }
+
+      scheduler.addInterval('clearinvites', config.scheduler.clearInvitesInterval, clearInvites(prisma));
+    }
 
     scheduler.start();
   });

tsup.config.ts

@@ -12,4 +12,15 @@ export default defineConfig([
     },
     outDir: 'build',
   },
+  {
+    platform: 'node',
+    format: 'cjs',
+    treeshake: true,
+    clean: false,
+    sourcemap: true,
+    entryPoints: {
+      thumbnails: 'src/offload/thumbnails.ts',
+    },
+    outDir: 'build/offload',
+  },
 ]);