feat: maintenance mode (#23431)

* feat: add a `maintenance.enabled` config flag

* feat: implement graceful restart
feat: restart when maintenance config is toggled

* feat: boot a stripped down maintenance api if enabled

* feat: cli command to toggle maintenance mode

* chore: fallback IMMICH_SERVER_URL environment variable in process

* chore: add additional routes to maintenance controller

* fix: don't wait for nest application to close to finish request response

* chore: add a failsafe on restart to prevent other exit codes from preventing restart

* feat: redirect into/from maintenance page

* refactor: use system metadata for maintenance status

* refactor: wait on WebSocket connection to refresh

* feat: broadcast websocket event on server restart
refactor: listen to WS instead of polling

* refactor: bubble up maintenance information instead of hijacking in fetch function
feat: show modal when server is restarting

* chore: increase timeout for ungraceful restart

* refactor: deduplicate code between api/maintenance workers

* fix: skip config check if database is not initialised

* fix: add `maintenanceMode` field to system config test

* refactor: move maintenance resolution code to static method in service

* chore: clean up linter issues

* chore: generate dart openapi

* refactor: use try{} block for maintenance mode check

* fix: logic error in server redirect

* chore: include `maintenanceMode` key in e2e test

* chore: add i18n entries for maintenance screens

* chore: remove negated condition from hook

* fix: should set default value not override in service

* fix: minor error in page

* feat: initial draft of maintenance module, repo., worker controller, worker service

* refactor: move broadcast code into notification service

* chore: connect websocket on client if in maintenance

* chore: set maintenance module app name

* refactor: rename repository to include worker
chore: configure websocket adapter

* feat: reimplement maintenance mode exit with new module

* refactor: add a constant enum for ExitCode

* refactor: remove redundant route for maintenance

* refactor: only spin up kysely on boot (rather than a Nest app)

* refactor(web): move redirect logic into +layout file where modal is setup

* feat: add Maintenance permission

* refactor: merge common code between api/maintenance

* fix: propagate changes from the CLI to servers

* feat: maintenance authentication guard

* refactor: unify maintenance code into repository
feat: add a step to generate maintenance mode token

* feat: jwt auth for maintenance

* refactor: switch from nest jwt to just jsonwebtokens

* feat: log into maintenance mode from CLI command

* refactor: use `secret` instead of `token` in jwt terminology
chore: log maintenance mode login URL on boot
chore: don't make CLI actions reload if already in target state

* docs: initial draft for maintenance mode page

* refactor: always validate the maintenance auth on the server

* feat: add a link to maintenance mode documentation

* feat: redirect users back to the last page they were on when exiting maintenance

* refactor: provide closeFn in both maintenance repos.

* refactor: ensure the user is also redirected by the server

* chore: swap jsonwebtoken for jose

* refactor: introduce AppRestartEvent w/o secret passing

* refactor: use navigation goto

* refactor: use `continue` instead of `next`

* chore: lint fixes for server

* chore: lint fixes for web

* test: add mock for maintenance repository

* test: add base service dependency to maintenance

* chore: remove @types/jsonwebtoken

* refactor: close database connection after startup check

* refactor: use `request#auth` key

* refactor: use service instead of repository
chore: read token from cookie if possible
chore: rename client event to AppRestartV1

* refactor: more concise redirect logic on web

* refactor: move redirect check into utils
refactor: update translation strings to be more sensible

* refactor: always validate login (i.e. check cookie)

* refactor: lint, open-api, remove old dto

* refactor: encode at point of usage

* refactor: remove business logic from repositories

* chore: fix server/web lints

* refactor: remove repository mock

* chore: fix formatting

* test: write service mocks for maintenance mode

* test: write cli service tests

* fix: catch errors when closing app

* fix: always report no maintenance when usual API is available

* test: api e2e maintenance spec

* chore: add response builder

* chore: add helper to set maint. auth cookie

* feat: add SSR to maintenance API

* test(e2e): write web spec for maintenance

* chore: clean up lint issues

* chore: format files

* feat: perform 302 redirect at server level during maintenance

* fix: keep trying to stop immich until it succeeds (CLI issue)

* chore: lint/format

* refactor: annotate references to other services in worker service

* chore: lint

* refactor: remove unnecessary await

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* refactor: move static methods into util

* refactor: assert secret exists in maintenance worker

* refactor: remove assertion which isn't necessary anymore

* refactor: remove assertion

* refactor: remove outer try {} catch block from loadMaintenanceAuth

* refactor: undo earlier change to vite.config.ts

* chore: update tests due to refactors

* revert: vite.config.ts

* test: expect string jwt

* chore: move blanket exceptions into controllers

* test: update tests according with last change

* refactor: use respondWithCookie
refactor: merge start/end into one route
refactor: rename MaintenanceRepository to AppRepository
chore: use new ApiTag/Endpoint
refactor: apply other requested changes

* chore: regenerate openapi

* chore: lint/format

* chore: remove secureOnly for maint. cookie

* refactor: move maintenance worker code into src/maintenance\nfix: various test fixes

* refactor: use `action` property for setting maint. mode

* refactor: remove Websocket#restartApp in favour of individual methods

* chore: incomplete commit

* chore: remove stray log

* fix: call exitApp from maintenance worker on exit

* fix: add app repository mock

* fix: ensure maintenance cookies are secure

* fix: run playwright tests over secure context (localhost)

* test: update other references to 127.0.0.1

* refactor: use serverSideEmitWithAck

* chore: correct the logic in tryTerminate

* test: juggle cookies ourselves

* chore: fix lint error for e2e spec

* chore: format e2e test

* fix: set cookie secure/non-secure depending on context

* chore: format files

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Paul Makles
2025-11-17 17:15:44 +00:00
committed by GitHub
parent ce82e27f4b
commit 15e00f82f0
73 changed files with 2592 additions and 136 deletions

View File

@@ -78,6 +78,7 @@
"handlebars": "^4.7.8",
"i18n-iso-countries": "^7.6.0",
"ioredis": "^5.8.2",
"jose": "^5.10.0",
"js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.2",
"kysely": "0.28.2",

87
server/src/app.common.ts Normal file
View File

@@ -0,0 +1,87 @@
import { NestExpressApplication } from '@nestjs/platform-express';
import { json } from 'body-parser';
import compression from 'compression';
import cookieParser from 'cookie-parser';
import { existsSync } from 'node:fs';
import sirv from 'sirv';
import { excludePaths, serverVersion } from 'src/constants';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
import { ApiService } from 'src/services/api.service';
import { useSwagger } from 'src/utils/misc';
export function configureTelemetry() {
const { telemetry } = new ConfigRepository().getEnv();
if (telemetry.metrics.size > 0) {
bootstrapTelemetry(telemetry.apiPort);
}
}
export async function configureExpress(
app: NestExpressApplication,
{
permitSwaggerWrite = true,
ssr,
}: {
/**
* Whether to allow swagger module to write to the specs.json
* This is not desirable when the API is not available
* @default true
*/
permitSwaggerWrite?: boolean;
/**
* Service to use for server-side rendering
*/
ssr: typeof ApiService | typeof MaintenanceWorkerService;
},
) {
const configRepository = app.get(ConfigRepository);
const { environment, host, port, resourcePaths, network } = configRepository.getEnv();
const logger = await app.resolve(LoggingRepository);
logger.setContext('Bootstrap');
app.useLogger(logger);
app.set('trust proxy', ['loopback', ...network.trustedProxies]);
app.set('etag', 'strong');
app.use(cookieParser());
app.use(json({ limit: '10mb' }));
if (configRepository.isDev()) {
app.enableCors();
}
app.setGlobalPrefix('api', { exclude: excludePaths });
app.useWebSocketAdapter(new WebSocketAdapter(app));
useSwagger(app, { write: configRepository.isDev() && permitSwaggerWrite });
if (existsSync(resourcePaths.web.root)) {
// copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46
// provides serving of precompressed assets and caching of immutable assets
app.use(
sirv(resourcePaths.web.root, {
etag: true,
gzip: true,
brotli: true,
extensions: [],
setHeaders: (res, pathname) => {
if (pathname.startsWith(`/_app/immutable`) && res.statusCode === 200) {
res.setHeader('cache-control', 'public,max-age=31536000,immutable');
}
},
}),
);
}
app.use(app.get(ssr).ssr(excludePaths));
app.use(compression());
const server = await (host ? app.listen(port, host) : app.listen(port));
server.requestTimeout = 24 * 60 * 60 * 1000;
logger.log(`Immich Server is listening on ${await app.getUrl()} [v${serverVersion}] [${environment}] `);
}

View File

@@ -9,15 +9,21 @@ import { commandsAndQuestions } from 'src/commands';
import { IWorker } from 'src/constants';
import { controllers } from 'src/controllers';
import { ImmichWorker } from 'src/enum';
import { MaintenanceAuthGuard } from 'src/maintenance/maintenance-auth.guard';
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
import { MaintenanceWorkerController } from 'src/maintenance/maintenance-worker.controller';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { AuthGuard } from 'src/middleware/auth.guard';
import { ErrorInterceptor } from 'src/middleware/error.interceptor';
import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor';
import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter';
import { LoggingInterceptor } from 'src/middleware/logging.interceptor';
import { repositories } from 'src/repositories';
import { AppRepository } from 'src/repositories/app.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { EventRepository } from 'src/repositories/event.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { teardownTelemetry, TelemetryRepository } from 'src/repositories/telemetry.repository';
import { WebsocketRepository } from 'src/repositories/websocket.repository';
import { services } from 'src/services';
@@ -28,27 +34,27 @@ import { getKyselyConfig } from 'src/utils/database';
const common = [...repositories, ...services, GlobalExceptionFilter];
export const middleware = [
FileUploadInterceptor,
const commonMiddleware = [
{ provide: APP_FILTER, useClass: GlobalExceptionFilter },
{ provide: APP_PIPE, useValue: new ValidationPipe({ transform: true, whitelist: true }) },
{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor },
{ provide: APP_INTERCEPTOR, useClass: ErrorInterceptor },
{ provide: APP_GUARD, useClass: AuthGuard },
];
const apiMiddleware = [FileUploadInterceptor, ...commonMiddleware, { provide: APP_GUARD, useClass: AuthGuard }];
const configRepository = new ConfigRepository();
const { bull, cls, database, otel } = configRepository.getEnv();
const imports = [
BullModule.forRoot(bull.config),
BullModule.registerQueue(...bull.queues),
const commonImports = [
ClsModule.forRoot(cls.config),
OpenTelemetryModule.forRoot(otel),
KyselyModule.forRoot(getKyselyConfig(database.config)),
OpenTelemetryModule.forRoot(otel),
];
class BaseModule implements OnModuleInit, OnModuleDestroy {
const bullImports = [BullModule.forRoot(bull.config), BullModule.registerQueue(...bull.queues)];
export class BaseModule implements OnModuleInit, OnModuleDestroy {
constructor(
@Inject(IWorker) private worker: ImmichWorker,
logger: LoggingRepository,
@@ -85,20 +91,44 @@ class BaseModule implements OnModuleInit, OnModuleDestroy {
}
@Module({
imports: [...imports, ScheduleModule.forRoot()],
imports: [...bullImports, ...commonImports, ScheduleModule.forRoot()],
controllers: [...controllers],
providers: [...common, ...middleware, { provide: IWorker, useValue: ImmichWorker.Api }],
providers: [...common, ...apiMiddleware, { provide: IWorker, useValue: ImmichWorker.Api }],
})
export class ApiModule extends BaseModule {}
@Module({
imports: [...imports],
imports: [...commonImports],
controllers: [MaintenanceWorkerController],
providers: [
ConfigRepository,
LoggingRepository,
SystemMetadataRepository,
AppRepository,
MaintenanceWebsocketRepository,
MaintenanceWorkerService,
...commonMiddleware,
{ provide: APP_GUARD, useClass: MaintenanceAuthGuard },
{ provide: IWorker, useValue: ImmichWorker.Maintenance },
],
})
export class MaintenanceModule {
constructor(
@Inject(IWorker) private worker: ImmichWorker,
logger: LoggingRepository,
) {
logger.setAppName(this.worker);
}
}
@Module({
imports: [...bullImports, ...commonImports],
providers: [...common, { provide: IWorker, useValue: ImmichWorker.Microservices }, SchedulerRegistry],
})
export class MicroservicesModule extends BaseModule {}
@Module({
imports: [...imports],
imports: [...bullImports, ...commonImports],
providers: [...common, ...commandsAndQuestions, SchedulerRegistry],
})
export class ImmichAdminModule implements OnModuleDestroy {

View File

@@ -1,5 +1,6 @@
import { GrantAdminCommand, PromptEmailQuestion, RevokeAdminCommand } from 'src/commands/grant-admin';
import { ListUsersCommand } from 'src/commands/list-users.command';
import { DisableMaintenanceModeCommand, EnableMaintenanceModeCommand } from 'src/commands/maintenance-mode';
import {
ChangeMediaLocationCommand,
PromptConfirmMoveQuestions,
@@ -16,6 +17,8 @@ export const commandsAndQuestions = [
PromptEmailQuestion,
EnablePasswordLoginCommand,
DisablePasswordLoginCommand,
EnableMaintenanceModeCommand,
DisableMaintenanceModeCommand,
EnableOAuthLogin,
DisableOAuthLogin,
ListUsersCommand,

View File

@@ -0,0 +1,37 @@
import { Command, CommandRunner } from 'nest-commander';
import { CliService } from 'src/services/cli.service';
@Command({
name: 'enable-maintenance-mode',
description: 'Enable maintenance mode or regenerate the maintenance token',
})
export class EnableMaintenanceModeCommand extends CommandRunner {
constructor(private service: CliService) {
super();
}
async run(): Promise<void> {
const { authUrl, alreadyEnabled } = await this.service.enableMaintenanceMode();
console.info(alreadyEnabled ? 'The server is already in maintenance mode!' : 'Maintenance mode has been enabled.');
console.info(`\nLog in using the following URL:\n${authUrl}`);
}
}
@Command({
name: 'disable-maintenance-mode',
description: 'Disable maintenance mode',
})
export class DisableMaintenanceModeCommand extends CommandRunner {
constructor(private service: CliService) {
super();
}
async run(): Promise<void> {
const { alreadyDisabled } = await this.service.disableMaintenanceMode();
console.log(
alreadyDisabled ? 'The server is already out of maintenance mode!' : 'Maintenance mode has been disabled.',
);
}
}

View File

@@ -150,6 +150,7 @@ export const endpointTags: Record<ApiTag, string> = {
'Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed.',
[ApiTag.Libraries]:
'An external library is made up of input file paths or expressions that are scanned for asset files. Discovered files are automatically imported. Assets much be unique within a library, but can be duplicated across libraries. Each user has a default upload library, and can have one or more external libraries.',
[ApiTag.Maintenance]: 'Maintenance mode allows you to put Immich in a read-only state to perform various operations.',
[ApiTag.Map]:
'Map endpoints include supplemental functionality related to geolocation, such as reverse geocoding and retrieving map markers for assets with geolocation data.',
[ApiTag.Memories]:

View File

@@ -11,6 +11,7 @@ import { DuplicateController } from 'src/controllers/duplicate.controller';
import { FaceController } from 'src/controllers/face.controller';
import { JobController } from 'src/controllers/job.controller';
import { LibraryController } from 'src/controllers/library.controller';
import { MaintenanceController } from 'src/controllers/maintenance.controller';
import { MapController } from 'src/controllers/map.controller';
import { MemoryController } from 'src/controllers/memory.controller';
import { NotificationAdminController } from 'src/controllers/notification-admin.controller';
@@ -49,6 +50,7 @@ export const controllers = [
FaceController,
JobController,
LibraryController,
MaintenanceController,
MapController,
MemoryController,
NotificationController,

View File

@@ -0,0 +1,49 @@
import { BadRequestException, Body, Controller, Post, Res } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Response } from 'express';
import { Endpoint, HistoryBuilder } from 'src/decorators';
import { AuthDto } from 'src/dtos/auth.dto';
import { MaintenanceAuthDto, MaintenanceLoginDto, SetMaintenanceModeDto } from 'src/dtos/maintenance.dto';
import { ApiTag, ImmichCookie, MaintenanceAction, Permission } from 'src/enum';
import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard';
import { LoginDetails } from 'src/services/auth.service';
import { MaintenanceService } from 'src/services/maintenance.service';
import { respondWithCookie } from 'src/utils/response';
@ApiTags(ApiTag.Maintenance)
@Controller('admin/maintenance')
export class MaintenanceController {
constructor(private service: MaintenanceService) {}
@Post('login')
@Endpoint({
summary: 'Log into maintenance mode',
description: 'Login with maintenance token or cookie to receive current information and perform further actions.',
history: new HistoryBuilder().added('v2.3.0').alpha('v2.3.0'),
})
maintenanceLogin(@Body() _dto: MaintenanceLoginDto): MaintenanceAuthDto {
throw new BadRequestException('Not in maintenance mode');
}
@Post()
@Endpoint({
summary: 'Set maintenance mode',
description: 'Put Immich into or take it out of maintenance mode',
history: new HistoryBuilder().added('v2.3.0').alpha('v2.3.0'),
})
@Authenticated({ permission: Permission.Maintenance, admin: true })
async setMaintenanceMode(
@Auth() auth: AuthDto,
@Body() dto: SetMaintenanceModeDto,
@GetLoginDetails() loginDetails: LoginDetails,
@Res({ passthrough: true }) res: Response,
): Promise<void> {
if (dto.action === MaintenanceAction.Start) {
const { jwt } = await this.service.startMaintenance(auth.user.name);
return respondWithCookie(res, undefined, {
isSecure: loginDetails.isSecure,
values: [{ key: ImmichCookie.MaintenanceToken, value: jwt }],
});
}
}
}

View File

@@ -0,0 +1,16 @@
import { MaintenanceAction } from 'src/enum';
import { ValidateEnum, ValidateString } from 'src/validation';
export class SetMaintenanceModeDto {
@ValidateEnum({ enum: MaintenanceAction, name: 'MaintenanceAction' })
action!: MaintenanceAction;
}
export class MaintenanceLoginDto {
@ValidateString({ optional: true })
token?: string;
}
export class MaintenanceAuthDto {
username!: string;
}

View File

@@ -154,6 +154,7 @@ export class ServerConfigDto {
publicUsers!: boolean;
mapDarkStyleUrl!: string;
mapLightStyleUrl!: string;
maintenanceMode!: boolean;
}
export class ServerFeaturesDto {

View File

@@ -5,6 +5,7 @@ export enum AuthType {
export enum ImmichCookie {
AccessToken = 'immich_access_token',
MaintenanceToken = 'immich_maintenance_token',
AuthType = 'immich_auth_type',
IsAuthenticated = 'immich_is_authenticated',
SharedLinkToken = 'immich_shared_link_token',
@@ -146,6 +147,8 @@ export enum Permission {
TimelineRead = 'timeline.read',
TimelineDownload = 'timeline.download',
Maintenance = 'maintenance',
MemoryCreate = 'memory.create',
MemoryRead = 'memory.read',
MemoryUpdate = 'memory.update',
@@ -285,6 +288,7 @@ export enum SystemMetadataKey {
FacialRecognitionState = 'facial-recognition-state',
MemoriesState = 'memories-state',
AdminOnboarding = 'admin-onboarding',
MaintenanceMode = 'maintenance-mode',
SystemConfig = 'system-config',
SystemFlags = 'system-flags',
VersionCheckState = 'version-check-state',
@@ -477,6 +481,7 @@ export enum ImmichEnvironment {
export enum ImmichWorker {
Api = 'api',
Maintenance = 'maintenance',
Microservices = 'microservices',
}
@@ -655,6 +660,15 @@ export enum DatabaseLock {
MemoryCreation = 777,
}
export enum MaintenanceAction {
Start = 'start',
End = 'end',
}
export enum ExitCode {
AppRestart = 7,
}
export enum SyncRequestType {
AlbumsV1 = 'AlbumsV1',
AlbumUsersV1 = 'AlbumUsersV1',
@@ -801,6 +815,7 @@ export enum ApiTag {
Faces = 'Faces',
Jobs = 'Jobs',
Libraries = 'Libraries',
Maintenance = 'Maintenance (admin)',
Map = 'Map',
Memories = 'Memories',
Notifications = 'Notifications',

View File

@@ -1,61 +1,151 @@
import { Kysely } from 'kysely';
import { CommandFactory } from 'nest-commander';
import { ChildProcess, fork } from 'node:child_process';
import { dirname, join } from 'node:path';
import { Worker } from 'node:worker_threads';
import { PostgresError } from 'postgres';
import { ImmichAdminModule } from 'src/app.module';
import { ImmichWorker, LogLevel } from 'src/enum';
import { ExitCode, ImmichWorker, LogLevel, SystemMetadataKey } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { type DB } from 'src/schema';
import { getKyselyConfig } from 'src/utils/database';
const immichApp = process.argv[2];
if (immichApp) {
process.argv.splice(2, 1);
}
/**
* Manages worker lifecycle
*/
class Workers {
/**
* Currently running workers
*/
workers: Partial<Record<ImmichWorker, { kill: (signal: NodeJS.Signals) => Promise<void> | void }>> = {};
let apiProcess: ChildProcess | undefined;
/**
* Fail-safe in case anything dies during restart
*/
restarting = false;
const onError = (name: string, error: Error) => {
console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
};
/**
* Boot all enabled workers
*/
async bootstrap() {
const isMaintenanceMode = await this.isMaintenanceMode();
const { workers } = new ConfigRepository().getEnv();
const onExit = (name: string, exitCode: number | null) => {
if (exitCode !== 0) {
console.error(`${name} worker exited with code ${exitCode}`);
if (apiProcess && name !== ImmichWorker.Api) {
console.error('Killing api process');
apiProcess.kill('SIGTERM');
apiProcess = undefined;
if (isMaintenanceMode) {
this.startWorker(ImmichWorker.Maintenance);
} else {
for (const worker of workers) {
this.startWorker(worker);
}
}
}
process.exit(exitCode);
};
/**
* Initialise a short-lived Nest application to build configuration
* @returns System configuration
*/
private async isMaintenanceMode(): Promise<boolean> {
const { database } = new ConfigRepository().getEnv();
const kysely = new Kysely<DB>(getKyselyConfig(database.config));
const systemMetadataRepository = new SystemMetadataRepository(kysely);
function bootstrapWorker(name: ImmichWorker) {
console.log(`Starting ${name} worker`);
try {
const value = await systemMetadataRepository.get(SystemMetadataKey.MaintenanceMode);
return value?.isMaintenanceMode || false;
} catch (error) {
// Table doesn't exist (migrations haven't run yet)
if (error instanceof PostgresError && error.code === '42P01') {
return false;
}
// eslint-disable-next-line unicorn/prefer-module
const basePath = dirname(__filename);
const workerFile = join(basePath, 'workers', `${name}.js`);
let worker: Worker | ChildProcess;
if (name === ImmichWorker.Api) {
worker = fork(workerFile, [], {
execArgv: process.execArgv.map((arg) => (arg.startsWith('--inspect') ? '--inspect=0.0.0.0:9231' : arg)),
});
apiProcess = worker;
} else {
worker = new Worker(workerFile);
throw error;
} finally {
await kysely.destroy();
}
}
worker.on('error', (error) => onError(name, error));
worker.on('exit', (exitCode) => onExit(name, exitCode));
/**
* Start an individual worker
* @param name Worker
*/
private startWorker(name: ImmichWorker) {
console.log(`Starting ${name} worker`);
// eslint-disable-next-line unicorn/prefer-module
const basePath = dirname(__filename);
const workerFile = join(basePath, 'workers', `${name}.js`);
let anyWorker: Worker | ChildProcess;
let kill: (signal?: NodeJS.Signals) => Promise<void> | void;
if (name === ImmichWorker.Api) {
const worker = fork(workerFile, [], {
execArgv: process.execArgv.map((arg) => (arg.startsWith('--inspect') ? '--inspect=0.0.0.0:9231' : arg)),
});
kill = (signal) => void worker.kill(signal);
anyWorker = worker;
} else {
const worker = new Worker(workerFile);
kill = async () => void (await worker.terminate());
anyWorker = worker;
}
anyWorker.on('error', (error) => this.onError(name, error));
anyWorker.on('exit', (exitCode) => this.onExit(name, exitCode));
this.workers[name] = { kill };
}
onError(name: ImmichWorker, error: Error) {
console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
}
onExit(name: ImmichWorker, exitCode: number | null) {
// restart immich server
if (exitCode === ExitCode.AppRestart || this.restarting) {
this.restarting = true;
console.info(`${name} worker shutdown for restart`);
delete this.workers[name];
// once all workers shut down, bootstrap again
if (Object.keys(this.workers).length === 0) {
void this.bootstrap();
this.restarting = false;
}
return;
}
// shutdown the entire process
delete this.workers[name];
if (exitCode !== 0) {
console.error(`${name} worker exited with code ${exitCode}`);
if (this.workers[ImmichWorker.Api] && name !== ImmichWorker.Api) {
console.error('Killing api process');
void this.workers[ImmichWorker.Api].kill('SIGTERM');
}
}
process.exit(exitCode);
}
}
function bootstrap() {
function main() {
const immichApp = process.argv[2];
if (immichApp) {
process.argv.splice(2, 1);
}
if (immichApp === 'immich-admin') {
process.title = 'immich_admin_cli';
process.env.IMMICH_LOG_LEVEL = LogLevel.Warn;
return CommandFactory.run(ImmichAdminModule);
}
@@ -72,10 +162,7 @@ function bootstrap() {
}
process.title = 'immich';
const { workers } = new ConfigRepository().getEnv();
for (const worker of workers) {
bootstrapWorker(worker);
}
void new Workers().bootstrap();
}
void bootstrap();
void main();

View File

@@ -0,0 +1,58 @@
import {
CanActivate,
ExecutionContext,
Injectable,
SetMetadata,
applyDecorators,
createParamDecorator,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import { Request } from 'express';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { MetadataKey } from 'src/enum';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { LoggingRepository } from 'src/repositories/logging.repository';
export const MaintenanceRoute = (options = {}): MethodDecorator => {
const decorators: MethodDecorator[] = [SetMetadata(MetadataKey.AuthRoute, options)];
return applyDecorators(...decorators);
};
export interface MaintenanceAuthRequest extends Request {
auth?: MaintenanceAuthDto;
}
export interface MaintenanceAuthenticatedRequest extends Request {
auth: MaintenanceAuthDto;
}
export const MaintenanceAuth = createParamDecorator((data, context: ExecutionContext): MaintenanceAuthDto => {
return context.switchToHttp().getRequest<MaintenanceAuthenticatedRequest>().auth;
});
@Injectable()
export class MaintenanceAuthGuard implements CanActivate {
constructor(
private logger: LoggingRepository,
private reflector: Reflector,
private service: MaintenanceWorkerService,
) {
this.logger.setContext(MaintenanceAuthGuard.name);
}
async canActivate(context: ExecutionContext): Promise<boolean> {
const targets = [context.getHandler()];
const options = this.reflector.getAllAndOverride<{ _emptyObject: never } | undefined>(
MetadataKey.AuthRoute,
targets,
);
if (!options) {
return true;
}
const request = context.switchToHttp().getRequest<MaintenanceAuthRequest>();
request.auth = await this.service.authenticate(request.headers);
return true;
}
}

View File

@@ -0,0 +1,59 @@
import { Injectable } from '@nestjs/common';
import {
OnGatewayConnection,
OnGatewayDisconnect,
OnGatewayInit,
WebSocketGateway,
WebSocketServer,
} from '@nestjs/websockets';
import { Server, Socket } from 'socket.io';
import { AppRepository } from 'src/repositories/app.repository';
import { AppRestartEvent, ArgsOf } from 'src/repositories/event.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
export const serverEvents = ['AppRestart'] as const;
export type ServerEvents = (typeof serverEvents)[number];
export interface ClientEventMap {
AppRestartV1: [AppRestartEvent];
}
@WebSocketGateway({
cors: true,
path: '/api/socket.io',
transports: ['websocket'],
})
@Injectable()
export class MaintenanceWebsocketRepository implements OnGatewayConnection, OnGatewayDisconnect, OnGatewayInit {
@WebSocketServer()
private websocketServer?: Server;
constructor(
private logger: LoggingRepository,
private appRepository: AppRepository,
) {
this.logger.setContext(MaintenanceWebsocketRepository.name);
}
afterInit(websocketServer: Server) {
this.logger.log('Initialized websocket server');
websocketServer.on('AppRestart', () => this.appRepository.exitApp());
}
clientBroadcast<T extends keyof ClientEventMap>(event: T, ...data: ClientEventMap[T]) {
this.websocketServer?.emit(event, ...data);
}
serverSend<T extends ServerEvents>(event: T, ...args: ArgsOf<T>): void {
this.logger.debug(`Server event: ${event} (send)`);
this.websocketServer?.serverSideEmit(event, ...args);
}
handleConnection(client: Socket) {
this.logger.log(`Websocket Connect: ${client.id}`);
}
handleDisconnect(client: Socket) {
this.logger.log(`Websocket Disconnect: ${client.id}`);
}
}

View File

@@ -0,0 +1,43 @@
import { Body, Controller, Get, Post, Req, Res } from '@nestjs/common';
import { Request, Response } from 'express';
import { MaintenanceAuthDto, MaintenanceLoginDto, SetMaintenanceModeDto } from 'src/dtos/maintenance.dto';
import { ServerConfigDto } from 'src/dtos/server.dto';
import { ImmichCookie, MaintenanceAction } from 'src/enum';
import { MaintenanceRoute } from 'src/maintenance/maintenance-auth.guard';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { GetLoginDetails } from 'src/middleware/auth.guard';
import { LoginDetails } from 'src/services/auth.service';
import { respondWithCookie } from 'src/utils/response';
@Controller()
export class MaintenanceWorkerController {
constructor(private service: MaintenanceWorkerService) {}
@Get('server/config')
getServerConfig(): Promise<ServerConfigDto> {
return this.service.getSystemConfig();
}
@Post('admin/maintenance/login')
async maintenanceLogin(
@Req() request: Request,
@Body() dto: MaintenanceLoginDto,
@GetLoginDetails() loginDetails: LoginDetails,
@Res({ passthrough: true }) res: Response,
): Promise<MaintenanceAuthDto> {
const token = dto.token ?? request.cookies[ImmichCookie.MaintenanceToken];
const auth = await this.service.login(token);
return respondWithCookie(res, auth, {
isSecure: loginDetails.isSecure,
values: [{ key: ImmichCookie.MaintenanceToken, value: token }],
});
}
@Post('admin/maintenance')
@MaintenanceRoute()
async setMaintenanceMode(@Body() dto: SetMaintenanceModeDto): Promise<void> {
if (dto.action === MaintenanceAction.End) {
await this.service.endMaintenance();
}
}
}

View File

@@ -0,0 +1,128 @@
import { UnauthorizedException } from '@nestjs/common';
import { SignJWT } from 'jose';
import { SystemMetadataKey } from 'src/enum';
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { automock, getMocks, ServiceMocks } from 'test/utils';
describe(MaintenanceWorkerService.name, () => {
let sut: MaintenanceWorkerService;
let mocks: ServiceMocks;
let maintenanceWorkerRepositoryMock: MaintenanceWebsocketRepository;
beforeEach(() => {
mocks = getMocks();
maintenanceWorkerRepositoryMock = automock(MaintenanceWebsocketRepository, { args: [mocks.logger], strict: false });
sut = new MaintenanceWorkerService(
mocks.logger as never,
mocks.app,
mocks.config,
mocks.systemMetadata as never,
maintenanceWorkerRepositoryMock,
);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('getSystemConfig', () => {
it('should respond the server is in maintenance mode', async () => {
await expect(sut.getSystemConfig()).resolves.toMatchObject(
expect.objectContaining({
maintenanceMode: true,
}),
);
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});
});
describe('logSecret', () => {
const RE_LOGIN_URL = /https:\/\/my.immich.app\/maintenance\?token=([A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*)/;
it('should log a valid login URL', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
await expect(sut.logSecret()).resolves.toBeUndefined();
expect(mocks.logger.log).toHaveBeenCalledWith(expect.stringMatching(RE_LOGIN_URL));
const [url] = mocks.logger.log.mock.lastCall!;
const token = RE_LOGIN_URL.exec(url)![1];
await expect(sut.login(token)).resolves.toEqual(
expect.objectContaining({
username: 'immich-admin',
}),
);
});
});
describe('authenticate', () => {
it('should fail without a cookie', async () => {
await expect(sut.authenticate({})).rejects.toThrowError(new UnauthorizedException('Missing JWT Token'));
});
it('should parse cookie properly', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
await expect(
sut.authenticate({
cookie: 'immich_maintenance_token=invalid-jwt',
}),
).rejects.toThrowError(new UnauthorizedException('Invalid JWT Token'));
});
});
describe('login', () => {
it('should fail without token', async () => {
await expect(sut.login()).rejects.toThrowError(new UnauthorizedException('Missing JWT Token'));
});
it('should fail with expired JWT', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
const jwt = await new SignJWT({})
.setProtectedHeader({ alg: 'HS256' })
.setIssuedAt()
.setExpirationTime('0s')
.sign(new TextEncoder().encode('secret'));
await expect(sut.login(jwt)).rejects.toThrowError(new UnauthorizedException('Invalid JWT Token'));
});
it('should succeed with valid JWT', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
const jwt = await new SignJWT({ _mockValue: true })
.setProtectedHeader({ alg: 'HS256' })
.setIssuedAt()
.setExpirationTime('4h')
.sign(new TextEncoder().encode('secret'));
await expect(sut.login(jwt)).resolves.toEqual(
expect.objectContaining({
_mockValue: true,
}),
);
});
});
describe('endMaintenance', () => {
it('should set maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.endMaintenance()).resolves.toBeUndefined();
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: false,
});
expect(maintenanceWorkerRepositoryMock.clientBroadcast).toHaveBeenCalledWith('AppRestartV1', {
isMaintenanceMode: false,
});
expect(maintenanceWorkerRepositoryMock.serverSend).toHaveBeenCalledWith('AppRestart', {
isMaintenanceMode: false,
});
});
});
});

View File

@@ -0,0 +1,161 @@
import { Injectable, UnauthorizedException } from '@nestjs/common';
import { parse } from 'cookie';
import { NextFunction, Request, Response } from 'express';
import { jwtVerify } from 'jose';
import { readFileSync } from 'node:fs';
import { IncomingHttpHeaders } from 'node:http';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { ImmichCookie, SystemMetadataKey } from 'src/enum';
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { type ApiService as _ApiService } from 'src/services/api.service';
import { type BaseService as _BaseService } from 'src/services/base.service';
import { type ServerService as _ServerService } from 'src/services/server.service';
import { MaintenanceModeState } from 'src/types';
import { getConfig } from 'src/utils/config';
import { createMaintenanceLoginUrl } from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
/**
* This service is available inside of maintenance mode to manage maintenance mode
*/
@Injectable()
export class MaintenanceWorkerService {
constructor(
protected logger: LoggingRepository,
private appRepository: AppRepository,
private configRepository: ConfigRepository,
private systemMetadataRepository: SystemMetadataRepository,
private maintenanceWorkerRepository: MaintenanceWebsocketRepository,
) {
this.logger.setContext(this.constructor.name);
}
/**
* {@link _BaseService.configRepos}
*/
private get configRepos() {
return {
configRepo: this.configRepository,
metadataRepo: this.systemMetadataRepository,
logger: this.logger,
};
}
/**
* {@link _BaseService.prototype.getConfig}
*/
private getConfig(options: { withCache: boolean }) {
return getConfig(this.configRepos, options);
}
/**
* {@link _ServerService.getSystemConfig}
*/
async getSystemConfig() {
const config = await this.getConfig({ withCache: false });
return {
loginPageMessage: config.server.loginPageMessage,
trashDays: config.trash.days,
userDeleteDelay: config.user.deleteDelay,
oauthButtonText: config.oauth.buttonText,
isInitialized: true,
isOnboarded: true,
externalDomain: config.server.externalDomain,
publicUsers: config.server.publicUsers,
mapDarkStyleUrl: config.map.darkStyle,
mapLightStyleUrl: config.map.lightStyle,
maintenanceMode: true,
};
}
/**
* {@link _ApiService.ssr}
*/
ssr(excludePaths: string[]) {
const { resourcePaths } = this.configRepository.getEnv();
let index = '';
try {
index = readFileSync(resourcePaths.web.indexHtml).toString();
} catch {
this.logger.warn(`Unable to open ${resourcePaths.web.indexHtml}, skipping SSR.`);
}
return (request: Request, res: Response, next: NextFunction) => {
if (
request.url.startsWith('/api') ||
request.method.toLowerCase() !== 'get' ||
excludePaths.some((item) => request.url.startsWith(item))
) {
return next();
}
const maintenancePath = '/maintenance';
if (!request.url.startsWith(maintenancePath)) {
const params = new URLSearchParams();
params.set('continue', request.path);
return res.redirect(`${maintenancePath}?${params}`);
}
res.status(200).type('text/html').header('Cache-Control', 'no-store').send(index);
};
}
private async secret(): Promise<string> {
const state = (await this.systemMetadataRepository.get(SystemMetadataKey.MaintenanceMode)) as {
secret: string;
};
return state.secret;
}
async logSecret(): Promise<void> {
const { server } = await this.getConfig({ withCache: true });
const baseUrl = getExternalDomain(server);
const url = await createMaintenanceLoginUrl(
baseUrl,
{
username: 'immich-admin',
},
await this.secret(),
);
this.logger.log(`\n\n🚧 Immich is in maintenance mode, you can log in using the following URL:\n${url}\n`);
}
async authenticate(headers: IncomingHttpHeaders): Promise<MaintenanceAuthDto> {
const jwtToken = parse(headers.cookie || '')[ImmichCookie.MaintenanceToken];
return this.login(jwtToken);
}
async login(jwt?: string): Promise<MaintenanceAuthDto> {
if (!jwt) {
throw new UnauthorizedException('Missing JWT Token');
}
const secret = await this.secret();
try {
const result = await jwtVerify<MaintenanceAuthDto>(jwt, new TextEncoder().encode(secret));
return result.payload;
} catch {
throw new UnauthorizedException('Invalid JWT Token');
}
}
async endMaintenance(): Promise<void> {
const state: MaintenanceModeState = { isMaintenanceMode: false as const };
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);
// => corresponds to notification.service.ts#onAppRestart
this.maintenanceWorkerRepository.clientBroadcast('AppRestartV1', state);
this.maintenanceWorkerRepository.serverSend('AppRestart', state);
this.appRepository.exitApp();
}
}

View File

@@ -0,0 +1,20 @@
import { Injectable } from '@nestjs/common';
import { ExitCode } from 'src/enum';
@Injectable()
export class AppRepository {
private closeFn?: () => Promise<void>;
exitApp() {
/* eslint-disable unicorn/no-process-exit */
void this.closeFn?.().finally(() => process.exit(ExitCode.AppRestart));
// in exceptional circumstance, the application may hang
setTimeout(() => process.exit(ExitCode.AppRestart), 2000);
/* eslint-enable unicorn/no-process-exit */
}
setCloseFn(fn: () => Promise<void>) {
this.closeFn = fn;
}
}

View File

@@ -26,6 +26,7 @@ type EventMap = {
// app events
AppBootstrap: [];
AppShutdown: [];
AppRestart: [AppRestartEvent];
ConfigInit: [{ newConfig: SystemConfig }];
// config events
@@ -96,6 +97,10 @@ type EventMap = {
WebsocketConnect: [{ userId: string }];
};
export type AppRestartEvent = {
isMaintenanceMode: boolean;
};
type JobSuccessEvent = { job: JobItem; response?: JobStatus };
type JobErrorEvent = { job: JobItem; error: Error | any };

View File

@@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
@@ -56,6 +57,7 @@ export const repositories = [
AlbumUserRepository,
AuditRepository,
ApiKeyRepository,
AppRepository,
AssetRepository,
AssetJobRepository,
ConfigRepository,

View File

@@ -12,11 +12,11 @@ import { AuthDto } from 'src/dtos/auth.dto';
import { NotificationDto } from 'src/dtos/notification.dto';
import { ReleaseNotification, ServerVersionResponseDto } from 'src/dtos/server.dto';
import { SyncAssetExifV1, SyncAssetV1 } from 'src/dtos/sync.dto';
import { ArgsOf, EventRepository } from 'src/repositories/event.repository';
import { AppRestartEvent, ArgsOf, EventRepository } from 'src/repositories/event.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { handlePromiseError } from 'src/utils/misc';
export const serverEvents = ['ConfigUpdate'] as const;
export const serverEvents = ['ConfigUpdate', 'AppRestart'] as const;
export type ServerEvents = (typeof serverEvents)[number];
export interface ClientEventMap {
@@ -36,6 +36,7 @@ export interface ClientEventMap {
on_session_delete: [string];
AssetUploadReadyV1: [{ asset: SyncAssetV1; exif: SyncAssetExifV1 }];
AppRestartV1: [AppRestartEvent];
}
export type AuthFn = (client: Socket) => Promise<AuthDto>;

View File

@@ -11,7 +11,7 @@ import { SharedLinkService } from 'src/services/shared-link.service';
import { VersionService } from 'src/services/version.service';
import { OpenGraphTags } from 'src/utils/misc';
const render = (index: string, meta: OpenGraphTags) => {
export const render = (index: string, meta: OpenGraphTags) => {
const [title, description, imageUrl] = [meta.title, meta.description, meta.imageUrl].map((item) =>
item ? sanitizeHtml(item, { allowedTags: [] }) : '',
);

View File

@@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
@@ -66,6 +67,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
AlbumRepository,
AlbumUserRepository,
ApiKeyRepository,
AppRepository,
AssetRepository,
AssetJobRepository,
AuditRepository,
@@ -123,6 +125,7 @@ export class BaseService {
protected albumRepository: AlbumRepository,
protected albumUserRepository: AlbumUserRepository,
protected apiKeyRepository: ApiKeyRepository,
protected appRepository: AppRepository,
protected assetRepository: AssetRepository,
protected assetJobRepository: AssetJobRepository,
protected auditRepository: AuditRepository,

View File

@@ -1,3 +1,5 @@
import { jwtVerify } from 'jose';
import { SystemMetadataKey } from 'src/enum';
import { CliService } from 'src/services/cli.service';
import { factory } from 'test/small.factory';
import { newTestService, ServiceMocks } from 'test/utils';
@@ -80,6 +82,82 @@ describe(CliService.name, () => {
});
});
describe('disableMaintenanceMode', () => {
it('should not do anything if not in maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.disableMaintenanceMode()).resolves.toEqual({
alreadyDisabled: true,
});
expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
expect(mocks.event.emit).toHaveBeenCalledTimes(0);
});
it('should disable maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
await expect(sut.disableMaintenanceMode()).resolves.toEqual({
alreadyDisabled: false,
});
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: false,
});
});
});
describe('enableMaintenanceMode', () => {
it('should not do anything if in maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
await expect(sut.enableMaintenanceMode()).resolves.toEqual(
expect.objectContaining({
alreadyEnabled: true,
}),
);
expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
expect(mocks.event.emit).toHaveBeenCalledTimes(0);
});
it('should enable maintenance mode', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.enableMaintenanceMode()).resolves.toEqual(
expect.objectContaining({
alreadyEnabled: false,
}),
);
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret: expect.stringMatching(/^\w{128}$/),
});
});
const RE_LOGIN_URL = /https:\/\/my.immich.app\/maintenance\?token=([A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*)/;
it('should return a valid login URL', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
const result = await sut.enableMaintenanceMode();
expect(result).toEqual(
expect.objectContaining({
authUrl: expect.stringMatching(RE_LOGIN_URL),
alreadyEnabled: true,
}),
);
const token = RE_LOGIN_URL.exec(result.authUrl)![1];
await expect(jwtVerify(token, new TextEncoder().encode('secret'))).resolves.toEqual(
expect.objectContaining({
payload: expect.objectContaining({
username: 'cli-admin',
}),
}),
);
});
});
describe('disableOAuthLogin', () => {
it('should disable oauth login', async () => {
await sut.disableOAuthLogin();

View File

@@ -1,8 +1,12 @@
import { Injectable } from '@nestjs/common';
import { isAbsolute } from 'node:path';
import { SALT_ROUNDS } from 'src/constants';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
import { SystemMetadataKey } from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { createMaintenanceLoginUrl, generateMaintenanceSecret, sendOneShotAppRestart } from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
@Injectable()
export class CliService extends BaseService {
@@ -38,6 +42,63 @@ export class CliService extends BaseService {
await this.updateConfig(config);
}
async disableMaintenanceMode(): Promise<{ alreadyDisabled: boolean }> {
const currentState = await this.systemMetadataRepository
.get(SystemMetadataKey.MaintenanceMode)
.then((state) => state ?? { isMaintenanceMode: false as const });
if (!currentState.isMaintenanceMode) {
return {
alreadyDisabled: true,
};
}
const state = { isMaintenanceMode: false as const };
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);
sendOneShotAppRestart(state);
return {
alreadyDisabled: false,
};
}
async enableMaintenanceMode(): Promise<{ authUrl: string; alreadyEnabled: boolean }> {
const { server } = await this.getConfig({ withCache: true });
const baseUrl = getExternalDomain(server);
const payload: MaintenanceAuthDto = {
username: 'cli-admin',
};
const state = await this.systemMetadataRepository
.get(SystemMetadataKey.MaintenanceMode)
.then((state) => state ?? { isMaintenanceMode: false as const });
if (state.isMaintenanceMode) {
return {
authUrl: await createMaintenanceLoginUrl(baseUrl, payload, state.secret),
alreadyEnabled: true,
};
}
const secret = generateMaintenanceSecret();
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret,
});
sendOneShotAppRestart({
isMaintenanceMode: true,
});
return {
authUrl: await createMaintenanceLoginUrl(baseUrl, payload, secret),
alreadyEnabled: false,
};
}
async grantAdminAccess(email: string): Promise<void> {
const user = await this.userRepository.getByEmail(email);
if (!user) {

View File

@@ -14,6 +14,7 @@ import { DownloadService } from 'src/services/download.service';
import { DuplicateService } from 'src/services/duplicate.service';
import { JobService } from 'src/services/job.service';
import { LibraryService } from 'src/services/library.service';
import { MaintenanceService } from 'src/services/maintenance.service';
import { MapService } from 'src/services/map.service';
import { MediaService } from 'src/services/media.service';
import { MemoryService } from 'src/services/memory.service';
@@ -63,6 +64,7 @@ export const services = [
DuplicateService,
JobService,
LibraryService,
MaintenanceService,
MapService,
MediaService,
MemoryService,

View File

@@ -0,0 +1,109 @@
import { SystemMetadataKey } from 'src/enum';
import { MaintenanceService } from 'src/services/maintenance.service';
import { newTestService, ServiceMocks } from 'test/utils';
describe(MaintenanceService.name, () => {
let sut: MaintenanceService;
let mocks: ServiceMocks;
beforeEach(() => {
({ sut, mocks } = newTestService(MaintenanceService));
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('getMaintenanceMode', () => {
it('should return false if state unknown', async () => {
mocks.systemMetadata.get.mockResolvedValue(null);
await expect(sut.getMaintenanceMode()).resolves.toEqual({
isMaintenanceMode: false,
});
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});
it('should return false if disabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.getMaintenanceMode()).resolves.toEqual({
isMaintenanceMode: false,
});
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});
it('should return true if enabled', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: '' });
await expect(sut.getMaintenanceMode()).resolves.toEqual({
isMaintenanceMode: true,
secret: '',
});
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});
});
describe('startMaintenance', () => {
it('should set maintenance mode and return a secret', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(sut.startMaintenance('admin')).resolves.toMatchObject({
jwt: expect.any(String),
});
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
isMaintenanceMode: true,
secret: expect.stringMatching(/^\w{128}$/),
});
expect(mocks.event.emit).toHaveBeenCalledWith('AppRestart', {
isMaintenanceMode: true,
});
});
});
describe('createLoginUrl', () => {
it('should fail outside of maintenance mode without secret', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
await expect(
sut.createLoginUrl({
username: '',
}),
).rejects.toThrowError('Not in maintenance mode');
});
it('should generate a login url with JWT', async () => {
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
await expect(
sut.createLoginUrl({
username: '',
}),
).resolves.toEqual(
expect.stringMatching(
/^https:\/\/my.immich.app\/maintenance\?token=[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*$/,
),
);
expect(mocks.systemMetadata.get).toHaveBeenCalledTimes(2);
});
it('should use the given secret', async () => {
await expect(
sut.createLoginUrl(
{
username: '',
},
'secret',
),
).resolves.toEqual(expect.stringMatching(/./));
expect(mocks.systemMetadata.get).toHaveBeenCalledTimes(1);
});
});
});

View File

@@ -0,0 +1,53 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from 'src/decorators';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { SystemMetadataKey } from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { MaintenanceModeState } from 'src/types';
import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance';
import { getExternalDomain } from 'src/utils/misc';
/**
* This service is available outside of maintenance mode to manage maintenance mode
*/
@Injectable()
export class MaintenanceService extends BaseService {
getMaintenanceMode(): Promise<MaintenanceModeState> {
return this.systemMetadataRepository
.get(SystemMetadataKey.MaintenanceMode)
.then((state) => state ?? { isMaintenanceMode: false });
}
async startMaintenance(username: string): Promise<{ jwt: string }> {
const secret = generateMaintenanceSecret();
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, { isMaintenanceMode: true, secret });
await this.eventRepository.emit('AppRestart', { isMaintenanceMode: true });
return {
jwt: await signMaintenanceJwt(secret, {
username,
}),
};
}
@OnEvent({ name: 'AppRestart', server: true })
onRestart(): void {
this.appRepository.exitApp();
}
async createLoginUrl(auth: MaintenanceAuthDto, secret?: string): Promise<string> {
const { server } = await this.getConfig({ withCache: true });
const baseUrl = getExternalDomain(server);
if (!secret) {
const state = await this.getMaintenanceMode();
if (!state.isMaintenanceMode) {
throw new Error('Not in maintenance mode');
}
secret = state.secret;
}
return await createMaintenanceLoginUrl(baseUrl, auth, secret);
}
}

View File

@@ -114,6 +114,15 @@ export class NotificationService extends BaseService {
this.websocketRepository.serverSend('ConfigUpdate', { oldConfig, newConfig });
}
@OnEvent({ name: 'AppRestart' })
onAppRestart(state: ArgOf<'AppRestart'>) {
this.websocketRepository.clientBroadcast('AppRestartV1', {
isMaintenanceMode: state.isMaintenanceMode,
});
this.websocketRepository.serverSend('AppRestart', state);
}
@OnEvent({ name: 'ConfigValidate', priority: -100 })
async onConfigValidate({ oldConfig, newConfig }: ArgOf<'ConfigValidate'>) {
try {

View File

@@ -166,6 +166,7 @@ describe(ServerService.name, () => {
publicUsers: true,
mapDarkStyleUrl: 'https://tiles.immich.cloud/v1/style/dark.json',
mapLightStyleUrl: 'https://tiles.immich.cloud/v1/style/light.json',
maintenanceMode: false,
});
expect(mocks.systemMetadata.get).toHaveBeenCalled();
});

View File

@@ -130,6 +130,7 @@ export class ServerService extends BaseService {
publicUsers: config.server.publicUsers,
mapDarkStyleUrl: config.map.darkStyle,
mapLightStyleUrl: config.map.lightStyle,
maintenanceMode: false,
};
}

View File

@@ -493,6 +493,7 @@ export interface MemoryData {
export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
export type SystemFlags = { mountChecks: Record<StorageFolder, boolean> };
export type MaintenanceModeState = { isMaintenanceMode: true; secret: string } | { isMaintenanceMode: false };
export type MemoriesState = {
/** memories have already been created through this date */
lastOnThisDayDate: string;
@@ -503,6 +504,7 @@ export interface SystemMetadata extends Record<SystemMetadataKey, Record<string,
[SystemMetadataKey.AdminOnboarding]: { isOnboarded: boolean };
[SystemMetadataKey.FacialRecognitionState]: { lastRun?: string };
[SystemMetadataKey.License]: { licenseKey: string; activationKey: string; activatedAt: Date };
[SystemMetadataKey.MaintenanceMode]: MaintenanceModeState;
[SystemMetadataKey.MediaLocation]: MediaLocation;
[SystemMetadataKey.ReverseGeocodingState]: { lastUpdate?: string; lastImportFileName?: string };
[SystemMetadataKey.SystemConfig]: DeepPartial<SystemConfig>;

View File

@@ -0,0 +1,74 @@
import { createAdapter } from '@socket.io/redis-adapter';
import Redis from 'ioredis';
import { SignJWT } from 'jose';
import { randomBytes } from 'node:crypto';
import { Server as SocketIO } from 'socket.io';
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
import { ConfigRepository } from 'src/repositories/config.repository';
import { AppRestartEvent } from 'src/repositories/event.repository';
export function sendOneShotAppRestart(state: AppRestartEvent): void {
const server = new SocketIO();
const { redis } = new ConfigRepository().getEnv();
const pubClient = new Redis(redis);
const subClient = pubClient.duplicate();
server.adapter(createAdapter(pubClient, subClient));
/**
* Keep trying until we manage to stop Immich
*
* Sometimes there appear to be communication
* issues between to the other servers.
*
* This issue only occurs with this method.
*/
async function tryTerminate() {
while (true) {
try {
const responses = await server.serverSideEmitWithAck('AppRestart', state);
if (responses.length > 0) {
return;
}
} catch (error) {
console.error(error);
console.error('Encountered an error while telling Immich to stop.');
}
console.info(
"\nIt doesn't appear that Immich stopped, trying again in a moment.\nIf Immich is already not running, you can ignore this error.",
);
await new Promise((r) => setTimeout(r, 1e3));
}
}
// => corresponds to notification.service.ts#onAppRestart
server.emit('AppRestartV1', state, () => {
void tryTerminate().finally(() => {
pubClient.disconnect();
subClient.disconnect();
});
});
}
export async function createMaintenanceLoginUrl(
baseUrl: string,
auth: MaintenanceAuthDto,
secret: string,
): Promise<string> {
return `${baseUrl}/maintenance?token=${await signMaintenanceJwt(secret, auth)}`;
}
export async function signMaintenanceJwt(secret: string, data: MaintenanceAuthDto): Promise<string> {
const alg = 'HS256';
return await new SignJWT({ ...data })
.setProtectedHeader({ alg })
.setIssuedAt()
.setExpirationTime('4h')
.sign(new TextEncoder().encode(secret));
}
export function generateMaintenanceSecret(): string {
return randomBytes(64).toString('hex');
}

View File

@@ -15,6 +15,7 @@ export const respondWithCookie = <T>(res: Response, body: T, { isSecure, values
const cookieOptions: Record<ImmichCookie, CookieOptions> = {
[ImmichCookie.AuthType]: defaults,
[ImmichCookie.AccessToken]: defaults,
[ImmichCookie.MaintenanceToken]: { ...defaults, maxAge: Duration.fromObject({ days: 1 }).toMillis() },
[ImmichCookie.OAuthState]: defaults,
[ImmichCookie.OAuthCodeVerifier]: defaults,
// no httpOnly so that the client can know the auth state

View File

@@ -1,69 +1,22 @@
import { NestFactory } from '@nestjs/core';
import { NestExpressApplication } from '@nestjs/platform-express';
import { json } from 'body-parser';
import compression from 'compression';
import cookieParser from 'cookie-parser';
import { existsSync } from 'node:fs';
import sirv from 'sirv';
import { configureExpress, configureTelemetry } from 'src/app.common';
import { ApiModule } from 'src/app.module';
import { excludePaths, serverVersion } from 'src/constants';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { ApiService } from 'src/services/api.service';
import { isStartUpError, useSwagger } from 'src/utils/misc';
import { isStartUpError } from 'src/utils/misc';
async function bootstrap() {
process.title = 'immich-api';
const { telemetry, network } = new ConfigRepository().getEnv();
if (telemetry.metrics.size > 0) {
bootstrapTelemetry(telemetry.apiPort);
}
configureTelemetry();
const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });
const logger = await app.resolve(LoggingRepository);
const configRepository = app.get(ConfigRepository);
app.get(AppRepository).setCloseFn(() => app.close());
const { environment, host, port, resourcePaths } = configRepository.getEnv();
logger.setContext('Bootstrap');
app.useLogger(logger);
app.set('trust proxy', ['loopback', ...network.trustedProxies]);
app.set('etag', 'strong');
app.use(cookieParser());
app.use(json({ limit: '10mb' }));
if (configRepository.isDev()) {
app.enableCors();
}
app.useWebSocketAdapter(new WebSocketAdapter(app));
useSwagger(app, { write: configRepository.isDev() });
app.setGlobalPrefix('api', { exclude: excludePaths });
if (existsSync(resourcePaths.web.root)) {
// copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46
// provides serving of precompressed assets and caching of immutable assets
app.use(
sirv(resourcePaths.web.root, {
etag: true,
gzip: true,
brotli: true,
extensions: [],
setHeaders: (res, pathname) => {
if (pathname.startsWith(`/_app/immutable`) && res.statusCode === 200) {
res.setHeader('cache-control', 'public,max-age=31536000,immutable');
}
},
}),
);
}
app.use(app.get(ApiService).ssr(excludePaths));
app.use(compression());
const server = await (host ? app.listen(port, host) : app.listen(port));
server.requestTimeout = 24 * 60 * 60 * 1000;
logger.log(`Immich Server is listening on ${await app.getUrl()} [v${serverVersion}] [${environment}] `);
void configureExpress(app, {
ssr: ApiService,
});
}
bootstrap().catch((error) => {

View File

@@ -0,0 +1,29 @@
import { NestFactory } from '@nestjs/core';
import { NestExpressApplication } from '@nestjs/platform-express';
import { configureExpress, configureTelemetry } from 'src/app.common';
import { MaintenanceModule } from 'src/app.module';
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
import { AppRepository } from 'src/repositories/app.repository';
import { isStartUpError } from 'src/utils/misc';
async function bootstrap() {
process.title = 'immich-maintenance';
configureTelemetry();
const app = await NestFactory.create<NestExpressApplication>(MaintenanceModule, { bufferLogs: true });
app.get(AppRepository).setCloseFn(() => app.close());
void configureExpress(app, {
permitSwaggerWrite: false,
ssr: MaintenanceWorkerService,
});
void app.get(MaintenanceWorkerService).logSecret();
}
bootstrap().catch((error) => {
if (!isStartUpError(error)) {
console.error(error);
}
// eslint-disable-next-line unicorn/no-process-exit
process.exit(1);
});

View File

@@ -3,6 +3,7 @@ import { isMainThread } from 'node:worker_threads';
import { MicroservicesModule } from 'src/app.module';
import { serverVersion } from 'src/constants';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { AppRepository } from 'src/repositories/app.repository';
import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
@@ -17,6 +18,7 @@ export async function bootstrap() {
const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
const logger = await app.resolve(LoggingRepository);
const configRepository = app.get(ConfigRepository);
app.get(AppRepository).setCloseFn(() => app.close());
const { environment, host } = configRepository.getEnv();

View File

@@ -19,6 +19,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
@@ -212,6 +213,7 @@ export type ServiceOverrides = {
album: AlbumRepository;
albumUser: AlbumUserRepository;
apiKey: ApiKeyRepository;
app: AppRepository;
audit: AuditRepository;
asset: AssetRepository;
assetJob: AssetJobRepository;
@@ -271,10 +273,7 @@ type Constructor<Type, Args extends Array<any>> = {
new (...deps: Args): Type;
};
export const newTestService = <T extends BaseService>(
Service: Constructor<T, BaseServiceArgs>,
overrides: Partial<ServiceOverrides> = {},
) => {
export const getMocks = () => {
const loggerMock = { setContext: () => {} };
const configMock = { getEnv: () => ({}) };
@@ -291,6 +290,7 @@ export const newTestService = <T extends BaseService>(
albumUser: automock(AlbumUserRepository),
asset: newAssetRepositoryMock(),
assetJob: automock(AssetJobRepository),
app: automock(AppRepository, { strict: false }),
config: newConfigRepositoryMock(),
database: newDatabaseRepositoryMock(),
downloadRepository: automock(DownloadRepository, { strict: false }),
@@ -338,6 +338,15 @@ export const newTestService = <T extends BaseService>(
workflow: automock(WorkflowRepository, { strict: true }),
};
return mocks;
};
export const newTestService = <T extends BaseService>(
Service: Constructor<T, BaseServiceArgs>,
overrides: Partial<ServiceOverrides> = {},
) => {
const mocks = getMocks();
const sut = new Service(
overrides.logger || (mocks.logger as As<LoggingRepository>),
overrides.access || (mocks.access as IAccessRepository as AccessRepository),
@@ -345,6 +354,7 @@ export const newTestService = <T extends BaseService>(
overrides.album || (mocks.album as As<AlbumRepository>),
overrides.albumUser || (mocks.albumUser as As<AlbumUserRepository>),
overrides.apiKey || (mocks.apiKey as As<ApiKeyRepository>),
overrides.app || (mocks.app as As<AppRepository>),
overrides.asset || (mocks.asset as As<AssetRepository>),
overrides.assetJob || (mocks.assetJob as As<AssetJobRepository>),
overrides.audit || (mocks.audit as As<AuditRepository>),