mirror of
https://github.com/immich-app/immich.git
synced 2025-12-12 15:50:43 -08:00
feat: maintenance mode (#23431)
* feat: add a `maintenance.enabled` config flag
* feat: implement graceful restart
feat: restart when maintenance config is toggled
* feat: boot a stripped down maintenance api if enabled
* feat: cli command to toggle maintenance mode
* chore: fallback IMMICH_SERVER_URL environment variable in process
* chore: add additional routes to maintenance controller
* fix: don't wait for nest application to close to finish request response
* chore: add a failsafe on restart to prevent other exit codes from preventing restart
* feat: redirect into/from maintenance page
* refactor: use system metadata for maintenance status
* refactor: wait on WebSocket connection to refresh
* feat: broadcast websocket event on server restart
refactor: listen to WS instead of polling
* refactor: bubble up maintenance information instead of hijacking in fetch function
feat: show modal when server is restarting
* chore: increase timeout for ungraceful restart
* refactor: deduplicate code between api/maintenance workers
* fix: skip config check if database is not initialised
* fix: add `maintenanceMode` field to system config test
* refactor: move maintenance resolution code to static method in service
* chore: clean up linter issues
* chore: generate dart openapi
* refactor: use try{} block for maintenance mode check
* fix: logic error in server redirect
* chore: include `maintenanceMode` key in e2e test
* chore: add i18n entries for maintenance screens
* chore: remove negated condition from hook
* fix: should set default value not override in service
* fix: minor error in page
* feat: initial draft of maintenance module, repo., worker controller, worker service
* refactor: move broadcast code into notification service
* chore: connect websocket on client if in maintenance
* chore: set maintenance module app name
* refactor: rename repository to include worker
chore: configure websocket adapter
* feat: reimplement maintenance mode exit with new module
* refactor: add a constant enum for ExitCode
* refactor: remove redundant route for maintenance
* refactor: only spin up kysely on boot (rather than a Nest app)
* refactor(web): move redirect logic into +layout file where modal is setup
* feat: add Maintenance permission
* refactor: merge common code between api/maintenance
* fix: propagate changes from the CLI to servers
* feat: maintenance authentication guard
* refactor: unify maintenance code into repository
feat: add a step to generate maintenance mode token
* feat: jwt auth for maintenance
* refactor: switch from nest jwt to just jsonwebtokens
* feat: log into maintenance mode from CLI command
* refactor: use `secret` instead of `token` in jwt terminology
chore: log maintenance mode login URL on boot
chore: don't make CLI actions reload if already in target state
* docs: initial draft for maintenance mode page
* refactor: always validate the maintenance auth on the server
* feat: add a link to maintenance mode documentation
* feat: redirect users back to the last page they were on when exiting maintenance
* refactor: provide closeFn in both maintenance repos.
* refactor: ensure the user is also redirected by the server
* chore: swap jsonwebtoken for jose
* refactor: introduce AppRestartEvent w/o secret passing
* refactor: use navigation goto
* refactor: use `continue` instead of `next`
* chore: lint fixes for server
* chore: lint fixes for web
* test: add mock for maintenance repository
* test: add base service dependency to maintenance
* chore: remove @types/jsonwebtoken
* refactor: close database connection after startup check
* refactor: use `request#auth` key
* refactor: use service instead of repository
chore: read token from cookie if possible
chore: rename client event to AppRestartV1
* refactor: more concise redirect logic on web
* refactor: move redirect check into utils
refactor: update translation strings to be more sensible
* refactor: always validate login (i.e. check cookie)
* refactor: lint, open-api, remove old dto
* refactor: encode at point of usage
* refactor: remove business logic from repositories
* chore: fix server/web lints
* refactor: remove repository mock
* chore: fix formatting
* test: write service mocks for maintenance mode
* test: write cli service tests
* fix: catch errors when closing app
* fix: always report no maintenance when usual API is available
* test: api e2e maintenance spec
* chore: add response builder
* chore: add helper to set maint. auth cookie
* feat: add SSR to maintenance API
* test(e2e): write web spec for maintenance
* chore: clean up lint issues
* chore: format files
* feat: perform 302 redirect at server level during maintenance
* fix: keep trying to stop immich until it succeeds (CLI issue)
* chore: lint/format
* refactor: annotate references to other services in worker service
* chore: lint
* refactor: remove unnecessary await
Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
* refactor: move static methods into util
* refactor: assert secret exists in maintenance worker
* refactor: remove assertion which isn't necessary anymore
* refactor: remove assertion
* refactor: remove outer try {} catch block from loadMaintenanceAuth
* refactor: undo earlier change to vite.config.ts
* chore: update tests due to refactors
* revert: vite.config.ts
* test: expect string jwt
* chore: move blanket exceptions into controllers
* test: update tests according with last change
* refactor: use respondWithCookie
refactor: merge start/end into one route
refactor: rename MaintenanceRepository to AppRepository
chore: use new ApiTag/Endpoint
refactor: apply other requested changes
* chore: regenerate openapi
* chore: lint/format
* chore: remove secureOnly for maint. cookie
* refactor: move maintenance worker code into src/maintenance\nfix: various test fixes
* refactor: use `action` property for setting maint. mode
* refactor: remove Websocket#restartApp in favour of individual methods
* chore: incomplete commit
* chore: remove stray log
* fix: call exitApp from maintenance worker on exit
* fix: add app repository mock
* fix: ensure maintenance cookies are secure
* fix: run playwright tests over secure context (localhost)
* test: update other references to 127.0.0.1
* refactor: use serverSideEmitWithAck
* chore: correct the logic in tryTerminate
* test: juggle cookies ourselves
* chore: fix lint error for e2e spec
* chore: format e2e test
* fix: set cookie secure/non-secure depending on context
* chore: format files
---------
Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
@@ -78,6 +78,7 @@
|
||||
"handlebars": "^4.7.8",
|
||||
"i18n-iso-countries": "^7.6.0",
|
||||
"ioredis": "^5.8.2",
|
||||
"jose": "^5.10.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"kysely": "0.28.2",
|
||||
|
||||
87
server/src/app.common.ts
Normal file
87
server/src/app.common.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import { NestExpressApplication } from '@nestjs/platform-express';
|
||||
import { json } from 'body-parser';
|
||||
import compression from 'compression';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import { existsSync } from 'node:fs';
|
||||
import sirv from 'sirv';
|
||||
import { excludePaths, serverVersion } from 'src/constants';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
|
||||
import { ApiService } from 'src/services/api.service';
|
||||
import { useSwagger } from 'src/utils/misc';
|
||||
|
||||
export function configureTelemetry() {
|
||||
const { telemetry } = new ConfigRepository().getEnv();
|
||||
if (telemetry.metrics.size > 0) {
|
||||
bootstrapTelemetry(telemetry.apiPort);
|
||||
}
|
||||
}
|
||||
|
||||
export async function configureExpress(
|
||||
app: NestExpressApplication,
|
||||
{
|
||||
permitSwaggerWrite = true,
|
||||
ssr,
|
||||
}: {
|
||||
/**
|
||||
* Whether to allow swagger module to write to the specs.json
|
||||
* This is not desirable when the API is not available
|
||||
* @default true
|
||||
*/
|
||||
permitSwaggerWrite?: boolean;
|
||||
/**
|
||||
* Service to use for server-side rendering
|
||||
*/
|
||||
ssr: typeof ApiService | typeof MaintenanceWorkerService;
|
||||
},
|
||||
) {
|
||||
const configRepository = app.get(ConfigRepository);
|
||||
const { environment, host, port, resourcePaths, network } = configRepository.getEnv();
|
||||
|
||||
const logger = await app.resolve(LoggingRepository);
|
||||
logger.setContext('Bootstrap');
|
||||
app.useLogger(logger);
|
||||
|
||||
app.set('trust proxy', ['loopback', ...network.trustedProxies]);
|
||||
app.set('etag', 'strong');
|
||||
app.use(cookieParser());
|
||||
app.use(json({ limit: '10mb' }));
|
||||
|
||||
if (configRepository.isDev()) {
|
||||
app.enableCors();
|
||||
}
|
||||
|
||||
app.setGlobalPrefix('api', { exclude: excludePaths });
|
||||
app.useWebSocketAdapter(new WebSocketAdapter(app));
|
||||
|
||||
useSwagger(app, { write: configRepository.isDev() && permitSwaggerWrite });
|
||||
|
||||
if (existsSync(resourcePaths.web.root)) {
|
||||
// copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46
|
||||
// provides serving of precompressed assets and caching of immutable assets
|
||||
app.use(
|
||||
sirv(resourcePaths.web.root, {
|
||||
etag: true,
|
||||
gzip: true,
|
||||
brotli: true,
|
||||
extensions: [],
|
||||
setHeaders: (res, pathname) => {
|
||||
if (pathname.startsWith(`/_app/immutable`) && res.statusCode === 200) {
|
||||
res.setHeader('cache-control', 'public,max-age=31536000,immutable');
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
app.use(app.get(ssr).ssr(excludePaths));
|
||||
app.use(compression());
|
||||
|
||||
const server = await (host ? app.listen(port, host) : app.listen(port));
|
||||
server.requestTimeout = 24 * 60 * 60 * 1000;
|
||||
|
||||
logger.log(`Immich Server is listening on ${await app.getUrl()} [v${serverVersion}] [${environment}] `);
|
||||
}
|
||||
@@ -9,15 +9,21 @@ import { commandsAndQuestions } from 'src/commands';
|
||||
import { IWorker } from 'src/constants';
|
||||
import { controllers } from 'src/controllers';
|
||||
import { ImmichWorker } from 'src/enum';
|
||||
import { MaintenanceAuthGuard } from 'src/maintenance/maintenance-auth.guard';
|
||||
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
|
||||
import { MaintenanceWorkerController } from 'src/maintenance/maintenance-worker.controller';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { AuthGuard } from 'src/middleware/auth.guard';
|
||||
import { ErrorInterceptor } from 'src/middleware/error.interceptor';
|
||||
import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor';
|
||||
import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter';
|
||||
import { LoggingInterceptor } from 'src/middleware/logging.interceptor';
|
||||
import { repositories } from 'src/repositories';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { EventRepository } from 'src/repositories/event.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
||||
import { teardownTelemetry, TelemetryRepository } from 'src/repositories/telemetry.repository';
|
||||
import { WebsocketRepository } from 'src/repositories/websocket.repository';
|
||||
import { services } from 'src/services';
|
||||
@@ -28,27 +34,27 @@ import { getKyselyConfig } from 'src/utils/database';
|
||||
|
||||
const common = [...repositories, ...services, GlobalExceptionFilter];
|
||||
|
||||
export const middleware = [
|
||||
FileUploadInterceptor,
|
||||
const commonMiddleware = [
|
||||
{ provide: APP_FILTER, useClass: GlobalExceptionFilter },
|
||||
{ provide: APP_PIPE, useValue: new ValidationPipe({ transform: true, whitelist: true }) },
|
||||
{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor },
|
||||
{ provide: APP_INTERCEPTOR, useClass: ErrorInterceptor },
|
||||
{ provide: APP_GUARD, useClass: AuthGuard },
|
||||
];
|
||||
|
||||
const apiMiddleware = [FileUploadInterceptor, ...commonMiddleware, { provide: APP_GUARD, useClass: AuthGuard }];
|
||||
|
||||
const configRepository = new ConfigRepository();
|
||||
const { bull, cls, database, otel } = configRepository.getEnv();
|
||||
|
||||
const imports = [
|
||||
BullModule.forRoot(bull.config),
|
||||
BullModule.registerQueue(...bull.queues),
|
||||
const commonImports = [
|
||||
ClsModule.forRoot(cls.config),
|
||||
OpenTelemetryModule.forRoot(otel),
|
||||
KyselyModule.forRoot(getKyselyConfig(database.config)),
|
||||
OpenTelemetryModule.forRoot(otel),
|
||||
];
|
||||
|
||||
class BaseModule implements OnModuleInit, OnModuleDestroy {
|
||||
const bullImports = [BullModule.forRoot(bull.config), BullModule.registerQueue(...bull.queues)];
|
||||
|
||||
export class BaseModule implements OnModuleInit, OnModuleDestroy {
|
||||
constructor(
|
||||
@Inject(IWorker) private worker: ImmichWorker,
|
||||
logger: LoggingRepository,
|
||||
@@ -85,20 +91,44 @@ class BaseModule implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
@Module({
|
||||
imports: [...imports, ScheduleModule.forRoot()],
|
||||
imports: [...bullImports, ...commonImports, ScheduleModule.forRoot()],
|
||||
controllers: [...controllers],
|
||||
providers: [...common, ...middleware, { provide: IWorker, useValue: ImmichWorker.Api }],
|
||||
providers: [...common, ...apiMiddleware, { provide: IWorker, useValue: ImmichWorker.Api }],
|
||||
})
|
||||
export class ApiModule extends BaseModule {}
|
||||
|
||||
@Module({
|
||||
imports: [...imports],
|
||||
imports: [...commonImports],
|
||||
controllers: [MaintenanceWorkerController],
|
||||
providers: [
|
||||
ConfigRepository,
|
||||
LoggingRepository,
|
||||
SystemMetadataRepository,
|
||||
AppRepository,
|
||||
MaintenanceWebsocketRepository,
|
||||
MaintenanceWorkerService,
|
||||
...commonMiddleware,
|
||||
{ provide: APP_GUARD, useClass: MaintenanceAuthGuard },
|
||||
{ provide: IWorker, useValue: ImmichWorker.Maintenance },
|
||||
],
|
||||
})
|
||||
export class MaintenanceModule {
|
||||
constructor(
|
||||
@Inject(IWorker) private worker: ImmichWorker,
|
||||
logger: LoggingRepository,
|
||||
) {
|
||||
logger.setAppName(this.worker);
|
||||
}
|
||||
}
|
||||
|
||||
@Module({
|
||||
imports: [...bullImports, ...commonImports],
|
||||
providers: [...common, { provide: IWorker, useValue: ImmichWorker.Microservices }, SchedulerRegistry],
|
||||
})
|
||||
export class MicroservicesModule extends BaseModule {}
|
||||
|
||||
@Module({
|
||||
imports: [...imports],
|
||||
imports: [...bullImports, ...commonImports],
|
||||
providers: [...common, ...commandsAndQuestions, SchedulerRegistry],
|
||||
})
|
||||
export class ImmichAdminModule implements OnModuleDestroy {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { GrantAdminCommand, PromptEmailQuestion, RevokeAdminCommand } from 'src/commands/grant-admin';
|
||||
import { ListUsersCommand } from 'src/commands/list-users.command';
|
||||
import { DisableMaintenanceModeCommand, EnableMaintenanceModeCommand } from 'src/commands/maintenance-mode';
|
||||
import {
|
||||
ChangeMediaLocationCommand,
|
||||
PromptConfirmMoveQuestions,
|
||||
@@ -16,6 +17,8 @@ export const commandsAndQuestions = [
|
||||
PromptEmailQuestion,
|
||||
EnablePasswordLoginCommand,
|
||||
DisablePasswordLoginCommand,
|
||||
EnableMaintenanceModeCommand,
|
||||
DisableMaintenanceModeCommand,
|
||||
EnableOAuthLogin,
|
||||
DisableOAuthLogin,
|
||||
ListUsersCommand,
|
||||
|
||||
37
server/src/commands/maintenance-mode.ts
Normal file
37
server/src/commands/maintenance-mode.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { Command, CommandRunner } from 'nest-commander';
|
||||
import { CliService } from 'src/services/cli.service';
|
||||
|
||||
@Command({
|
||||
name: 'enable-maintenance-mode',
|
||||
description: 'Enable maintenance mode or regenerate the maintenance token',
|
||||
})
|
||||
export class EnableMaintenanceModeCommand extends CommandRunner {
|
||||
constructor(private service: CliService) {
|
||||
super();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { authUrl, alreadyEnabled } = await this.service.enableMaintenanceMode();
|
||||
|
||||
console.info(alreadyEnabled ? 'The server is already in maintenance mode!' : 'Maintenance mode has been enabled.');
|
||||
console.info(`\nLog in using the following URL:\n${authUrl}`);
|
||||
}
|
||||
}
|
||||
|
||||
@Command({
|
||||
name: 'disable-maintenance-mode',
|
||||
description: 'Disable maintenance mode',
|
||||
})
|
||||
export class DisableMaintenanceModeCommand extends CommandRunner {
|
||||
constructor(private service: CliService) {
|
||||
super();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
const { alreadyDisabled } = await this.service.disableMaintenanceMode();
|
||||
|
||||
console.log(
|
||||
alreadyDisabled ? 'The server is already out of maintenance mode!' : 'Maintenance mode has been disabled.',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -150,6 +150,7 @@ export const endpointTags: Record<ApiTag, string> = {
|
||||
'Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed.',
|
||||
[ApiTag.Libraries]:
|
||||
'An external library is made up of input file paths or expressions that are scanned for asset files. Discovered files are automatically imported. Assets much be unique within a library, but can be duplicated across libraries. Each user has a default upload library, and can have one or more external libraries.',
|
||||
[ApiTag.Maintenance]: 'Maintenance mode allows you to put Immich in a read-only state to perform various operations.',
|
||||
[ApiTag.Map]:
|
||||
'Map endpoints include supplemental functionality related to geolocation, such as reverse geocoding and retrieving map markers for assets with geolocation data.',
|
||||
[ApiTag.Memories]:
|
||||
|
||||
@@ -11,6 +11,7 @@ import { DuplicateController } from 'src/controllers/duplicate.controller';
|
||||
import { FaceController } from 'src/controllers/face.controller';
|
||||
import { JobController } from 'src/controllers/job.controller';
|
||||
import { LibraryController } from 'src/controllers/library.controller';
|
||||
import { MaintenanceController } from 'src/controllers/maintenance.controller';
|
||||
import { MapController } from 'src/controllers/map.controller';
|
||||
import { MemoryController } from 'src/controllers/memory.controller';
|
||||
import { NotificationAdminController } from 'src/controllers/notification-admin.controller';
|
||||
@@ -49,6 +50,7 @@ export const controllers = [
|
||||
FaceController,
|
||||
JobController,
|
||||
LibraryController,
|
||||
MaintenanceController,
|
||||
MapController,
|
||||
MemoryController,
|
||||
NotificationController,
|
||||
|
||||
49
server/src/controllers/maintenance.controller.ts
Normal file
49
server/src/controllers/maintenance.controller.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { BadRequestException, Body, Controller, Post, Res } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { Response } from 'express';
|
||||
import { Endpoint, HistoryBuilder } from 'src/decorators';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { MaintenanceAuthDto, MaintenanceLoginDto, SetMaintenanceModeDto } from 'src/dtos/maintenance.dto';
|
||||
import { ApiTag, ImmichCookie, MaintenanceAction, Permission } from 'src/enum';
|
||||
import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard';
|
||||
import { LoginDetails } from 'src/services/auth.service';
|
||||
import { MaintenanceService } from 'src/services/maintenance.service';
|
||||
import { respondWithCookie } from 'src/utils/response';
|
||||
|
||||
@ApiTags(ApiTag.Maintenance)
|
||||
@Controller('admin/maintenance')
|
||||
export class MaintenanceController {
|
||||
constructor(private service: MaintenanceService) {}
|
||||
|
||||
@Post('login')
|
||||
@Endpoint({
|
||||
summary: 'Log into maintenance mode',
|
||||
description: 'Login with maintenance token or cookie to receive current information and perform further actions.',
|
||||
history: new HistoryBuilder().added('v2.3.0').alpha('v2.3.0'),
|
||||
})
|
||||
maintenanceLogin(@Body() _dto: MaintenanceLoginDto): MaintenanceAuthDto {
|
||||
throw new BadRequestException('Not in maintenance mode');
|
||||
}
|
||||
|
||||
@Post()
|
||||
@Endpoint({
|
||||
summary: 'Set maintenance mode',
|
||||
description: 'Put Immich into or take it out of maintenance mode',
|
||||
history: new HistoryBuilder().added('v2.3.0').alpha('v2.3.0'),
|
||||
})
|
||||
@Authenticated({ permission: Permission.Maintenance, admin: true })
|
||||
async setMaintenanceMode(
|
||||
@Auth() auth: AuthDto,
|
||||
@Body() dto: SetMaintenanceModeDto,
|
||||
@GetLoginDetails() loginDetails: LoginDetails,
|
||||
@Res({ passthrough: true }) res: Response,
|
||||
): Promise<void> {
|
||||
if (dto.action === MaintenanceAction.Start) {
|
||||
const { jwt } = await this.service.startMaintenance(auth.user.name);
|
||||
return respondWithCookie(res, undefined, {
|
||||
isSecure: loginDetails.isSecure,
|
||||
values: [{ key: ImmichCookie.MaintenanceToken, value: jwt }],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
16
server/src/dtos/maintenance.dto.ts
Normal file
16
server/src/dtos/maintenance.dto.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { MaintenanceAction } from 'src/enum';
|
||||
import { ValidateEnum, ValidateString } from 'src/validation';
|
||||
|
||||
export class SetMaintenanceModeDto {
|
||||
@ValidateEnum({ enum: MaintenanceAction, name: 'MaintenanceAction' })
|
||||
action!: MaintenanceAction;
|
||||
}
|
||||
|
||||
export class MaintenanceLoginDto {
|
||||
@ValidateString({ optional: true })
|
||||
token?: string;
|
||||
}
|
||||
|
||||
export class MaintenanceAuthDto {
|
||||
username!: string;
|
||||
}
|
||||
@@ -154,6 +154,7 @@ export class ServerConfigDto {
|
||||
publicUsers!: boolean;
|
||||
mapDarkStyleUrl!: string;
|
||||
mapLightStyleUrl!: string;
|
||||
maintenanceMode!: boolean;
|
||||
}
|
||||
|
||||
export class ServerFeaturesDto {
|
||||
|
||||
@@ -5,6 +5,7 @@ export enum AuthType {
|
||||
|
||||
export enum ImmichCookie {
|
||||
AccessToken = 'immich_access_token',
|
||||
MaintenanceToken = 'immich_maintenance_token',
|
||||
AuthType = 'immich_auth_type',
|
||||
IsAuthenticated = 'immich_is_authenticated',
|
||||
SharedLinkToken = 'immich_shared_link_token',
|
||||
@@ -146,6 +147,8 @@ export enum Permission {
|
||||
TimelineRead = 'timeline.read',
|
||||
TimelineDownload = 'timeline.download',
|
||||
|
||||
Maintenance = 'maintenance',
|
||||
|
||||
MemoryCreate = 'memory.create',
|
||||
MemoryRead = 'memory.read',
|
||||
MemoryUpdate = 'memory.update',
|
||||
@@ -285,6 +288,7 @@ export enum SystemMetadataKey {
|
||||
FacialRecognitionState = 'facial-recognition-state',
|
||||
MemoriesState = 'memories-state',
|
||||
AdminOnboarding = 'admin-onboarding',
|
||||
MaintenanceMode = 'maintenance-mode',
|
||||
SystemConfig = 'system-config',
|
||||
SystemFlags = 'system-flags',
|
||||
VersionCheckState = 'version-check-state',
|
||||
@@ -477,6 +481,7 @@ export enum ImmichEnvironment {
|
||||
|
||||
export enum ImmichWorker {
|
||||
Api = 'api',
|
||||
Maintenance = 'maintenance',
|
||||
Microservices = 'microservices',
|
||||
}
|
||||
|
||||
@@ -655,6 +660,15 @@ export enum DatabaseLock {
|
||||
MemoryCreation = 777,
|
||||
}
|
||||
|
||||
export enum MaintenanceAction {
|
||||
Start = 'start',
|
||||
End = 'end',
|
||||
}
|
||||
|
||||
export enum ExitCode {
|
||||
AppRestart = 7,
|
||||
}
|
||||
|
||||
export enum SyncRequestType {
|
||||
AlbumsV1 = 'AlbumsV1',
|
||||
AlbumUsersV1 = 'AlbumUsersV1',
|
||||
@@ -801,6 +815,7 @@ export enum ApiTag {
|
||||
Faces = 'Faces',
|
||||
Jobs = 'Jobs',
|
||||
Libraries = 'Libraries',
|
||||
Maintenance = 'Maintenance (admin)',
|
||||
Map = 'Map',
|
||||
Memories = 'Memories',
|
||||
Notifications = 'Notifications',
|
||||
|
||||
@@ -1,61 +1,151 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { CommandFactory } from 'nest-commander';
|
||||
import { ChildProcess, fork } from 'node:child_process';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
import { PostgresError } from 'postgres';
|
||||
import { ImmichAdminModule } from 'src/app.module';
|
||||
import { ImmichWorker, LogLevel } from 'src/enum';
|
||||
import { ExitCode, ImmichWorker, LogLevel, SystemMetadataKey } from 'src/enum';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
||||
import { type DB } from 'src/schema';
|
||||
import { getKyselyConfig } from 'src/utils/database';
|
||||
|
||||
const immichApp = process.argv[2];
|
||||
if (immichApp) {
|
||||
process.argv.splice(2, 1);
|
||||
}
|
||||
/**
|
||||
* Manages worker lifecycle
|
||||
*/
|
||||
class Workers {
|
||||
/**
|
||||
* Currently running workers
|
||||
*/
|
||||
workers: Partial<Record<ImmichWorker, { kill: (signal: NodeJS.Signals) => Promise<void> | void }>> = {};
|
||||
|
||||
let apiProcess: ChildProcess | undefined;
|
||||
/**
|
||||
* Fail-safe in case anything dies during restart
|
||||
*/
|
||||
restarting = false;
|
||||
|
||||
const onError = (name: string, error: Error) => {
|
||||
console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
|
||||
};
|
||||
/**
|
||||
* Boot all enabled workers
|
||||
*/
|
||||
async bootstrap() {
|
||||
const isMaintenanceMode = await this.isMaintenanceMode();
|
||||
const { workers } = new ConfigRepository().getEnv();
|
||||
|
||||
const onExit = (name: string, exitCode: number | null) => {
|
||||
if (exitCode !== 0) {
|
||||
console.error(`${name} worker exited with code ${exitCode}`);
|
||||
|
||||
if (apiProcess && name !== ImmichWorker.Api) {
|
||||
console.error('Killing api process');
|
||||
apiProcess.kill('SIGTERM');
|
||||
apiProcess = undefined;
|
||||
if (isMaintenanceMode) {
|
||||
this.startWorker(ImmichWorker.Maintenance);
|
||||
} else {
|
||||
for (const worker of workers) {
|
||||
this.startWorker(worker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(exitCode);
|
||||
};
|
||||
/**
|
||||
* Initialise a short-lived Nest application to build configuration
|
||||
* @returns System configuration
|
||||
*/
|
||||
private async isMaintenanceMode(): Promise<boolean> {
|
||||
const { database } = new ConfigRepository().getEnv();
|
||||
const kysely = new Kysely<DB>(getKyselyConfig(database.config));
|
||||
const systemMetadataRepository = new SystemMetadataRepository(kysely);
|
||||
|
||||
function bootstrapWorker(name: ImmichWorker) {
|
||||
console.log(`Starting ${name} worker`);
|
||||
try {
|
||||
const value = await systemMetadataRepository.get(SystemMetadataKey.MaintenanceMode);
|
||||
return value?.isMaintenanceMode || false;
|
||||
} catch (error) {
|
||||
// Table doesn't exist (migrations haven't run yet)
|
||||
if (error instanceof PostgresError && error.code === '42P01') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
const basePath = dirname(__filename);
|
||||
const workerFile = join(basePath, 'workers', `${name}.js`);
|
||||
|
||||
let worker: Worker | ChildProcess;
|
||||
if (name === ImmichWorker.Api) {
|
||||
worker = fork(workerFile, [], {
|
||||
execArgv: process.execArgv.map((arg) => (arg.startsWith('--inspect') ? '--inspect=0.0.0.0:9231' : arg)),
|
||||
});
|
||||
apiProcess = worker;
|
||||
} else {
|
||||
worker = new Worker(workerFile);
|
||||
throw error;
|
||||
} finally {
|
||||
await kysely.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
worker.on('error', (error) => onError(name, error));
|
||||
worker.on('exit', (exitCode) => onExit(name, exitCode));
|
||||
/**
|
||||
* Start an individual worker
|
||||
* @param name Worker
|
||||
*/
|
||||
private startWorker(name: ImmichWorker) {
|
||||
console.log(`Starting ${name} worker`);
|
||||
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
const basePath = dirname(__filename);
|
||||
const workerFile = join(basePath, 'workers', `${name}.js`);
|
||||
|
||||
let anyWorker: Worker | ChildProcess;
|
||||
let kill: (signal?: NodeJS.Signals) => Promise<void> | void;
|
||||
|
||||
if (name === ImmichWorker.Api) {
|
||||
const worker = fork(workerFile, [], {
|
||||
execArgv: process.execArgv.map((arg) => (arg.startsWith('--inspect') ? '--inspect=0.0.0.0:9231' : arg)),
|
||||
});
|
||||
|
||||
kill = (signal) => void worker.kill(signal);
|
||||
anyWorker = worker;
|
||||
} else {
|
||||
const worker = new Worker(workerFile);
|
||||
|
||||
kill = async () => void (await worker.terminate());
|
||||
anyWorker = worker;
|
||||
}
|
||||
|
||||
anyWorker.on('error', (error) => this.onError(name, error));
|
||||
anyWorker.on('exit', (exitCode) => this.onExit(name, exitCode));
|
||||
|
||||
this.workers[name] = { kill };
|
||||
}
|
||||
|
||||
onError(name: ImmichWorker, error: Error) {
|
||||
console.error(`${name} worker error: ${error}, stack: ${error.stack}`);
|
||||
}
|
||||
|
||||
onExit(name: ImmichWorker, exitCode: number | null) {
|
||||
// restart immich server
|
||||
if (exitCode === ExitCode.AppRestart || this.restarting) {
|
||||
this.restarting = true;
|
||||
|
||||
console.info(`${name} worker shutdown for restart`);
|
||||
delete this.workers[name];
|
||||
|
||||
// once all workers shut down, bootstrap again
|
||||
if (Object.keys(this.workers).length === 0) {
|
||||
void this.bootstrap();
|
||||
this.restarting = false;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// shutdown the entire process
|
||||
delete this.workers[name];
|
||||
|
||||
if (exitCode !== 0) {
|
||||
console.error(`${name} worker exited with code ${exitCode}`);
|
||||
|
||||
if (this.workers[ImmichWorker.Api] && name !== ImmichWorker.Api) {
|
||||
console.error('Killing api process');
|
||||
void this.workers[ImmichWorker.Api].kill('SIGTERM');
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(exitCode);
|
||||
}
|
||||
}
|
||||
|
||||
function bootstrap() {
|
||||
function main() {
|
||||
const immichApp = process.argv[2];
|
||||
if (immichApp) {
|
||||
process.argv.splice(2, 1);
|
||||
}
|
||||
|
||||
if (immichApp === 'immich-admin') {
|
||||
process.title = 'immich_admin_cli';
|
||||
process.env.IMMICH_LOG_LEVEL = LogLevel.Warn;
|
||||
|
||||
return CommandFactory.run(ImmichAdminModule);
|
||||
}
|
||||
|
||||
@@ -72,10 +162,7 @@ function bootstrap() {
|
||||
}
|
||||
|
||||
process.title = 'immich';
|
||||
const { workers } = new ConfigRepository().getEnv();
|
||||
for (const worker of workers) {
|
||||
bootstrapWorker(worker);
|
||||
}
|
||||
void new Workers().bootstrap();
|
||||
}
|
||||
|
||||
void bootstrap();
|
||||
void main();
|
||||
|
||||
58
server/src/maintenance/maintenance-auth.guard.ts
Normal file
58
server/src/maintenance/maintenance-auth.guard.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import {
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
Injectable,
|
||||
SetMetadata,
|
||||
applyDecorators,
|
||||
createParamDecorator,
|
||||
} from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import { Request } from 'express';
|
||||
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
|
||||
import { MetadataKey } from 'src/enum';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
|
||||
export const MaintenanceRoute = (options = {}): MethodDecorator => {
|
||||
const decorators: MethodDecorator[] = [SetMetadata(MetadataKey.AuthRoute, options)];
|
||||
return applyDecorators(...decorators);
|
||||
};
|
||||
|
||||
export interface MaintenanceAuthRequest extends Request {
|
||||
auth?: MaintenanceAuthDto;
|
||||
}
|
||||
|
||||
export interface MaintenanceAuthenticatedRequest extends Request {
|
||||
auth: MaintenanceAuthDto;
|
||||
}
|
||||
|
||||
export const MaintenanceAuth = createParamDecorator((data, context: ExecutionContext): MaintenanceAuthDto => {
|
||||
return context.switchToHttp().getRequest<MaintenanceAuthenticatedRequest>().auth;
|
||||
});
|
||||
|
||||
@Injectable()
|
||||
export class MaintenanceAuthGuard implements CanActivate {
|
||||
constructor(
|
||||
private logger: LoggingRepository,
|
||||
private reflector: Reflector,
|
||||
private service: MaintenanceWorkerService,
|
||||
) {
|
||||
this.logger.setContext(MaintenanceAuthGuard.name);
|
||||
}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const targets = [context.getHandler()];
|
||||
const options = this.reflector.getAllAndOverride<{ _emptyObject: never } | undefined>(
|
||||
MetadataKey.AuthRoute,
|
||||
targets,
|
||||
);
|
||||
if (!options) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const request = context.switchToHttp().getRequest<MaintenanceAuthRequest>();
|
||||
request.auth = await this.service.authenticate(request.headers);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
59
server/src/maintenance/maintenance-websocket.repository.ts
Normal file
59
server/src/maintenance/maintenance-websocket.repository.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
OnGatewayConnection,
|
||||
OnGatewayDisconnect,
|
||||
OnGatewayInit,
|
||||
WebSocketGateway,
|
||||
WebSocketServer,
|
||||
} from '@nestjs/websockets';
|
||||
import { Server, Socket } from 'socket.io';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { AppRestartEvent, ArgsOf } from 'src/repositories/event.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
|
||||
export const serverEvents = ['AppRestart'] as const;
|
||||
export type ServerEvents = (typeof serverEvents)[number];
|
||||
|
||||
export interface ClientEventMap {
|
||||
AppRestartV1: [AppRestartEvent];
|
||||
}
|
||||
|
||||
@WebSocketGateway({
|
||||
cors: true,
|
||||
path: '/api/socket.io',
|
||||
transports: ['websocket'],
|
||||
})
|
||||
@Injectable()
|
||||
export class MaintenanceWebsocketRepository implements OnGatewayConnection, OnGatewayDisconnect, OnGatewayInit {
|
||||
@WebSocketServer()
|
||||
private websocketServer?: Server;
|
||||
|
||||
constructor(
|
||||
private logger: LoggingRepository,
|
||||
private appRepository: AppRepository,
|
||||
) {
|
||||
this.logger.setContext(MaintenanceWebsocketRepository.name);
|
||||
}
|
||||
|
||||
afterInit(websocketServer: Server) {
|
||||
this.logger.log('Initialized websocket server');
|
||||
websocketServer.on('AppRestart', () => this.appRepository.exitApp());
|
||||
}
|
||||
|
||||
clientBroadcast<T extends keyof ClientEventMap>(event: T, ...data: ClientEventMap[T]) {
|
||||
this.websocketServer?.emit(event, ...data);
|
||||
}
|
||||
|
||||
serverSend<T extends ServerEvents>(event: T, ...args: ArgsOf<T>): void {
|
||||
this.logger.debug(`Server event: ${event} (send)`);
|
||||
this.websocketServer?.serverSideEmit(event, ...args);
|
||||
}
|
||||
|
||||
handleConnection(client: Socket) {
|
||||
this.logger.log(`Websocket Connect: ${client.id}`);
|
||||
}
|
||||
|
||||
handleDisconnect(client: Socket) {
|
||||
this.logger.log(`Websocket Disconnect: ${client.id}`);
|
||||
}
|
||||
}
|
||||
43
server/src/maintenance/maintenance-worker.controller.ts
Normal file
43
server/src/maintenance/maintenance-worker.controller.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { Body, Controller, Get, Post, Req, Res } from '@nestjs/common';
|
||||
import { Request, Response } from 'express';
|
||||
import { MaintenanceAuthDto, MaintenanceLoginDto, SetMaintenanceModeDto } from 'src/dtos/maintenance.dto';
|
||||
import { ServerConfigDto } from 'src/dtos/server.dto';
|
||||
import { ImmichCookie, MaintenanceAction } from 'src/enum';
|
||||
import { MaintenanceRoute } from 'src/maintenance/maintenance-auth.guard';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { GetLoginDetails } from 'src/middleware/auth.guard';
|
||||
import { LoginDetails } from 'src/services/auth.service';
|
||||
import { respondWithCookie } from 'src/utils/response';
|
||||
|
||||
@Controller()
|
||||
export class MaintenanceWorkerController {
|
||||
constructor(private service: MaintenanceWorkerService) {}
|
||||
|
||||
@Get('server/config')
|
||||
getServerConfig(): Promise<ServerConfigDto> {
|
||||
return this.service.getSystemConfig();
|
||||
}
|
||||
|
||||
@Post('admin/maintenance/login')
|
||||
async maintenanceLogin(
|
||||
@Req() request: Request,
|
||||
@Body() dto: MaintenanceLoginDto,
|
||||
@GetLoginDetails() loginDetails: LoginDetails,
|
||||
@Res({ passthrough: true }) res: Response,
|
||||
): Promise<MaintenanceAuthDto> {
|
||||
const token = dto.token ?? request.cookies[ImmichCookie.MaintenanceToken];
|
||||
const auth = await this.service.login(token);
|
||||
return respondWithCookie(res, auth, {
|
||||
isSecure: loginDetails.isSecure,
|
||||
values: [{ key: ImmichCookie.MaintenanceToken, value: token }],
|
||||
});
|
||||
}
|
||||
|
||||
@Post('admin/maintenance')
|
||||
@MaintenanceRoute()
|
||||
async setMaintenanceMode(@Body() dto: SetMaintenanceModeDto): Promise<void> {
|
||||
if (dto.action === MaintenanceAction.End) {
|
||||
await this.service.endMaintenance();
|
||||
}
|
||||
}
|
||||
}
|
||||
128
server/src/maintenance/maintenance-worker.service.spec.ts
Normal file
128
server/src/maintenance/maintenance-worker.service.spec.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import { UnauthorizedException } from '@nestjs/common';
|
||||
import { SignJWT } from 'jose';
|
||||
import { SystemMetadataKey } from 'src/enum';
|
||||
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { automock, getMocks, ServiceMocks } from 'test/utils';
|
||||
|
||||
describe(MaintenanceWorkerService.name, () => {
|
||||
let sut: MaintenanceWorkerService;
|
||||
let mocks: ServiceMocks;
|
||||
let maintenanceWorkerRepositoryMock: MaintenanceWebsocketRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
mocks = getMocks();
|
||||
maintenanceWorkerRepositoryMock = automock(MaintenanceWebsocketRepository, { args: [mocks.logger], strict: false });
|
||||
sut = new MaintenanceWorkerService(
|
||||
mocks.logger as never,
|
||||
mocks.app,
|
||||
mocks.config,
|
||||
mocks.systemMetadata as never,
|
||||
maintenanceWorkerRepositoryMock,
|
||||
);
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getSystemConfig', () => {
|
||||
it('should respond the server is in maintenance mode', async () => {
|
||||
await expect(sut.getSystemConfig()).resolves.toMatchObject(
|
||||
expect.objectContaining({
|
||||
maintenanceMode: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('logSecret', () => {
|
||||
const RE_LOGIN_URL = /https:\/\/my.immich.app\/maintenance\?token=([A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*)/;
|
||||
|
||||
it('should log a valid login URL', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
await expect(sut.logSecret()).resolves.toBeUndefined();
|
||||
expect(mocks.logger.log).toHaveBeenCalledWith(expect.stringMatching(RE_LOGIN_URL));
|
||||
|
||||
const [url] = mocks.logger.log.mock.lastCall!;
|
||||
const token = RE_LOGIN_URL.exec(url)![1];
|
||||
|
||||
await expect(sut.login(token)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
username: 'immich-admin',
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('authenticate', () => {
|
||||
it('should fail without a cookie', async () => {
|
||||
await expect(sut.authenticate({})).rejects.toThrowError(new UnauthorizedException('Missing JWT Token'));
|
||||
});
|
||||
|
||||
it('should parse cookie properly', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
|
||||
await expect(
|
||||
sut.authenticate({
|
||||
cookie: 'immich_maintenance_token=invalid-jwt',
|
||||
}),
|
||||
).rejects.toThrowError(new UnauthorizedException('Invalid JWT Token'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('login', () => {
|
||||
it('should fail without token', async () => {
|
||||
await expect(sut.login()).rejects.toThrowError(new UnauthorizedException('Missing JWT Token'));
|
||||
});
|
||||
|
||||
it('should fail with expired JWT', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
|
||||
const jwt = await new SignJWT({})
|
||||
.setProtectedHeader({ alg: 'HS256' })
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('0s')
|
||||
.sign(new TextEncoder().encode('secret'));
|
||||
|
||||
await expect(sut.login(jwt)).rejects.toThrowError(new UnauthorizedException('Invalid JWT Token'));
|
||||
});
|
||||
|
||||
it('should succeed with valid JWT', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
|
||||
const jwt = await new SignJWT({ _mockValue: true })
|
||||
.setProtectedHeader({ alg: 'HS256' })
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('4h')
|
||||
.sign(new TextEncoder().encode('secret'));
|
||||
|
||||
await expect(sut.login(jwt)).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
_mockValue: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('endMaintenance', () => {
|
||||
it('should set maintenance mode', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
await expect(sut.endMaintenance()).resolves.toBeUndefined();
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
|
||||
expect(maintenanceWorkerRepositoryMock.clientBroadcast).toHaveBeenCalledWith('AppRestartV1', {
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
|
||||
expect(maintenanceWorkerRepositoryMock.serverSend).toHaveBeenCalledWith('AppRestart', {
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
161
server/src/maintenance/maintenance-worker.service.ts
Normal file
161
server/src/maintenance/maintenance-worker.service.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
import { Injectable, UnauthorizedException } from '@nestjs/common';
|
||||
import { parse } from 'cookie';
|
||||
import { NextFunction, Request, Response } from 'express';
|
||||
import { jwtVerify } from 'jose';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { IncomingHttpHeaders } from 'node:http';
|
||||
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
|
||||
import { ImmichCookie, SystemMetadataKey } from 'src/enum';
|
||||
import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
|
||||
import { type ApiService as _ApiService } from 'src/services/api.service';
|
||||
import { type BaseService as _BaseService } from 'src/services/base.service';
|
||||
import { type ServerService as _ServerService } from 'src/services/server.service';
|
||||
import { MaintenanceModeState } from 'src/types';
|
||||
import { getConfig } from 'src/utils/config';
|
||||
import { createMaintenanceLoginUrl } from 'src/utils/maintenance';
|
||||
import { getExternalDomain } from 'src/utils/misc';
|
||||
|
||||
/**
|
||||
* This service is available inside of maintenance mode to manage maintenance mode
|
||||
*/
|
||||
@Injectable()
|
||||
export class MaintenanceWorkerService {
|
||||
constructor(
|
||||
protected logger: LoggingRepository,
|
||||
private appRepository: AppRepository,
|
||||
private configRepository: ConfigRepository,
|
||||
private systemMetadataRepository: SystemMetadataRepository,
|
||||
private maintenanceWorkerRepository: MaintenanceWebsocketRepository,
|
||||
) {
|
||||
this.logger.setContext(this.constructor.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link _BaseService.configRepos}
|
||||
*/
|
||||
private get configRepos() {
|
||||
return {
|
||||
configRepo: this.configRepository,
|
||||
metadataRepo: this.systemMetadataRepository,
|
||||
logger: this.logger,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link _BaseService.prototype.getConfig}
|
||||
*/
|
||||
private getConfig(options: { withCache: boolean }) {
|
||||
return getConfig(this.configRepos, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link _ServerService.getSystemConfig}
|
||||
*/
|
||||
async getSystemConfig() {
|
||||
const config = await this.getConfig({ withCache: false });
|
||||
|
||||
return {
|
||||
loginPageMessage: config.server.loginPageMessage,
|
||||
trashDays: config.trash.days,
|
||||
userDeleteDelay: config.user.deleteDelay,
|
||||
oauthButtonText: config.oauth.buttonText,
|
||||
isInitialized: true,
|
||||
isOnboarded: true,
|
||||
externalDomain: config.server.externalDomain,
|
||||
publicUsers: config.server.publicUsers,
|
||||
mapDarkStyleUrl: config.map.darkStyle,
|
||||
mapLightStyleUrl: config.map.lightStyle,
|
||||
maintenanceMode: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link _ApiService.ssr}
|
||||
*/
|
||||
ssr(excludePaths: string[]) {
|
||||
const { resourcePaths } = this.configRepository.getEnv();
|
||||
|
||||
let index = '';
|
||||
try {
|
||||
index = readFileSync(resourcePaths.web.indexHtml).toString();
|
||||
} catch {
|
||||
this.logger.warn(`Unable to open ${resourcePaths.web.indexHtml}, skipping SSR.`);
|
||||
}
|
||||
|
||||
return (request: Request, res: Response, next: NextFunction) => {
|
||||
if (
|
||||
request.url.startsWith('/api') ||
|
||||
request.method.toLowerCase() !== 'get' ||
|
||||
excludePaths.some((item) => request.url.startsWith(item))
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const maintenancePath = '/maintenance';
|
||||
if (!request.url.startsWith(maintenancePath)) {
|
||||
const params = new URLSearchParams();
|
||||
params.set('continue', request.path);
|
||||
return res.redirect(`${maintenancePath}?${params}`);
|
||||
}
|
||||
|
||||
res.status(200).type('text/html').header('Cache-Control', 'no-store').send(index);
|
||||
};
|
||||
}
|
||||
|
||||
private async secret(): Promise<string> {
|
||||
const state = (await this.systemMetadataRepository.get(SystemMetadataKey.MaintenanceMode)) as {
|
||||
secret: string;
|
||||
};
|
||||
|
||||
return state.secret;
|
||||
}
|
||||
|
||||
async logSecret(): Promise<void> {
|
||||
const { server } = await this.getConfig({ withCache: true });
|
||||
|
||||
const baseUrl = getExternalDomain(server);
|
||||
const url = await createMaintenanceLoginUrl(
|
||||
baseUrl,
|
||||
{
|
||||
username: 'immich-admin',
|
||||
},
|
||||
await this.secret(),
|
||||
);
|
||||
|
||||
this.logger.log(`\n\n🚧 Immich is in maintenance mode, you can log in using the following URL:\n${url}\n`);
|
||||
}
|
||||
|
||||
async authenticate(headers: IncomingHttpHeaders): Promise<MaintenanceAuthDto> {
|
||||
const jwtToken = parse(headers.cookie || '')[ImmichCookie.MaintenanceToken];
|
||||
return this.login(jwtToken);
|
||||
}
|
||||
|
||||
async login(jwt?: string): Promise<MaintenanceAuthDto> {
|
||||
if (!jwt) {
|
||||
throw new UnauthorizedException('Missing JWT Token');
|
||||
}
|
||||
|
||||
const secret = await this.secret();
|
||||
|
||||
try {
|
||||
const result = await jwtVerify<MaintenanceAuthDto>(jwt, new TextEncoder().encode(secret));
|
||||
return result.payload;
|
||||
} catch {
|
||||
throw new UnauthorizedException('Invalid JWT Token');
|
||||
}
|
||||
}
|
||||
|
||||
async endMaintenance(): Promise<void> {
|
||||
const state: MaintenanceModeState = { isMaintenanceMode: false as const };
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);
|
||||
|
||||
// => corresponds to notification.service.ts#onAppRestart
|
||||
this.maintenanceWorkerRepository.clientBroadcast('AppRestartV1', state);
|
||||
this.maintenanceWorkerRepository.serverSend('AppRestart', state);
|
||||
this.appRepository.exitApp();
|
||||
}
|
||||
}
|
||||
20
server/src/repositories/app.repository.ts
Normal file
20
server/src/repositories/app.repository.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { ExitCode } from 'src/enum';
|
||||
|
||||
@Injectable()
|
||||
export class AppRepository {
|
||||
private closeFn?: () => Promise<void>;
|
||||
|
||||
exitApp() {
|
||||
/* eslint-disable unicorn/no-process-exit */
|
||||
void this.closeFn?.().finally(() => process.exit(ExitCode.AppRestart));
|
||||
|
||||
// in exceptional circumstance, the application may hang
|
||||
setTimeout(() => process.exit(ExitCode.AppRestart), 2000);
|
||||
/* eslint-enable unicorn/no-process-exit */
|
||||
}
|
||||
|
||||
setCloseFn(fn: () => Promise<void>) {
|
||||
this.closeFn = fn;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ type EventMap = {
|
||||
// app events
|
||||
AppBootstrap: [];
|
||||
AppShutdown: [];
|
||||
AppRestart: [AppRestartEvent];
|
||||
|
||||
ConfigInit: [{ newConfig: SystemConfig }];
|
||||
// config events
|
||||
@@ -96,6 +97,10 @@ type EventMap = {
|
||||
WebsocketConnect: [{ userId: string }];
|
||||
};
|
||||
|
||||
export type AppRestartEvent = {
|
||||
isMaintenanceMode: boolean;
|
||||
};
|
||||
|
||||
type JobSuccessEvent = { job: JobItem; response?: JobStatus };
|
||||
type JobErrorEvent = { job: JobItem; error: Error | any };
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
|
||||
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
|
||||
import { AlbumRepository } from 'src/repositories/album.repository';
|
||||
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
|
||||
import { AssetRepository } from 'src/repositories/asset.repository';
|
||||
import { AuditRepository } from 'src/repositories/audit.repository';
|
||||
@@ -56,6 +57,7 @@ export const repositories = [
|
||||
AlbumUserRepository,
|
||||
AuditRepository,
|
||||
ApiKeyRepository,
|
||||
AppRepository,
|
||||
AssetRepository,
|
||||
AssetJobRepository,
|
||||
ConfigRepository,
|
||||
|
||||
@@ -12,11 +12,11 @@ import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { NotificationDto } from 'src/dtos/notification.dto';
|
||||
import { ReleaseNotification, ServerVersionResponseDto } from 'src/dtos/server.dto';
|
||||
import { SyncAssetExifV1, SyncAssetV1 } from 'src/dtos/sync.dto';
|
||||
import { ArgsOf, EventRepository } from 'src/repositories/event.repository';
|
||||
import { AppRestartEvent, ArgsOf, EventRepository } from 'src/repositories/event.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { handlePromiseError } from 'src/utils/misc';
|
||||
|
||||
export const serverEvents = ['ConfigUpdate'] as const;
|
||||
export const serverEvents = ['ConfigUpdate', 'AppRestart'] as const;
|
||||
export type ServerEvents = (typeof serverEvents)[number];
|
||||
|
||||
export interface ClientEventMap {
|
||||
@@ -36,6 +36,7 @@ export interface ClientEventMap {
|
||||
on_session_delete: [string];
|
||||
|
||||
AssetUploadReadyV1: [{ asset: SyncAssetV1; exif: SyncAssetExifV1 }];
|
||||
AppRestartV1: [AppRestartEvent];
|
||||
}
|
||||
|
||||
export type AuthFn = (client: Socket) => Promise<AuthDto>;
|
||||
|
||||
@@ -11,7 +11,7 @@ import { SharedLinkService } from 'src/services/shared-link.service';
|
||||
import { VersionService } from 'src/services/version.service';
|
||||
import { OpenGraphTags } from 'src/utils/misc';
|
||||
|
||||
const render = (index: string, meta: OpenGraphTags) => {
|
||||
export const render = (index: string, meta: OpenGraphTags) => {
|
||||
const [title, description, imageUrl] = [meta.title, meta.description, meta.imageUrl].map((item) =>
|
||||
item ? sanitizeHtml(item, { allowedTags: [] }) : '',
|
||||
);
|
||||
|
||||
@@ -10,6 +10,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
|
||||
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
|
||||
import { AlbumRepository } from 'src/repositories/album.repository';
|
||||
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
|
||||
import { AssetRepository } from 'src/repositories/asset.repository';
|
||||
import { AuditRepository } from 'src/repositories/audit.repository';
|
||||
@@ -66,6 +67,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
|
||||
AlbumRepository,
|
||||
AlbumUserRepository,
|
||||
ApiKeyRepository,
|
||||
AppRepository,
|
||||
AssetRepository,
|
||||
AssetJobRepository,
|
||||
AuditRepository,
|
||||
@@ -123,6 +125,7 @@ export class BaseService {
|
||||
protected albumRepository: AlbumRepository,
|
||||
protected albumUserRepository: AlbumUserRepository,
|
||||
protected apiKeyRepository: ApiKeyRepository,
|
||||
protected appRepository: AppRepository,
|
||||
protected assetRepository: AssetRepository,
|
||||
protected assetJobRepository: AssetJobRepository,
|
||||
protected auditRepository: AuditRepository,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { jwtVerify } from 'jose';
|
||||
import { SystemMetadataKey } from 'src/enum';
|
||||
import { CliService } from 'src/services/cli.service';
|
||||
import { factory } from 'test/small.factory';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
@@ -80,6 +82,82 @@ describe(CliService.name, () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('disableMaintenanceMode', () => {
|
||||
it('should not do anything if not in maintenance mode', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
await expect(sut.disableMaintenanceMode()).resolves.toEqual({
|
||||
alreadyDisabled: true,
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
|
||||
expect(mocks.event.emit).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should disable maintenance mode', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
await expect(sut.disableMaintenanceMode()).resolves.toEqual({
|
||||
alreadyDisabled: false,
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('enableMaintenanceMode', () => {
|
||||
it('should not do anything if in maintenance mode', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
await expect(sut.enableMaintenanceMode()).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
alreadyEnabled: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
|
||||
expect(mocks.event.emit).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should enable maintenance mode', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
await expect(sut.enableMaintenanceMode()).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
alreadyEnabled: false,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
|
||||
isMaintenanceMode: true,
|
||||
secret: expect.stringMatching(/^\w{128}$/),
|
||||
});
|
||||
});
|
||||
|
||||
const RE_LOGIN_URL = /https:\/\/my.immich.app\/maintenance\?token=([A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*)/;
|
||||
|
||||
it('should return a valid login URL', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
|
||||
const result = await sut.enableMaintenanceMode();
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
authUrl: expect.stringMatching(RE_LOGIN_URL),
|
||||
alreadyEnabled: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const token = RE_LOGIN_URL.exec(result.authUrl)![1];
|
||||
|
||||
await expect(jwtVerify(token, new TextEncoder().encode('secret'))).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
username: 'cli-admin',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('disableOAuthLogin', () => {
|
||||
it('should disable oauth login', async () => {
|
||||
await sut.disableOAuthLogin();
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { isAbsolute } from 'node:path';
|
||||
import { SALT_ROUNDS } from 'src/constants';
|
||||
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
|
||||
import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
|
||||
import { SystemMetadataKey } from 'src/enum';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { createMaintenanceLoginUrl, generateMaintenanceSecret, sendOneShotAppRestart } from 'src/utils/maintenance';
|
||||
import { getExternalDomain } from 'src/utils/misc';
|
||||
|
||||
@Injectable()
|
||||
export class CliService extends BaseService {
|
||||
@@ -38,6 +42,63 @@ export class CliService extends BaseService {
|
||||
await this.updateConfig(config);
|
||||
}
|
||||
|
||||
async disableMaintenanceMode(): Promise<{ alreadyDisabled: boolean }> {
|
||||
const currentState = await this.systemMetadataRepository
|
||||
.get(SystemMetadataKey.MaintenanceMode)
|
||||
.then((state) => state ?? { isMaintenanceMode: false as const });
|
||||
|
||||
if (!currentState.isMaintenanceMode) {
|
||||
return {
|
||||
alreadyDisabled: true,
|
||||
};
|
||||
}
|
||||
|
||||
const state = { isMaintenanceMode: false as const };
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);
|
||||
|
||||
sendOneShotAppRestart(state);
|
||||
|
||||
return {
|
||||
alreadyDisabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
async enableMaintenanceMode(): Promise<{ authUrl: string; alreadyEnabled: boolean }> {
|
||||
const { server } = await this.getConfig({ withCache: true });
|
||||
const baseUrl = getExternalDomain(server);
|
||||
|
||||
const payload: MaintenanceAuthDto = {
|
||||
username: 'cli-admin',
|
||||
};
|
||||
|
||||
const state = await this.systemMetadataRepository
|
||||
.get(SystemMetadataKey.MaintenanceMode)
|
||||
.then((state) => state ?? { isMaintenanceMode: false as const });
|
||||
|
||||
if (state.isMaintenanceMode) {
|
||||
return {
|
||||
authUrl: await createMaintenanceLoginUrl(baseUrl, payload, state.secret),
|
||||
alreadyEnabled: true,
|
||||
};
|
||||
}
|
||||
|
||||
const secret = generateMaintenanceSecret();
|
||||
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, {
|
||||
isMaintenanceMode: true,
|
||||
secret,
|
||||
});
|
||||
|
||||
sendOneShotAppRestart({
|
||||
isMaintenanceMode: true,
|
||||
});
|
||||
|
||||
return {
|
||||
authUrl: await createMaintenanceLoginUrl(baseUrl, payload, secret),
|
||||
alreadyEnabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
async grantAdminAccess(email: string): Promise<void> {
|
||||
const user = await this.userRepository.getByEmail(email);
|
||||
if (!user) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import { DownloadService } from 'src/services/download.service';
|
||||
import { DuplicateService } from 'src/services/duplicate.service';
|
||||
import { JobService } from 'src/services/job.service';
|
||||
import { LibraryService } from 'src/services/library.service';
|
||||
import { MaintenanceService } from 'src/services/maintenance.service';
|
||||
import { MapService } from 'src/services/map.service';
|
||||
import { MediaService } from 'src/services/media.service';
|
||||
import { MemoryService } from 'src/services/memory.service';
|
||||
@@ -63,6 +64,7 @@ export const services = [
|
||||
DuplicateService,
|
||||
JobService,
|
||||
LibraryService,
|
||||
MaintenanceService,
|
||||
MapService,
|
||||
MediaService,
|
||||
MemoryService,
|
||||
|
||||
109
server/src/services/maintenance.service.spec.ts
Normal file
109
server/src/services/maintenance.service.spec.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import { SystemMetadataKey } from 'src/enum';
|
||||
import { MaintenanceService } from 'src/services/maintenance.service';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
|
||||
describe(MaintenanceService.name, () => {
|
||||
let sut: MaintenanceService;
|
||||
let mocks: ServiceMocks;
|
||||
|
||||
beforeEach(() => {
|
||||
({ sut, mocks } = newTestService(MaintenanceService));
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getMaintenanceMode', () => {
|
||||
it('should return false if state unknown', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue(null);
|
||||
|
||||
await expect(sut.getMaintenanceMode()).resolves.toEqual({
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return false if disabled', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
|
||||
await expect(sut.getMaintenanceMode()).resolves.toEqual({
|
||||
isMaintenanceMode: false,
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return true if enabled', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: '' });
|
||||
|
||||
await expect(sut.getMaintenanceMode()).resolves.toEqual({
|
||||
isMaintenanceMode: true,
|
||||
secret: '',
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('startMaintenance', () => {
|
||||
it('should set maintenance mode and return a secret', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
|
||||
await expect(sut.startMaintenance('admin')).resolves.toMatchObject({
|
||||
jwt: expect.any(String),
|
||||
});
|
||||
|
||||
expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
|
||||
isMaintenanceMode: true,
|
||||
secret: expect.stringMatching(/^\w{128}$/),
|
||||
});
|
||||
|
||||
expect(mocks.event.emit).toHaveBeenCalledWith('AppRestart', {
|
||||
isMaintenanceMode: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('createLoginUrl', () => {
|
||||
it('should fail outside of maintenance mode without secret', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: false });
|
||||
|
||||
await expect(
|
||||
sut.createLoginUrl({
|
||||
username: '',
|
||||
}),
|
||||
).rejects.toThrowError('Not in maintenance mode');
|
||||
});
|
||||
|
||||
it('should generate a login url with JWT', async () => {
|
||||
mocks.systemMetadata.get.mockResolvedValue({ isMaintenanceMode: true, secret: 'secret' });
|
||||
|
||||
await expect(
|
||||
sut.createLoginUrl({
|
||||
username: '',
|
||||
}),
|
||||
).resolves.toEqual(
|
||||
expect.stringMatching(
|
||||
/^https:\/\/my.immich.app\/maintenance\?token=[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*\.[A-Za-z0-9-_]*$/,
|
||||
),
|
||||
);
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should use the given secret', async () => {
|
||||
await expect(
|
||||
sut.createLoginUrl(
|
||||
{
|
||||
username: '',
|
||||
},
|
||||
'secret',
|
||||
),
|
||||
).resolves.toEqual(expect.stringMatching(/./));
|
||||
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
53
server/src/services/maintenance.service.ts
Normal file
53
server/src/services/maintenance.service.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { OnEvent } from 'src/decorators';
|
||||
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
|
||||
import { SystemMetadataKey } from 'src/enum';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { MaintenanceModeState } from 'src/types';
|
||||
import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance';
|
||||
import { getExternalDomain } from 'src/utils/misc';
|
||||
|
||||
/**
|
||||
* This service is available outside of maintenance mode to manage maintenance mode
|
||||
*/
|
||||
@Injectable()
|
||||
export class MaintenanceService extends BaseService {
|
||||
getMaintenanceMode(): Promise<MaintenanceModeState> {
|
||||
return this.systemMetadataRepository
|
||||
.get(SystemMetadataKey.MaintenanceMode)
|
||||
.then((state) => state ?? { isMaintenanceMode: false });
|
||||
}
|
||||
|
||||
async startMaintenance(username: string): Promise<{ jwt: string }> {
|
||||
const secret = generateMaintenanceSecret();
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, { isMaintenanceMode: true, secret });
|
||||
await this.eventRepository.emit('AppRestart', { isMaintenanceMode: true });
|
||||
|
||||
return {
|
||||
jwt: await signMaintenanceJwt(secret, {
|
||||
username,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'AppRestart', server: true })
|
||||
onRestart(): void {
|
||||
this.appRepository.exitApp();
|
||||
}
|
||||
|
||||
async createLoginUrl(auth: MaintenanceAuthDto, secret?: string): Promise<string> {
|
||||
const { server } = await this.getConfig({ withCache: true });
|
||||
const baseUrl = getExternalDomain(server);
|
||||
|
||||
if (!secret) {
|
||||
const state = await this.getMaintenanceMode();
|
||||
if (!state.isMaintenanceMode) {
|
||||
throw new Error('Not in maintenance mode');
|
||||
}
|
||||
|
||||
secret = state.secret;
|
||||
}
|
||||
|
||||
return await createMaintenanceLoginUrl(baseUrl, auth, secret);
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,15 @@ export class NotificationService extends BaseService {
|
||||
this.websocketRepository.serverSend('ConfigUpdate', { oldConfig, newConfig });
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'AppRestart' })
|
||||
onAppRestart(state: ArgOf<'AppRestart'>) {
|
||||
this.websocketRepository.clientBroadcast('AppRestartV1', {
|
||||
isMaintenanceMode: state.isMaintenanceMode,
|
||||
});
|
||||
|
||||
this.websocketRepository.serverSend('AppRestart', state);
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'ConfigValidate', priority: -100 })
|
||||
async onConfigValidate({ oldConfig, newConfig }: ArgOf<'ConfigValidate'>) {
|
||||
try {
|
||||
|
||||
@@ -166,6 +166,7 @@ describe(ServerService.name, () => {
|
||||
publicUsers: true,
|
||||
mapDarkStyleUrl: 'https://tiles.immich.cloud/v1/style/dark.json',
|
||||
mapLightStyleUrl: 'https://tiles.immich.cloud/v1/style/light.json',
|
||||
maintenanceMode: false,
|
||||
});
|
||||
expect(mocks.systemMetadata.get).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -130,6 +130,7 @@ export class ServerService extends BaseService {
|
||||
publicUsers: config.server.publicUsers,
|
||||
mapDarkStyleUrl: config.map.darkStyle,
|
||||
mapLightStyleUrl: config.map.lightStyle,
|
||||
maintenanceMode: false,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -493,6 +493,7 @@ export interface MemoryData {
|
||||
|
||||
export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
|
||||
export type SystemFlags = { mountChecks: Record<StorageFolder, boolean> };
|
||||
export type MaintenanceModeState = { isMaintenanceMode: true; secret: string } | { isMaintenanceMode: false };
|
||||
export type MemoriesState = {
|
||||
/** memories have already been created through this date */
|
||||
lastOnThisDayDate: string;
|
||||
@@ -503,6 +504,7 @@ export interface SystemMetadata extends Record<SystemMetadataKey, Record<string,
|
||||
[SystemMetadataKey.AdminOnboarding]: { isOnboarded: boolean };
|
||||
[SystemMetadataKey.FacialRecognitionState]: { lastRun?: string };
|
||||
[SystemMetadataKey.License]: { licenseKey: string; activationKey: string; activatedAt: Date };
|
||||
[SystemMetadataKey.MaintenanceMode]: MaintenanceModeState;
|
||||
[SystemMetadataKey.MediaLocation]: MediaLocation;
|
||||
[SystemMetadataKey.ReverseGeocodingState]: { lastUpdate?: string; lastImportFileName?: string };
|
||||
[SystemMetadataKey.SystemConfig]: DeepPartial<SystemConfig>;
|
||||
|
||||
74
server/src/utils/maintenance.ts
Normal file
74
server/src/utils/maintenance.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { createAdapter } from '@socket.io/redis-adapter';
|
||||
import Redis from 'ioredis';
|
||||
import { SignJWT } from 'jose';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { Server as SocketIO } from 'socket.io';
|
||||
import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { AppRestartEvent } from 'src/repositories/event.repository';
|
||||
|
||||
export function sendOneShotAppRestart(state: AppRestartEvent): void {
|
||||
const server = new SocketIO();
|
||||
const { redis } = new ConfigRepository().getEnv();
|
||||
const pubClient = new Redis(redis);
|
||||
const subClient = pubClient.duplicate();
|
||||
server.adapter(createAdapter(pubClient, subClient));
|
||||
|
||||
/**
|
||||
* Keep trying until we manage to stop Immich
|
||||
*
|
||||
* Sometimes there appear to be communication
|
||||
* issues between to the other servers.
|
||||
*
|
||||
* This issue only occurs with this method.
|
||||
*/
|
||||
async function tryTerminate() {
|
||||
while (true) {
|
||||
try {
|
||||
const responses = await server.serverSideEmitWithAck('AppRestart', state);
|
||||
if (responses.length > 0) {
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
console.error('Encountered an error while telling Immich to stop.');
|
||||
}
|
||||
|
||||
console.info(
|
||||
"\nIt doesn't appear that Immich stopped, trying again in a moment.\nIf Immich is already not running, you can ignore this error.",
|
||||
);
|
||||
|
||||
await new Promise((r) => setTimeout(r, 1e3));
|
||||
}
|
||||
}
|
||||
|
||||
// => corresponds to notification.service.ts#onAppRestart
|
||||
server.emit('AppRestartV1', state, () => {
|
||||
void tryTerminate().finally(() => {
|
||||
pubClient.disconnect();
|
||||
subClient.disconnect();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function createMaintenanceLoginUrl(
|
||||
baseUrl: string,
|
||||
auth: MaintenanceAuthDto,
|
||||
secret: string,
|
||||
): Promise<string> {
|
||||
return `${baseUrl}/maintenance?token=${await signMaintenanceJwt(secret, auth)}`;
|
||||
}
|
||||
|
||||
export async function signMaintenanceJwt(secret: string, data: MaintenanceAuthDto): Promise<string> {
|
||||
const alg = 'HS256';
|
||||
|
||||
return await new SignJWT({ ...data })
|
||||
.setProtectedHeader({ alg })
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('4h')
|
||||
.sign(new TextEncoder().encode(secret));
|
||||
}
|
||||
|
||||
export function generateMaintenanceSecret(): string {
|
||||
return randomBytes(64).toString('hex');
|
||||
}
|
||||
@@ -15,6 +15,7 @@ export const respondWithCookie = <T>(res: Response, body: T, { isSecure, values
|
||||
const cookieOptions: Record<ImmichCookie, CookieOptions> = {
|
||||
[ImmichCookie.AuthType]: defaults,
|
||||
[ImmichCookie.AccessToken]: defaults,
|
||||
[ImmichCookie.MaintenanceToken]: { ...defaults, maxAge: Duration.fromObject({ days: 1 }).toMillis() },
|
||||
[ImmichCookie.OAuthState]: defaults,
|
||||
[ImmichCookie.OAuthCodeVerifier]: defaults,
|
||||
// no httpOnly so that the client can know the auth state
|
||||
|
||||
@@ -1,69 +1,22 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { NestExpressApplication } from '@nestjs/platform-express';
|
||||
import { json } from 'body-parser';
|
||||
import compression from 'compression';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import { existsSync } from 'node:fs';
|
||||
import sirv from 'sirv';
|
||||
import { configureExpress, configureTelemetry } from 'src/app.common';
|
||||
import { ApiModule } from 'src/app.module';
|
||||
import { excludePaths, serverVersion } from 'src/constants';
|
||||
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { ApiService } from 'src/services/api.service';
|
||||
import { isStartUpError, useSwagger } from 'src/utils/misc';
|
||||
import { isStartUpError } from 'src/utils/misc';
|
||||
|
||||
async function bootstrap() {
|
||||
process.title = 'immich-api';
|
||||
|
||||
const { telemetry, network } = new ConfigRepository().getEnv();
|
||||
if (telemetry.metrics.size > 0) {
|
||||
bootstrapTelemetry(telemetry.apiPort);
|
||||
}
|
||||
configureTelemetry();
|
||||
|
||||
const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });
|
||||
const logger = await app.resolve(LoggingRepository);
|
||||
const configRepository = app.get(ConfigRepository);
|
||||
app.get(AppRepository).setCloseFn(() => app.close());
|
||||
|
||||
const { environment, host, port, resourcePaths } = configRepository.getEnv();
|
||||
|
||||
logger.setContext('Bootstrap');
|
||||
app.useLogger(logger);
|
||||
app.set('trust proxy', ['loopback', ...network.trustedProxies]);
|
||||
app.set('etag', 'strong');
|
||||
app.use(cookieParser());
|
||||
app.use(json({ limit: '10mb' }));
|
||||
if (configRepository.isDev()) {
|
||||
app.enableCors();
|
||||
}
|
||||
app.useWebSocketAdapter(new WebSocketAdapter(app));
|
||||
useSwagger(app, { write: configRepository.isDev() });
|
||||
|
||||
app.setGlobalPrefix('api', { exclude: excludePaths });
|
||||
if (existsSync(resourcePaths.web.root)) {
|
||||
// copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46
|
||||
// provides serving of precompressed assets and caching of immutable assets
|
||||
app.use(
|
||||
sirv(resourcePaths.web.root, {
|
||||
etag: true,
|
||||
gzip: true,
|
||||
brotli: true,
|
||||
extensions: [],
|
||||
setHeaders: (res, pathname) => {
|
||||
if (pathname.startsWith(`/_app/immutable`) && res.statusCode === 200) {
|
||||
res.setHeader('cache-control', 'public,max-age=31536000,immutable');
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
app.use(app.get(ApiService).ssr(excludePaths));
|
||||
app.use(compression());
|
||||
|
||||
const server = await (host ? app.listen(port, host) : app.listen(port));
|
||||
server.requestTimeout = 24 * 60 * 60 * 1000;
|
||||
|
||||
logger.log(`Immich Server is listening on ${await app.getUrl()} [v${serverVersion}] [${environment}] `);
|
||||
void configureExpress(app, {
|
||||
ssr: ApiService,
|
||||
});
|
||||
}
|
||||
|
||||
bootstrap().catch((error) => {
|
||||
|
||||
29
server/src/workers/maintenance.ts
Normal file
29
server/src/workers/maintenance.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { NestExpressApplication } from '@nestjs/platform-express';
|
||||
import { configureExpress, configureTelemetry } from 'src/app.common';
|
||||
import { MaintenanceModule } from 'src/app.module';
|
||||
import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { isStartUpError } from 'src/utils/misc';
|
||||
|
||||
async function bootstrap() {
|
||||
process.title = 'immich-maintenance';
|
||||
configureTelemetry();
|
||||
|
||||
const app = await NestFactory.create<NestExpressApplication>(MaintenanceModule, { bufferLogs: true });
|
||||
app.get(AppRepository).setCloseFn(() => app.close());
|
||||
void configureExpress(app, {
|
||||
permitSwaggerWrite: false,
|
||||
ssr: MaintenanceWorkerService,
|
||||
});
|
||||
|
||||
void app.get(MaintenanceWorkerService).logSecret();
|
||||
}
|
||||
|
||||
bootstrap().catch((error) => {
|
||||
if (!isStartUpError(error)) {
|
||||
console.error(error);
|
||||
}
|
||||
// eslint-disable-next-line unicorn/no-process-exit
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { isMainThread } from 'node:worker_threads';
|
||||
import { MicroservicesModule } from 'src/app.module';
|
||||
import { serverVersion } from 'src/constants';
|
||||
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { ConfigRepository } from 'src/repositories/config.repository';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { bootstrapTelemetry } from 'src/repositories/telemetry.repository';
|
||||
@@ -17,6 +18,7 @@ export async function bootstrap() {
|
||||
const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
|
||||
const logger = await app.resolve(LoggingRepository);
|
||||
const configRepository = app.get(ConfigRepository);
|
||||
app.get(AppRepository).setCloseFn(() => app.close());
|
||||
|
||||
const { environment, host } = configRepository.getEnv();
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import { ActivityRepository } from 'src/repositories/activity.repository';
|
||||
import { AlbumUserRepository } from 'src/repositories/album-user.repository';
|
||||
import { AlbumRepository } from 'src/repositories/album.repository';
|
||||
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
|
||||
import { AppRepository } from 'src/repositories/app.repository';
|
||||
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
|
||||
import { AssetRepository } from 'src/repositories/asset.repository';
|
||||
import { AuditRepository } from 'src/repositories/audit.repository';
|
||||
@@ -212,6 +213,7 @@ export type ServiceOverrides = {
|
||||
album: AlbumRepository;
|
||||
albumUser: AlbumUserRepository;
|
||||
apiKey: ApiKeyRepository;
|
||||
app: AppRepository;
|
||||
audit: AuditRepository;
|
||||
asset: AssetRepository;
|
||||
assetJob: AssetJobRepository;
|
||||
@@ -271,10 +273,7 @@ type Constructor<Type, Args extends Array<any>> = {
|
||||
new (...deps: Args): Type;
|
||||
};
|
||||
|
||||
export const newTestService = <T extends BaseService>(
|
||||
Service: Constructor<T, BaseServiceArgs>,
|
||||
overrides: Partial<ServiceOverrides> = {},
|
||||
) => {
|
||||
export const getMocks = () => {
|
||||
const loggerMock = { setContext: () => {} };
|
||||
const configMock = { getEnv: () => ({}) };
|
||||
|
||||
@@ -291,6 +290,7 @@ export const newTestService = <T extends BaseService>(
|
||||
albumUser: automock(AlbumUserRepository),
|
||||
asset: newAssetRepositoryMock(),
|
||||
assetJob: automock(AssetJobRepository),
|
||||
app: automock(AppRepository, { strict: false }),
|
||||
config: newConfigRepositoryMock(),
|
||||
database: newDatabaseRepositoryMock(),
|
||||
downloadRepository: automock(DownloadRepository, { strict: false }),
|
||||
@@ -338,6 +338,15 @@ export const newTestService = <T extends BaseService>(
|
||||
workflow: automock(WorkflowRepository, { strict: true }),
|
||||
};
|
||||
|
||||
return mocks;
|
||||
};
|
||||
|
||||
export const newTestService = <T extends BaseService>(
|
||||
Service: Constructor<T, BaseServiceArgs>,
|
||||
overrides: Partial<ServiceOverrides> = {},
|
||||
) => {
|
||||
const mocks = getMocks();
|
||||
|
||||
const sut = new Service(
|
||||
overrides.logger || (mocks.logger as As<LoggingRepository>),
|
||||
overrides.access || (mocks.access as IAccessRepository as AccessRepository),
|
||||
@@ -345,6 +354,7 @@ export const newTestService = <T extends BaseService>(
|
||||
overrides.album || (mocks.album as As<AlbumRepository>),
|
||||
overrides.albumUser || (mocks.albumUser as As<AlbumUserRepository>),
|
||||
overrides.apiKey || (mocks.apiKey as As<ApiKeyRepository>),
|
||||
overrides.app || (mocks.app as As<AppRepository>),
|
||||
overrides.asset || (mocks.asset as As<AssetRepository>),
|
||||
overrides.assetJob || (mocks.assetJob as As<AssetJobRepository>),
|
||||
overrides.audit || (mocks.audit as As<AuditRepository>),
|
||||
|
||||
Reference in New Issue
Block a user