From 1c339ff85beefa8f3d2680f2041d0baba85f4708 Mon Sep 17 00:00:00 2001 From: Min Idzelis Date: Thu, 26 Jun 2025 22:41:24 +0000 Subject: [PATCH] more progress - dockerfile --- .devcontainer/server/container-common.sh | 3 +- .../server/container-compose-overrides.yml | 6 +- .dockerignore | 1 + Makefile | 14 +- docker/docker-compose.dev.yml | 2 +- package-lock.json | 15 ++ package.json | 11 +- pnpm-lock.yaml | 17 +- pnpm-workspace.yaml | 1 + server/Dockerfile | 202 ++++++++++++------ server/pnpm-workspace.yaml | 2 + web/package.json | 2 +- web/pnpm-workspace.yaml | 2 + 13 files changed, 178 insertions(+), 100 deletions(-) create mode 100644 package-lock.json diff --git a/.devcontainer/server/container-common.sh b/.devcontainer/server/container-common.sh index 03c3cfe80d..ebe6e6b09d 100755 --- a/.devcontainer/server/container-common.sh +++ b/.devcontainer/server/container-common.sh @@ -73,7 +73,8 @@ install_dependencies() { log "Installing dependencies" ( cd "${IMMICH_WORKSPACE}" || exit 1 - run_cmd make install-server install-sdk build-sdk install-web + # CI=1 run_cmd make clean install-server install-sdk build-sdk install-web + CI=1 run_cmd make install-all ) log "" } diff --git a/.devcontainer/server/container-compose-overrides.yml b/.devcontainer/server/container-compose-overrides.yml index eb8e66a3d3..14a73dedc3 100644 --- a/.devcontainer/server/container-compose-overrides.yml +++ b/.devcontainer/server/container-compose-overrides.yml @@ -3,15 +3,11 @@ services: build: target: dev-container-server env_file: !reset [] + hostname: immich-dev environment: - IMMICH_SERVER_URL=http://127.0.0.1:2283/ volumes: !override - ..:/workspaces/immich - - cli_node_modules:/workspaces/immich/cli/node_modules - - e2e_node_modules:/workspaces/immich/e2e/node_modules - - open_api_node_modules:/workspaces/immich/open-api/typescript-sdk/node_modules - - server_node_modules:/workspaces/immich/server/node_modules - - web_node_modules:/workspaces/immich/web/node_modules - ${UPLOAD_LOCATION:-upload1-devcontainer-volume}${UPLOAD_LOCATION:+/photos}:/workspaces/immich/server/upload - ${UPLOAD_LOCATION:-upload2-devcontainer-volume}${UPLOAD_LOCATION:+/photos/upload}:/workspaces/immich/server/upload/upload - /etc/localtime:/etc/localtime:ro diff --git a/.dockerignore b/.dockerignore index e182865ae0..3b85a7d1ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ design/ docker/ +Dockerfile !docker/scripts docs/ e2e/ diff --git a/Makefile b/Makefile index b5e7d999e7..117e41d844 100644 --- a/Makefile +++ b/Makefile @@ -92,23 +92,23 @@ test-medium-dev: docker exec -it immich_server /bin/sh -c "pnpm run test:medium" install-all: - pnpm install + pnpm -r --filter '!documentation' install build-all: $(foreach M,$(filter-out e2e docs .github,$(MODULES)),build-$M) ; check-all: - pnpm -r --filter '!docs' run "/^(check|check\:svelte|check\:typescript)$/" + pnpm -r --filter '!documentation' run "/^(check|check\:svelte|check\:typescript)$/" lint-all: - pnpm -r --filter '!docs' run lint:fix + pnpm -r --filter '!documentation' run lint:fix format-all: - pnpm -r --filter '!docs' run format:fix + pnpm -r --filter '!documentation' run format:fix audit-all: - pnpm -r --filter '!docs' audit fix + pnpm -r --filter '!documentation' audit fix hygiene-all: audit-all - pnpm -r --filter '!docs' run "/(format:fix|check|check:svelte|check:typescript|sql)/" + pnpm -r --filter '!documentation' run "/(format:fix|check|check:svelte|check:typescript|sql)/" test-all: - pnpm -r --filter '!docs' run "/^test/" + pnpm -r --filter '!documentation' run "/^test/" prune: pnpm store prune diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 94920f02bf..810fe8dc39 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -24,7 +24,7 @@ services: build: context: ../ dockerfile: server/Dockerfile - target: dev + target: dev-docker restart: unless-stopped volumes: - ../server:/usr/src/app diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..0372edbe62 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,15 @@ +{ + "name": "immich-monorepo", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "immich-monorepo", + "version": "0.0.1", + "engines": { + "pnpm": ">=10.0.0" + } + } + } +} diff --git a/package.json b/package.json index 82707b1cbe..88272956e3 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,10 @@ { - "name": "immich-workspace", - "version": "1.135.3", + "name": "immich-monorepo", + "version": "0.0.1", + "description": "monorepo for immich and friends", "private": true, - "description": "Immich workspace root", - "devDependencies": { - "pnpm": "^10.12.1" + "packageManager": "pnpm@10.12.3+sha512.467df2c586056165580ad6dfb54ceaad94c5a30f80893ebdec5a44c5aa73c205ae4a5bb9d5ed6bb84ea7c249ece786642bbb49d06a307df218d03da41c317417", + "engines": { + "pnpm": ">=10.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6942bc008f..9b60ae1782 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,19 +6,4 @@ settings: importers: - .: - devDependencies: - pnpm: - specifier: ^10.12.1 - version: 10.12.2 - -packages: - - pnpm@10.12.2: - resolution: {integrity: sha512-oyVAGFuWTuMLtOl55AWtxq9ZImtDjuTMGfnodzZnpm0wL1v+5go508rGnjXkuW5winHdACt+k1nEESoXIqwyPw==} - engines: {node: '>=18.12'} - hasBin: true - -snapshots: - - pnpm@10.12.2: {} + .: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b51e501571..f648d6af4e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,3 +18,4 @@ onlyBuiltDependencies: sharedWorkspaceLockfile: false shamefullyHoist: true dedupePeerDependents: false +preferWorkspacePackages: true diff --git a/server/Dockerfile b/server/Dockerfile index 6094929684..cb67050324 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,47 +1,82 @@ # dev build FROM ghcr.io/immich-app/base-server-dev:202505131114@sha256:cf4507bbbf307e9b6d8ee9418993321f2b85867da8ce14d0a20ccaf9574cb995 AS dev -ENV COREPACK_ENABLE_AUTO_PIN=0 -ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 -RUN corepack enable && corepack install -g pnpm && apt-get install --no-install-recommends -yqq tini -RUN mkdir -p /pnpm/store && chmod 777 /pnpm/store && mkdir -p /usr/local/etc && echo "store-dir=/pnpm/store" >> /usr/local/etc/npmrc - -WORKDIR /usr/src/app -COPY server/package.json server/pnpm-lock.yaml server/pnpm-workspace.yaml ./ -COPY server/patches ./patches -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install && \ - # exiftool-vendored.pl, sharp-linux-x64 and sharp-linux-arm64 are the only ones we need - # they're marked as optional dependencies, so we need to copy them manually after pruning - rm -rf node_modules/@img/sharp-libvips* && \ - rm -rf node_modules/@img/sharp-linuxmusl-x64 ENV PATH="${PATH}:/usr/src/app/bin" \ IMMICH_ENV=development \ NVIDIA_DRIVER_CAPABILITIES=all \ - NVIDIA_VISIBLE_DEVICES=all + NVIDIA_VISIBLE_DEVICES=all \ + COREPACK_ENABLE_AUTO_PIN=0 \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + npm_config_devdir=/buildcache/node_gyp + +RUN corepack enable && \ + corepack install -g pnpm && \ + apt-get install --no-install-recommends -yqq tini + +RUN mkdir -p /buildcache/pnpm_store && \ + chown -R node:node /buildcache && \ + mkdir -p /usr/local/etc && \ + echo "store-dir=/buildcache/pnpm_store" >> /usr/local/etc/npmrc + +RUN rm -rf /usr/src/app && \ + mkdir -p /usr/src/app && \ + chown node:node /usr/src/app + +USER node +WORKDIR /usr/src/app +COPY --chown=node:node server/package.json server/pnpm-lock.yaml server/pnpm-workspace.yaml ./ +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 pnpm fetch + ENTRYPOINT ["tini", "--", "/bin/sh"] +FROM dev AS dev-docker +WORKDIR /usr/src/app +# Run this without build-cache, so these are cached in image itself +# This will also build node-gyp binaries, like sharp/canvas +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 pnpm install --offline + FROM dev AS dev-container-server +USER root +RUN rm -rf /usr/src/app + RUN apt-get update && \ apt-get install sudo inetutils-ping openjdk-11-jre-headless \ - vim nano \ - -y --no-install-recommends --fix-missing + # these are build-dependencies for 'canvas' used by web testing libs + pango1.0 \ + vim nano -y --no-install-recommends --fix-missing -RUN usermod -aG sudo node -RUN echo "node ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -RUN mkdir -p /workspaces/immich -RUN chown node -R /workspaces +RUN usermod -aG sudo node && \ + echo "node ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +USER node + +RUN sudo mkdir -p /workspaces/immich && \ + sudo chown node -R /workspaces && \ + sudo mkdir /immich-devcontainer && \ + sudo chown node -R /immich-devcontainer -RUN mkdir /immich-devcontainer && chown node -R /immich-devcontainer COPY --chmod=777 ../.devcontainer/server/*.sh /immich-devcontainer/ +COPY --chown=node:node package.json pnpm-lock.yaml* pnpm-workspace.yaml* /tmp/build/ +# note: e2e is part of dockerignore, so it is not copied here +COPY --chown=node:node web/package.json web/pnpm-lock.yaml* web/pnpm-workspace.yaml* /tmp/build/web/ +COPY --chown=node:node cli/package.json cli/pnpm-lock.yaml* cli/pnpm-workspace.yaml* /tmp/build/cli/ +COPY --chown=node:node server/package.json server/pnpm-lock.yaml* server/pnpm-workspace.yaml* /tmp/build/server/ +# note: docs is part of dockerignore, so it is not copied here +COPY --chown=node:node open-api/typescript-sdk/package.json open-api/typescript-sdk/pnpm-lock.yaml* open-api/typescript-sdk/pnpm-workspace.yaml* /tmp/build/open-api/typescript-sdk/ + +# # This will cache all dependencies +RUN cd /tmp/build && \ + pnpm fetch && \ + rm -rf /tmp/build + FROM dev-container-server AS dev-container-mobile # Enable multiarch for arm64 if necessary RUN if [ "$(dpkg --print-architecture)" = "arm64" ]; then \ - dpkg --add-architecture amd64 && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ + sudo dpkg --add-architecture amd64 && \ + sudo apt-get install -y --no-install-recommends \ qemu-user-static \ libc6:amd64 \ libstdc++6:amd64 \ @@ -56,15 +91,13 @@ ENV FLUTTER_HOME=/flutter ENV PATH=${PATH}:${FLUTTER_HOME}/bin # Flutter SDK -RUN mkdir -p ${FLUTTER_HOME} \ - && curl -C - --output flutter.tar.xz https://storage.googleapis.com/flutter_infra_release/releases/${FLUTTER_CHANNEL}/linux/flutter_linux_${FLUTTER_VERSION}-${FLUTTER_CHANNEL}.tar.xz \ - && tar -xf flutter.tar.xz --strip-components=1 -C ${FLUTTER_HOME} \ - && rm flutter.tar.xz \ - && chown -R node ${FLUTTER_HOME} +RUN sudo mkdir -p ${FLUTTER_HOME} \ + && sudo curl -C - --output flutter.tar.xz https://storage.googleapis.com/flutter_infra_release/releases/${FLUTTER_CHANNEL}/linux/flutter_linux_${FLUTTER_VERSION}-${FLUTTER_CHANNEL}.tar.xz \ + && sudo tar -xf flutter.tar.xz --strip-components=1 -C ${FLUTTER_HOME} \ + && sudo rm flutter.tar.xz \ + && sudo chown -R node ${FLUTTER_HOME} -USER node -RUN sudo apt-get update \ - && wget -qO- https://dcm.dev/pgp-key.public | sudo gpg --dearmor -o /usr/share/keyrings/dcm.gpg \ +RUN wget -qO- https://dcm.dev/pgp-key.public | sudo gpg --dearmor -o /usr/share/keyrings/dcm.gpg \ && echo 'deb [signed-by=/usr/share/keyrings/dcm.gpg arch=amd64] https://dcm.dev/debian stable main' | sudo tee /etc/apt/sources.list.d/dart_stable.list \ && sudo apt-get update \ && sudo apt-get install dcm -y @@ -73,55 +106,96 @@ COPY --chmod=777 ../.devcontainer/mobile/container-mobile-post-create.sh /immich RUN dart --disable-analytics -COPY . /tmp/immich -WORKDIR /tmp/immich -RUN make install-sdk build-sdk install-cli install-e2e install-web install-server -WORKDIR /usr/src/app -RUN rm -rf /tmp/immich - -FROM dev AS prod +# server production build +FROM dev-container-server AS prod +USER root +RUN chown node:node /usr/src/app +USER node COPY server . -RUN pnpm run build -RUN pnpm prune --prod --no-optional -COPY --from=dev /usr/src/app/node_modules/@img ./node_modules/@img -COPY --from=dev /usr/src/app/node_modules/exiftool-vendored.pl ./node_modules/exiftool-vendored.pl +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --offline && \ + pnpm build && pnpm prune --prod --no-optional +# && \ +# cp -R /usr/src/app/node_modules/@img /tmp/optionals && \ +# cp -R /usr/src/app/node_modules/exiftool-vendored.pl /tmp/optionals && \ +# rm -rf /tmp/optionals/node_modules/@img/*musl* +# RUN pnpm prune --prod --no-optional && \ +# mv /tmp/optionals/* ./node_modules/ && \ +# rm -rf /tmp/optionals -# web build -FROM node:22.16.0-alpine3.20@sha256:2289fb1fba0f4633b08ec47b94a89c7e20b829fc5679f9b7b298eaa2f1ed8b7e AS web +# web production build +FROM dev AS web +COPY --chown=node:node open-api/typescript-sdk/ ../open-api/typescript-sdk/ WORKDIR /usr/src/open-api/typescript-sdk -COPY open-api/typescript-sdk/package*.json open-api/typescript-sdk/tsconfig*.json ./ -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY open-api/typescript-sdk/ ./ -RUN pnpm run build +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --offline --no-optional && \ + pnpm build && pnpm prune --prod --no-optional +COPY --chown=node:node web /usr/src/app +COPY --chown=node:node i18n /usr/src/i18n WORKDIR /usr/src/app -COPY web/package*.json web/svelte.config.js ./ -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY web ./ -COPY i18n ../i18n -RUN pnpm run build +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --offline --force && \ + pnpm build && pnpm prune --prod --no-optional +FROM dev AS cli + +COPY --chown=node:node open-api/typescript-sdk/ ../open-api/typescript-sdk/ +WORKDIR /usr/src/open-api/typescript-sdk +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --offline --no-optional && \ + pnpm build && pnpm prune --prod --no-optional + +COPY --chown=node:node cli /usr/src/app +WORKDIR /usr/src/app +# the following command does not use --offline, because the cache created in +# the 'dev' stage did not includ the cli depenencies +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --force && \ + pnpm build && pnpm prune --prod --no-optional + +RUN touch CLI_SUCCESS # prod build FROM ghcr.io/immich-app/base-server-prod:202505061115@sha256:9971d3a089787f0bd01f4682141d3665bcf5efb3e101a88e394ffd25bee4eedb +RUN corepack enable && \ + corepack install -g pnpm + WORKDIR /usr/src/app ENV NODE_ENV=production \ NVIDIA_DRIVER_CAPABILITIES=all \ - NVIDIA_VISIBLE_DEVICES=all -COPY --from=prod /usr/src/app/node_modules ./node_modules -COPY --from=prod /usr/src/app/dist ./dist -COPY --from=prod /usr/src/app/bin ./bin -COPY --from=web /usr/src/app/build /build/www -COPY server/resources resources -COPY server/package.json server/package-lock.json ./ -COPY server/start*.sh ./ -COPY "docker/scripts/get-cpus.sh" ./ -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install -g @immich/cli + NVIDIA_VISIBLE_DEVICES=all \ + npm_config_devdir=/buildcache/node_gyp \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + +RUN mkdir -p /buildcache/pnpm_store && \ + chown -R node:node /buildcache && \ + mkdir -p /usr/local/etc && \ + echo "store-dir=/buildcache/pnpm_store" >> /usr/local/etc/npmrc && \ + mkdir -p /usr/src/app/upload && \ + chown -R node:node /usr/src/app && \ + chmod 755 /usr/src/app + +COPY --chown=node:node --from=prod /usr/src/app/dist ./dist +COPY --chown=node:node --from=prod /usr/src/app/bin ./bin +COPY --chown=node:node --from=web /usr/src/app/build /build/www +COPY --chown=node:node --from=cli /usr/src/app/dist ./cli +COPY --chown=node:node server/resources ./resources/ +COPY --chown=node:node server/package.json server/pnpm-lock.yaml server/pnpm-workspace.yaml server/start*.sh \ + docker/scripts/get-cpus.sh ./ COPY LICENSE /licenses/LICENSE.txt COPY LICENSE /LICENSE + +USER node +RUN --mount=type=cache,id=pnpm,target=/buildcache,uid=1000,gid=1000 \ + pnpm install --frozen-lockfile --offline --prod --no-optional && \ + echo '#!/usr/bin/env node' > /usr/src/app/bin/immich && \ + echo 'require("../cli/index.js");' >> /usr/src/app/bin/immich && \ + chmod +x /usr/src/app/bin/immich + ENV PATH="${PATH}:/usr/src/app/bin" ARG BUILD_ID diff --git a/server/pnpm-workspace.yaml b/server/pnpm-workspace.yaml index e689873197..c43e5fa425 100644 --- a/server/pnpm-workspace.yaml +++ b/server/pnpm-workspace.yaml @@ -6,3 +6,5 @@ ignoredBuiltDependencies: onlyBuiltDependencies: - canvas - sharp + +preferWorkspacePackages: true \ No newline at end of file diff --git a/web/package.json b/web/package.json index 388333d0eb..9ca5c6d6c4 100644 --- a/web/package.json +++ b/web/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:cov": "vitest --coverage", "test:watch": "vitest dev", - "prepare": "svelte-kit sync" + "prepare": "node -e \"try { require.resolve('@sveltejs/kit/package.json'); } catch(e) { process.exit(1); }\" && svelte-kit sync || echo 'Skipping svelte-kit sync (devDependency not available)'" }, "dependencies": { "@formatjs/icu-messageformat-parser": "^2.9.8", diff --git a/web/pnpm-workspace.yaml b/web/pnpm-workspace.yaml index e689873197..c43e5fa425 100644 --- a/web/pnpm-workspace.yaml +++ b/web/pnpm-workspace.yaml @@ -6,3 +6,5 @@ ignoredBuiltDependencies: onlyBuiltDependencies: - canvas - sharp + +preferWorkspacePackages: true \ No newline at end of file