feat: ocr (#18836)

* feat: add OCR functionality and related configurations

* chore: update labeler configuration for machine learning files

* feat(i18n): enhance OCR model descriptions and add orientation classification and unwarping features

* chore: update Dockerfile to include ccache for improved build performance

* feat(ocr): enhance OCR model configuration with orientation classification and unwarping options, update PaddleOCR integration, and improve response structure

* refactor(ocr): remove OCR_CLEANUP job from enum and type definitions

* refactor(ocr): remove obsolete OCR entity and migration files, and update asset job status and schema to accommodate new OCR table structure

* refactor(ocr): update OCR schema and response structure to use individual coordinates instead of bounding box, and adjust related service and repository files

* feat: enhance OCR configuration and functionality

- Updated OCR settings to include minimum detection box score, minimum detection score, and minimum recognition score.
- Refactored PaddleOCRecognizer to utilize new scoring parameters.
- Introduced new database tables for asset OCR data and search functionality.
- Modified related services and repositories to support the new OCR features.
- Updated translations for improved clarity in settings UI.

* sql changes

* use rapidocr

* change dto

* update web

* update lock

* update api

* store positions as normalized floats

* match column order in db

* update admin ui settings descriptions

fix max resolution key

set min threshold to 0.1

fix bind

* apply config correctly, adjust defaults

* unnecessary model type

* unnecessary sources

* fix(ocr): switch RapidOCR lang type from LangDet to LangRec

* fix(ocr): expose lang_type (LangRec.CH) and font_path on OcrOptions for RapidOCR

* fix(ocr): make OCR text search case- and accent-insensitive using ILIKE + unaccent

* fix(ocr): add OCR search fields

* fix: Add OCR database migration and update ML prediction logic.

* trigrams are already case insensitive

* add tests

* format

* update migrations

* wrong uuid function

* linting

* maybe fix medium tests

* formatting

* fix weblate check

* openapi

* sql

* minor fixes

* maybe fix medium tests part 2

* passing medium tests

* format web

* readd sql

* format dart

* disabled in e2e

* chore: translation ordering

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

machine-learning/Dockerfile

@@ -141,7 +141,7 @@ FROM prod-${DEVICE} AS prod
 ARG DEVICE
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends tini $(if ! [ "$DEVICE" = "openvino" ] && ! [ "$DEVICE" = "rocm" ]; then echo "libmimalloc2.0"; fi) && \
+    apt-get install -y --no-install-recommends tini ccache libgl1 libglib2.0-0 libgomp1 $(if ! [ "$DEVICE" = "openvino" ] && ! [ "$DEVICE" = "rocm" ]; then echo "libmimalloc2.0"; fi) && \
     apt-get autoremove -yqq && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*

machine-learning/immich_ml/config.py

@@ -41,6 +41,7 @@ class PreloadModelData(BaseModel):
 
 class MaxBatchSize(BaseModel):
     facial_recognition: int | None = None
+    text_recognition: int | None = None
 
 
 class Settings(BaseSettings):

machine-learning/immich_ml/main.py

@@ -183,7 +183,9 @@ async def run_inference(payload: Image | str, entries: InferenceEntries) -> Infe
     response: InferenceResponse = {}
 
     async def _run_inference(entry: InferenceEntry) -> None:
-        model = await model_cache.get(entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl)
+        model = await model_cache.get(
+            entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl, **entry["options"]
+        )
         inputs = [payload]
         for dep in model.depends:
             try:

machine-learning/immich_ml/models/__init__.py

@@ -3,6 +3,8 @@ from typing import Any
 from immich_ml.models.base import InferenceModel
 from immich_ml.models.clip.textual import MClipTextualEncoder, OpenClipTextualEncoder
 from immich_ml.models.clip.visual import OpenClipVisualEncoder
+from immich_ml.models.ocr.detection import TextDetector
+from immich_ml.models.ocr.recognition import TextRecognizer
 from immich_ml.schemas import ModelSource, ModelTask, ModelType
 
 from .constants import get_model_source
@@ -28,6 +30,12 @@ def get_model_class(model_name: str, model_type: ModelType, model_task: ModelTas
         case ModelSource.INSIGHTFACE, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION:
             return FaceRecognizer
 
+        case ModelSource.PADDLE, ModelType.DETECTION, ModelTask.OCR:
+            return TextDetector
+
+        case ModelSource.PADDLE, ModelType.RECOGNITION, ModelTask.OCR:
+            return TextRecognizer
+
         case _:
             raise ValueError(f"Unknown model combination: {source}, {model_type}, {model_task}")
 

machine-learning/immich_ml/models/base.py

@@ -38,9 +38,8 @@ class InferenceModel(ABC):
 
     def download(self) -> None:
         if not self.cached:
-            log.info(
-                f"Downloading {self.model_type.replace('-', ' ')} model '{self.model_name}'. This may take a while."
-            )
+            model_type = self.model_type.replace("-", " ")
+            log.info(f"Downloading {model_type} model '{self.model_name}' to {self.model_path}. This may take a while.")
             self._download()
 
     def load(self) -> None:
@@ -58,7 +57,7 @@ class InferenceModel(ABC):
         self.load()
         if model_kwargs:
             self.configure(**model_kwargs)
-        return self._predict(*inputs, **model_kwargs)
+        return self._predict(*inputs)
 
     @abstractmethod
     def _predict(self, *inputs: Any, **model_kwargs: Any) -> Any: ...

machine-learning/immich_ml/models/clip/textual.py

@@ -19,7 +19,7 @@ class BaseCLIPTextualEncoder(InferenceModel):
     depends = []
     identity = (ModelType.TEXTUAL, ModelTask.SEARCH)
 
-    def _predict(self, inputs: str, language: str | None = None, **kwargs: Any) -> str:
+    def _predict(self, inputs: str, language: str | None = None) -> str:
         tokens = self.tokenize(inputs, language=language)
         res: NDArray[np.float32] = self.session.run(None, tokens)[0][0]
         return serialize_np_array(res)

machine-learning/immich_ml/models/clip/visual.py

@@ -26,7 +26,7 @@ class BaseCLIPVisualEncoder(InferenceModel):
     depends = []
     identity = (ModelType.VISUAL, ModelTask.SEARCH)
 
-    def _predict(self, inputs: Image.Image | bytes, **kwargs: Any) -> str:
+    def _predict(self, inputs: Image.Image | bytes) -> str:
         image = decode_pil(inputs)
         res: NDArray[np.float32] = self.session.run(None, self.transform(image))[0][0]
         return serialize_np_array(res)

machine-learning/immich_ml/models/constants.py

@@ -75,6 +75,11 @@ _INSIGHTFACE_MODELS = {
 }
 
 
+_PADDLE_MODELS = {
+    "PP-OCRv5_server",
+    "PP-OCRv5_mobile",
+}
+
 SUPPORTED_PROVIDERS = [
     "CUDAExecutionProvider",
     "ROCMExecutionProvider",
@@ -159,4 +164,7 @@ def get_model_source(model_name: str) -> ModelSource | None:
     if cleaned_name in _OPENCLIP_MODELS:
         return ModelSource.OPENCLIP
 
+    if cleaned_name in _PADDLE_MODELS:
+        return ModelSource.PADDLE
+
     return None

machine-learning/immich_ml/models/facial_recognition/detection.py

@@ -24,7 +24,7 @@ class FaceDetector(InferenceModel):
 
         return session
 
-    def _predict(self, inputs: NDArray[np.uint8] | bytes, **kwargs: Any) -> FaceDetectionOutput:
+    def _predict(self, inputs: NDArray[np.uint8] | bytes) -> FaceDetectionOutput:
         inputs = decode_cv2(inputs)
 
         bboxes, landmarks = self._detect(inputs)

machine-learning/immich_ml/models/facial_recognition/recognition.py

@@ -44,7 +44,7 @@ class FaceRecognizer(InferenceModel):
         return session
 
     def _predict(
-        self, inputs: NDArray[np.uint8] | bytes | Image.Image, faces: FaceDetectionOutput, **kwargs: Any
+        self, inputs: NDArray[np.uint8] | bytes | Image.Image, faces: FaceDetectionOutput
     ) -> FacialRecognitionOutput:
         if faces["boxes"].shape[0] == 0:
             return []

machine-learning/immich_ml/models/ocr/detection.py

@@ -0,0 +1,86 @@
+from typing import Any
+
+import numpy as np
+from PIL import Image
+from rapidocr.ch_ppocr_det import TextDetector as RapidTextDetector
+from rapidocr.inference_engine.base import FileInfo, InferSession
+from rapidocr.utils import DownloadFile, DownloadFileInput
+from rapidocr.utils.typings import EngineType, LangDet, OCRVersion, TaskType
+from rapidocr.utils.typings import ModelType as RapidModelType
+
+from immich_ml.config import log
+from immich_ml.models.base import InferenceModel
+from immich_ml.models.transforms import decode_cv2
+from immich_ml.schemas import ModelFormat, ModelSession, ModelTask, ModelType
+from immich_ml.sessions.ort import OrtSession
+
+from .schemas import OcrOptions, TextDetectionOutput
+
+
+class TextDetector(InferenceModel):
+    depends = []
+    identity = (ModelType.DETECTION, ModelTask.OCR)
+
+    def __init__(self, model_name: str, **model_kwargs: Any) -> None:
+        super().__init__(model_name, **model_kwargs, model_format=ModelFormat.ONNX)
+        self.max_resolution = 736
+        self.min_score = 0.5
+        self.score_mode = "fast"
+        self._empty: TextDetectionOutput = {
+            "image": np.empty(0, dtype=np.float32),
+            "boxes": np.empty(0, dtype=np.float32),
+            "scores": np.empty(0, dtype=np.float32),
+        }
+
+    def _download(self) -> None:
+        model_info = InferSession.get_model_url(
+            FileInfo(
+                engine_type=EngineType.ONNXRUNTIME,
+                ocr_version=OCRVersion.PPOCRV5,
+                task_type=TaskType.DET,
+                lang_type=LangDet.CH,
+                model_type=RapidModelType.MOBILE if "mobile" in self.model_name else RapidModelType.SERVER,
+            )
+        )
+        download_params = DownloadFileInput(
+            file_url=model_info["model_dir"],
+            sha256=model_info["SHA256"],
+            save_path=self.model_path,
+            logger=log,
+        )
+        DownloadFile.run(download_params)
+
+    def _load(self) -> ModelSession:
+        # TODO: support other runtime sessions
+        session = OrtSession(self.model_path)
+        self.model = RapidTextDetector(
+            OcrOptions(
+                session=session.session,
+                limit_side_len=self.max_resolution,
+                limit_type="min",
+                box_thresh=self.min_score,
+                score_mode=self.score_mode,
+            )
+        )
+        return session
+
+    def _predict(self, inputs: bytes | Image.Image) -> TextDetectionOutput:
+        results = self.model(decode_cv2(inputs))
+        if results.boxes is None or results.scores is None or results.img is None:
+            return self._empty
+        return {
+            "image": results.img,
+            "boxes": np.array(results.boxes, dtype=np.float32),
+            "scores": np.array(results.scores, dtype=np.float32),
+        }
+
+    def configure(self, **kwargs: Any) -> None:
+        if (max_resolution := kwargs.get("maxResolution")) is not None:
+            self.max_resolution = max_resolution
+            self.model.limit_side_len = max_resolution
+        if (min_score := kwargs.get("minScore")) is not None:
+            self.min_score = min_score
+            self.model.postprocess_op.box_thresh = min_score
+        if (score_mode := kwargs.get("scoreMode")) is not None:
+            self.score_mode = score_mode
+            self.model.postprocess_op.score_mode = score_mode

machine-learning/immich_ml/models/ocr/recognition.py

@@ -0,0 +1,117 @@
+from typing import Any
+
+import cv2
+import numpy as np
+from numpy.typing import NDArray
+from PIL.Image import Image
+from rapidocr.ch_ppocr_rec import TextRecInput
+from rapidocr.ch_ppocr_rec import TextRecognizer as RapidTextRecognizer
+from rapidocr.inference_engine.base import FileInfo, InferSession
+from rapidocr.utils import DownloadFile, DownloadFileInput
+from rapidocr.utils.typings import EngineType, LangRec, OCRVersion, TaskType
+from rapidocr.utils.typings import ModelType as RapidModelType
+
+from immich_ml.config import log, settings
+from immich_ml.models.base import InferenceModel
+from immich_ml.schemas import ModelFormat, ModelSession, ModelTask, ModelType
+from immich_ml.sessions.ort import OrtSession
+
+from .schemas import OcrOptions, TextDetectionOutput, TextRecognitionOutput
+
+
+class TextRecognizer(InferenceModel):
+    depends = [(ModelType.DETECTION, ModelTask.OCR)]
+    identity = (ModelType.RECOGNITION, ModelTask.OCR)
+
+    def __init__(self, model_name: str, **model_kwargs: Any) -> None:
+        self.min_score = model_kwargs.get("minScore", 0.9)
+        self._empty: TextRecognitionOutput = {
+            "box": np.empty(0, dtype=np.float32),
+            "boxScore": np.empty(0, dtype=np.float32),
+            "text": [],
+            "textScore": np.empty(0, dtype=np.float32),
+        }
+        super().__init__(model_name, **model_kwargs, model_format=ModelFormat.ONNX)
+
+    def _download(self) -> None:
+        model_info = InferSession.get_model_url(
+            FileInfo(
+                engine_type=EngineType.ONNXRUNTIME,
+                ocr_version=OCRVersion.PPOCRV5,
+                task_type=TaskType.REC,
+                lang_type=LangRec.CH,
+                model_type=RapidModelType.MOBILE if "mobile" in self.model_name else RapidModelType.SERVER,
+            )
+        )
+        download_params = DownloadFileInput(
+            file_url=model_info["model_dir"],
+            sha256=model_info["SHA256"],
+            save_path=self.model_path,
+            logger=log,
+        )
+        DownloadFile.run(download_params)
+
+    def _load(self) -> ModelSession:
+        # TODO: support other runtimes
+        session = OrtSession(self.model_path)
+        self.model = RapidTextRecognizer(
+            OcrOptions(
+                session=session.session,
+                rec_batch_num=settings.max_batch_size.text_recognition if settings.max_batch_size is not None else 6,
+                rec_img_shape=(3, 48, 320),
+            )
+        )
+        return session
+
+    def _predict(self, _: Image, texts: TextDetectionOutput) -> TextRecognitionOutput:
+        boxes, img, box_scores = texts["boxes"], texts["image"], texts["scores"]
+        if boxes.shape[0] == 0:
+            return self._empty
+        rec = self.model(TextRecInput(img=self.get_crop_img_list(img, boxes)))
+        if rec.txts is None:
+            return self._empty
+
+        height, width = img.shape[0:2]
+        boxes[:, :, 0] /= width
+        boxes[:, :, 1] /= height
+
+        text_scores = np.array(rec.scores)
+        valid_text_score_idx = text_scores > self.min_score
+        valid_score_idx_list = valid_text_score_idx.tolist()
+        return {
+            "box": boxes.reshape(-1, 8)[valid_text_score_idx].reshape(-1),
+            "text": [rec.txts[i] for i in range(len(rec.txts)) if valid_score_idx_list[i]],
+            "boxScore": box_scores[valid_text_score_idx],
+            "textScore": text_scores[valid_text_score_idx],
+        }
+
+    def get_crop_img_list(self, img: NDArray[np.float32], boxes: NDArray[np.float32]) -> list[NDArray[np.float32]]:
+        img_crop_width = np.maximum(
+            np.linalg.norm(boxes[:, 1] - boxes[:, 0], axis=1), np.linalg.norm(boxes[:, 2] - boxes[:, 3], axis=1)
+        ).astype(np.int32)
+        img_crop_height = np.maximum(
+            np.linalg.norm(boxes[:, 0] - boxes[:, 3], axis=1), np.linalg.norm(boxes[:, 1] - boxes[:, 2], axis=1)
+        ).astype(np.int32)
+        pts_std = np.zeros((img_crop_width.shape[0], 4, 2), dtype=np.float32)
+        pts_std[:, 1:3, 0] = img_crop_width[:, None]
+        pts_std[:, 2:4, 1] = img_crop_height[:, None]
+
+        img_crop_sizes = np.stack([img_crop_width, img_crop_height], axis=1).tolist()
+        imgs: list[NDArray[np.float32]] = []
+        for box, pts_std, dst_size in zip(list(boxes), list(pts_std), img_crop_sizes):
+            M = cv2.getPerspectiveTransform(box, pts_std)
+            dst_img: NDArray[np.float32] = cv2.warpPerspective(
+                img,
+                M,
+                dst_size,
+                borderMode=cv2.BORDER_REPLICATE,
+                flags=cv2.INTER_CUBIC,
+            )  # type: ignore
+            dst_height, dst_width = dst_img.shape[0:2]
+            if dst_height * 1.0 / dst_width >= 1.5:
+                dst_img = np.rot90(dst_img)
+            imgs.append(dst_img)
+        return imgs
+
+    def configure(self, **kwargs: Any) -> None:
+        self.min_score = kwargs.get("minScore", self.min_score)

machine-learning/immich_ml/models/ocr/schemas.py

@@ -0,0 +1,28 @@
+from typing import Any, Iterable
+
+import numpy as np
+import numpy.typing as npt
+from rapidocr.utils.typings import EngineType, LangRec
+from typing_extensions import TypedDict
+
+
+class TextDetectionOutput(TypedDict):
+    image: npt.NDArray[np.float32]
+    boxes: npt.NDArray[np.float32]
+    scores: npt.NDArray[np.float32]
+
+
+class TextRecognitionOutput(TypedDict):
+    box: npt.NDArray[np.float32]
+    boxScore: npt.NDArray[np.float32]
+    text: Iterable[str]
+    textScore: npt.NDArray[np.float32]
+
+
+# RapidOCR expects `engine_type`, `lang_type`, and `font_path` to be attributes
+class OcrOptions(dict[str, Any]):
+    def __init__(self, **options: Any) -> None:
+        super().__init__(**options)
+        self.engine_type = EngineType.ONNXRUNTIME
+        self.lang_type = LangRec.CH
+        self.font_path = None

machine-learning/immich_ml/schemas.py

@@ -23,6 +23,7 @@ class BoundingBox(TypedDict):
 class ModelTask(StrEnum):
     FACIAL_RECOGNITION = "facial-recognition"
     SEARCH = "clip"
+    OCR = "ocr"
 
 
 class ModelType(StrEnum):
@@ -42,6 +43,7 @@ class ModelSource(StrEnum):
     INSIGHTFACE = "insightface"
     MCLIP = "mclip"
     OPENCLIP = "openclip"
+    PADDLE = "paddle"
 
 
 ModelIdentity = tuple[ModelType, ModelTask]

machine-learning/immich_ml/sessions/ort.py

@@ -14,6 +14,8 @@ from ..config import log, settings
 
 
 class OrtSession:
+    session: ort.InferenceSession
+
     def __init__(
         self,
         model_path: Path | str,

machine-learning/pyproject.toml

@@ -22,6 +22,8 @@ dependencies = [
     "rich>=13.4.2",
     "tokenizers>=0.15.0,<1.0",
     "uvicorn[standard]>=0.22.0,<1.0",
+    "setuptools>=78.1.0",
+    "rapidocr>=3.1.0",
 ]
 
 [dependency-groups]