fix(server): don't crash when refreshing large libraries (#7934)

* add job to check for offline files

* fix lint

* only check for offline when using checkForOffline

* improve tests

* remove old test

* wip

* remove trie

* refactor batches

* also check offline status

* fix spelling

* don't do offline scan

* rename scan to check

* fix job statuses

* fix lint

* cleanup

* add test

* open-api

* fix test

* fix spinner

* reset text

* don't double batch

* fix comments from mert

* remove tries

* fix tests

* fix e2e

* fix test

* fix test

* add tests

* fix lint

* fix e2e

* interweave scans

* fix errors

* fix messages

* fix test

* add mock

* fix sql

* fix e2e

* use library batch size

* save -> update

* add file extensions

* update specs

* test for import paths

* check import paths when testing offline

* fix lint

* normalize import path

* remove console logs

* decrease batch size to 1000

* add test for import path

* add test for already-online assets

* fix merge

* fix lint

* add library job back

* add offline job to correct queue

* library spec compiles now

* move one test to new e2e

* fix comments

* fix comments

* fix lint

* refactor path validation

* fix loop bug

* remove logging

* expect responses

* fix asset mock

* take the straightforward approach

* use generator correctly

* fix vitest on file edit

* bump vitest to 1.6.0

* test for offline check

* add e2e tests for offlining assets depending on import path

* cleanup e2e test after finish

* cleanup library service

* paginate the walk generator

* fix tests

* fix typo

* refactoring handleOfflineCheck

* better testing of handleOfflineCheck

* fix lint

* handle large library deletions

* dont check if library is deleted

* fix mock

* add a 100k page size to library

* fix loading animation

* better log messages

* Better logging for offline asset removal

* fix sql and tests

* fix number format

* Remove submodule

* fix format

* chore: cleanup

* chore: fix tests

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>

server/src/interfaces/asset.interface.ts

@@ -36,6 +36,7 @@ export enum WithoutProperty {
 
 export enum WithProperty {
   SIDECAR = 'sidecar',
+  IS_ONLINE = 'isOnline',
   IS_OFFLINE = 'isOffline',
 }
 

server/src/interfaces/job.interface.ts

@@ -76,6 +76,7 @@ export enum JobName {
   LIBRARY_SCAN = 'library-refresh',
   LIBRARY_SCAN_ASSET = 'library-refresh-asset',
   LIBRARY_REMOVE_OFFLINE = 'library-remove-offline',
+  LIBRARY_CHECK_OFFLINE = 'library-check-offline',
   LIBRARY_DELETE = 'library-delete',
   LIBRARY_QUEUE_SCAN_ALL = 'library-queue-all-refresh',
   LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup',
@@ -110,6 +111,7 @@ export enum JobName {
 }
 
 export const JOBS_ASSET_PAGINATION_SIZE = 1000;
+export const JOBS_LIBRARY_PAGINATION_SIZE = 100_000;
 
 export interface IBaseJob {
   force?: boolean;
@@ -129,6 +131,10 @@ export interface ILibraryFileJob extends IEntityJob {
   assetPath: string;
 }
 
+export interface ILibraryOfflineJob extends IEntityJob {
+  importPaths: string[];
+}
+
 export interface ILibraryRefreshJob extends IEntityJob {
   refreshModifiedFiles: boolean;
   refreshAllFiles: boolean;
@@ -264,6 +270,7 @@ export type JobItem =
   | { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
   | { name: JobName.LIBRARY_DELETE; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
+  | { name: JobName.LIBRARY_CHECK_OFFLINE; data: IEntityJob }
   | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
 
   // Notification

server/src/interfaces/library.interface.ts

@@ -12,5 +12,4 @@ export interface ILibraryRepository {
   softDelete(id: string): Promise<void>;
   update(library: Partial<LibraryEntity>): Promise<LibraryEntity>;
   getStatistics(id: string): Promise<LibraryStatsResponseDto | undefined>;
-  getAssetIds(id: string, withDeleted?: boolean): Promise<string[]>;
 }

server/src/interfaces/storage.interface.ts

@@ -2,7 +2,7 @@ import { WatchOptions } from 'chokidar';
 import { Stats } from 'node:fs';
 import { FileReadOptions } from 'node:fs/promises';
 import { Readable } from 'node:stream';
-import { CrawlOptionsDto } from 'src/dtos/library.dto';
+import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto';
 
 export interface ImmichReadStream {
   stream: Readable;
@@ -45,8 +45,8 @@ export interface IStorageRepository {
   checkDiskUsage(folder: string): Promise<DiskUsage>;
   readdir(folder: string): Promise<string[]>;
   stat(filepath: string): Promise<Stats>;
-  crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
-  walk(crawlOptions: CrawlOptionsDto): AsyncGenerator<string>;
+  crawl(options: CrawlOptionsDto): Promise<string[]>;
+  walk(options: WalkOptionsDto): AsyncGenerator<string[]>;
   copyFile(source: string, target: string): Promise<void>;
   rename(source: string, target: string): Promise<void>;
   watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>): () => Promise<void>;