feat(cli): add --delete-duplicates option (#20035)

* Add --delete-duplicates option to delete local assets that already exist on the server, fixes #12181

* Update docs

* Fix `--delete-duplicates` implying `--delete`

* fix the test, break the english

* format

* also ran the formatter on the e2e folder :)

* early return, fewer allocations

* add back log

---------

Co-authored-by: Robin Jacobs <robin.jacobs@beeline.com>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

cli/src/commands/asset.spec.ts

@@ -271,7 +271,7 @@ describe('startWatch', () => {
     });
   });
 
-  it('should filger out ignored patterns', async () => {
+  it('should filter out ignored patterns', async () => {
     const testFilePath = path.join(testFolder, 'test.jpg');
     const ignoredPattern = 'ignored';
     const ignoredFolder = path.join(testFolder, ignoredPattern);

cli/src/commands/asset.ts

@@ -37,6 +37,7 @@ export interface UploadOptionsDto {
   dryRun?: boolean;
   skipHash?: boolean;
   delete?: boolean;
+  deleteDuplicates?: boolean;
   album?: boolean;
   albumName?: string;
   includeHidden?: boolean;
@@ -70,10 +71,8 @@ const uploadBatch = async (files: string[], options: UploadOptionsDto) => {
     console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4));
   }
   await updateAlbums([...newAssets, ...duplicates], options);
-  await deleteFiles(
-    newAssets.map(({ filepath }) => filepath),
-    options,
-  );
+
+  await deleteFiles(newAssets, duplicates, options);
 };
 
 export const startWatch = async (
@@ -406,28 +405,46 @@ const uploadFile = async (input: string, stats: Stats): Promise<AssetMediaRespon
   return response.json();
 };
 
-const deleteFiles = async (files: string[], options: UploadOptionsDto): Promise<void> => {
-  if (!options.delete) {
-    return;
+const deleteFiles = async (uploaded: Asset[], duplicates: Asset[], options: UploadOptionsDto): Promise<void> => {
+  let fileCount = 0;
+  if (options.delete) {
+    fileCount += uploaded.length;
+  }
+
+  if (options.deleteDuplicates) {
+    fileCount += duplicates.length;
   }
 
   if (options.dryRun) {
-    console.log(`Would have deleted ${files.length} local asset${s(files.length)}`);
+    console.log(`Would have deleted ${fileCount} local asset${s(fileCount)}`);
+    return;
+  }
+
+  if (fileCount === 0) {
     return;
   }
 
   console.log('Deleting assets that have been uploaded...');
-
   const deletionProgress = new SingleBar(
     { format: 'Deleting local assets | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' },
     Presets.shades_classic,
   );
-  deletionProgress.start(files.length, 0);
+  deletionProgress.start(fileCount, 0);
+
+  const chunkDelete = async (files: Asset[]) => {
+    for (const assetBatch of chunk(files, options.concurrency)) {
+      await Promise.all(assetBatch.map((input: Asset) => unlink(input.filepath)));
+      deletionProgress.update(assetBatch.length);
+    }
+  };
 
   try {
-    for (const assetBatch of chunk(files, options.concurrency)) {
-      await Promise.all(assetBatch.map((input: string) => unlink(input)));
-      deletionProgress.update(assetBatch.length);
+    if (options.delete) {
+      await chunkDelete(uploaded);
+    }
+
+    if (options.deleteDuplicates) {
+      await chunkDelete(duplicates);
     }
   } finally {
     deletionProgress.stop();

cli/src/index.ts

@@ -75,6 +75,11 @@ program
       .default(false),
   )
   .addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS'))
+  .addOption(
+    new Option('--delete-duplicates', 'Delete local assets that are duplicates (already exist on server)').env(
+      'IMMICH_DELETE_DUPLICATES',
+    ),
+  )
   .addOption(new Option('--no-progress', 'Hide progress bars').env('IMMICH_PROGRESS_BAR').default(true))
   .addOption(
     new Option('--watch', 'Watch for changes and upload automatically')

docs/docs/features/command-line-interface.md

@@ -103,6 +103,7 @@ Options:
   -c, --concurrency <number>  Number of assets to upload at the same time (default: 4, env: IMMICH_UPLOAD_CONCURRENCY)
   -j, --json-output           Output detailed information in json format (default: false, env: IMMICH_JSON_OUTPUT)
   --delete                    Delete local assets after upload (env: IMMICH_DELETE_ASSETS)
+  --delete-duplicates         Delete local assets that are duplicates (already exist on server) (env: IMMICH_DELETE_DUPLICATES)
   --no-progress               Hide progress bars (env: IMMICH_PROGRESS_BAR)
   --watch                     Watch for changes and upload automatically (default: false, env: IMMICH_WATCH_CHANGES)
   --help                      display help for command

e2e/src/cli/specs/upload.e2e-spec.ts

@@ -442,6 +442,176 @@ describe(`immich upload`, () => {
     });
   });
 
+  describe('immich upload --delete-duplicates', () => {
+    it('should delete local duplicate files', async () => {
+      const {
+        stderr: firstStderr,
+        stdout: firstStdout,
+        exitCode: firstExitCode,
+      } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
+      expect(firstStderr).toContain('{message}');
+      expect(firstStdout.split('\n')).toEqual(
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
+      );
+      expect(firstExitCode).toBe(0);
+
+      await mkdir(`/tmp/albums/nature`, { recursive: true });
+      await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`);
+
+      // Upload with --delete-duplicates flag
+      const { stderr, stdout, exitCode } = await immichCli([
+        'upload',
+        `/tmp/albums/nature/silver_fir.jpg`,
+        '--delete-duplicates',
+      ]);
+
+      // Check that the duplicate file was deleted
+      const files = await readdir(`/tmp/albums/nature`);
+      await rm(`/tmp/albums/nature`, { recursive: true });
+      expect(files.length).toBe(0);
+
+      expect(stdout.split('\n')).toEqual(
+        expect.arrayContaining([
+          expect.stringContaining('Found 0 new files and 1 duplicate'),
+          expect.stringContaining('All assets were already uploaded, nothing to do'),
+        ]),
+      );
+      expect(stderr).toContain('{message}');
+      expect(exitCode).toBe(0);
+
+      // Verify no new assets were uploaded
+      const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
+      expect(assets.total).toBe(1);
+    });
+
+    it('should have accurate dry run with --delete-duplicates', async () => {
+      const {
+        stderr: firstStderr,
+        stdout: firstStdout,
+        exitCode: firstExitCode,
+      } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
+      expect(firstStderr).toContain('{message}');
+      expect(firstStdout.split('\n')).toEqual(
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
+      );
+      expect(firstExitCode).toBe(0);
+
+      await mkdir(`/tmp/albums/nature`, { recursive: true });
+      await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`);
+
+      // Upload with --delete-duplicates and --dry-run flags
+      const { stderr, stdout, exitCode } = await immichCli([
+        'upload',
+        `/tmp/albums/nature/silver_fir.jpg`,
+        '--delete-duplicates',
+        '--dry-run',
+      ]);
+
+      // Check that the duplicate file was NOT deleted in dry run mode
+      const files = await readdir(`/tmp/albums/nature`);
+      await rm(`/tmp/albums/nature`, { recursive: true });
+      expect(files.length).toBe(1);
+
+      expect(stdout.split('\n')).toEqual(
+        expect.arrayContaining([
+          expect.stringContaining('Found 0 new files and 1 duplicate'),
+          expect.stringContaining('Would have deleted 1 local asset'),
+        ]),
+      );
+      expect(stderr).toContain('{message}');
+      expect(exitCode).toBe(0);
+
+      // Verify no new assets were uploaded
+      const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
+      expect(assets.total).toBe(1);
+    });
+
+    it('should work with both --delete and --delete-duplicates flags', async () => {
+      // First, upload a file to create a duplicate on the server
+      const {
+        stderr: firstStderr,
+        stdout: firstStdout,
+        exitCode: firstExitCode,
+      } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
+      expect(firstStderr).toContain('{message}');
+      expect(firstStdout.split('\n')).toEqual(
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
+      );
+      expect(firstExitCode).toBe(0);
+
+      // Both new and duplicate files
+      await mkdir(`/tmp/albums/nature`, { recursive: true });
+      await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate
+      await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new
+
+      // Upload with both --delete and --delete-duplicates flags
+      const { stderr, stdout, exitCode } = await immichCli([
+        'upload',
+        `/tmp/albums/nature`,
+        '--delete',
+        '--delete-duplicates',
+      ]);
+
+      // Check that both files were deleted (new file due to --delete, duplicate due to --delete-duplicates)
+      const files = await readdir(`/tmp/albums/nature`);
+      await rm(`/tmp/albums/nature`, { recursive: true });
+      expect(files.length).toBe(0);
+
+      expect(stdout.split('\n')).toEqual(
+        expect.arrayContaining([
+          expect.stringContaining('Found 1 new files and 1 duplicate'),
+          expect.stringContaining('Successfully uploaded 1 new asset'),
+          expect.stringContaining('Deleting assets that have been uploaded'),
+        ]),
+      );
+      expect(stderr).toContain('{message}');
+      expect(exitCode).toBe(0);
+
+      // Verify one new asset was uploaded (total should be 2 now)
+      const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
+      expect(assets.total).toBe(2);
+    });
+
+    it('should only delete duplicates when --delete-duplicates is used without --delete', async () => {
+      const {
+        stderr: firstStderr,
+        stdout: firstStdout,
+        exitCode: firstExitCode,
+      } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]);
+      expect(firstStderr).toContain('{message}');
+      expect(firstStdout.split('\n')).toEqual(
+        expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]),
+      );
+      expect(firstExitCode).toBe(0);
+
+      // Both new and duplicate files
+      await mkdir(`/tmp/albums/nature`, { recursive: true });
+      await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate
+      await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new
+
+      // Upload with only --delete-duplicates flag
+      const { stderr, stdout, exitCode } = await immichCli(['upload', `/tmp/albums/nature`, '--delete-duplicates']);
+
+      // Check that only the duplicate was deleted, new file should remain
+      const files = await readdir(`/tmp/albums/nature`);
+      await rm(`/tmp/albums/nature`, { recursive: true });
+      expect(files).toEqual(['el_torcal_rocks.jpg']);
+
+      expect(stdout.split('\n')).toEqual(
+        expect.arrayContaining([
+          expect.stringContaining('Found 1 new files and 1 duplicate'),
+          expect.stringContaining('Successfully uploaded 1 new asset'),
+        ]),
+      );
+      expect(stderr).toContain('{message}');
+      expect(exitCode).toBe(0);
+
+      // Verify one new asset was uploaded (total should be 2 now)
+      const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) });
+      expect(assets.total).toBe(2);
+    });
+  });
+
   describe('immich upload --skip-hash', () => {
     it('should skip hashing', async () => {
       const filename = `albums/nature/silver_fir.jpg`;