mirror of
https://github.com/immich-app/immich.git
synced 2025-11-23 23:00:42 +09:00
refactor(cli): organize files, simplify types, use @immich/sdk (#6747)
This commit is contained in:
Jason Rasmussen
GitHub
287
cli/src/commands/upload.command.ts
Normal file
287
cli/src/commands/upload.command.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
|
||||
import byteSize from 'byte-size';
|
||||
import cliProgress from 'cli-progress';
|
||||
import FormData from 'form-data';
|
||||
import fs, { ReadStream, createReadStream } from 'node:fs';
|
||||
import { CrawlService } from '../services/crawl.service';
|
||||
import { BaseCommand } from './base-command';
|
||||
import { basename } from 'node:path';
|
||||
import { access, constants, stat, unlink } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import Os from 'os';
|
||||
|
||||
class Asset {
|
||||
readonly path: string;
|
||||
readonly deviceId!: string;
|
||||
|
||||
deviceAssetId?: string;
|
||||
fileCreatedAt?: string;
|
||||
fileModifiedAt?: string;
|
||||
sidecarPath?: string;
|
||||
fileSize!: number;
|
||||
albumName?: string;
|
||||
|
||||
constructor(path: string) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
async prepare() {
|
||||
const stats = await stat(this.path);
|
||||
this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, '');
|
||||
this.fileCreatedAt = stats.mtime.toISOString();
|
||||
this.fileModifiedAt = stats.mtime.toISOString();
|
||||
this.fileSize = stats.size;
|
||||
this.albumName = this.extractAlbumName();
|
||||
}
|
||||
|
||||
async getUploadFormData(): Promise<FormData> {
|
||||
if (!this.deviceAssetId) throw new Error('Device asset id not set');
|
||||
if (!this.fileCreatedAt) throw new Error('File created at not set');
|
||||
if (!this.fileModifiedAt) throw new Error('File modified at not set');
|
||||
|
||||
// TODO: doesn't xmp replace the file extension? Will need investigation
|
||||
const sideCarPath = `${this.path}.xmp`;
|
||||
let sidecarData: ReadStream | undefined = undefined;
|
||||
try {
|
||||
await access(sideCarPath, constants.R_OK);
|
||||
sidecarData = createReadStream(sideCarPath);
|
||||
} catch (error) {}
|
||||
|
||||
const data: any = {
|
||||
assetData: createReadStream(this.path),
|
||||
deviceAssetId: this.deviceAssetId,
|
||||
deviceId: 'CLI',
|
||||
fileCreatedAt: this.fileCreatedAt,
|
||||
fileModifiedAt: this.fileModifiedAt,
|
||||
isFavorite: String(false),
|
||||
};
|
||||
const formData = new FormData();
|
||||
|
||||
for (const prop in data) {
|
||||
formData.append(prop, data[prop]);
|
||||
}
|
||||
|
||||
if (sidecarData) {
|
||||
formData.append('sidecarData', sidecarData);
|
||||
}
|
||||
|
||||
return formData;
|
||||
}
|
||||
|
||||
async delete(): Promise<void> {
|
||||
return unlink(this.path);
|
||||
}
|
||||
|
||||
public async hash(): Promise<string> {
|
||||
const sha1 = (filePath: string) => {
|
||||
const hash = createHash('sha1');
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const rs = createReadStream(filePath);
|
||||
rs.on('error', reject);
|
||||
rs.on('data', (chunk) => hash.update(chunk));
|
||||
rs.on('end', () => resolve(hash.digest('hex')));
|
||||
});
|
||||
};
|
||||
|
||||
return await sha1(this.path);
|
||||
}
|
||||
|
||||
private extractAlbumName(): string {
|
||||
if (Os.platform() === 'win32') {
|
||||
return this.path.split('\\').slice(-2)[0];
|
||||
} else {
|
||||
return this.path.split('/').slice(-2)[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class UploadOptionsDto {
|
||||
recursive? = false;
|
||||
exclusionPatterns?: string[] = [];
|
||||
dryRun? = false;
|
||||
skipHash? = false;
|
||||
delete? = false;
|
||||
album? = false;
|
||||
albumName? = '';
|
||||
includeHidden? = false;
|
||||
}
|
||||
|
||||
export class UploadCommand extends BaseCommand {
|
||||
uploadLength!: number;
|
||||
|
||||
public async run(paths: string[], options: UploadOptionsDto): Promise<void> {
|
||||
await this.connect();
|
||||
|
||||
const formatResponse = await this.immichApi.serverInfoApi.getSupportedMediaTypes();
|
||||
const crawlService = new CrawlService(formatResponse.data.image, formatResponse.data.video);
|
||||
|
||||
const inputFiles: string[] = [];
|
||||
for (const pathArgument of paths) {
|
||||
const fileStat = await fs.promises.lstat(pathArgument);
|
||||
if (fileStat.isFile()) {
|
||||
inputFiles.push(pathArgument);
|
||||
}
|
||||
}
|
||||
|
||||
const files: string[] = await crawlService.crawl({
|
||||
pathsToCrawl: paths,
|
||||
recursive: options.recursive,
|
||||
exclusionPatterns: options.exclusionPatterns,
|
||||
includeHidden: options.includeHidden,
|
||||
});
|
||||
|
||||
files.push(...inputFiles);
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No assets found, exiting');
|
||||
return;
|
||||
}
|
||||
|
||||
const assetsToUpload = files.map((path) => new Asset(path));
|
||||
|
||||
const uploadProgress = new cliProgress.SingleBar(
|
||||
{
|
||||
format: '{bar} | {percentage}% | ETA: {eta_formatted} | {value_formatted}/{total_formatted}: {filename}',
|
||||
},
|
||||
cliProgress.Presets.shades_classic,
|
||||
);
|
||||
|
||||
let totalSize = 0;
|
||||
let sizeSoFar = 0;
|
||||
|
||||
let totalSizeUploaded = 0;
|
||||
let uploadCounter = 0;
|
||||
|
||||
for (const asset of assetsToUpload) {
|
||||
// Compute total size first
|
||||
await asset.prepare();
|
||||
totalSize += asset.fileSize;
|
||||
|
||||
if (options.albumName) {
|
||||
asset.albumName = options.albumName;
|
||||
}
|
||||
}
|
||||
|
||||
const existingAlbums = (await this.immichApi.albumApi.getAllAlbums()).data;
|
||||
|
||||
uploadProgress.start(totalSize, 0);
|
||||
uploadProgress.update({ value_formatted: 0, total_formatted: byteSize(totalSize) });
|
||||
|
||||
try {
|
||||
for (const asset of assetsToUpload) {
|
||||
uploadProgress.update({
|
||||
filename: asset.path,
|
||||
});
|
||||
|
||||
let skipUpload = false;
|
||||
|
||||
let skipAsset = false;
|
||||
let existingAssetId: string | undefined = undefined;
|
||||
|
||||
if (!options.skipHash) {
|
||||
const assetBulkUploadCheckDto = { assets: [{ id: asset.path, checksum: await asset.hash() }] };
|
||||
|
||||
const checkResponse = await this.immichApi.assetApi.checkBulkUpload({
|
||||
assetBulkUploadCheckDto,
|
||||
});
|
||||
|
||||
skipUpload = checkResponse.data.results[0].action === 'reject';
|
||||
|
||||
const isDuplicate = checkResponse.data.results[0].reason === 'duplicate';
|
||||
if (isDuplicate) {
|
||||
existingAssetId = checkResponse.data.results[0].assetId;
|
||||
}
|
||||
|
||||
skipAsset = skipUpload && !isDuplicate;
|
||||
}
|
||||
|
||||
if (!skipAsset) {
|
||||
if (!options.dryRun) {
|
||||
if (!skipUpload) {
|
||||
const formData = await asset.getUploadFormData();
|
||||
const res = await this.uploadAsset(formData);
|
||||
existingAssetId = res.data.id;
|
||||
uploadCounter++;
|
||||
totalSizeUploaded += asset.fileSize;
|
||||
}
|
||||
|
||||
if ((options.album || options.albumName) && asset.albumName !== undefined) {
|
||||
let album = existingAlbums.find((album) => album.albumName === asset.albumName);
|
||||
if (!album) {
|
||||
const res = await this.immichApi.albumApi.createAlbum({
|
||||
createAlbumDto: { albumName: asset.albumName },
|
||||
});
|
||||
album = res.data;
|
||||
existingAlbums.push(album);
|
||||
}
|
||||
|
||||
if (existingAssetId) {
|
||||
await this.immichApi.albumApi.addAssetsToAlbum({
|
||||
id: album.id,
|
||||
bulkIdsDto: { ids: [existingAssetId] },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sizeSoFar += asset.fileSize;
|
||||
|
||||
uploadProgress.update(sizeSoFar, { value_formatted: byteSize(sizeSoFar) });
|
||||
}
|
||||
} finally {
|
||||
uploadProgress.stop();
|
||||
}
|
||||
|
||||
let messageStart;
|
||||
if (options.dryRun) {
|
||||
messageStart = 'Would have';
|
||||
} else {
|
||||
messageStart = 'Successfully';
|
||||
}
|
||||
|
||||
if (uploadCounter === 0) {
|
||||
console.log('All assets were already uploaded, nothing to do.');
|
||||
} else {
|
||||
console.log(`${messageStart} uploaded ${uploadCounter} assets (${byteSize(totalSizeUploaded)})`);
|
||||
}
|
||||
if (options.delete) {
|
||||
if (options.dryRun) {
|
||||
console.log(`Would now have deleted assets, but skipped due to dry run`);
|
||||
} else {
|
||||
console.log('Deleting assets that have been uploaded...');
|
||||
const deletionProgress = new cliProgress.SingleBar(cliProgress.Presets.shades_classic);
|
||||
deletionProgress.start(files.length, 0);
|
||||
|
||||
for (const asset of assetsToUpload) {
|
||||
if (!options.dryRun) {
|
||||
await asset.delete();
|
||||
}
|
||||
deletionProgress.increment();
|
||||
}
|
||||
deletionProgress.stop();
|
||||
console.log('Deletion complete');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async uploadAsset(data: FormData): Promise<AxiosResponse> {
|
||||
const url = this.immichApi.instanceUrl + '/asset/upload';
|
||||
|
||||
const config: AxiosRequestConfig = {
|
||||
method: 'post',
|
||||
maxRedirects: 0,
|
||||
url,
|
||||
headers: {
|
||||
'x-api-key': this.immichApi.apiKey,
|
||||
...data.getHeaders(),
|
||||
},
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
data,
|
||||
};
|
||||
|
||||
const res = await axios(config);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user