mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-23 07:46:42 +02:00
feat(storage): add upload hooks, metrics integration, and presigned multipart
Upload hooks: - StorageHooks class with fire-and-forget event emitter pattern - Events: upload, upload:error, delete, delete:error, download - All StorageClient operations now emit appropriate events - Unsubscribe functions for cleanup Metrics: - StorageMetricsCollector interface (decoupled from prom-client) - InMemoryMetrics for testing and local dev - attachMetrics() wires hooks to any collector automatically - Backends can create a Prometheus collector via MetricsService Presigned multipart upload (browser direct-upload): - createMultipartUpload() initiates and returns uploadId - getMultipartUploadUrls() generates presigned PUT URLs per part - completeMultipartUpload() finalizes with part ETags - abortMultipartUpload() for cleanup on abandoned uploads 90 tests passing across 5 test files. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
822e75368a
commit
b0e5a9c5ff
8 changed files with 708 additions and 28 deletions
|
|
@ -6,15 +6,22 @@ import {
|
|||
DeleteObjectsCommand,
|
||||
ListObjectsV2Command,
|
||||
HeadObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
type PutObjectCommandInput,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { StorageHooks } from './hooks';
|
||||
import type {
|
||||
StorageConfig,
|
||||
BucketConfig,
|
||||
UploadOptions,
|
||||
PresignedUrlOptions,
|
||||
MultipartUploadInit,
|
||||
MultipartUploadPart,
|
||||
UploadResult,
|
||||
FileInfo,
|
||||
} from './types';
|
||||
|
|
@ -26,8 +33,10 @@ export class StorageClient {
|
|||
private client: S3Client;
|
||||
private presignClient: S3Client;
|
||||
private bucket: BucketConfig;
|
||||
readonly hooks: StorageHooks;
|
||||
|
||||
constructor(config: StorageConfig, bucket: BucketConfig) {
|
||||
this.hooks = new StorageHooks();
|
||||
// Main client for internal operations (upload, download, delete, etc.)
|
||||
this.client = new S3Client({
|
||||
endpoint: config.endpoint,
|
||||
|
|
@ -86,13 +95,27 @@ export class StorageClient {
|
|||
}
|
||||
|
||||
const command = new PutObjectCommand(input);
|
||||
const result = await this.client.send(command);
|
||||
|
||||
return {
|
||||
key,
|
||||
url: this.getPublicUrl(key),
|
||||
etag: result.ETag,
|
||||
};
|
||||
try {
|
||||
const result = await this.client.send(command);
|
||||
const uploadResult: UploadResult = { key, url: this.getPublicUrl(key), etag: result.ETag };
|
||||
const sizeBytes =
|
||||
typeof body !== 'string' && !(body instanceof ReadableStream) ? body.byteLength : undefined;
|
||||
this.hooks.emit('upload', {
|
||||
bucket: this.bucket.name,
|
||||
key,
|
||||
sizeBytes,
|
||||
contentType: options.contentType,
|
||||
result: uploadResult,
|
||||
});
|
||||
return uploadResult;
|
||||
} catch (err) {
|
||||
this.hooks.emit('upload:error', {
|
||||
bucket: this.bucket.name,
|
||||
key,
|
||||
error: err instanceof Error ? err : new Error(String(err)),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -114,28 +137,41 @@ export class StorageClient {
|
|||
}
|
||||
}
|
||||
|
||||
const upload = new Upload({
|
||||
client: this.client,
|
||||
params: {
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: options.contentType,
|
||||
CacheControl: options.cacheControl,
|
||||
Metadata: options.metadata,
|
||||
...(options.public ? { ACL: 'public-read' as const } : {}),
|
||||
},
|
||||
queueSize: 4,
|
||||
partSize: 10 * 1024 * 1024, // 10MB parts
|
||||
});
|
||||
try {
|
||||
const upload = new Upload({
|
||||
client: this.client,
|
||||
params: {
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: options.contentType,
|
||||
CacheControl: options.cacheControl,
|
||||
Metadata: options.metadata,
|
||||
...(options.public ? { ACL: 'public-read' as const } : {}),
|
||||
},
|
||||
queueSize: 4,
|
||||
partSize: 10 * 1024 * 1024, // 10MB parts
|
||||
});
|
||||
|
||||
const result = await upload.done();
|
||||
|
||||
return {
|
||||
key,
|
||||
url: this.getPublicUrl(key),
|
||||
etag: result.ETag,
|
||||
};
|
||||
const result = await upload.done();
|
||||
const uploadResult: UploadResult = { key, url: this.getPublicUrl(key), etag: result.ETag };
|
||||
const sizeBytes = !(body instanceof ReadableStream) ? body.byteLength : undefined;
|
||||
this.hooks.emit('upload', {
|
||||
bucket: this.bucket.name,
|
||||
key,
|
||||
sizeBytes,
|
||||
contentType: options.contentType,
|
||||
result: uploadResult,
|
||||
});
|
||||
return uploadResult;
|
||||
} catch (err) {
|
||||
this.hooks.emit('upload:error', {
|
||||
bucket: this.bucket.name,
|
||||
key,
|
||||
error: err instanceof Error ? err : new Error(String(err)),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -153,6 +189,8 @@ export class StorageClient {
|
|||
throw new Error(`File not found: ${key}`);
|
||||
}
|
||||
|
||||
this.hooks.emit('download', { bucket: this.bucket.name, key });
|
||||
|
||||
// Convert stream to buffer
|
||||
const chunks: Uint8Array[] = [];
|
||||
const stream = response.Body as AsyncIterable<Uint8Array>;
|
||||
|
|
@ -190,6 +228,7 @@ export class StorageClient {
|
|||
});
|
||||
|
||||
await this.client.send(command);
|
||||
this.hooks.emit('delete', { bucket: this.bucket.name, keys: [key] });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -211,6 +250,7 @@ export class StorageClient {
|
|||
});
|
||||
await this.client.send(command);
|
||||
}
|
||||
this.hooks.emit('delete', { bucket: this.bucket.name, keys });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -329,4 +369,108 @@ export class StorageClient {
|
|||
getBucketName(): string {
|
||||
return this.bucket.name;
|
||||
}
|
||||
|
||||
// ── Presigned Multipart Upload (browser direct-upload) ──────────────
|
||||
|
||||
/**
|
||||
* Initiate a multipart upload and return the upload ID.
|
||||
* The browser uses this ID to upload parts directly via presigned URLs.
|
||||
*/
|
||||
async createMultipartUpload(key: string, contentType?: string): Promise<MultipartUploadInit> {
|
||||
const command = new CreateMultipartUploadCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
});
|
||||
|
||||
const response = await this.client.send(command);
|
||||
if (!response.UploadId) {
|
||||
throw new Error('Failed to create multipart upload — no UploadId returned');
|
||||
}
|
||||
|
||||
return { uploadId: response.UploadId, key };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate presigned URLs for each part of a multipart upload.
|
||||
* The browser PUTs each chunk to the corresponding URL.
|
||||
*
|
||||
* @param key - Object key
|
||||
* @param uploadId - From createMultipartUpload()
|
||||
* @param parts - Number of parts to generate URLs for
|
||||
* @param expiresIn - URL expiration in seconds (default: 3600)
|
||||
*/
|
||||
async getMultipartUploadUrls(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: number,
|
||||
expiresIn = 3600
|
||||
): Promise<string[]> {
|
||||
const urls: string[] = [];
|
||||
|
||||
for (let partNumber = 1; partNumber <= parts; partNumber++) {
|
||||
const command = new UploadPartCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
});
|
||||
|
||||
const url = await getSignedUrl(this.presignClient, command, { expiresIn });
|
||||
urls.push(url);
|
||||
}
|
||||
|
||||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete a multipart upload after all parts have been uploaded.
|
||||
* The browser sends the ETag of each part (from the PUT response headers).
|
||||
*/
|
||||
async completeMultipartUpload(
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: MultipartUploadPart[]
|
||||
): Promise<UploadResult> {
|
||||
const command = new CompleteMultipartUploadCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: {
|
||||
Parts: parts.map((p) => ({
|
||||
PartNumber: p.partNumber,
|
||||
ETag: p.etag,
|
||||
})),
|
||||
},
|
||||
});
|
||||
|
||||
const result = await this.client.send(command);
|
||||
|
||||
const uploadResult: UploadResult = {
|
||||
key,
|
||||
url: this.getPublicUrl(key),
|
||||
etag: result.ETag,
|
||||
};
|
||||
|
||||
this.hooks.emit('upload', {
|
||||
bucket: this.bucket.name,
|
||||
key,
|
||||
result: uploadResult,
|
||||
});
|
||||
|
||||
return uploadResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort a multipart upload (cleanup if the browser abandons the upload).
|
||||
*/
|
||||
async abortMultipartUpload(key: string, uploadId: string): Promise<void> {
|
||||
const command = new AbortMultipartUploadCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
});
|
||||
|
||||
await this.client.send(command);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue