mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-21 06:46:42 +02:00
feat(storage): improve shared-storage robustness, scalability, and DX
- Fix exists() to only catch 404/NotFound, rethrow real errors - Add downloadStream() for memory-efficient large file downloads - Add uploadMultipart() using @aws-sdk/lib-storage for large files - Add automatic pagination to list() via continuation tokens - Add CDN URL support (cdnUrl in BucketConfig, getCdnUrl() method) - Reduce factory boilerplate with generic createStorage() function - Add MinIO lifecycle rules for tmp/ prefixes (chat 90d, calendar 30d, picture 7d) - Add vitest setup with 56 tests covering client, factory, and utils Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ab42c265e1
commit
41fbd2f035
12 changed files with 1450 additions and 783 deletions
|
|
@ -7,6 +7,7 @@ import {
|
|||
HeadObjectCommand,
|
||||
type PutObjectCommandInput,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import type {
|
||||
StorageConfig,
|
||||
|
|
@ -85,7 +86,41 @@ export class StorageClient {
|
|||
}
|
||||
|
||||
/**
|
||||
* Download a file from the bucket
|
||||
* Upload a large file using multipart upload.
|
||||
* Automatically splits the file into parts and uploads them in parallel.
|
||||
* Use this for files >100MB or when uploading over unstable connections.
|
||||
*/
|
||||
async uploadMultipart(
|
||||
key: string,
|
||||
body: Buffer | Uint8Array | ReadableStream,
|
||||
options: UploadOptions = {}
|
||||
): Promise<UploadResult> {
|
||||
const upload = new Upload({
|
||||
client: this.client,
|
||||
params: {
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
Body: body,
|
||||
ContentType: options.contentType,
|
||||
CacheControl: options.cacheControl,
|
||||
Metadata: options.metadata,
|
||||
...(options.public ? { ACL: 'public-read' as const } : {}),
|
||||
},
|
||||
queueSize: 4,
|
||||
partSize: 10 * 1024 * 1024, // 10MB parts
|
||||
});
|
||||
|
||||
const result = await upload.done();
|
||||
|
||||
return {
|
||||
key,
|
||||
url: this.getPublicUrl(key),
|
||||
etag: result.ETag,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file from the bucket (loads entire file into memory)
|
||||
*/
|
||||
async download(key: string): Promise<Buffer> {
|
||||
const command = new GetObjectCommand({
|
||||
|
|
@ -108,6 +143,24 @@ export class StorageClient {
|
|||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file as a readable stream (memory-efficient for large files)
|
||||
*/
|
||||
async downloadStream(key: string): Promise<ReadableStream> {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
});
|
||||
|
||||
const response = await this.client.send(command);
|
||||
|
||||
if (!response.Body) {
|
||||
throw new Error(`File not found: ${key}`);
|
||||
}
|
||||
|
||||
return response.Body.transformToWebStream();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from the bucket
|
||||
*/
|
||||
|
|
@ -131,29 +184,46 @@ export class StorageClient {
|
|||
});
|
||||
await this.client.send(command);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof Error) {
|
||||
if (err.name === 'NotFound') return false;
|
||||
const metadata = (err as Error & { $metadata?: { httpStatusCode?: number } }).$metadata;
|
||||
if (metadata?.httpStatusCode === 404) return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in the bucket with optional prefix
|
||||
* List files in the bucket with optional prefix.
|
||||
* Automatically paginates through all results if the bucket contains more than maxKeys items.
|
||||
*/
|
||||
async list(prefix?: string, maxKeys = 1000): Promise<FileInfo[]> {
|
||||
const command = new ListObjectsV2Command({
|
||||
Bucket: this.bucket.name,
|
||||
Prefix: prefix,
|
||||
MaxKeys: maxKeys,
|
||||
});
|
||||
const allFiles: FileInfo[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
const response = await this.client.send(command);
|
||||
do {
|
||||
const command = new ListObjectsV2Command({
|
||||
Bucket: this.bucket.name,
|
||||
Prefix: prefix,
|
||||
MaxKeys: maxKeys,
|
||||
ContinuationToken: continuationToken,
|
||||
});
|
||||
|
||||
return (response.Contents ?? []).map((item) => ({
|
||||
key: item.Key!,
|
||||
size: item.Size ?? 0,
|
||||
lastModified: item.LastModified ?? new Date(),
|
||||
etag: item.ETag,
|
||||
}));
|
||||
const response = await this.client.send(command);
|
||||
|
||||
const files = (response.Contents ?? []).map((item) => ({
|
||||
key: item.Key ?? '',
|
||||
size: item.Size ?? 0,
|
||||
lastModified: item.LastModified ?? new Date(),
|
||||
etag: item.ETag,
|
||||
}));
|
||||
|
||||
allFiles.push(...files);
|
||||
continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -196,6 +266,16 @@ export class StorageClient {
|
|||
return `${this.bucket.publicUrl}/${key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the CDN URL for a file. Falls back to publicUrl if no CDN is configured.
|
||||
*/
|
||||
getCdnUrl(key: string): string | undefined {
|
||||
if (this.bucket.cdnUrl) {
|
||||
return `${this.bucket.cdnUrl}/${key}`;
|
||||
}
|
||||
return this.getPublicUrl(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the underlying S3 client for advanced operations
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue