mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 17:41:09 +02:00
feat(storage): cleanup buckets, add file-size validation and bulk delete
- Remove archived LIGHTWRITE and NUTRIPHI from BUCKETS - Add missing PLANTA and PROJECTDOC buckets (were in Docker init but not in code) - Add maxSizeBytes option to upload() and uploadMultipart() for size enforcement - Add deleteMany() with automatic batching (1000 keys per S3 request) - Add factories for createPlantaStorage() and createProjectDocStorage() - Update tests (62 passing) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
720602343e
commit
01c4d3a9d1
6 changed files with 106 additions and 8 deletions
|
|
@ -22,6 +22,9 @@ vi.mock('@aws-sdk/client-s3', () => ({
|
|||
DeleteObjectCommand: vi.fn(function (this: any, input: any) {
|
||||
Object.assign(this, input);
|
||||
}),
|
||||
DeleteObjectsCommand: vi.fn(function (this: any, input: any) {
|
||||
Object.assign(this, input);
|
||||
}),
|
||||
ListObjectsV2Command: vi.fn(function (this: any, input: any) {
|
||||
Object.assign(this, input);
|
||||
}),
|
||||
|
|
@ -79,6 +82,19 @@ describe('StorageClient', () => {
|
|||
expect(result.etag).toBe('"abc123"');
|
||||
});
|
||||
|
||||
it('throws when file exceeds maxSizeBytes', async () => {
|
||||
const bigBuffer = Buffer.alloc(1024);
|
||||
await expect(storage.upload('file.png', bigBuffer, { maxSizeBytes: 512 })).rejects.toThrow(
|
||||
'File size 1024 bytes exceeds maximum allowed 512 bytes'
|
||||
);
|
||||
});
|
||||
|
||||
it('allows file within maxSizeBytes', async () => {
|
||||
mockSend.mockResolvedValue({ ETag: '"ok"' });
|
||||
const result = await storage.upload('file.png', Buffer.alloc(100), { maxSizeBytes: 512 });
|
||||
expect(result.key).toBe('file.png');
|
||||
});
|
||||
|
||||
it('sets ACL to public-read when public option is true', async () => {
|
||||
mockSend.mockResolvedValue({ ETag: '"abc"' });
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
|
|
@ -92,6 +108,13 @@ describe('StorageClient', () => {
|
|||
});
|
||||
|
||||
describe('uploadMultipart', () => {
|
||||
it('throws when file exceeds maxSizeBytes', async () => {
|
||||
const bigBuffer = Buffer.alloc(2048);
|
||||
await expect(
|
||||
storage.uploadMultipart('big.zip', bigBuffer, { maxSizeBytes: 1024 })
|
||||
).rejects.toThrow('File size 2048 bytes exceeds maximum allowed 1024 bytes');
|
||||
});
|
||||
|
||||
it('uses Upload from lib-storage', async () => {
|
||||
const { Upload } = await import('@aws-sdk/lib-storage');
|
||||
|
||||
|
|
@ -163,6 +186,39 @@ describe('StorageClient', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('deleteMany', () => {
|
||||
it('deletes multiple files in one request', async () => {
|
||||
mockSend.mockResolvedValue({});
|
||||
const { DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
|
||||
|
||||
await storage.deleteMany(['a.png', 'b.png', 'c.png']);
|
||||
|
||||
expect(DeleteObjectsCommand).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
Bucket: 'test-bucket',
|
||||
Delete: {
|
||||
Objects: [{ Key: 'a.png' }, { Key: 'b.png' }, { Key: 'c.png' }],
|
||||
Quiet: true,
|
||||
},
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('does nothing for empty array', async () => {
|
||||
await storage.deleteMany([]);
|
||||
expect(mockSend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('batches requests for >1000 keys', async () => {
|
||||
mockSend.mockResolvedValue({});
|
||||
const keys = Array.from({ length: 1500 }, (_, i) => `file-${i}.png`);
|
||||
|
||||
await storage.deleteMany(keys);
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('exists', () => {
|
||||
it('returns true when file exists', async () => {
|
||||
mockSend.mockResolvedValue({});
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import {
|
|||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
DeleteObjectsCommand,
|
||||
ListObjectsV2Command,
|
||||
HeadObjectCommand,
|
||||
type PutObjectCommandInput,
|
||||
|
|
@ -62,6 +63,15 @@ export class StorageClient {
|
|||
body: Buffer | Uint8Array | string | ReadableStream,
|
||||
options: UploadOptions = {}
|
||||
): Promise<UploadResult> {
|
||||
if (options.maxSizeBytes && typeof body !== 'string' && !(body instanceof ReadableStream)) {
|
||||
const size = body.byteLength;
|
||||
if (size > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const input: PutObjectCommandInput = {
|
||||
Bucket: this.bucket.name,
|
||||
Key: key,
|
||||
|
|
@ -95,6 +105,15 @@ export class StorageClient {
|
|||
body: Buffer | Uint8Array | ReadableStream,
|
||||
options: UploadOptions = {}
|
||||
): Promise<UploadResult> {
|
||||
if (options.maxSizeBytes && !(body instanceof ReadableStream)) {
|
||||
const size = body.byteLength;
|
||||
if (size > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const upload = new Upload({
|
||||
client: this.client,
|
||||
params: {
|
||||
|
|
@ -173,6 +192,27 @@ export class StorageClient {
|
|||
await this.client.send(command);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete multiple files from the bucket in a single request.
|
||||
* S3 supports up to 1000 keys per request; this method batches automatically.
|
||||
*/
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
if (keys.length === 0) return;
|
||||
|
||||
const BATCH_SIZE = 1000;
|
||||
for (let i = 0; i < keys.length; i += BATCH_SIZE) {
|
||||
const batch = keys.slice(i, i + BATCH_SIZE);
|
||||
const command = new DeleteObjectsCommand({
|
||||
Bucket: this.bucket.name,
|
||||
Delete: {
|
||||
Objects: batch.map((key) => ({ Key: key })),
|
||||
Quiet: true,
|
||||
},
|
||||
});
|
||||
await this.client.send(command);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ const MINIO_DEFAULTS: StorageConfig = {
|
|||
const PUBLIC_URL_ENV: Partial<Record<keyof typeof BUCKETS, string>> = {
|
||||
MANACORE: 'MANACORE_STORAGE_PUBLIC_URL',
|
||||
PICTURE: 'PICTURE_STORAGE_PUBLIC_URL',
|
||||
NUTRIPHI: 'NUTRIPHI_S3_PUBLIC_URL',
|
||||
STORAGE: 'STORAGE_S3_PUBLIC_URL',
|
||||
INVENTORY: 'INVENTORY_S3_PUBLIC_URL',
|
||||
PLANTA: 'PLANTA_STORAGE_PUBLIC_URL',
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -104,12 +104,12 @@ export const createManaCoreStorage = (publicUrl?: string) => createStorage('MANA
|
|||
export const createPictureStorage = (publicUrl?: string) => createStorage('PICTURE', publicUrl);
|
||||
export const createChatStorage = () => createStorage('CHAT');
|
||||
export const createManaDeckStorage = () => createStorage('MANADECK');
|
||||
export const createNutriPhiStorage = (publicUrl?: string) => createStorage('NUTRIPHI', publicUrl);
|
||||
export const createPresiStorage = () => createStorage('PRESI');
|
||||
export const createCalendarStorage = () => createStorage('CALENDAR');
|
||||
export const createContactsStorage = () => createStorage('CONTACTS');
|
||||
export const createStorageStorage = (publicUrl?: string) => createStorage('STORAGE', publicUrl);
|
||||
export const createMailStorage = () => createStorage('MAIL');
|
||||
export const createInventoryStorage = (publicUrl?: string) => createStorage('INVENTORY', publicUrl);
|
||||
export const createLightWriteStorage = () => createStorage('LIGHTWRITE');
|
||||
export const createMukkeStorage = () => createStorage('MUKKE');
|
||||
export const createPlantaStorage = (publicUrl?: string) => createStorage('PLANTA', publicUrl);
|
||||
export const createProjectDocStorage = () => createStorage('PROJECTDOC');
|
||||
|
|
|
|||
|
|
@ -10,15 +10,15 @@ export {
|
|||
createPictureStorage,
|
||||
createChatStorage,
|
||||
createManaDeckStorage,
|
||||
createNutriPhiStorage,
|
||||
createPresiStorage,
|
||||
createCalendarStorage,
|
||||
createContactsStorage,
|
||||
createStorageStorage,
|
||||
createMailStorage,
|
||||
createInventoryStorage,
|
||||
createLightWriteStorage,
|
||||
createMukkeStorage,
|
||||
createPlantaStorage,
|
||||
createProjectDocStorage,
|
||||
} from './factory';
|
||||
|
||||
// Utilities
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ export interface UploadOptions {
|
|||
metadata?: Record<string, string>;
|
||||
/** Make the object publicly readable */
|
||||
public?: boolean;
|
||||
/** Maximum allowed file size in bytes. Throws if body exceeds this limit. */
|
||||
maxSizeBytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -92,15 +94,15 @@ export const BUCKETS = {
|
|||
PICTURE: 'picture-storage',
|
||||
CHAT: 'chat-storage',
|
||||
MANADECK: 'manadeck-storage',
|
||||
NUTRIPHI: 'nutriphi-storage',
|
||||
PRESI: 'presi-storage',
|
||||
CALENDAR: 'calendar-storage',
|
||||
CONTACTS: 'contacts-storage',
|
||||
STORAGE: 'storage-storage',
|
||||
MAIL: 'mail-storage',
|
||||
INVENTORY: 'inventory-storage',
|
||||
LIGHTWRITE: 'lightwrite-storage',
|
||||
MUKKE: 'mukke-storage',
|
||||
PLANTA: 'planta-storage',
|
||||
PROJECTDOC: 'projectdoc-storage',
|
||||
} as const;
|
||||
|
||||
export type BucketName = (typeof BUCKETS)[keyof typeof BUCKETS];
|
||||
|
|
|
|||
|
|
@ -11,6 +11,6 @@
|
|||
"outDir": "./dist",
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src/**/*", "vitest.config.ts"],
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue