feat(storage): cleanup buckets, add file-size validation and bulk delete

- Remove archived LIGHTWRITE and NUTRIPHI from BUCKETS
- Add missing PLANTA and PROJECTDOC buckets (were in Docker init but not in code)
- Add maxSizeBytes option to upload() and uploadMultipart() for size enforcement
- Add deleteMany() with automatic batching (1000 keys per S3 request)
- Add factories for createPlantaStorage() and createProjectDocStorage()
- Update tests (62 passing)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS 2026-03-20 19:27:42 +01:00
parent 720602343e
commit 01c4d3a9d1
6 changed files with 106 additions and 8 deletions

View file

@ -22,6 +22,9 @@ vi.mock('@aws-sdk/client-s3', () => ({
DeleteObjectCommand: vi.fn(function (this: any, input: any) {
Object.assign(this, input);
}),
DeleteObjectsCommand: vi.fn(function (this: any, input: any) {
Object.assign(this, input);
}),
ListObjectsV2Command: vi.fn(function (this: any, input: any) {
Object.assign(this, input);
}),
@ -79,6 +82,19 @@ describe('StorageClient', () => {
expect(result.etag).toBe('"abc123"');
});
it('throws when file exceeds maxSizeBytes', async () => {
const bigBuffer = Buffer.alloc(1024);
await expect(storage.upload('file.png', bigBuffer, { maxSizeBytes: 512 })).rejects.toThrow(
'File size 1024 bytes exceeds maximum allowed 512 bytes'
);
});
it('allows file within maxSizeBytes', async () => {
mockSend.mockResolvedValue({ ETag: '"ok"' });
const result = await storage.upload('file.png', Buffer.alloc(100), { maxSizeBytes: 512 });
expect(result.key).toBe('file.png');
});
it('sets ACL to public-read when public option is true', async () => {
mockSend.mockResolvedValue({ ETag: '"abc"' });
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
@ -92,6 +108,13 @@ describe('StorageClient', () => {
});
describe('uploadMultipart', () => {
it('throws when file exceeds maxSizeBytes', async () => {
const bigBuffer = Buffer.alloc(2048);
await expect(
storage.uploadMultipart('big.zip', bigBuffer, { maxSizeBytes: 1024 })
).rejects.toThrow('File size 2048 bytes exceeds maximum allowed 1024 bytes');
});
it('uses Upload from lib-storage', async () => {
const { Upload } = await import('@aws-sdk/lib-storage');
@ -163,6 +186,39 @@ describe('StorageClient', () => {
});
});
describe('deleteMany', () => {
it('deletes multiple files in one request', async () => {
mockSend.mockResolvedValue({});
const { DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
await storage.deleteMany(['a.png', 'b.png', 'c.png']);
expect(DeleteObjectsCommand).toHaveBeenCalledWith(
expect.objectContaining({
Bucket: 'test-bucket',
Delete: {
Objects: [{ Key: 'a.png' }, { Key: 'b.png' }, { Key: 'c.png' }],
Quiet: true,
},
})
);
});
it('does nothing for empty array', async () => {
await storage.deleteMany([]);
expect(mockSend).not.toHaveBeenCalled();
});
it('batches requests for >1000 keys', async () => {
mockSend.mockResolvedValue({});
const keys = Array.from({ length: 1500 }, (_, i) => `file-${i}.png`);
await storage.deleteMany(keys);
expect(mockSend).toHaveBeenCalledTimes(2);
});
});
describe('exists', () => {
it('returns true when file exists', async () => {
mockSend.mockResolvedValue({});

View file

@ -3,6 +3,7 @@ import {
PutObjectCommand,
GetObjectCommand,
DeleteObjectCommand,
DeleteObjectsCommand,
ListObjectsV2Command,
HeadObjectCommand,
type PutObjectCommandInput,
@ -62,6 +63,15 @@ export class StorageClient {
body: Buffer | Uint8Array | string | ReadableStream,
options: UploadOptions = {}
): Promise<UploadResult> {
if (options.maxSizeBytes && typeof body !== 'string' && !(body instanceof ReadableStream)) {
const size = body.byteLength;
if (size > options.maxSizeBytes) {
throw new Error(
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
);
}
}
const input: PutObjectCommandInput = {
Bucket: this.bucket.name,
Key: key,
@ -95,6 +105,15 @@ export class StorageClient {
body: Buffer | Uint8Array | ReadableStream,
options: UploadOptions = {}
): Promise<UploadResult> {
if (options.maxSizeBytes && !(body instanceof ReadableStream)) {
const size = body.byteLength;
if (size > options.maxSizeBytes) {
throw new Error(
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
);
}
}
const upload = new Upload({
client: this.client,
params: {
@ -173,6 +192,27 @@ export class StorageClient {
await this.client.send(command);
}
/**
* Delete multiple files from the bucket in a single request.
* S3 supports up to 1000 keys per request; this method batches automatically.
*/
async deleteMany(keys: string[]): Promise<void> {
if (keys.length === 0) return;
const BATCH_SIZE = 1000;
for (let i = 0; i < keys.length; i += BATCH_SIZE) {
const batch = keys.slice(i, i + BATCH_SIZE);
const command = new DeleteObjectsCommand({
Bucket: this.bucket.name,
Delete: {
Objects: batch.map((key) => ({ Key: key })),
Quiet: true,
},
});
await this.client.send(command);
}
}
/**
* Check if a file exists
*/

View file

@ -30,9 +30,9 @@ const MINIO_DEFAULTS: StorageConfig = {
const PUBLIC_URL_ENV: Partial<Record<keyof typeof BUCKETS, string>> = {
MANACORE: 'MANACORE_STORAGE_PUBLIC_URL',
PICTURE: 'PICTURE_STORAGE_PUBLIC_URL',
NUTRIPHI: 'NUTRIPHI_S3_PUBLIC_URL',
STORAGE: 'STORAGE_S3_PUBLIC_URL',
INVENTORY: 'INVENTORY_S3_PUBLIC_URL',
PLANTA: 'PLANTA_STORAGE_PUBLIC_URL',
};
/**
@ -104,12 +104,12 @@ export const createManaCoreStorage = (publicUrl?: string) => createStorage('MANA
export const createPictureStorage = (publicUrl?: string) => createStorage('PICTURE', publicUrl);
export const createChatStorage = () => createStorage('CHAT');
export const createManaDeckStorage = () => createStorage('MANADECK');
export const createNutriPhiStorage = (publicUrl?: string) => createStorage('NUTRIPHI', publicUrl);
export const createPresiStorage = () => createStorage('PRESI');
export const createCalendarStorage = () => createStorage('CALENDAR');
export const createContactsStorage = () => createStorage('CONTACTS');
export const createStorageStorage = (publicUrl?: string) => createStorage('STORAGE', publicUrl);
export const createMailStorage = () => createStorage('MAIL');
export const createInventoryStorage = (publicUrl?: string) => createStorage('INVENTORY', publicUrl);
export const createLightWriteStorage = () => createStorage('LIGHTWRITE');
export const createMukkeStorage = () => createStorage('MUKKE');
export const createPlantaStorage = (publicUrl?: string) => createStorage('PLANTA', publicUrl);
export const createProjectDocStorage = () => createStorage('PROJECTDOC');

View file

@ -10,15 +10,15 @@ export {
createPictureStorage,
createChatStorage,
createManaDeckStorage,
createNutriPhiStorage,
createPresiStorage,
createCalendarStorage,
createContactsStorage,
createStorageStorage,
createMailStorage,
createInventoryStorage,
createLightWriteStorage,
createMukkeStorage,
createPlantaStorage,
createProjectDocStorage,
} from './factory';
// Utilities

View file

@ -48,6 +48,8 @@ export interface UploadOptions {
metadata?: Record<string, string>;
/** Make the object publicly readable */
public?: boolean;
/** Maximum allowed file size in bytes. Throws if body exceeds this limit. */
maxSizeBytes?: number;
}
/**
@ -92,15 +94,15 @@ export const BUCKETS = {
PICTURE: 'picture-storage',
CHAT: 'chat-storage',
MANADECK: 'manadeck-storage',
NUTRIPHI: 'nutriphi-storage',
PRESI: 'presi-storage',
CALENDAR: 'calendar-storage',
CONTACTS: 'contacts-storage',
STORAGE: 'storage-storage',
MAIL: 'mail-storage',
INVENTORY: 'inventory-storage',
LIGHTWRITE: 'lightwrite-storage',
MUKKE: 'mukke-storage',
PLANTA: 'planta-storage',
PROJECTDOC: 'projectdoc-storage',
} as const;
export type BucketName = (typeof BUCKETS)[keyof typeof BUCKETS];

View file

@ -11,6 +11,6 @@
"outDir": "./dist",
"rootDir": "./src"
},
"include": ["src/**/*", "vitest.config.ts"],
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}