mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 19:01:08 +02:00
feat(storage): cleanup buckets, add file-size validation and bulk delete
- Remove archived LIGHTWRITE and NUTRIPHI from BUCKETS - Add missing PLANTA and PROJECTDOC buckets (were in Docker init but not in code) - Add maxSizeBytes option to upload() and uploadMultipart() for size enforcement - Add deleteMany() with automatic batching (1000 keys per S3 request) - Add factories for createPlantaStorage() and createProjectDocStorage() - Update tests (62 passing) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
720602343e
commit
01c4d3a9d1
6 changed files with 106 additions and 8 deletions
|
|
@ -22,6 +22,9 @@ vi.mock('@aws-sdk/client-s3', () => ({
|
||||||
DeleteObjectCommand: vi.fn(function (this: any, input: any) {
|
DeleteObjectCommand: vi.fn(function (this: any, input: any) {
|
||||||
Object.assign(this, input);
|
Object.assign(this, input);
|
||||||
}),
|
}),
|
||||||
|
DeleteObjectsCommand: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
ListObjectsV2Command: vi.fn(function (this: any, input: any) {
|
ListObjectsV2Command: vi.fn(function (this: any, input: any) {
|
||||||
Object.assign(this, input);
|
Object.assign(this, input);
|
||||||
}),
|
}),
|
||||||
|
|
@ -79,6 +82,19 @@ describe('StorageClient', () => {
|
||||||
expect(result.etag).toBe('"abc123"');
|
expect(result.etag).toBe('"abc123"');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('throws when file exceeds maxSizeBytes', async () => {
|
||||||
|
const bigBuffer = Buffer.alloc(1024);
|
||||||
|
await expect(storage.upload('file.png', bigBuffer, { maxSizeBytes: 512 })).rejects.toThrow(
|
||||||
|
'File size 1024 bytes exceeds maximum allowed 512 bytes'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('allows file within maxSizeBytes', async () => {
|
||||||
|
mockSend.mockResolvedValue({ ETag: '"ok"' });
|
||||||
|
const result = await storage.upload('file.png', Buffer.alloc(100), { maxSizeBytes: 512 });
|
||||||
|
expect(result.key).toBe('file.png');
|
||||||
|
});
|
||||||
|
|
||||||
it('sets ACL to public-read when public option is true', async () => {
|
it('sets ACL to public-read when public option is true', async () => {
|
||||||
mockSend.mockResolvedValue({ ETag: '"abc"' });
|
mockSend.mockResolvedValue({ ETag: '"abc"' });
|
||||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||||
|
|
@ -92,6 +108,13 @@ describe('StorageClient', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('uploadMultipart', () => {
|
describe('uploadMultipart', () => {
|
||||||
|
it('throws when file exceeds maxSizeBytes', async () => {
|
||||||
|
const bigBuffer = Buffer.alloc(2048);
|
||||||
|
await expect(
|
||||||
|
storage.uploadMultipart('big.zip', bigBuffer, { maxSizeBytes: 1024 })
|
||||||
|
).rejects.toThrow('File size 2048 bytes exceeds maximum allowed 1024 bytes');
|
||||||
|
});
|
||||||
|
|
||||||
it('uses Upload from lib-storage', async () => {
|
it('uses Upload from lib-storage', async () => {
|
||||||
const { Upload } = await import('@aws-sdk/lib-storage');
|
const { Upload } = await import('@aws-sdk/lib-storage');
|
||||||
|
|
||||||
|
|
@ -163,6 +186,39 @@ describe('StorageClient', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('deleteMany', () => {
|
||||||
|
it('deletes multiple files in one request', async () => {
|
||||||
|
mockSend.mockResolvedValue({});
|
||||||
|
const { DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
|
||||||
|
|
||||||
|
await storage.deleteMany(['a.png', 'b.png', 'c.png']);
|
||||||
|
|
||||||
|
expect(DeleteObjectsCommand).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
Bucket: 'test-bucket',
|
||||||
|
Delete: {
|
||||||
|
Objects: [{ Key: 'a.png' }, { Key: 'b.png' }, { Key: 'c.png' }],
|
||||||
|
Quiet: true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does nothing for empty array', async () => {
|
||||||
|
await storage.deleteMany([]);
|
||||||
|
expect(mockSend).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('batches requests for >1000 keys', async () => {
|
||||||
|
mockSend.mockResolvedValue({});
|
||||||
|
const keys = Array.from({ length: 1500 }, (_, i) => `file-${i}.png`);
|
||||||
|
|
||||||
|
await storage.deleteMany(keys);
|
||||||
|
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('exists', () => {
|
describe('exists', () => {
|
||||||
it('returns true when file exists', async () => {
|
it('returns true when file exists', async () => {
|
||||||
mockSend.mockResolvedValue({});
|
mockSend.mockResolvedValue({});
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import {
|
||||||
PutObjectCommand,
|
PutObjectCommand,
|
||||||
GetObjectCommand,
|
GetObjectCommand,
|
||||||
DeleteObjectCommand,
|
DeleteObjectCommand,
|
||||||
|
DeleteObjectsCommand,
|
||||||
ListObjectsV2Command,
|
ListObjectsV2Command,
|
||||||
HeadObjectCommand,
|
HeadObjectCommand,
|
||||||
type PutObjectCommandInput,
|
type PutObjectCommandInput,
|
||||||
|
|
@ -62,6 +63,15 @@ export class StorageClient {
|
||||||
body: Buffer | Uint8Array | string | ReadableStream,
|
body: Buffer | Uint8Array | string | ReadableStream,
|
||||||
options: UploadOptions = {}
|
options: UploadOptions = {}
|
||||||
): Promise<UploadResult> {
|
): Promise<UploadResult> {
|
||||||
|
if (options.maxSizeBytes && typeof body !== 'string' && !(body instanceof ReadableStream)) {
|
||||||
|
const size = body.byteLength;
|
||||||
|
if (size > options.maxSizeBytes) {
|
||||||
|
throw new Error(
|
||||||
|
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const input: PutObjectCommandInput = {
|
const input: PutObjectCommandInput = {
|
||||||
Bucket: this.bucket.name,
|
Bucket: this.bucket.name,
|
||||||
Key: key,
|
Key: key,
|
||||||
|
|
@ -95,6 +105,15 @@ export class StorageClient {
|
||||||
body: Buffer | Uint8Array | ReadableStream,
|
body: Buffer | Uint8Array | ReadableStream,
|
||||||
options: UploadOptions = {}
|
options: UploadOptions = {}
|
||||||
): Promise<UploadResult> {
|
): Promise<UploadResult> {
|
||||||
|
if (options.maxSizeBytes && !(body instanceof ReadableStream)) {
|
||||||
|
const size = body.byteLength;
|
||||||
|
if (size > options.maxSizeBytes) {
|
||||||
|
throw new Error(
|
||||||
|
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const upload = new Upload({
|
const upload = new Upload({
|
||||||
client: this.client,
|
client: this.client,
|
||||||
params: {
|
params: {
|
||||||
|
|
@ -173,6 +192,27 @@ export class StorageClient {
|
||||||
await this.client.send(command);
|
await this.client.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete multiple files from the bucket in a single request.
|
||||||
|
* S3 supports up to 1000 keys per request; this method batches automatically.
|
||||||
|
*/
|
||||||
|
async deleteMany(keys: string[]): Promise<void> {
|
||||||
|
if (keys.length === 0) return;
|
||||||
|
|
||||||
|
const BATCH_SIZE = 1000;
|
||||||
|
for (let i = 0; i < keys.length; i += BATCH_SIZE) {
|
||||||
|
const batch = keys.slice(i, i + BATCH_SIZE);
|
||||||
|
const command = new DeleteObjectsCommand({
|
||||||
|
Bucket: this.bucket.name,
|
||||||
|
Delete: {
|
||||||
|
Objects: batch.map((key) => ({ Key: key })),
|
||||||
|
Quiet: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await this.client.send(command);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a file exists
|
* Check if a file exists
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -30,9 +30,9 @@ const MINIO_DEFAULTS: StorageConfig = {
|
||||||
const PUBLIC_URL_ENV: Partial<Record<keyof typeof BUCKETS, string>> = {
|
const PUBLIC_URL_ENV: Partial<Record<keyof typeof BUCKETS, string>> = {
|
||||||
MANACORE: 'MANACORE_STORAGE_PUBLIC_URL',
|
MANACORE: 'MANACORE_STORAGE_PUBLIC_URL',
|
||||||
PICTURE: 'PICTURE_STORAGE_PUBLIC_URL',
|
PICTURE: 'PICTURE_STORAGE_PUBLIC_URL',
|
||||||
NUTRIPHI: 'NUTRIPHI_S3_PUBLIC_URL',
|
|
||||||
STORAGE: 'STORAGE_S3_PUBLIC_URL',
|
STORAGE: 'STORAGE_S3_PUBLIC_URL',
|
||||||
INVENTORY: 'INVENTORY_S3_PUBLIC_URL',
|
INVENTORY: 'INVENTORY_S3_PUBLIC_URL',
|
||||||
|
PLANTA: 'PLANTA_STORAGE_PUBLIC_URL',
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -104,12 +104,12 @@ export const createManaCoreStorage = (publicUrl?: string) => createStorage('MANA
|
||||||
export const createPictureStorage = (publicUrl?: string) => createStorage('PICTURE', publicUrl);
|
export const createPictureStorage = (publicUrl?: string) => createStorage('PICTURE', publicUrl);
|
||||||
export const createChatStorage = () => createStorage('CHAT');
|
export const createChatStorage = () => createStorage('CHAT');
|
||||||
export const createManaDeckStorage = () => createStorage('MANADECK');
|
export const createManaDeckStorage = () => createStorage('MANADECK');
|
||||||
export const createNutriPhiStorage = (publicUrl?: string) => createStorage('NUTRIPHI', publicUrl);
|
|
||||||
export const createPresiStorage = () => createStorage('PRESI');
|
export const createPresiStorage = () => createStorage('PRESI');
|
||||||
export const createCalendarStorage = () => createStorage('CALENDAR');
|
export const createCalendarStorage = () => createStorage('CALENDAR');
|
||||||
export const createContactsStorage = () => createStorage('CONTACTS');
|
export const createContactsStorage = () => createStorage('CONTACTS');
|
||||||
export const createStorageStorage = (publicUrl?: string) => createStorage('STORAGE', publicUrl);
|
export const createStorageStorage = (publicUrl?: string) => createStorage('STORAGE', publicUrl);
|
||||||
export const createMailStorage = () => createStorage('MAIL');
|
export const createMailStorage = () => createStorage('MAIL');
|
||||||
export const createInventoryStorage = (publicUrl?: string) => createStorage('INVENTORY', publicUrl);
|
export const createInventoryStorage = (publicUrl?: string) => createStorage('INVENTORY', publicUrl);
|
||||||
export const createLightWriteStorage = () => createStorage('LIGHTWRITE');
|
|
||||||
export const createMukkeStorage = () => createStorage('MUKKE');
|
export const createMukkeStorage = () => createStorage('MUKKE');
|
||||||
|
export const createPlantaStorage = (publicUrl?: string) => createStorage('PLANTA', publicUrl);
|
||||||
|
export const createProjectDocStorage = () => createStorage('PROJECTDOC');
|
||||||
|
|
|
||||||
|
|
@ -10,15 +10,15 @@ export {
|
||||||
createPictureStorage,
|
createPictureStorage,
|
||||||
createChatStorage,
|
createChatStorage,
|
||||||
createManaDeckStorage,
|
createManaDeckStorage,
|
||||||
createNutriPhiStorage,
|
|
||||||
createPresiStorage,
|
createPresiStorage,
|
||||||
createCalendarStorage,
|
createCalendarStorage,
|
||||||
createContactsStorage,
|
createContactsStorage,
|
||||||
createStorageStorage,
|
createStorageStorage,
|
||||||
createMailStorage,
|
createMailStorage,
|
||||||
createInventoryStorage,
|
createInventoryStorage,
|
||||||
createLightWriteStorage,
|
|
||||||
createMukkeStorage,
|
createMukkeStorage,
|
||||||
|
createPlantaStorage,
|
||||||
|
createProjectDocStorage,
|
||||||
} from './factory';
|
} from './factory';
|
||||||
|
|
||||||
// Utilities
|
// Utilities
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,8 @@ export interface UploadOptions {
|
||||||
metadata?: Record<string, string>;
|
metadata?: Record<string, string>;
|
||||||
/** Make the object publicly readable */
|
/** Make the object publicly readable */
|
||||||
public?: boolean;
|
public?: boolean;
|
||||||
|
/** Maximum allowed file size in bytes. Throws if body exceeds this limit. */
|
||||||
|
maxSizeBytes?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -92,15 +94,15 @@ export const BUCKETS = {
|
||||||
PICTURE: 'picture-storage',
|
PICTURE: 'picture-storage',
|
||||||
CHAT: 'chat-storage',
|
CHAT: 'chat-storage',
|
||||||
MANADECK: 'manadeck-storage',
|
MANADECK: 'manadeck-storage',
|
||||||
NUTRIPHI: 'nutriphi-storage',
|
|
||||||
PRESI: 'presi-storage',
|
PRESI: 'presi-storage',
|
||||||
CALENDAR: 'calendar-storage',
|
CALENDAR: 'calendar-storage',
|
||||||
CONTACTS: 'contacts-storage',
|
CONTACTS: 'contacts-storage',
|
||||||
STORAGE: 'storage-storage',
|
STORAGE: 'storage-storage',
|
||||||
MAIL: 'mail-storage',
|
MAIL: 'mail-storage',
|
||||||
INVENTORY: 'inventory-storage',
|
INVENTORY: 'inventory-storage',
|
||||||
LIGHTWRITE: 'lightwrite-storage',
|
|
||||||
MUKKE: 'mukke-storage',
|
MUKKE: 'mukke-storage',
|
||||||
|
PLANTA: 'planta-storage',
|
||||||
|
PROJECTDOC: 'projectdoc-storage',
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export type BucketName = (typeof BUCKETS)[keyof typeof BUCKETS];
|
export type BucketName = (typeof BUCKETS)[keyof typeof BUCKETS];
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,6 @@
|
||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"rootDir": "./src"
|
"rootDir": "./src"
|
||||||
},
|
},
|
||||||
"include": ["src/**/*", "vitest.config.ts"],
|
"include": ["src/**/*"],
|
||||||
"exclude": ["node_modules", "dist"]
|
"exclude": ["node_modules", "dist"]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue