mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 19:01:08 +02:00
feat(storage): improve shared-storage robustness, scalability, and DX
- Fix exists() to only catch 404/NotFound, rethrow real errors - Add downloadStream() for memory-efficient large file downloads - Add uploadMultipart() using @aws-sdk/lib-storage for large files - Add automatic pagination to list() via continuation tokens - Add CDN URL support (cdnUrl in BucketConfig, getCdnUrl() method) - Reduce factory boilerplate with generic createStorage() function - Add MinIO lifecycle rules for tmp/ prefixes (chat 90d, calendar 30d, picture 7d) - Add vitest setup with 56 tests covering client, factory, and utils Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ab42c265e1
commit
41fbd2f035
12 changed files with 1450 additions and 783 deletions
|
|
@ -93,7 +93,10 @@ services:
|
||||||
mc anonymous set download myminio/picture-storage;
|
mc anonymous set download myminio/picture-storage;
|
||||||
mc anonymous set download myminio/planta-storage;
|
mc anonymous set download myminio/planta-storage;
|
||||||
mc anonymous set download myminio/inventory-storage;
|
mc anonymous set download myminio/inventory-storage;
|
||||||
echo 'Buckets created successfully';
|
mc ilm rule add --expire-days 90 myminio/chat-storage --prefix 'tmp/' 2>/dev/null || true;
|
||||||
|
mc ilm rule add --expire-days 30 myminio/calendar-storage --prefix 'tmp/' 2>/dev/null || true;
|
||||||
|
mc ilm rule add --expire-days 7 myminio/picture-storage --prefix 'tmp/' 2>/dev/null || true;
|
||||||
|
echo 'Buckets and lifecycle rules created successfully';
|
||||||
exit 0;
|
exit 0;
|
||||||
"
|
"
|
||||||
networks:
|
networks:
|
||||||
|
|
|
||||||
|
|
@ -17,15 +17,19 @@
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"type-check": "tsc --noEmit",
|
"type-check": "tsc --noEmit",
|
||||||
|
"test": "vitest run",
|
||||||
|
"test:watch": "vitest",
|
||||||
"clean": "rm -rf dist",
|
"clean": "rm -rf dist",
|
||||||
"lint": "eslint ."
|
"lint": "eslint ."
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.700.0",
|
"@aws-sdk/client-s3": "^3.700.0",
|
||||||
|
"@aws-sdk/lib-storage": "^3.700.0",
|
||||||
"@aws-sdk/s3-request-presigner": "^3.700.0"
|
"@aws-sdk/s3-request-presigner": "^3.700.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^24.10.1",
|
"@types/node": "^24.10.1",
|
||||||
"typescript": "^5.9.3"
|
"typescript": "^5.9.3",
|
||||||
|
"vitest": "^4.1.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
292
packages/shared-storage/src/client.spec.ts
Normal file
292
packages/shared-storage/src/client.spec.ts
Normal file
|
|
@ -0,0 +1,292 @@
|
||||||
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||||
|
import type { StorageConfig, BucketConfig } from './types';
|
||||||
|
|
||||||
|
const { mockSend, mockUploadDone, mockGetSignedUrl } = vi.hoisted(() => ({
|
||||||
|
mockSend: vi.fn(),
|
||||||
|
mockUploadDone: vi.fn().mockResolvedValue({ ETag: '"multipart-etag"' }),
|
||||||
|
mockGetSignedUrl: vi.fn().mockResolvedValue('https://signed.url/file'),
|
||||||
|
}));
|
||||||
|
|
||||||
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
|
// Mock AWS SDK — all classes must use function() for `new` support
|
||||||
|
vi.mock('@aws-sdk/client-s3', () => ({
|
||||||
|
S3Client: vi.fn(function (this: any) {
|
||||||
|
this.send = mockSend;
|
||||||
|
}),
|
||||||
|
PutObjectCommand: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
|
GetObjectCommand: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
|
DeleteObjectCommand: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
|
ListObjectsV2Command: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
|
HeadObjectCommand: vi.fn(function (this: any, input: any) {
|
||||||
|
Object.assign(this, input);
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@aws-sdk/lib-storage', () => ({
|
||||||
|
Upload: vi.fn(function (this: Record<string, unknown>) {
|
||||||
|
this.done = mockUploadDone;
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@aws-sdk/s3-request-presigner', () => ({
|
||||||
|
getSignedUrl: mockGetSignedUrl,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Import after mocks
|
||||||
|
import { StorageClient } from './client';
|
||||||
|
|
||||||
|
const TEST_CONFIG: StorageConfig = {
|
||||||
|
endpoint: 'http://localhost:9000',
|
||||||
|
region: 'us-east-1',
|
||||||
|
accessKeyId: 'test-key',
|
||||||
|
secretAccessKey: 'test-secret',
|
||||||
|
forcePathStyle: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const TEST_BUCKET: BucketConfig = {
|
||||||
|
name: 'test-bucket',
|
||||||
|
publicUrl: 'http://localhost:9000/test-bucket',
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('StorageClient', () => {
|
||||||
|
let storage: StorageClient;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
mockSend.mockReset();
|
||||||
|
mockUploadDone.mockReset().mockResolvedValue({ ETag: '"multipart-etag"' });
|
||||||
|
mockGetSignedUrl.mockReset().mockResolvedValue('https://signed.url/file');
|
||||||
|
storage = new StorageClient(TEST_CONFIG, TEST_BUCKET);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('upload', () => {
|
||||||
|
it('uploads a file and returns key and url', async () => {
|
||||||
|
mockSend.mockResolvedValue({ ETag: '"abc123"' });
|
||||||
|
|
||||||
|
const result = await storage.upload('test.png', Buffer.from('data'), {
|
||||||
|
contentType: 'image/png',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.key).toBe('test.png');
|
||||||
|
expect(result.url).toBe('http://localhost:9000/test-bucket/test.png');
|
||||||
|
expect(result.etag).toBe('"abc123"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sets ACL to public-read when public option is true', async () => {
|
||||||
|
mockSend.mockResolvedValue({ ETag: '"abc"' });
|
||||||
|
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||||
|
|
||||||
|
await storage.upload('file.png', Buffer.from('data'), { public: true });
|
||||||
|
|
||||||
|
expect(PutObjectCommand).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ ACL: 'public-read' })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('uploadMultipart', () => {
|
||||||
|
it('uses Upload from lib-storage', async () => {
|
||||||
|
const { Upload } = await import('@aws-sdk/lib-storage');
|
||||||
|
|
||||||
|
const result = await storage.uploadMultipart('big-file.zip', Buffer.from('data'));
|
||||||
|
|
||||||
|
expect(Upload).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
queueSize: 4,
|
||||||
|
partSize: 10 * 1024 * 1024,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(result.key).toBe('big-file.zip');
|
||||||
|
expect(result.etag).toBe('"multipart-etag"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('download', () => {
|
||||||
|
it('downloads and returns a buffer', async () => {
|
||||||
|
const chunks = [new Uint8Array([1, 2]), new Uint8Array([3, 4])];
|
||||||
|
mockSend.mockResolvedValue({
|
||||||
|
Body: (async function* () {
|
||||||
|
for (const chunk of chunks) yield chunk;
|
||||||
|
})(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await storage.download('file.bin');
|
||||||
|
|
||||||
|
expect(result).toEqual(Buffer.from([1, 2, 3, 4]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when Body is missing', async () => {
|
||||||
|
mockSend.mockResolvedValue({ Body: null });
|
||||||
|
|
||||||
|
await expect(storage.download('missing.bin')).rejects.toThrow('File not found: missing.bin');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('downloadStream', () => {
|
||||||
|
it('returns a ReadableStream', async () => {
|
||||||
|
const mockStream = new ReadableStream();
|
||||||
|
mockSend.mockResolvedValue({
|
||||||
|
Body: { transformToWebStream: () => mockStream },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await storage.downloadStream('file.bin');
|
||||||
|
|
||||||
|
expect(result).toBe(mockStream);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when Body is missing', async () => {
|
||||||
|
mockSend.mockResolvedValue({ Body: null });
|
||||||
|
|
||||||
|
await expect(storage.downloadStream('missing.bin')).rejects.toThrow(
|
||||||
|
'File not found: missing.bin'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('delete', () => {
|
||||||
|
it('sends DeleteObjectCommand', async () => {
|
||||||
|
mockSend.mockResolvedValue({});
|
||||||
|
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||||
|
|
||||||
|
await storage.delete('file.png');
|
||||||
|
|
||||||
|
expect(DeleteObjectCommand).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ Bucket: 'test-bucket', Key: 'file.png' })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('exists', () => {
|
||||||
|
it('returns true when file exists', async () => {
|
||||||
|
mockSend.mockResolvedValue({});
|
||||||
|
|
||||||
|
expect(await storage.exists('file.png')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false on NotFound error', async () => {
|
||||||
|
const error = new Error('Not Found');
|
||||||
|
error.name = 'NotFound';
|
||||||
|
mockSend.mockRejectedValue(error);
|
||||||
|
|
||||||
|
expect(await storage.exists('missing.png')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false on 404 status code', async () => {
|
||||||
|
const error: Error & { $metadata?: { httpStatusCode: number } } = new Error('Not Found');
|
||||||
|
error.$metadata = { httpStatusCode: 404 };
|
||||||
|
mockSend.mockRejectedValue(error);
|
||||||
|
|
||||||
|
expect(await storage.exists('missing.png')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rethrows non-404 errors', async () => {
|
||||||
|
const error = new Error('Network failure');
|
||||||
|
error.name = 'NetworkError';
|
||||||
|
mockSend.mockRejectedValue(error);
|
||||||
|
|
||||||
|
await expect(storage.exists('file.png')).rejects.toThrow('Network failure');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('list', () => {
|
||||||
|
it('returns files from single page', async () => {
|
||||||
|
mockSend.mockResolvedValue({
|
||||||
|
Contents: [
|
||||||
|
{ Key: 'a.png', Size: 100, LastModified: new Date('2024-01-01'), ETag: '"aaa"' },
|
||||||
|
{ Key: 'b.png', Size: 200, LastModified: new Date('2024-01-02'), ETag: '"bbb"' },
|
||||||
|
],
|
||||||
|
IsTruncated: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const files = await storage.list('users/');
|
||||||
|
|
||||||
|
expect(files).toHaveLength(2);
|
||||||
|
expect(files[0].key).toBe('a.png');
|
||||||
|
expect(files[1].size).toBe(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('paginates through multiple pages', async () => {
|
||||||
|
mockSend
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
Contents: [{ Key: 'a.png', Size: 100, LastModified: new Date() }],
|
||||||
|
IsTruncated: true,
|
||||||
|
NextContinuationToken: 'token-1',
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
Contents: [{ Key: 'b.png', Size: 200, LastModified: new Date() }],
|
||||||
|
IsTruncated: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const files = await storage.list();
|
||||||
|
|
||||||
|
expect(files).toHaveLength(2);
|
||||||
|
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array when no contents', async () => {
|
||||||
|
mockSend.mockResolvedValue({ Contents: undefined, IsTruncated: false });
|
||||||
|
|
||||||
|
const files = await storage.list();
|
||||||
|
|
||||||
|
expect(files).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getPublicUrl', () => {
|
||||||
|
it('returns url with publicUrl configured', () => {
|
||||||
|
expect(storage.getPublicUrl('users/avatar.png')).toBe(
|
||||||
|
'http://localhost:9000/test-bucket/users/avatar.png'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns undefined without publicUrl', () => {
|
||||||
|
const client = new StorageClient(TEST_CONFIG, { name: 'private-bucket' });
|
||||||
|
expect(client.getPublicUrl('file.png')).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getCdnUrl', () => {
|
||||||
|
it('returns CDN url when configured', () => {
|
||||||
|
const client = new StorageClient(TEST_CONFIG, {
|
||||||
|
name: 'test-bucket',
|
||||||
|
cdnUrl: 'https://cdn.example.com',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(client.getCdnUrl('file.png')).toBe('https://cdn.example.com/file.png');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to publicUrl when no CDN', () => {
|
||||||
|
expect(storage.getCdnUrl('file.png')).toBe('http://localhost:9000/test-bucket/file.png');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns undefined when neither CDN nor publicUrl set', () => {
|
||||||
|
const client = new StorageClient(TEST_CONFIG, { name: 'bare-bucket' });
|
||||||
|
expect(client.getCdnUrl('file.png')).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getBucketName', () => {
|
||||||
|
it('returns the bucket name', () => {
|
||||||
|
expect(storage.getBucketName()).toBe('test-bucket');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('presigned URLs', () => {
|
||||||
|
it('getUploadUrl returns a signed URL', async () => {
|
||||||
|
const url = await storage.getUploadUrl('upload.png');
|
||||||
|
expect(url).toBe('https://signed.url/file');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('getDownloadUrl returns a signed URL', async () => {
|
||||||
|
const url = await storage.getDownloadUrl('download.png');
|
||||||
|
expect(url).toBe('https://signed.url/file');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
HeadObjectCommand,
|
HeadObjectCommand,
|
||||||
type PutObjectCommandInput,
|
type PutObjectCommandInput,
|
||||||
} from '@aws-sdk/client-s3';
|
} from '@aws-sdk/client-s3';
|
||||||
|
import { Upload } from '@aws-sdk/lib-storage';
|
||||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||||
import type {
|
import type {
|
||||||
StorageConfig,
|
StorageConfig,
|
||||||
|
|
@ -85,7 +86,41 @@ export class StorageClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Download a file from the bucket
|
* Upload a large file using multipart upload.
|
||||||
|
* Automatically splits the file into parts and uploads them in parallel.
|
||||||
|
* Use this for files >100MB or when uploading over unstable connections.
|
||||||
|
*/
|
||||||
|
async uploadMultipart(
|
||||||
|
key: string,
|
||||||
|
body: Buffer | Uint8Array | ReadableStream,
|
||||||
|
options: UploadOptions = {}
|
||||||
|
): Promise<UploadResult> {
|
||||||
|
const upload = new Upload({
|
||||||
|
client: this.client,
|
||||||
|
params: {
|
||||||
|
Bucket: this.bucket.name,
|
||||||
|
Key: key,
|
||||||
|
Body: body,
|
||||||
|
ContentType: options.contentType,
|
||||||
|
CacheControl: options.cacheControl,
|
||||||
|
Metadata: options.metadata,
|
||||||
|
...(options.public ? { ACL: 'public-read' as const } : {}),
|
||||||
|
},
|
||||||
|
queueSize: 4,
|
||||||
|
partSize: 10 * 1024 * 1024, // 10MB parts
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await upload.done();
|
||||||
|
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
url: this.getPublicUrl(key),
|
||||||
|
etag: result.ETag,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download a file from the bucket (loads entire file into memory)
|
||||||
*/
|
*/
|
||||||
async download(key: string): Promise<Buffer> {
|
async download(key: string): Promise<Buffer> {
|
||||||
const command = new GetObjectCommand({
|
const command = new GetObjectCommand({
|
||||||
|
|
@ -108,6 +143,24 @@ export class StorageClient {
|
||||||
return Buffer.concat(chunks);
|
return Buffer.concat(chunks);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download a file as a readable stream (memory-efficient for large files)
|
||||||
|
*/
|
||||||
|
async downloadStream(key: string): Promise<ReadableStream> {
|
||||||
|
const command = new GetObjectCommand({
|
||||||
|
Bucket: this.bucket.name,
|
||||||
|
Key: key,
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await this.client.send(command);
|
||||||
|
|
||||||
|
if (!response.Body) {
|
||||||
|
throw new Error(`File not found: ${key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Body.transformToWebStream();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a file from the bucket
|
* Delete a file from the bucket
|
||||||
*/
|
*/
|
||||||
|
|
@ -131,29 +184,46 @@ export class StorageClient {
|
||||||
});
|
});
|
||||||
await this.client.send(command);
|
await this.client.send(command);
|
||||||
return true;
|
return true;
|
||||||
} catch {
|
} catch (err: unknown) {
|
||||||
return false;
|
if (err instanceof Error) {
|
||||||
|
if (err.name === 'NotFound') return false;
|
||||||
|
const metadata = (err as Error & { $metadata?: { httpStatusCode?: number } }).$metadata;
|
||||||
|
if (metadata?.httpStatusCode === 404) return false;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List files in the bucket with optional prefix
|
* List files in the bucket with optional prefix.
|
||||||
|
* Automatically paginates through all results if the bucket contains more than maxKeys items.
|
||||||
*/
|
*/
|
||||||
async list(prefix?: string, maxKeys = 1000): Promise<FileInfo[]> {
|
async list(prefix?: string, maxKeys = 1000): Promise<FileInfo[]> {
|
||||||
const command = new ListObjectsV2Command({
|
const allFiles: FileInfo[] = [];
|
||||||
Bucket: this.bucket.name,
|
let continuationToken: string | undefined;
|
||||||
Prefix: prefix,
|
|
||||||
MaxKeys: maxKeys,
|
|
||||||
});
|
|
||||||
|
|
||||||
const response = await this.client.send(command);
|
do {
|
||||||
|
const command = new ListObjectsV2Command({
|
||||||
|
Bucket: this.bucket.name,
|
||||||
|
Prefix: prefix,
|
||||||
|
MaxKeys: maxKeys,
|
||||||
|
ContinuationToken: continuationToken,
|
||||||
|
});
|
||||||
|
|
||||||
return (response.Contents ?? []).map((item) => ({
|
const response = await this.client.send(command);
|
||||||
key: item.Key!,
|
|
||||||
size: item.Size ?? 0,
|
const files = (response.Contents ?? []).map((item) => ({
|
||||||
lastModified: item.LastModified ?? new Date(),
|
key: item.Key ?? '',
|
||||||
etag: item.ETag,
|
size: item.Size ?? 0,
|
||||||
}));
|
lastModified: item.LastModified ?? new Date(),
|
||||||
|
etag: item.ETag,
|
||||||
|
}));
|
||||||
|
|
||||||
|
allFiles.push(...files);
|
||||||
|
continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
|
||||||
|
} while (continuationToken);
|
||||||
|
|
||||||
|
return allFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -196,6 +266,16 @@ export class StorageClient {
|
||||||
return `${this.bucket.publicUrl}/${key}`;
|
return `${this.bucket.publicUrl}/${key}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the CDN URL for a file. Falls back to publicUrl if no CDN is configured.
|
||||||
|
*/
|
||||||
|
getCdnUrl(key: string): string | undefined {
|
||||||
|
if (this.bucket.cdnUrl) {
|
||||||
|
return `${this.bucket.cdnUrl}/${key}`;
|
||||||
|
}
|
||||||
|
return this.getPublicUrl(key);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the underlying S3 client for advanced operations
|
* Get the underlying S3 client for advanced operations
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
132
packages/shared-storage/src/factory.spec.ts
Normal file
132
packages/shared-storage/src/factory.spec.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
||||||
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||||
|
|
||||||
|
// Mock the client module — StorageClient must be a class
|
||||||
|
vi.mock('./client', () => ({
|
||||||
|
StorageClient: vi.fn(function (this: Record<string, unknown>) {
|
||||||
|
this.upload = vi.fn();
|
||||||
|
this.getBucketName = vi.fn();
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import {
|
||||||
|
createStorage,
|
||||||
|
createStorageClient,
|
||||||
|
createPictureStorage,
|
||||||
|
createChatStorage,
|
||||||
|
} from './factory';
|
||||||
|
import { BUCKETS } from './types';
|
||||||
|
import { StorageClient } from './client';
|
||||||
|
|
||||||
|
describe('createStorage', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
process.env.NODE_ENV = 'development';
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
delete process.env.PICTURE_STORAGE_PUBLIC_URL;
|
||||||
|
delete process.env.PICTURE_CDN_URL;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('creates a client with the correct bucket name', () => {
|
||||||
|
createStorage('PICTURE');
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ name: BUCKETS.PICTURE })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves public URL from environment', () => {
|
||||||
|
process.env.PICTURE_STORAGE_PUBLIC_URL = 'https://cdn.example.com/picture';
|
||||||
|
|
||||||
|
createStorage('PICTURE');
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ publicUrl: 'https://cdn.example.com/picture' })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves CDN URL from environment', () => {
|
||||||
|
process.env.PICTURE_CDN_URL = 'https://cdn.fast.com/picture';
|
||||||
|
|
||||||
|
createStorage('PICTURE');
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ cdnUrl: 'https://cdn.fast.com/picture' })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('allows explicit publicUrl override', () => {
|
||||||
|
process.env.PICTURE_STORAGE_PUBLIC_URL = 'https://from-env.com';
|
||||||
|
|
||||||
|
createStorage('PICTURE', 'https://explicit.com');
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ publicUrl: 'https://explicit.com' })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('convenience aliases', () => {
|
||||||
|
it('createPictureStorage creates PICTURE bucket', () => {
|
||||||
|
createPictureStorage();
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ name: BUCKETS.PICTURE })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('createChatStorage creates CHAT bucket', () => {
|
||||||
|
createChatStorage();
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.any(Object),
|
||||||
|
expect.objectContaining({ name: BUCKETS.CHAT })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('createStorageClient', () => {
|
||||||
|
afterEach(() => {
|
||||||
|
process.env.NODE_ENV = 'development';
|
||||||
|
delete process.env.S3_ENDPOINT;
|
||||||
|
delete process.env.S3_ACCESS_KEY;
|
||||||
|
delete process.env.S3_SECRET_KEY;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when endpoint is missing in production', () => {
|
||||||
|
process.env.NODE_ENV = 'production';
|
||||||
|
process.env.S3_ENDPOINT = '';
|
||||||
|
|
||||||
|
expect(() => createStorageClient(BUCKETS.CHAT)).toThrow('S3_ENDPOINT is required');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when credentials are missing in production', () => {
|
||||||
|
process.env.NODE_ENV = 'production';
|
||||||
|
process.env.S3_ENDPOINT = 'https://s3.example.com';
|
||||||
|
process.env.S3_ACCESS_KEY = '';
|
||||||
|
process.env.S3_SECRET_KEY = '';
|
||||||
|
|
||||||
|
expect(() => createStorageClient(BUCKETS.CHAT)).toThrow(
|
||||||
|
'S3_ACCESS_KEY and S3_SECRET_KEY are required'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses MinIO defaults in development', () => {
|
||||||
|
process.env.NODE_ENV = 'development';
|
||||||
|
|
||||||
|
createStorageClient(BUCKETS.CHAT);
|
||||||
|
|
||||||
|
expect(StorageClient).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
endpoint: 'http://localhost:9000',
|
||||||
|
accessKeyId: 'minioadmin',
|
||||||
|
}),
|
||||||
|
expect.any(Object)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -24,6 +24,17 @@ const MINIO_DEFAULTS: StorageConfig = {
|
||||||
forcePathStyle: true,
|
forcePathStyle: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mapping of bucket keys to their public URL environment variable names
|
||||||
|
*/
|
||||||
|
const PUBLIC_URL_ENV: Partial<Record<keyof typeof BUCKETS, string>> = {
|
||||||
|
MANACORE: 'MANACORE_STORAGE_PUBLIC_URL',
|
||||||
|
PICTURE: 'PICTURE_STORAGE_PUBLIC_URL',
|
||||||
|
NUTRIPHI: 'NUTRIPHI_S3_PUBLIC_URL',
|
||||||
|
STORAGE: 'STORAGE_S3_PUBLIC_URL',
|
||||||
|
INVENTORY: 'INVENTORY_S3_PUBLIC_URL',
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get storage configuration from environment variables
|
* Get storage configuration from environment variables
|
||||||
* Falls back to MinIO defaults in development
|
* Falls back to MinIO defaults in development
|
||||||
|
|
@ -70,107 +81,35 @@ export function createStorageClient(
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a storage client for the Mana Core Auth project (avatars, etc.)
|
* Create a storage client for a project by bucket key.
|
||||||
|
* Automatically resolves the public URL and CDN URL from environment variables.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* const storage = createStorage('PICTURE');
|
||||||
|
* const storage = createStorage('CHAT');
|
||||||
*/
|
*/
|
||||||
export function createManaCoreStorage(publicUrl?: string): StorageClient {
|
export function createStorage(bucketKey: keyof typeof BUCKETS, publicUrl?: string): StorageClient {
|
||||||
|
const envKey = PUBLIC_URL_ENV[bucketKey];
|
||||||
|
const cdnEnvKey = `${bucketKey}_CDN_URL`;
|
||||||
return createStorageClient({
|
return createStorageClient({
|
||||||
name: BUCKETS.MANACORE,
|
name: BUCKETS[bucketKey],
|
||||||
publicUrl: publicUrl ?? process.env.MANACORE_STORAGE_PUBLIC_URL,
|
publicUrl: publicUrl ?? (envKey ? process.env[envKey] : undefined),
|
||||||
|
cdnUrl: process.env[cdnEnvKey],
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// Convenience aliases for backward compatibility
|
||||||
* Create a storage client for the Picture project
|
|
||||||
*/
|
|
||||||
export function createPictureStorage(publicUrl?: string): StorageClient {
|
|
||||||
return createStorageClient({
|
|
||||||
name: BUCKETS.PICTURE,
|
|
||||||
publicUrl: publicUrl ?? process.env.PICTURE_STORAGE_PUBLIC_URL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
export const createManaCoreStorage = (publicUrl?: string) => createStorage('MANACORE', publicUrl);
|
||||||
* Create a storage client for the Chat project
|
export const createPictureStorage = (publicUrl?: string) => createStorage('PICTURE', publicUrl);
|
||||||
*/
|
export const createChatStorage = () => createStorage('CHAT');
|
||||||
export function createChatStorage(): StorageClient {
|
export const createManaDeckStorage = () => createStorage('MANADECK');
|
||||||
return createStorageClient({ name: BUCKETS.CHAT });
|
export const createNutriPhiStorage = (publicUrl?: string) => createStorage('NUTRIPHI', publicUrl);
|
||||||
}
|
export const createPresiStorage = () => createStorage('PRESI');
|
||||||
|
export const createCalendarStorage = () => createStorage('CALENDAR');
|
||||||
/**
|
export const createContactsStorage = () => createStorage('CONTACTS');
|
||||||
* Create a storage client for the ManaDeck project
|
export const createStorageStorage = (publicUrl?: string) => createStorage('STORAGE', publicUrl);
|
||||||
*/
|
export const createMailStorage = () => createStorage('MAIL');
|
||||||
export function createManaDeckStorage(): StorageClient {
|
export const createInventoryStorage = (publicUrl?: string) => createStorage('INVENTORY', publicUrl);
|
||||||
return createStorageClient({ name: BUCKETS.MANADECK });
|
export const createLightWriteStorage = () => createStorage('LIGHTWRITE');
|
||||||
}
|
export const createMukkeStorage = () => createStorage('MUKKE');
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the NutriPhi project
|
|
||||||
*/
|
|
||||||
export function createNutriPhiStorage(publicUrl?: string): StorageClient {
|
|
||||||
return createStorageClient({
|
|
||||||
name: BUCKETS.NUTRIPHI,
|
|
||||||
publicUrl: publicUrl ?? process.env.NUTRIPHI_S3_PUBLIC_URL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Presi project
|
|
||||||
*/
|
|
||||||
export function createPresiStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.PRESI });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Calendar project
|
|
||||||
*/
|
|
||||||
export function createCalendarStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.CALENDAR });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Contacts project
|
|
||||||
*/
|
|
||||||
export function createContactsStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.CONTACTS });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Storage project (cloud drive)
|
|
||||||
*/
|
|
||||||
export function createStorageStorage(publicUrl?: string): StorageClient {
|
|
||||||
return createStorageClient({
|
|
||||||
name: BUCKETS.STORAGE,
|
|
||||||
publicUrl: publicUrl ?? process.env.STORAGE_S3_PUBLIC_URL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Mail project
|
|
||||||
*/
|
|
||||||
export function createMailStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.MAIL });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Inventory project
|
|
||||||
*/
|
|
||||||
export function createInventoryStorage(publicUrl?: string): StorageClient {
|
|
||||||
return createStorageClient({
|
|
||||||
name: BUCKETS.INVENTORY,
|
|
||||||
publicUrl: publicUrl ?? process.env.INVENTORY_S3_PUBLIC_URL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the LightWrite project
|
|
||||||
*/
|
|
||||||
export function createLightWriteStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.LIGHTWRITE });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a storage client for the Mukke project
|
|
||||||
*/
|
|
||||||
export function createMukkeStorage(): StorageClient {
|
|
||||||
return createStorageClient({ name: BUCKETS.MUKKE });
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ export { StorageClient } from './client';
|
||||||
|
|
||||||
// Factory functions
|
// Factory functions
|
||||||
export {
|
export {
|
||||||
|
createStorage,
|
||||||
createStorageClient,
|
createStorageClient,
|
||||||
getStorageConfig,
|
getStorageConfig,
|
||||||
createManaCoreStorage,
|
createManaCoreStorage,
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,11 @@ export interface BucketConfig {
|
||||||
name: string;
|
name: string;
|
||||||
/** Public URL for accessing files (optional, for CDN/public buckets) */
|
/** Public URL for accessing files (optional, for CDN/public buckets) */
|
||||||
publicUrl?: string;
|
publicUrl?: string;
|
||||||
|
/**
|
||||||
|
* CDN URL prefix for serving files (e.g., https://cdn.mana.how/picture).
|
||||||
|
* When set, getCdnUrl() returns URLs through the CDN instead of direct S3 access.
|
||||||
|
*/
|
||||||
|
cdnUrl?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
91
packages/shared-storage/src/utils.spec.ts
Normal file
91
packages/shared-storage/src/utils.spec.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import {
|
||||||
|
generateFileKey,
|
||||||
|
generateUserFileKey,
|
||||||
|
getContentType,
|
||||||
|
validateFileSize,
|
||||||
|
validateFileExtension,
|
||||||
|
} from './utils';
|
||||||
|
|
||||||
|
describe('generateFileKey', () => {
|
||||||
|
it('generates a UUID-based key with extension', () => {
|
||||||
|
const key = generateFileKey('photo.png');
|
||||||
|
expect(key).toMatch(/^[0-9a-f-]{36}\.png$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes folder path', () => {
|
||||||
|
const key = generateFileKey('image.jpg', 'uploads', '2024');
|
||||||
|
expect(key).toMatch(/^uploads\/2024\/[0-9a-f-]{36}\.jpg$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles files without extension', () => {
|
||||||
|
const key = generateFileKey('Dockerfile');
|
||||||
|
expect(key).toMatch(/^[0-9a-f-]{36}$/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('generateUserFileKey', () => {
|
||||||
|
it('generates a user-scoped key', () => {
|
||||||
|
const key = generateUserFileKey('user-123', 'avatar.png');
|
||||||
|
expect(key).toMatch(/^users\/user-123\/[0-9a-f-]{36}\.png$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('includes subfolder when provided', () => {
|
||||||
|
const key = generateUserFileKey('user-123', 'photo.jpg', 'avatars');
|
||||||
|
expect(key).toMatch(/^users\/user-123\/avatars\/[0-9a-f-]{36}\.jpg$/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getContentType', () => {
|
||||||
|
it.each([
|
||||||
|
['image.jpg', 'image/jpeg'],
|
||||||
|
['image.jpeg', 'image/jpeg'],
|
||||||
|
['image.png', 'image/png'],
|
||||||
|
['image.gif', 'image/gif'],
|
||||||
|
['image.webp', 'image/webp'],
|
||||||
|
['image.svg', 'image/svg+xml'],
|
||||||
|
['doc.pdf', 'application/pdf'],
|
||||||
|
['song.mp3', 'audio/mpeg'],
|
||||||
|
['video.mp4', 'video/mp4'],
|
||||||
|
['data.json', 'application/json'],
|
||||||
|
['archive.zip', 'application/zip'],
|
||||||
|
])('returns correct type for %s', (filename, expected) => {
|
||||||
|
expect(getContentType(filename)).toBe(expected);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns octet-stream for unknown extensions', () => {
|
||||||
|
expect(getContentType('file.xyz')).toBe('application/octet-stream');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is case-insensitive', () => {
|
||||||
|
expect(getContentType('FILE.PNG')).toBe('image/png');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('validateFileSize', () => {
|
||||||
|
it('returns true when within limit', () => {
|
||||||
|
expect(validateFileSize(5 * 1024 * 1024, 10)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true at exact limit', () => {
|
||||||
|
expect(validateFileSize(10 * 1024 * 1024, 10)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false when over limit', () => {
|
||||||
|
expect(validateFileSize(11 * 1024 * 1024, 10)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('validateFileExtension', () => {
|
||||||
|
it('returns true for allowed extension', () => {
|
||||||
|
expect(validateFileExtension('photo.png', ['.png', '.jpg'])).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for disallowed extension', () => {
|
||||||
|
expect(validateFileExtension('script.exe', ['.png', '.jpg'])).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is case-insensitive', () => {
|
||||||
|
expect(validateFileExtension('photo.PNG', ['.png'])).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -11,6 +11,6 @@
|
||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"rootDir": "./src"
|
"rootDir": "./src"
|
||||||
},
|
},
|
||||||
"include": ["src/**/*"],
|
"include": ["src/**/*", "vitest.config.ts"],
|
||||||
"exclude": ["node_modules", "dist"]
|
"exclude": ["node_modules", "dist"]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
12
packages/shared-storage/vitest.config.ts
Normal file
12
packages/shared-storage/vitest.config.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
include: ['src/**/*.{test,spec}.ts'],
|
||||||
|
environment: 'node',
|
||||||
|
globals: true,
|
||||||
|
clearMocks: true,
|
||||||
|
mockReset: true,
|
||||||
|
restoreMocks: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
1440
pnpm-lock.yaml
generated
1440
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue