mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-22 05:26:41 +02:00
perf(auth): replace bcrypt with bcryptjs (pure JS, no native build tools)
- Switch from bcrypt (native C++ addon) to bcryptjs (pure JavaScript) - Remove python3/make/g++ build tools from Dockerfile builder stage - bcryptjs is 100% hash-compatible with bcrypt - Smaller builder image and faster Docker builds Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
aeabdcaf8e
commit
8c2aa261e8
11 changed files with 314 additions and 146 deletions
|
|
@ -110,6 +110,23 @@ describe('StorageClient', () => {
|
|||
expect(result.key).toBe('file.png');
|
||||
});
|
||||
|
||||
it('wraps ReadableStream with size constraint when maxSizeBytes set', async () => {
|
||||
// Stream that produces 2 chunks of 512 bytes = 1024 total, limit is 768
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(512));
|
||||
controller.enqueue(new Uint8Array(512));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
// The stream constraint happens during S3 transfer — since we mock send,
|
||||
// we verify the upload doesn't throw synchronously (constraint is lazy)
|
||||
mockSend.mockResolvedValue({ ETag: '"ok"' });
|
||||
// With a stream, the constraint wraps it but errors happen during read
|
||||
const result = await storage.upload('file.png', stream, { maxSizeBytes: 2048 });
|
||||
expect(result.key).toBe('file.png');
|
||||
});
|
||||
|
||||
it('sets ACL to public-read when public option is true', async () => {
|
||||
mockSend.mockResolvedValue({ ETag: '"abc"' });
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
|
|
@ -475,6 +492,20 @@ describe('StorageClient', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('move', () => {
|
||||
it('copies then deletes source', async () => {
|
||||
mockSend
|
||||
.mockResolvedValueOnce({ CopyObjectResult: { ETag: '"moved"' } }) // copy
|
||||
.mockResolvedValueOnce({}); // delete
|
||||
|
||||
const result = await storage.move('old/file.png', 'new/file.png');
|
||||
|
||||
expect(result.key).toBe('new/file.png');
|
||||
expect(result.etag).toBe('"moved"');
|
||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMetadata', () => {
|
||||
it('returns file metadata', async () => {
|
||||
mockSend.mockResolvedValue({
|
||||
|
|
|
|||
|
|
@ -28,6 +28,39 @@ import type {
|
|||
FileMetadata,
|
||||
} from './types';
|
||||
|
||||
/**
|
||||
* Wraps a ReadableStream to enforce a maximum byte size.
|
||||
* Throws if the stream exceeds the limit mid-transfer.
|
||||
*/
|
||||
function constrainStream(stream: ReadableStream, maxBytes: number): ReadableStream {
|
||||
const reader = stream.getReader();
|
||||
let bytesRead = 0;
|
||||
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
bytesRead += value.byteLength;
|
||||
if (bytesRead > maxBytes) {
|
||||
controller.error(
|
||||
new Error(
|
||||
`Stream size ${bytesRead} bytes exceeds maximum allowed ${maxBytes} bytes`
|
||||
)
|
||||
);
|
||||
reader.cancel();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(value);
|
||||
},
|
||||
cancel(reason) {
|
||||
reader.cancel(reason);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* S3-compatible storage client for MinIO (local) and Hetzner Object Storage (production)
|
||||
*/
|
||||
|
|
@ -74,12 +107,15 @@ export class StorageClient {
|
|||
body: Buffer | Uint8Array | string | ReadableStream,
|
||||
options: UploadOptions = {}
|
||||
): Promise<UploadResult> {
|
||||
if (options.maxSizeBytes && typeof body !== 'string' && !(body instanceof ReadableStream)) {
|
||||
const size = body.byteLength;
|
||||
if (size > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
if (options.maxSizeBytes) {
|
||||
if (typeof body !== 'string' && !(body instanceof ReadableStream)) {
|
||||
if (body.byteLength > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${body.byteLength} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
}
|
||||
} else if (body instanceof ReadableStream) {
|
||||
body = constrainStream(body, options.maxSizeBytes);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,12 +166,15 @@ export class StorageClient {
|
|||
body: Buffer | Uint8Array | ReadableStream,
|
||||
options: UploadOptions = {}
|
||||
): Promise<UploadResult> {
|
||||
if (options.maxSizeBytes && !(body instanceof ReadableStream)) {
|
||||
const size = body.byteLength;
|
||||
if (size > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${size} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
if (options.maxSizeBytes) {
|
||||
if (!(body instanceof ReadableStream)) {
|
||||
if (body.byteLength > options.maxSizeBytes) {
|
||||
throw new Error(
|
||||
`File size ${body.byteLength} bytes exceeds maximum allowed ${options.maxSizeBytes} bytes`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
body = constrainStream(body, options.maxSizeBytes);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -385,9 +424,17 @@ export class StorageClient {
|
|||
return keys.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a file within the same bucket (copy + delete source).
|
||||
*/
|
||||
async move(sourceKey: string, destKey: string): Promise<UploadResult> {
|
||||
const result = await this.copy(sourceKey, destKey);
|
||||
await this.delete(sourceKey);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a file within the same bucket.
|
||||
* For move operations, call copy() then delete() the source.
|
||||
*/
|
||||
async copy(sourceKey: string, destKey: string): Promise<UploadResult> {
|
||||
const command = new CopyObjectCommand({
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ export type {
|
|||
} from './hooks';
|
||||
|
||||
// Metrics
|
||||
export { InMemoryMetrics, attachMetrics } from './metrics';
|
||||
export type { StorageMetricsCollector } from './metrics';
|
||||
export { InMemoryMetrics, attachMetrics, createPrometheusCollector } from './metrics';
|
||||
export type { StorageMetricsCollector, MetricsFactory } from './metrics';
|
||||
|
||||
// Utilities
|
||||
export {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { StorageHooks } from './hooks';
|
||||
import { InMemoryMetrics, attachMetrics } from './metrics';
|
||||
import { InMemoryMetrics, attachMetrics, createPrometheusCollector } from './metrics';
|
||||
import type { MetricsFactory } from './metrics';
|
||||
|
||||
describe('InMemoryMetrics', () => {
|
||||
let metrics: InMemoryMetrics;
|
||||
|
|
@ -111,3 +112,104 @@ describe('attachMetrics', () => {
|
|||
expect(metrics.sizes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createPrometheusCollector', () => {
|
||||
function createMockFactory(): MetricsFactory & {
|
||||
counters: Map<string, { inc: ReturnType<typeof vi.fn> }>;
|
||||
histograms: Map<string, { observe: ReturnType<typeof vi.fn> }>;
|
||||
} {
|
||||
const counters = new Map<string, { inc: ReturnType<typeof vi.fn> }>();
|
||||
const histograms = new Map<string, { observe: ReturnType<typeof vi.fn> }>();
|
||||
|
||||
return {
|
||||
counters,
|
||||
histograms,
|
||||
createCounter(name: string) {
|
||||
const counter = { inc: vi.fn() };
|
||||
counters.set(name, counter);
|
||||
return counter;
|
||||
},
|
||||
createHistogram(name: string) {
|
||||
const histogram = { observe: vi.fn() };
|
||||
histograms.set(name, histogram);
|
||||
return histogram;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it('creates expected metrics', () => {
|
||||
const factory = createMockFactory();
|
||||
createPrometheusCollector(factory);
|
||||
|
||||
expect(factory.counters.has('storage_uploads_total')).toBe(true);
|
||||
expect(factory.counters.has('storage_upload_errors_total')).toBe(true);
|
||||
expect(factory.counters.has('storage_deletes_total')).toBe(true);
|
||||
expect(factory.counters.has('storage_downloads_total')).toBe(true);
|
||||
expect(factory.histograms.has('storage_upload_size_bytes')).toBe(true);
|
||||
});
|
||||
|
||||
it('increments upload counter with labels', () => {
|
||||
const factory = createMockFactory();
|
||||
const collector = createPrometheusCollector(factory);
|
||||
|
||||
collector.incrementUploads('picture-storage', 'image/png');
|
||||
|
||||
const counter = factory.counters.get('storage_uploads_total');
|
||||
expect(counter?.inc).toHaveBeenCalledWith({
|
||||
bucket: 'picture-storage',
|
||||
content_type: 'image/png',
|
||||
});
|
||||
});
|
||||
|
||||
it('uses "unknown" for missing content type', () => {
|
||||
const factory = createMockFactory();
|
||||
const collector = createPrometheusCollector(factory);
|
||||
|
||||
collector.incrementUploads('chat-storage');
|
||||
|
||||
const counter = factory.counters.get('storage_uploads_total');
|
||||
expect(counter?.inc).toHaveBeenCalledWith({
|
||||
bucket: 'chat-storage',
|
||||
content_type: 'unknown',
|
||||
});
|
||||
});
|
||||
|
||||
it('observes upload size in histogram', () => {
|
||||
const factory = createMockFactory();
|
||||
const collector = createPrometheusCollector(factory);
|
||||
|
||||
collector.observeUploadSize('picture-storage', 1048576);
|
||||
|
||||
const histogram = factory.histograms.get('storage_upload_size_bytes');
|
||||
expect(histogram?.observe).toHaveBeenCalledWith({ bucket: 'picture-storage' }, 1048576);
|
||||
});
|
||||
|
||||
it('increments deletes with count', () => {
|
||||
const factory = createMockFactory();
|
||||
const collector = createPrometheusCollector(factory);
|
||||
|
||||
collector.incrementDeletes('chat-storage', 5);
|
||||
|
||||
const counter = factory.counters.get('storage_deletes_total');
|
||||
expect(counter?.inc).toHaveBeenCalledWith({ bucket: 'chat-storage' }, 5);
|
||||
});
|
||||
|
||||
it('works end-to-end with hooks', () => {
|
||||
const factory = createMockFactory();
|
||||
const collector = createPrometheusCollector(factory);
|
||||
const hooks = new StorageHooks();
|
||||
attachMetrics(hooks, collector);
|
||||
|
||||
hooks.emit('upload', { bucket: 'test', key: 'f.png', sizeBytes: 512, contentType: 'image/png' });
|
||||
hooks.emit('download', { bucket: 'test', key: 'f.png' });
|
||||
hooks.emit('delete', { bucket: 'test', keys: ['a', 'b'] });
|
||||
|
||||
expect(factory.counters.get('storage_uploads_total')?.inc).toHaveBeenCalledTimes(1);
|
||||
expect(factory.counters.get('storage_downloads_total')?.inc).toHaveBeenCalledTimes(1);
|
||||
expect(factory.counters.get('storage_deletes_total')?.inc).toHaveBeenCalledWith({ bucket: 'test' }, 2);
|
||||
expect(factory.histograms.get('storage_upload_size_bytes')?.observe).toHaveBeenCalledWith(
|
||||
{ bucket: 'test' },
|
||||
512
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,27 @@
|
|||
import type { StorageHooks } from './hooks';
|
||||
|
||||
/**
|
||||
* Minimal interface matching MetricsService.createCounter/createHistogram.
|
||||
* This avoids a hard dependency on @manacore/shared-nestjs-metrics or prom-client.
|
||||
*/
|
||||
export interface MetricsFactory {
|
||||
createCounter(name: string, help: string, labelNames?: string[]): CounterLike;
|
||||
createHistogram(
|
||||
name: string,
|
||||
help: string,
|
||||
labelNames?: string[],
|
||||
buckets?: number[]
|
||||
): HistogramLike;
|
||||
}
|
||||
|
||||
interface CounterLike {
|
||||
inc(labels?: Record<string, string | number>, value?: number): void;
|
||||
}
|
||||
|
||||
interface HistogramLike {
|
||||
observe(labels: Record<string, string | number>, value: number): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for a metrics collector — decoupled from prom-client so shared-storage
|
||||
* stays dependency-free. NestJS backends wire this up with their MetricsService.
|
||||
|
|
@ -80,3 +102,67 @@ export function attachMetrics(hooks: StorageHooks, collector: StorageMetricsColl
|
|||
|
||||
return () => unsubs.forEach((unsub) => unsub());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a StorageMetricsCollector backed by Prometheus counters/histograms.
|
||||
* Pass your NestJS MetricsService (or anything that matches MetricsFactory).
|
||||
*
|
||||
* @example
|
||||
* // In a NestJS service
|
||||
* import { MetricsService } from '@manacore/shared-nestjs-metrics';
|
||||
* import { createPrometheusCollector, attachMetrics } from '@manacore/shared-storage';
|
||||
*
|
||||
* const storage = createPictureStorage();
|
||||
* const collector = createPrometheusCollector(metricsService);
|
||||
* attachMetrics(storage.hooks, collector);
|
||||
*/
|
||||
export function createPrometheusCollector(factory: MetricsFactory): StorageMetricsCollector {
|
||||
const uploadsCounter = factory.createCounter(
|
||||
'storage_uploads_total',
|
||||
'Total storage upload operations',
|
||||
['bucket', 'content_type']
|
||||
);
|
||||
|
||||
const uploadErrorsCounter = factory.createCounter(
|
||||
'storage_upload_errors_total',
|
||||
'Total storage upload errors',
|
||||
['bucket']
|
||||
);
|
||||
|
||||
const deletesCounter = factory.createCounter(
|
||||
'storage_deletes_total',
|
||||
'Total storage delete operations',
|
||||
['bucket']
|
||||
);
|
||||
|
||||
const downloadsCounter = factory.createCounter(
|
||||
'storage_downloads_total',
|
||||
'Total storage download operations',
|
||||
['bucket']
|
||||
);
|
||||
|
||||
const uploadSizeHistogram = factory.createHistogram(
|
||||
'storage_upload_size_bytes',
|
||||
'Upload file sizes in bytes',
|
||||
['bucket'],
|
||||
[1024, 10240, 102400, 1048576, 10485760, 104857600] // 1KB, 10KB, 100KB, 1MB, 10MB, 100MB
|
||||
);
|
||||
|
||||
return {
|
||||
incrementUploads(bucket: string, contentType?: string): void {
|
||||
uploadsCounter.inc({ bucket, content_type: contentType ?? 'unknown' });
|
||||
},
|
||||
incrementUploadErrors(bucket: string): void {
|
||||
uploadErrorsCounter.inc({ bucket });
|
||||
},
|
||||
incrementDeletes(bucket: string, count: number): void {
|
||||
deletesCounter.inc({ bucket }, count);
|
||||
},
|
||||
incrementDownloads(bucket: string): void {
|
||||
downloadsCounter.inc({ bucket });
|
||||
},
|
||||
observeUploadSize(bucket: string, sizeBytes: number): void {
|
||||
uploadSizeHistogram.observe({ bucket }, sizeBytes);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue