/** * SpiralDB - Main Database Class * Manages the spiral pixel database with CRUD operations */ import type { SpiralDBOptions, SchemaDefinition, SpiralImage, MasterIndex, IndexEntry, RecordStatus, SpiralRecord, WriteResult, ReadResult, ColorIndex, BitStream, } from './types.js'; import { MAGIC_VALID, FLAG_READABLE, RING_HEADER, RING_SCHEMA, RING_INDEX, RING_DATA_START, STATUS_BITS, BITS_TO_STATUS, END_MARKER, MAX_RECORD_LENGTH, } from './constants.js'; import { createBitStream, writeBits, readBits, bitsToPixels, pixelsToBits, encodeString, decodeString, encodeInt, decodeInt, encodeBool, decodeBool, encodeTimestamp, decodeTimestamp, encodeIntArray, decodeIntArray, } from './encoding.js'; import { createImageForRing, getPixelByIndex, setPixelByIndex, readPixelRange, writePixelRange, expandImage, } from './image.js'; import { getRingInfo, findSpaceForRecord, getTotalPixelsForRing } from './spiral.js'; import { encodeSchema, validateRecord } from './schema.js'; export class SpiralDB> { private image: SpiralImage; private schema: SchemaDefinition; private index: MasterIndex; private dataStartRing: number; private currentRing: number; private currentOffset: number; private compression: boolean; constructor(options: SpiralDBOptions) { this.schema = options.schema; this.compression = options.compression ?? false; // Initialize with minimum size for header + schema + index this.dataStartRing = RING_DATA_START; const initialRing = Math.max(this.dataStartRing, options.initialSize ?? this.dataStartRing); this.image = createImageForRing(initialRing); // Initialize empty index this.index = { records: [], deletedIds: new Set(), nextId: 0, }; // Start writing data after index ring this.currentRing = this.dataStartRing; this.currentOffset = 0; // Write initial structure this.initializeDatabase(); } /** * Initialize the database structure (magic, header, schema) */ private initializeDatabase(): void { // Ring 0: Magic byte setPixelByIndex(this.image, 0, MAGIC_VALID as ColorIndex); // Ring 1: Header this.writeHeader(); // Ring 2: Schema this.writeSchema(); // Ring 3: Index (initially empty) this.writeIndex(); } /** * Write the database header to Ring 1 */ private writeHeader(): void { const stream = createBitStream(); // Version (9 bits) writeBits(stream, this.schema.version, 9); // Flags (3 bits) writeBits(stream, FLAG_READABLE, 3); // Record count (12 bits) writeBits(stream, this.index.records.length, 12); const pixels = bitsToPixels(stream.bits); const ringInfo = getRingInfo(RING_HEADER); writePixelRange(this.image, ringInfo.startIndex, pixels); } /** * Write the schema to Ring 2 */ private writeSchema(): void { const schemaPixels = encodeSchema(this.schema); const ringInfo = getRingInfo(RING_SCHEMA); writePixelRange(this.image, ringInfo.startIndex, schemaPixels); } /** * Calculate how many rings the index needs for a given record count. */ private getIndexRingsNeeded(recordCount: number): number { // Header: 12 bits (count) + 12 bits (nextId) = 24 bits // Per entry: 12+8+8+9+3 = 40 bits const totalBits = 24 + recordCount * 40; const totalPixels = Math.ceil(totalBits / 3); let ringsNeeded = 0; let pixelsAvailable = 0; let ring = RING_INDEX; while (pixelsAvailable < totalPixels) { pixelsAvailable += getRingInfo(ring).pixelCount; ring++; ringsNeeded++; } return ringsNeeded; } /** * Ensure there's room for additional index entries without overlapping data. * If the index would overflow into the data region, rebuild the database * with a higher dataStartRing. */ private ensureIndexCapacity(additionalEntries: number): void { const futureCount = this.index.records.length + additionalEntries; const ringsNeeded = this.getIndexRingsNeeded(futureCount); const requiredDataStart = RING_INDEX + ringsNeeded; if (requiredDataStart > this.dataStartRing) { // Need to rebuild: collect all current records, recreate with more index space const activeRecords = this.getAll('active'); const completedRecords = this.getAll('completed'); const allRecords = [...activeRecords, ...completedRecords]; // Reset with new data start this.dataStartRing = requiredDataStart; this.image = createImageForRing(this.dataStartRing); this.index = { records: [], deletedIds: new Set(), nextId: 0 }; this.currentRing = this.dataStartRing; this.currentOffset = 0; this.initializeDatabase(); // Re-insert all records for (const record of allRecords) { const id = this.index.nextId++; const pixels = this.serializeRecord(id, record.meta.status, record.data); const space = findSpaceForRecord(this.currentRing, this.currentOffset, pixels.length); if (space.ring > Math.floor(this.image.width / 2)) { this.image = expandImage(this.image, space.ring); } const ringInfo = getRingInfo(space.ring); const startIndex = ringInfo.startIndex + space.offset; writePixelRange(this.image, startIndex, pixels); this.index.records.push({ id, ring: space.ring, offset: space.offset, length: pixels.length, status: record.meta.status, }); this.currentRing = space.ring; this.currentOffset = space.offset + pixels.length; if (this.currentOffset >= ringInfo.pixelCount) { this.currentRing++; this.currentOffset = 0; } } this.writeHeader(); this.writeIndex(); } } /** * Write the master index to Ring 3+ * Index can span multiple rings if needed */ private writeIndex(): void { const stream = createBitStream(); // Record count (12 bits) writeBits(stream, this.index.records.length, 12); // Next ID (12 bits) writeBits(stream, this.index.nextId, 12); // Each index entry: [id:12][ring:8][offset:8][length:9][status:3] = 40 bits for (const entry of this.index.records) { writeBits(stream, entry.id, 12); writeBits(stream, entry.ring, 8); writeBits(stream, entry.offset, 8); writeBits(stream, entry.length, 9); writeBits(stream, STATUS_BITS[entry.status], 3); } const pixels = bitsToPixels(stream.bits); // Write index pixels starting at Ring 3 let pixelIndex = 0; let currentRing = RING_INDEX; while (pixelIndex < pixels.length) { // Expand image if this ring doesn't fit const maxRing = Math.floor(this.image.width / 2); if (currentRing > maxRing) { this.image = expandImage(this.image, currentRing); } const ringInfo = getRingInfo(currentRing); const pixelsInRing = Math.min(pixels.length - pixelIndex, ringInfo.pixelCount); writePixelRange( this.image, ringInfo.startIndex, pixels.slice(pixelIndex, pixelIndex + pixelsInRing) ); pixelIndex += pixelsInRing; currentRing++; } // Store how many rings the index spans (for loading) // Encode as 2 pixels (6 bits, max 63 rings) at end of Ring 2 const indexRingCount = currentRing - RING_INDEX; const ring2Info = getRingInfo(RING_SCHEMA); const countPixelIndex = ring2Info.startIndex + ring2Info.pixelCount - 2; // High 3 bits in second-to-last pixel, low 3 bits in last pixel setPixelByIndex(this.image, countPixelIndex, ((indexRingCount >> 3) & 0x7) as ColorIndex); setPixelByIndex(this.image, countPixelIndex + 1, (indexRingCount & 0x7) as ColorIndex); } /** * Serialize a record to pixels */ private serializeRecord(id: number, status: RecordStatus, data: T): ColorIndex[] { const stream = createBitStream(); // Record ID (12 bits) writeBits(stream, id, 12); // Status (3 bits) writeBits(stream, STATUS_BITS[status], 3); // Encode each field according to schema for (const field of this.schema.fields) { const value = (data as Record)[field.name]; // Null flag for nullable fields if (field.nullable) { const isNull = value === null || value === undefined; encodeBool(stream, isNull); if (isNull) continue; } switch (field.type) { case 'int': encodeInt(stream, value as number, field.maxLength); break; case 'string': encodeString(stream, value as string, this.compression); break; case 'bool': encodeBool(stream, value as boolean); break; case 'timestamp': encodeTimestamp(stream, value as Date | null); break; case 'array': if (Array.isArray(value) && typeof value[0] === 'number') { encodeIntArray(stream, value as number[], 12); } else if (Array.isArray(value) && typeof value[0] === 'string') { // For string arrays, encode each string writeBits(stream, value.length, 8); for (const str of value as string[]) { encodeString(stream, str, this.compression); } } else { writeBits(stream, 0, 8); // Empty array } break; } } // End marker writeBits(stream, END_MARKER, 3); return bitsToPixels(stream.bits); } /** * Deserialize pixels to a record */ private deserializeRecord(pixels: ColorIndex[]): { id: number; status: RecordStatus; data: T } { const bits = pixelsToBits(pixels); const stream: BitStream = { bits, position: 0 }; // Record ID const id = readBits(stream, 12); // Status const statusBits = readBits(stream, 3); const status = BITS_TO_STATUS[statusBits] || 'active'; // Decode each field const data: Record = {}; for (const field of this.schema.fields) { // Null flag for nullable fields if (field.nullable) { const isNull = decodeBool(stream); if (isNull) { data[field.name] = null; continue; } } switch (field.type) { case 'int': data[field.name] = decodeInt(stream, field.maxLength); break; case 'string': data[field.name] = decodeString(stream); break; case 'bool': data[field.name] = decodeBool(stream); break; case 'timestamp': data[field.name] = decodeTimestamp(stream); break; case 'array': if (field.name === 'tags') { data[field.name] = decodeIntArray(stream, 12); } else { const count = readBits(stream, 8); const arr: string[] = []; for (let i = 0; i < count; i++) { arr.push(decodeString(stream)); } data[field.name] = arr; } break; } } return { id, status, data: data as T }; } /** * Insert a new record */ insert(data: T): WriteResult { try { // Validate record against schema before writing const validation = validateRecord(this.schema, data as unknown as Record); if (!validation.valid) { return { success: false, error: `Validation failed: ${validation.errors.join('; ')}` }; } // Ensure index has room for one more entry without overlapping data this.ensureIndexCapacity(1); const id = this.index.nextId++; const pixels = this.serializeRecord(id, 'active', data); if (pixels.length > MAX_RECORD_LENGTH) { return { success: false, error: 'Record too large' }; } // Find space for the record const space = findSpaceForRecord(this.currentRing, this.currentOffset, pixels.length); // Expand image if needed (check both ring advancement and image bounds) const maxRing = Math.floor(this.image.width / 2); if (space.ring > maxRing) { this.image = expandImage(this.image, space.ring); } // Calculate absolute pixel index const ringInfo = getRingInfo(space.ring); const startIndex = ringInfo.startIndex + space.offset; // Write record pixels writePixelRange(this.image, startIndex, pixels); // Update index const entry: IndexEntry = { id, ring: space.ring, offset: space.offset, length: pixels.length, status: 'active', }; this.index.records.push(entry); // Update position this.currentRing = space.ring; this.currentOffset = space.offset + pixels.length; // Check if we filled this ring if (this.currentOffset >= ringInfo.pixelCount) { this.currentRing++; this.currentOffset = 0; } // Update header and index in image this.writeHeader(); this.writeIndex(); return { success: true, recordId: id, newImageSize: space.needsExpansion ? this.image.width : undefined, }; } catch (error) { return { success: false, error: String(error) }; } } /** * Read a record by ID */ read(id: number): ReadResult { // Find the latest non-deleted entry for this ID // (update creates a new entry with the same ID, so we search from the end) let entry: IndexEntry | undefined; for (let i = this.index.records.length - 1; i >= 0; i--) { if (this.index.records[i].id === id) { entry = this.index.records[i]; break; } } if (!entry) { return { success: false, error: 'Record not found' }; } if (entry.status === 'deleted') { return { success: false, error: 'Record has been deleted' }; } try { const ringInfo = getRingInfo(entry.ring); const startIndex = ringInfo.startIndex + entry.offset; const pixels = readPixelRange(this.image, startIndex, entry.length); const { id: recordId, status, data } = this.deserializeRecord(pixels); const record: SpiralRecord = { meta: { id: recordId, status, createdAt: entry.ring, ringStart: entry.ring, pixelOffset: entry.offset, length: entry.length, }, data, }; return { success: true, record }; } catch (error) { return { success: false, error: String(error) }; } } /** * Update a record (creates new version, marks old as deleted) */ update(id: number, data: Partial): WriteResult { const readResult = this.read(id); if (!readResult.success || !readResult.record) { return { success: false, error: readResult.error }; } // Mark old record as deleted this.delete(id); // Insert new record with same ID (reuse ID) this.index.nextId--; // Revert increment from delete const mergedData = { ...readResult.record.data, ...data } as T; // Need to manually set the ID const pixels = this.serializeRecord(id, 'active', mergedData); const space = findSpaceForRecord(this.currentRing, this.currentOffset, pixels.length); if (space.ring > Math.floor(this.image.width / 2)) { this.image = expandImage(this.image, space.ring); } const ringInfo = getRingInfo(space.ring); const startIndex = ringInfo.startIndex + space.offset; writePixelRange(this.image, startIndex, pixels); // Add new index entry const entry: IndexEntry = { id, ring: space.ring, offset: space.offset, length: pixels.length, status: 'active', }; this.index.records.push(entry); this.currentRing = space.ring; this.currentOffset = space.offset + pixels.length; this.writeHeader(); this.writeIndex(); return { success: true, recordId: id }; } /** * Delete a record (marks as deleted, doesn't remove) */ delete(id: number): WriteResult { const entryIndex = this.index.records.findIndex((r) => r.id === id && r.status !== 'deleted'); if (entryIndex === -1) { return { success: false, error: 'Record not found' }; } // Update status in index this.index.records[entryIndex].status = 'deleted'; this.index.deletedIds.add(id); // Update status pixel in record const entry = this.index.records[entryIndex]; const ringInfo = getRingInfo(entry.ring); const statusPixelIndex = ringInfo.startIndex + entry.offset + 4; // After 12-bit ID setPixelByIndex(this.image, statusPixelIndex, STATUS_BITS['deleted'] as ColorIndex); this.writeHeader(); this.writeIndex(); return { success: true, recordId: id }; } /** * Mark a record as completed */ complete(id: number): WriteResult { const entryIndex = this.index.records.findIndex((r) => r.id === id && r.status === 'active'); if (entryIndex === -1) { return { success: false, error: 'Active record not found' }; } this.index.records[entryIndex].status = 'completed'; // Update status pixel const entry = this.index.records[entryIndex]; const ringInfo = getRingInfo(entry.ring); const statusPixelIndex = ringInfo.startIndex + entry.offset + 4; setPixelByIndex(this.image, statusPixelIndex, STATUS_BITS['completed'] as ColorIndex); this.writeIndex(); return { success: true, recordId: id }; } /** * Get all records (optionally filtered by status) */ getAll(status?: RecordStatus): SpiralRecord[] { const records: SpiralRecord[] = []; for (const entry of this.index.records) { if (status && entry.status !== status) continue; if (entry.status === 'deleted') continue; const result = this.read(entry.id); if (result.success && result.record) { records.push(result.record); } } return records; } /** * Get the current image */ getImage(): SpiralImage { return this.image; } /** * Get database statistics */ getStats(): { imageSize: number; totalPixels: number; usedPixels: number; totalRecords: number; activeRecords: number; deletedRecords: number; currentRing: number; } { const activeRecords = this.index.records.filter((r) => r.status === 'active').length; const deletedRecords = this.index.records.filter((r) => r.status === 'deleted').length; const usedPixels = this.index.records.reduce((sum, r) => sum + r.length, 0); const headerPixels = getTotalPixelsForRing(this.dataStartRing - 1); return { imageSize: this.image.width, totalPixels: this.image.width * this.image.height, usedPixels: usedPixels + headerPixels, totalRecords: this.index.records.length, activeRecords, deletedRecords, currentRing: this.currentRing, }; } /** * Compact the database (remove deleted records) */ compact(): SpiralImage { const activeRecords = this.getAll('active'); const completedRecords = this.getAll('completed'); const allRecords = [...activeRecords, ...completedRecords]; // Create new database with same schema const newDb = new SpiralDB({ schema: this.schema, compression: this.compression, }); // Re-insert all records for (const record of allRecords) { newDb.insert(record.data); if (record.meta.status === 'completed') { newDb.complete(record.meta.id); } } this.image = newDb.image; this.index = newDb.index; this.dataStartRing = newDb.dataStartRing; this.currentRing = newDb.currentRing; this.currentOffset = newDb.currentOffset; return this.image; } /** * Load database from an existing image */ static fromImage(image: SpiralImage, schema: SchemaDefinition): SpiralDB { const db = new SpiralDB({ schema }); db.image = image; // Validate magic byte const magic = getPixelByIndex(image, 0); if (magic !== MAGIC_VALID) { throw new Error('Invalid SpiralDB image (magic byte mismatch)'); } // Read index from Ring 3 db.loadIndex(); return db; } /** * Load index from image */ private loadIndex(): void { // Read index ring count from last 2 pixels of Ring 2 (6 bits, max 63) const ring2Info = getRingInfo(RING_SCHEMA); const countPixelIndex = ring2Info.startIndex + ring2Info.pixelCount - 2; const highBits = getPixelByIndex(this.image, countPixelIndex); const lowBits = getPixelByIndex(this.image, countPixelIndex + 1); const indexRingCount = (highBits << 3) | lowBits || 1; // Set data start ring based on stored index size this.dataStartRing = RING_INDEX + indexRingCount; // Read pixels from all index rings const allPixels: ColorIndex[] = []; for (let r = 0; r < indexRingCount; r++) { const ringInfo = getRingInfo(RING_INDEX + r); const ringPixels = readPixelRange(this.image, ringInfo.startIndex, ringInfo.pixelCount); allPixels.push(...ringPixels); } const bits = pixelsToBits(allPixels); const stream: BitStream = { bits, position: 0 }; const recordCount = readBits(stream, 12); const nextId = readBits(stream, 12); this.index = { records: [], deletedIds: new Set(), nextId, }; for (let i = 0; i < recordCount; i++) { const id = readBits(stream, 12); const ring = readBits(stream, 8); const offset = readBits(stream, 8); const length = readBits(stream, 9); const statusBits = readBits(stream, 3); const status = BITS_TO_STATUS[statusBits] || 'active'; this.index.records.push({ id, ring, offset, length, status }); if (status === 'deleted') { this.index.deletedIds.add(id); } } // Find current write position if (this.index.records.length > 0) { const lastRecord = this.index.records[this.index.records.length - 1]; this.currentRing = lastRecord.ring; this.currentOffset = lastRecord.offset + lastRecord.length; const lastRingInfo = getRingInfo(this.currentRing); if (this.currentOffset >= lastRingInfo.pixelCount) { this.currentRing++; this.currentOffset = 0; } } else { this.currentRing = this.dataStartRing; this.currentOffset = 0; } } }