feat(local-first): add local-first architecture with Dexie.js, Go sync server, and Todo pilot

Implement the foundational local-first data layer for ManaCore apps:

- New @manacore/local-store package (Dexie.js IndexedDB, sync engine, Svelte 5 reactive queries)
- New mana-sync Go service (sync protocol, WebSocket push, field-level LWW conflict resolution)
- Todo app migrated as pilot: stores read/write IndexedDB, guest mode with onboarding seed data
- PillNavigation: prominent login pill for unauthenticated users
- SyncIndicator component showing local/syncing/offline status
- GuestWelcomeModal on first visit for Todo app
- Removed demo-mode auth_required checks from Todo components (all writes are now local)
- CSP fix for local development (localhost:3001, localhost:3050)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS 2026-03-27 11:17:58 +01:00
parent 4ddff8485b
commit 2e4bb9bad7
41 changed files with 4388 additions and 340 deletions

View file

@ -0,0 +1,28 @@
{
"name": "@manacore/local-store",
"version": "0.1.0",
"private": true,
"description": "Local-first data layer with Dexie.js, reactive Svelte 5 queries, and sync engine",
"main": "./src/index.ts",
"types": "./src/index.ts",
"exports": {
".": "./src/index.ts",
"./svelte": "./src/svelte/index.ts",
"./sync": "./src/sync/index.ts"
},
"scripts": {
"type-check": "tsc --noEmit",
"clean": "rm -rf dist"
},
"dependencies": {
"dexie": "^4.4.1"
},
"devDependencies": {
"@types/node": "^24.10.1",
"svelte": "^5.0.0",
"typescript": "^5.9.3"
},
"peerDependencies": {
"svelte": "^5.0.0"
}
}

View file

@ -0,0 +1,304 @@
/**
* LocalCollection typed, reactive collection backed by IndexedDB.
*
* Wraps a Dexie table with:
* - Change tracking (writes are recorded as PendingChanges for sync)
* - Soft-delete (deletedAt instead of hard delete)
* - Timestamp management (updatedAt per record, field_timestamps for LWW)
*
* All writes are synchronous from the caller's perspective they return
* immediately after queuing the IndexedDB write. No network call is needed.
*/
import type Dexie from 'dexie';
import type { Table } from 'dexie';
import type { LocalDatabase } from './database.js';
import type { BaseRecord, ChangeOp, FieldChange, PendingChange, QueryOptions } from './types.js';
export class LocalCollection<T extends BaseRecord> {
readonly name: string;
private readonly _db: LocalDatabase;
private readonly _table: Table<T, string>;
/** Called after every write to notify the sync engine. Set by LocalStore. */
onWrite: (() => void) | null = null;
constructor(db: LocalDatabase, name: string) {
this.name = name;
this._db = db;
this._table = db.table(name);
}
/** Access the underlying Dexie table for advanced queries. */
get table(): Table<T, string> {
return this._table;
}
/** Access the parent database. */
get db(): LocalDatabase {
return this._db;
}
// ─── Reads ──────────────────────────────────────────────────
/**
* Get a single record by ID. Returns undefined if not found or soft-deleted.
*/
async get(id: string): Promise<T | undefined> {
const record = await this._table.get(id);
if (!record || record.deletedAt) return undefined;
return record;
}
/**
* Get all non-deleted records, optionally filtered and sorted.
*/
async getAll(filter?: Partial<T>, options?: QueryOptions<T>): Promise<T[]> {
let collection: Dexie.Collection<T, string>;
if (filter && Object.keys(filter).length > 0) {
// Use the first filter key as an indexed where clause
const entries = Object.entries(filter);
const [firstKey, firstValue] = entries[0];
collection = this._table.where(firstKey).equals(firstValue as string);
// Apply remaining filters as JS filters
for (let i = 1; i < entries.length; i++) {
const [key, value] = entries[i];
collection = collection.and((item) => (item as Record<string, unknown>)[key] === value);
}
} else {
collection = this._table.toCollection();
}
// Exclude soft-deleted
collection = collection.filter((item) => !item.deletedAt);
let results: T[];
if (options?.sortBy) {
// Dexie doesn't support sorting on filtered collections directly,
// so we get all matching, then sort in JS
results = await collection.toArray();
const key = options.sortBy as string;
const dir = options.sortDirection === 'desc' ? -1 : 1;
results.sort((a, b) => {
const aVal = (a as Record<string, unknown>)[key];
const bVal = (b as Record<string, unknown>)[key];
if (aVal == null && bVal == null) return 0;
if (aVal == null) return dir;
if (bVal == null) return -dir;
return aVal < bVal ? -dir : aVal > bVal ? dir : 0;
});
} else {
results = await collection.toArray();
}
if (options?.offset) {
results = results.slice(options.offset);
}
if (options?.limit) {
results = results.slice(0, options.limit);
}
return results;
}
/**
* Count non-deleted records matching an optional filter.
*/
async count(filter?: Partial<T>): Promise<number> {
if (!filter) {
return this._table.filter((item) => !item.deletedAt).count();
}
const results = await this.getAll(filter);
return results.length;
}
// ─── Writes ─────────────────────────────────────────────────
/**
* Insert a new record. Generates timestamps and tracks the change.
*/
async insert(record: T): Promise<T> {
const now = new Date().toISOString();
const withMeta: T = {
...record,
createdAt: record.createdAt ?? now,
updatedAt: now,
deletedAt: null,
};
await this._db.transaction('rw', [this._table, this._db._pendingChanges], async () => {
await this._table.put(withMeta);
await this._trackChange(record.id, 'insert', undefined, withMeta);
});
this.onWrite?.();
return withMeta;
}
/**
* Insert multiple records in a single transaction.
*/
async bulkInsert(records: T[]): Promise<T[]> {
const now = new Date().toISOString();
const withMeta = records.map((r) => ({
...r,
createdAt: r.createdAt ?? now,
updatedAt: now,
deletedAt: null,
}));
await this._db.transaction('rw', [this._table, this._db._pendingChanges], async () => {
await this._table.bulkPut(withMeta);
for (const record of withMeta) {
await this._trackChange(record.id, 'insert', undefined, record);
}
});
this.onWrite?.();
return withMeta;
}
/**
* Update specific fields of a record. Only changed fields are tracked.
*/
async update(id: string, changes: Partial<T>): Promise<T | undefined> {
const now = new Date().toISOString();
// Remove meta fields from changes — we manage those
const {
id: _id,
createdAt: _c,
updatedAt: _u,
deletedAt: _d,
...fieldChanges
} = changes as Record<string, unknown>;
const fields: Record<string, FieldChange> = {};
for (const [key, value] of Object.entries(fieldChanges)) {
fields[key] = { value, updatedAt: now };
}
let updated: T | undefined;
await this._db.transaction('rw', [this._table, this._db._pendingChanges], async () => {
const existing = await this._table.get(id);
if (!existing || existing.deletedAt) return;
updated = { ...existing, ...fieldChanges, updatedAt: now } as T;
await this._table.put(updated);
await this._trackChange(id, 'update', fields);
});
this.onWrite?.();
return updated;
}
/**
* Soft-delete a record. The record stays in IndexedDB but is excluded from queries.
*/
async delete(id: string): Promise<void> {
const now = new Date().toISOString();
await this._db.transaction('rw', [this._table, this._db._pendingChanges], async () => {
const existing = await this._table.get(id);
if (!existing || existing.deletedAt) return;
const deleted = { ...existing, deletedAt: now, updatedAt: now };
await this._table.put(deleted);
await this._trackChange(id, 'delete', undefined, undefined, now);
});
this.onWrite?.();
}
/**
* Hard-delete a record. Used for purging old soft-deleted records.
*/
async purge(id: string): Promise<void> {
await this._table.delete(id);
}
// ─── Sync Helpers ───────────────────────────────────────────
/**
* Apply a server change to the local collection (used by SyncEngine).
* Does NOT create a PendingChange (to avoid re-syncing back to server).
*/
async applyServerChange(change: {
id: string;
op: ChangeOp;
data?: Record<string, unknown>;
fields?: Record<string, FieldChange>;
deletedAt?: string;
}): Promise<void> {
switch (change.op) {
case 'insert': {
if (change.data) {
await this._table.put(change.data as T);
}
break;
}
case 'update': {
if (change.fields) {
const existing = await this._table.get(change.id);
if (!existing) {
// Record doesn't exist locally — treat as insert if we have full data
if (change.data) {
await this._table.put(change.data as T);
}
break;
}
const updates: Record<string, unknown> = {};
for (const [key, fc] of Object.entries(change.fields)) {
updates[key] = fc.value;
}
updates['updatedAt'] = Object.values(change.fields).reduce(
(latest, fc) => (fc.updatedAt > latest ? fc.updatedAt : latest),
existing.updatedAt ?? ''
);
await this._table.put({ ...existing, ...updates } as T);
}
break;
}
case 'delete': {
const now = change.deletedAt ?? new Date().toISOString();
const toDelete = await this._table.get(change.id);
if (toDelete) {
await this._table.put({ ...toDelete, deletedAt: now, updatedAt: now });
}
break;
}
}
}
/**
* Get all records modified since a timestamp (for building changesets).
*/
async getModifiedSince(since: string): Promise<T[]> {
return this._table.where('updatedAt').above(since).toArray();
}
// ─── Internal ───────────────────────────────────────────────
private async _trackChange(
recordId: string,
op: ChangeOp,
fields?: Record<string, FieldChange>,
data?: T,
deletedAt?: string
): Promise<void> {
const pending: PendingChange = {
collection: this.name,
recordId,
op,
fields,
data: data as unknown as Record<string, unknown>,
deletedAt,
createdAt: new Date().toISOString(),
};
await this._db._pendingChanges.add(pending);
}
}

View file

@ -0,0 +1,153 @@
/**
* Local database management via Dexie.js.
*
* Each app gets its own IndexedDB database with:
* - App-specific collections (tables)
* - A shared _pendingChanges table for sync tracking
* - A shared _syncMeta table for sync cursors
*/
import Dexie from 'dexie';
import type { BaseRecord, CollectionConfig, PendingChange, SyncMeta } from './types.js';
/**
* Creates a Dexie database for an app with the given collections.
*
* @example
* ```ts
* const db = createDatabase('todo', [
* { name: 'tasks', indexes: ['projectId', 'dueDate', '[isCompleted+dueDate]'] },
* { name: 'projects', indexes: ['order'] },
* { name: 'labels', indexes: [] },
* ]);
* ```
*/
export function createDatabase(
appId: string,
collections: CollectionConfig<BaseRecord>[]
): LocalDatabase {
const db = new LocalDatabase(appId, collections);
return db;
}
export class LocalDatabase extends Dexie {
/** Pending changes waiting to be synced to the server. */
_pendingChanges!: Dexie.Table<PendingChange, number>;
/** Sync metadata per collection (last sync timestamp, etc.). */
_syncMeta!: Dexie.Table<SyncMeta, string>;
private readonly _appId: string;
private readonly _collections: CollectionConfig<BaseRecord>[];
private _seeded = false;
constructor(appId: string, collections: CollectionConfig<BaseRecord>[]) {
super(`manacore-${appId}`);
this._appId = appId;
this._collections = collections;
// Build Dexie schema from collection configs
const schema: Record<string, string> = {
// Internal tables
_pendingChanges: '++id, collection, recordId, createdAt',
_syncMeta: 'collection',
};
for (const col of collections) {
// Primary key is always 'id', plus any additional indexes
const indexes = ['id', ...(col.indexes ?? [])];
// Add updatedAt and deletedAt for sync queries
indexes.push('updatedAt', 'deletedAt');
schema[col.name] = indexes.join(', ');
}
this.version(1).stores(schema);
}
get appId(): string {
return this._appId;
}
/**
* Load guest seed data into empty collections.
* Only runs once per database lifetime.
*/
async seedGuestData(): Promise<void> {
if (this._seeded) return;
this._seeded = true;
for (const col of this._collections) {
if (!col.guestSeed || col.guestSeed.length === 0) continue;
const table = this.table(col.name);
const count = await table.count();
if (count === 0) {
const now = new Date().toISOString();
const records = col.guestSeed.map((record) => ({
...record,
createdAt: record.createdAt ?? now,
updatedAt: record.updatedAt ?? now,
deletedAt: null,
}));
await table.bulkPut(records);
}
}
}
/**
* Get the sync cursor (last synced timestamp) for a collection.
*/
async getSyncCursor(collection: string): Promise<string> {
const meta = await this._syncMeta.get(collection);
// Default: epoch — pull everything on first sync
return meta?.lastSyncedAt ?? '1970-01-01T00:00:00.000Z';
}
/**
* Update the sync cursor after a successful sync.
*/
async setSyncCursor(collection: string, syncedUntil: string): Promise<void> {
const pendingCount = await this._pendingChanges.where('collection').equals(collection).count();
await this._syncMeta.put({
collection,
lastSyncedAt: syncedUntil,
pendingCount,
});
}
/**
* Get the number of pending (un-synced) changes across all collections.
*/
async getPendingCount(): Promise<number> {
return this._pendingChanges.count();
}
/**
* Get pending changes for a specific collection, ordered by creation time.
*/
async getPendingChanges(collection: string): Promise<PendingChange[]> {
return this._pendingChanges.where('collection').equals(collection).sortBy('createdAt');
}
/**
* Clear pending changes that have been successfully synced.
*/
async clearPendingChanges(ids: number[]): Promise<void> {
await this._pendingChanges.bulkDelete(ids);
}
/**
* Wipe all data and re-seed. Used for recovery from corruption.
*/
async reset(): Promise<void> {
for (const col of this._collections) {
await this.table(col.name).clear();
}
await this._pendingChanges.clear();
await this._syncMeta.clear();
this._seeded = false;
await this.seedGuestData();
}
}

View file

@ -0,0 +1,28 @@
// Core
export { createLocalStore, LocalStore } from './store.js';
export type { LocalStoreConfig } from './store.js';
// Database
export { createDatabase, LocalDatabase } from './database.js';
// Collection
export { LocalCollection } from './collection.js';
// Types
export type {
BaseRecord,
Change,
ChangeOp,
Changeset,
CollectionConfig,
ConflictStrategy,
FieldChange,
PendingChange,
QueryOptions,
SortDirection,
SyncConfig,
SyncConflict,
SyncMeta,
SyncResponse,
SyncStatus,
} from './types.js';

View file

@ -0,0 +1,220 @@
/**
* LocalStore the main entry point for apps.
*
* Creates a complete local-first data layer for an app:
* - IndexedDB database (via Dexie.js)
* - Typed collections with change tracking
* - Sync engine (started/stopped based on auth state)
*
* @example
* ```ts
* import { createLocalStore } from '@manacore/local-store';
*
* const store = createLocalStore({
* appId: 'todo',
* collections: [
* { name: 'tasks', indexes: ['projectId', 'dueDate'] },
* { name: 'projects', indexes: ['order'] },
* { name: 'labels' },
* ],
* sync: {
* serverUrl: 'http://localhost:3050',
* },
* });
*
* // Get typed collections
* const tasks = store.collection<Task>('tasks');
* const projects = store.collection<Project>('projects');
*
* // Guest mode: just use collections, no sync
* await tasks.insert({ id: crypto.randomUUID(), title: 'Hello', ... });
*
* // After login: start sync
* store.startSync(() => authStore.getValidToken());
*
* // On logout: stop sync
* store.stopSync();
* ```
*/
import { LocalCollection } from './collection.js';
import { createDatabase, type LocalDatabase } from './database.js';
import { SyncEngine } from './sync/engine.js';
import type { BaseRecord, CollectionConfig, SyncStatus } from './types.js';
/** Client ID persisted in localStorage for device identification. */
function getOrCreateClientId(): string {
const key = 'manacore-client-id';
if (typeof localStorage === 'undefined') return 'ssr-' + Math.random().toString(36).slice(2);
let clientId = localStorage.getItem(key);
if (!clientId) {
clientId = crypto.randomUUID();
localStorage.setItem(key, clientId);
}
return clientId;
}
export interface LocalStoreConfig {
/** App identifier (e.g. 'todo', 'contacts'). Used as IndexedDB database name. */
appId: string;
/** Collection (table) definitions. */
collections: CollectionConfig<BaseRecord>[];
/** Sync server configuration. If omitted, sync is disabled (pure offline). */
sync?: {
/** Sync server base URL (e.g. 'http://localhost:3050'). */
serverUrl: string;
/** Debounce before pushing changes. Default: 1000ms. */
pushDebounce?: number;
/** Pull interval. Default: 30000ms. */
pullInterval?: number;
/** WebSocket URL (defaults to serverUrl with ws:// protocol). */
wsUrl?: string;
};
}
export class LocalStore {
readonly db: LocalDatabase;
readonly appId: string;
private readonly _collections: Map<string, LocalCollection<BaseRecord>> = new Map();
private _syncEngine: SyncEngine | null = null;
private readonly _syncConfig: LocalStoreConfig['sync'];
constructor(config: LocalStoreConfig) {
this.appId = config.appId;
this._syncConfig = config.sync;
// Create database
this.db = createDatabase(config.appId, config.collections);
// Create collections with write notifications
for (const colConfig of config.collections) {
const collection = new LocalCollection(this.db, colConfig.name);
collection.onWrite = () => this.schedulePush();
this._collections.set(colConfig.name, collection);
}
}
/**
* Initialize the store: open database and seed guest data.
* Call this once on app startup.
*/
async initialize(): Promise<void> {
await this.db.open();
await this.db.seedGuestData();
}
/**
* Get a typed collection by name.
*/
collection<T extends BaseRecord>(name: string): LocalCollection<T> {
const col = this._collections.get(name);
if (!col) {
throw new Error(`[LocalStore] Collection "${name}" not found in app "${this.appId}"`);
}
return col as unknown as LocalCollection<T>;
}
/**
* Start syncing to the server. Call after user authenticates.
*
* @param getAuthToken function that returns a valid JWT (or null).
*/
startSync(getAuthToken: () => Promise<string | null>): void {
if (!this._syncConfig) {
console.warn('[LocalStore] Sync not configured. Skipping startSync().');
return;
}
if (this._syncEngine) {
// Already running
return;
}
this._syncEngine = new SyncEngine(this.db, {
serverUrl: this._syncConfig.serverUrl,
appId: this.appId,
clientId: getOrCreateClientId(),
getAuthToken,
pushDebounce: this._syncConfig.pushDebounce,
pullInterval: this._syncConfig.pullInterval,
wsUrl: this._syncConfig.wsUrl,
});
// Register all collections
for (const col of this._collections.values()) {
this._syncEngine.registerCollection(col);
}
this._syncEngine.start();
}
/**
* Stop syncing. Call on sign-out.
*/
stopSync(): void {
this._syncEngine?.stop();
this._syncEngine = null;
}
/**
* Get the sync engine (or null if not syncing).
* Used by useSyncStatus() Svelte hook.
*/
get syncEngine(): SyncEngine | null {
return this._syncEngine;
}
/**
* Current sync status.
*/
get syncStatus(): SyncStatus {
return this._syncEngine?.status ?? 'idle';
}
/**
* Whether the sync engine is running.
*/
get isSyncing(): boolean {
return this._syncEngine?.enabled ?? false;
}
/**
* Schedule a push of local changes to the server.
* Called automatically by collections on write, but can be triggered manually.
*/
schedulePush(): void {
this._syncEngine?.schedulePush();
}
/**
* Trigger an immediate full sync.
*/
async sync(): Promise<void> {
await this._syncEngine?.sync();
}
/**
* Wipe all local data and re-seed. Use for recovery or sign-out cleanup.
*/
async reset(): Promise<void> {
this.stopSync();
await this.db.reset();
}
/**
* Close the database connection.
*/
close(): void {
this.stopSync();
this.db.close();
}
}
/**
* Create a LocalStore instance.
*/
export function createLocalStore(config: LocalStoreConfig): LocalStore {
return new LocalStore(config);
}

View file

@ -0,0 +1,2 @@
export { useLiveQuery, useLiveQueryWithDefault } from './reactive.svelte.js';
export { useSyncStatus } from './useSyncStatus.svelte.js';

View file

@ -0,0 +1,113 @@
/**
* Svelte 5 reactive bindings for LocalCollection.
*
* Uses Dexie's liveQuery() to create reactive queries that automatically
* update when the underlying IndexedDB data changes whether from local
* writes, sync engine updates, or other browser tabs.
*
* @example
* ```svelte
* <script lang="ts">
* import { useLiveQuery } from '@manacore/local-store/svelte';
*
* const tasks = useLiveQuery(() => taskCollection.getAll({ isCompleted: false }));
* </script>
*
* {#each tasks.value ?? [] as task}
* <div>{task.title}</div>
* {/each}
* ```
*/
import { liveQuery, type Observable } from 'dexie';
import { onDestroy } from 'svelte';
interface LiveQueryResult<T> {
/** The current query result. Undefined while loading. */
readonly value: T | undefined;
/** Whether the query is still loading its first result. */
readonly loading: boolean;
/** Error from the last query execution, if any. */
readonly error: unknown;
}
/**
* Create a reactive query that subscribes to IndexedDB changes.
*
* The querier function is re-executed whenever the underlying Dexie tables
* it reads from are modified. This works across tabs and from sync updates.
*
* Must be called during component initialization (like onMount).
*/
export function useLiveQuery<T>(querier: () => T | Promise<T>): LiveQueryResult<T> {
let value = $state<T | undefined>(undefined);
let loading = $state(true);
let error = $state<unknown>(undefined);
const observable: Observable<T> = liveQuery(querier);
const subscription = observable.subscribe({
next: (result) => {
value = result;
loading = false;
error = undefined;
},
error: (err) => {
error = err;
loading = false;
},
});
onDestroy(() => {
subscription.unsubscribe();
});
return {
get value() {
return value;
},
get loading() {
return loading;
},
get error() {
return error;
},
};
}
/**
* Create a reactive query with an initial value (no undefined/loading state).
* Useful when you have sensible defaults.
*/
export function useLiveQueryWithDefault<T>(
querier: () => T | Promise<T>,
defaultValue: T
): { readonly value: T; readonly error: unknown } {
let value = $state<T>(defaultValue);
let error = $state<unknown>(undefined);
const observable: Observable<T> = liveQuery(querier);
const subscription = observable.subscribe({
next: (result) => {
value = result;
error = undefined;
},
error: (err) => {
error = err;
},
});
onDestroy(() => {
subscription.unsubscribe();
});
return {
get value() {
return value;
},
get error() {
return error;
},
};
}

View file

@ -0,0 +1,71 @@
/**
* Reactive sync status for Svelte 5 components.
*
* @example
* ```svelte
* <script lang="ts">
* import { useSyncStatus } from '@manacore/local-store/svelte';
* const sync = useSyncStatus(syncEngine);
* </script>
*
* {#if sync.status === 'offline'}
* <span>Offline</span>
* {:else if sync.status === 'syncing'}
* <span>Syncing...</span>
* {:else if sync.pendingCount > 0}
* <span>{sync.pendingCount} pending</span>
* {/if}
* ```
*/
import { onDestroy, onMount } from 'svelte';
import type { SyncEngine } from '../sync/engine.js';
import type { SyncStatus } from '../types.js';
interface SyncStatusState {
readonly status: SyncStatus;
readonly pendingCount: number;
readonly isOnline: boolean;
readonly isSyncing: boolean;
}
export function useSyncStatus(engine: SyncEngine): SyncStatusState {
let status = $state<SyncStatus>(engine.status);
let pendingCount = $state(0);
let unsubscribe: (() => void) | undefined;
let pendingInterval: ReturnType<typeof setInterval> | undefined;
onMount(() => {
unsubscribe = engine.onStatusChange((newStatus) => {
status = newStatus;
});
// Poll pending count every 2s (cheap IndexedDB query)
const updatePending = async () => {
pendingCount = await engine.getPendingCount();
};
updatePending();
pendingInterval = setInterval(updatePending, 2000);
});
onDestroy(() => {
unsubscribe?.();
if (pendingInterval) clearInterval(pendingInterval);
});
return {
get status() {
return status;
},
get pendingCount() {
return pendingCount;
},
get isOnline() {
return status !== 'offline';
},
get isSyncing() {
return status === 'syncing';
},
};
}

View file

@ -0,0 +1,410 @@
/**
* SyncEngine orchestrates bidirectional sync between IndexedDB and the server.
*
* Push: Collects PendingChanges sends as Changeset clears on success
* Pull: Fetches server delta since last cursor applies to local collections
* WebSocket: Listens for push notifications triggers immediate pull
*
* The engine is designed to be resilient:
* - Offline: queues changes, retries when online
* - Partial failure: individual collection syncs are independent
* - Duplicate safety: pending changes are only cleared after server confirms
*/
import type { LocalDatabase } from '../database.js';
import type { LocalCollection } from '../collection.js';
import type {
BaseRecord,
Change,
Changeset,
SyncConfig,
SyncResponse,
SyncStatus,
} from '../types.js';
export class SyncEngine {
private readonly _db: LocalDatabase;
private readonly _config: SyncConfig;
private readonly _collections: Map<string, LocalCollection<BaseRecord>> = new Map();
private _status: SyncStatus = 'idle';
private _statusListeners: Set<(status: SyncStatus) => void> = new Set();
private _pushTimer: ReturnType<typeof setTimeout> | null = null;
private _pullTimer: ReturnType<typeof setInterval> | null = null;
private _ws: WebSocket | null = null;
private _enabled = false;
private _online = true;
constructor(db: LocalDatabase, config: SyncConfig) {
this._db = db;
this._config = {
conflictStrategy: 'field-level-lww',
pushDebounce: 1000,
pullInterval: 30_000,
...config,
};
}
// ─── Public API ─────────────────────────────────────────────
/** Current sync status. */
get status(): SyncStatus {
return this._status;
}
/** Whether the sync engine is enabled (user is authenticated). */
get enabled(): boolean {
return this._enabled;
}
/**
* Register a collection with the sync engine.
*/
registerCollection(collection: LocalCollection<BaseRecord>): void {
this._collections.set(collection.name, collection);
}
/**
* Start the sync engine. Call this after user authenticates.
*/
start(): void {
if (this._enabled) return;
this._enabled = true;
// Listen for online/offline events
if (typeof window !== 'undefined') {
window.addEventListener('online', this._handleOnline);
window.addEventListener('offline', this._handleOffline);
this._online = navigator.onLine;
}
// Initial sync
this._doSync();
// Start pull interval (fallback to WebSocket)
this._pullTimer = setInterval(() => {
if (this._online) this._doPull();
}, this._config.pullInterval!);
// Connect WebSocket
this._connectWebSocket();
}
/**
* Stop the sync engine. Call this on sign-out.
*/
stop(): void {
this._enabled = false;
if (typeof window !== 'undefined') {
window.removeEventListener('online', this._handleOnline);
window.removeEventListener('offline', this._handleOffline);
}
if (this._pushTimer) {
clearTimeout(this._pushTimer);
this._pushTimer = null;
}
if (this._pullTimer) {
clearInterval(this._pullTimer);
this._pullTimer = null;
}
this._disconnectWebSocket();
this._setStatus('idle');
}
/**
* Schedule a push of local changes. Debounced to avoid hammering the server.
*/
schedulePush(): void {
if (!this._enabled || !this._online) return;
if (this._pushTimer) {
clearTimeout(this._pushTimer);
}
this._pushTimer = setTimeout(() => {
this._doPush();
}, this._config.pushDebounce!);
}
/**
* Trigger an immediate full sync (push + pull).
*/
async sync(): Promise<void> {
if (!this._enabled) return;
await this._doSync();
}
/**
* Listen for sync status changes.
*/
onStatusChange(listener: (status: SyncStatus) => void): () => void {
this._statusListeners.add(listener);
return () => this._statusListeners.delete(listener);
}
/**
* Get the total number of pending changes.
*/
async getPendingCount(): Promise<number> {
return this._db.getPendingCount();
}
// ─── Internal: Sync Operations ──────────────────────────────
private async _doSync(): Promise<void> {
if (!this._online) {
this._setStatus('offline');
return;
}
this._setStatus('syncing');
try {
await this._doPush();
await this._doPull();
this._setStatus('synced');
} catch (err) {
console.error('[SyncEngine] sync failed:', err);
this._setStatus('error');
}
}
/**
* Push local pending changes to the server.
*/
private async _doPush(): Promise<void> {
const allPending = await this._db._pendingChanges.orderBy('createdAt').toArray();
if (allPending.length === 0) return;
// Group by collection
const byCollection = new Map<string, typeof allPending>();
for (const p of allPending) {
const list = byCollection.get(p.collection) ?? [];
list.push(p);
byCollection.set(p.collection, list);
}
// Build changeset
const changes: Change[] = [];
for (const [collection, pending] of byCollection) {
// Deduplicate: for the same recordId, keep only the latest change
const latest = new Map<string, (typeof pending)[0]>();
for (const p of pending) {
const existing = latest.get(p.recordId);
if (!existing || p.createdAt > existing.createdAt) {
// Merge fields if both are updates
if (
existing &&
existing.op === 'update' &&
p.op === 'update' &&
existing.fields &&
p.fields
) {
p.fields = { ...existing.fields, ...p.fields };
}
latest.set(p.recordId, p);
}
}
for (const [recordId, p] of latest) {
changes.push({
table: collection,
id: recordId,
op: p.op,
fields: p.fields,
data: p.data,
deletedAt: p.deletedAt,
});
}
}
const since = await this._getOldestSyncCursor();
const changeset: Changeset = {
clientId: this._config.clientId,
appId: this._config.appId,
since,
changes,
};
const response = await this._sendChangeset(changeset);
if (!response) return;
// Apply server changes
await this._applyServerChanges(response.serverChanges);
// Clear successfully synced pending changes
const ids = allPending.map((p) => p.id!).filter(Boolean);
await this._db.clearPendingChanges(ids);
// Update sync cursors
for (const collection of this._collections.keys()) {
await this._db.setSyncCursor(collection, response.syncedUntil);
}
}
/**
* Pull server changes for all collections.
*/
private async _doPull(): Promise<void> {
for (const [name] of this._collections) {
const since = await this._db.getSyncCursor(name);
const url = new URL(`/sync/${this._config.appId}/pull`, this._config.serverUrl);
url.searchParams.set('collection', name);
url.searchParams.set('since', since);
try {
const response = await this._fetch(url.toString(), { method: 'GET' });
if (!response.ok) continue;
const data: SyncResponse = await response.json();
await this._applyServerChanges(data.serverChanges);
await this._db.setSyncCursor(name, data.syncedUntil);
} catch {
// Pull failures are non-critical, will retry on next interval
}
}
}
/**
* Send a changeset to the sync server.
*/
private async _sendChangeset(changeset: Changeset): Promise<SyncResponse | null> {
const url = `${this._config.serverUrl}/sync/${this._config.appId}`;
try {
const response = await this._fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(changeset),
});
if (!response.ok) {
console.error('[SyncEngine] push failed:', response.status, await response.text());
return null;
}
return response.json();
} catch (err) {
console.error('[SyncEngine] push network error:', err);
return null;
}
}
/**
* Apply server changes to local collections.
*/
private async _applyServerChanges(changes: Change[]): Promise<void> {
for (const change of changes) {
const collection = this._collections.get(change.table);
if (!collection) continue;
await collection.applyServerChange(change);
}
}
// ─── Internal: WebSocket ────────────────────────────────────
private _connectWebSocket(): void {
if (!this._online) return;
const baseUrl = this._config.wsUrl ?? this._config.serverUrl;
const wsUrl = baseUrl.replace(/^http/, 'ws') + `/ws/${this._config.appId}`;
try {
this._ws = new WebSocket(wsUrl);
this._ws.onopen = async () => {
// Authenticate the WebSocket connection
const token = await this._config.getAuthToken?.();
if (token && this._ws?.readyState === WebSocket.OPEN) {
this._ws.send(JSON.stringify({ type: 'auth', token }));
}
};
this._ws.onmessage = (event) => {
try {
const msg = JSON.parse(event.data);
if (msg.type === 'sync-available') {
// Server has new changes — trigger immediate pull
this._doPull();
}
} catch {
// Ignore malformed messages
}
};
this._ws.onclose = () => {
this._ws = null;
// Reconnect after delay if still enabled
if (this._enabled && this._online) {
setTimeout(() => this._connectWebSocket(), 5000);
}
};
this._ws.onerror = () => {
this._ws?.close();
};
} catch {
// WebSocket not available (e.g. SSR)
}
}
private _disconnectWebSocket(): void {
if (this._ws) {
this._ws.onclose = null; // Prevent auto-reconnect
this._ws.close();
this._ws = null;
}
}
// ─── Internal: Helpers ──────────────────────────────────────
/**
* Fetch with auth token injection.
*/
private async _fetch(url: string, init: RequestInit = {}): Promise<Response> {
const token = await this._config.getAuthToken?.();
const headers = new Headers(init.headers);
if (token) {
headers.set('Authorization', `Bearer ${token}`);
}
headers.set('X-Client-Id', this._config.clientId);
return fetch(url, { ...init, headers });
}
/**
* Get the oldest sync cursor across all collections.
*/
private async _getOldestSyncCursor(): Promise<string> {
let oldest = new Date().toISOString();
for (const name of this._collections.keys()) {
const cursor = await this._db.getSyncCursor(name);
if (cursor < oldest) oldest = cursor;
}
return oldest;
}
private _setStatus(status: SyncStatus): void {
if (this._status === status) return;
this._status = status;
for (const listener of this._statusListeners) {
listener(status);
}
}
private _handleOnline = (): void => {
this._online = true;
this._connectWebSocket();
this._doSync();
};
private _handleOffline = (): void => {
this._online = false;
this._disconnectWebSocket();
this._setStatus('offline');
};
}

View file

@ -0,0 +1 @@
export { SyncEngine } from './engine.js';

View file

@ -0,0 +1,144 @@
/**
* Core types for the local-first data layer.
*/
/** Base record that all local-store entities must extend. */
export interface BaseRecord {
id: string;
createdAt?: string;
updatedAt?: string;
deletedAt?: string | null;
}
/** Sync status of a collection or the entire store. */
export type SyncStatus = 'idle' | 'syncing' | 'synced' | 'offline' | 'error';
/** A single field-level change for conflict resolution. */
export interface FieldChange {
value: unknown;
updatedAt: string;
}
/** Operations that can be applied to a record. */
export type ChangeOp = 'insert' | 'update' | 'delete';
/** A single change within a changeset. */
export interface Change {
table: string;
id: string;
op: ChangeOp;
/** Field-level values with timestamps (for update ops). */
fields?: Record<string, FieldChange>;
/** Full record data (for insert ops). */
data?: Record<string, unknown>;
/** Soft-delete timestamp (for delete ops). */
deletedAt?: string;
}
/** A batch of changes sent to/from the sync server. */
export interface Changeset {
clientId: string;
appId: string;
/** ISO timestamp — sync changes since this point. */
since: string;
changes: Change[];
}
/** Response from the sync server after processing a changeset. */
export interface SyncResponse {
/** Changes from the server that the client doesn't have yet. */
serverChanges: Change[];
/** Conflicts that couldn't be auto-resolved (empty with field-level LWW). */
conflicts: SyncConflict[];
/** New sync cursor — use as `since` in the next request. */
syncedUntil: string;
}
/** A conflict the server couldn't auto-resolve. */
export interface SyncConflict {
table: string;
id: string;
field: string;
clientValue: unknown;
clientTimestamp: string;
serverValue: unknown;
serverTimestamp: string;
}
/** Conflict resolution strategy. */
export type ConflictStrategy = 'field-level-lww' | 'client-wins' | 'server-wins';
/** Configuration for the sync engine. */
export interface SyncConfig {
/** Base URL of the sync server (e.g. http://localhost:3050). */
serverUrl: string;
/** App identifier (e.g. 'todo', 'contacts'). */
appId: string;
/** Unique device identifier (persisted in localStorage). */
clientId: string;
/** Conflict resolution strategy. Default: 'field-level-lww'. */
conflictStrategy?: ConflictStrategy;
/** Debounce time in ms before pushing local changes. Default: 1000. */
pushDebounce?: number;
/** Interval in ms for pulling server changes (fallback to WebSocket). Default: 30000. */
pullInterval?: number;
/** Function to get the current auth token (or null for guests). */
getAuthToken?: () => Promise<string | null>;
/** WebSocket URL (defaults to serverUrl with ws:// protocol). */
wsUrl?: string;
}
/** Configuration for a single collection (table). */
export interface CollectionConfig<T extends BaseRecord> {
/** Table/collection name (e.g. 'tasks', 'projects'). */
name: string;
/** Dexie index definitions (e.g. ['projectId', 'dueDate', '[isCompleted+dueDate]']). */
indexes?: string[];
/** Default seed data for guest mode (loaded when DB is empty). */
guestSeed?: T[];
}
/** Metadata stored per collection for sync tracking. */
export interface SyncMeta {
/** Collection name. */
collection: string;
/** Last successful sync timestamp (ISO). */
lastSyncedAt: string;
/** Number of pending (un-synced) changes. */
pendingCount: number;
}
/** A pending change waiting to be synced. */
export interface PendingChange {
/** Auto-incremented ID. */
id?: number;
/** Collection name. */
collection: string;
/** Record ID. */
recordId: string;
/** Operation type. */
op: ChangeOp;
/** Changed fields with timestamps. */
fields?: Record<string, FieldChange>;
/** Full record (for inserts). */
data?: Record<string, unknown>;
/** Soft-delete timestamp (for delete ops). */
deletedAt?: string;
/** When this change was made locally. */
createdAt: string;
}
/** Sort direction for queries. */
export type SortDirection = 'asc' | 'desc';
/** Query options for collection.query(). */
export interface QueryOptions<T> {
/** Sort by field name. */
sortBy?: keyof T & string;
/** Sort direction. Default: 'asc'. */
sortDirection?: SortDirection;
/** Maximum number of results. */
limit?: number;
/** Number of results to skip. */
offset?: number;
}

View file

@ -0,0 +1,18 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"lib": ["ES2022", "DOM"],
"types": ["node"],
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"verbatimModuleSyntax": true,
"noEmit": true
},
"include": ["src/**/*"],
"exclude": ["node_modules"]
}