feat(sync): add pull pagination with hasMore flag

Server now returns hasMore: true when there are more than 1000 changes
pending for a collection. Client continues pulling in a loop until
hasMore is false, using the last row's timestamp as cursor.

Prevents data loss after long offline periods where >1000 changes
accumulated for a single collection.

Server changes (Go):
- GetChangesSince() accepts limit parameter
- HandlePull() fetches limit+1, trims, sets hasMore
- SyncedUntil uses last row's timestamp when paginating

Client changes (TypeScript):
- Pull loop: while (hasMore) { fetch → apply → advance cursor }
- Cursor only persisted after all pages fetched

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS 2026-04-02 22:17:20 +02:00
parent 03434c2802
commit f7f5c9eb3a
4 changed files with 52 additions and 27 deletions

View file

@ -201,30 +201,39 @@ export function createUnifiedSync(serverUrl: string, getToken: () => Promise<str
try {
for (const tableName of channel.tables) {
const syncName = toSyncName(tableName);
const cursor = await getSyncCursor(appId, tableName);
let cursor = await getSyncCursor(appId, tableName);
let hasMore = true;
const res = await fetch(
`${serverUrl}/sync/${appId}/pull?collection=${encodeURIComponent(syncName)}&since=${encodeURIComponent(cursor)}`,
{
headers: {
Authorization: `Bearer ${token}`,
'X-Client-Id': clientId,
},
// Paginated pull: continue fetching until server signals no more data
while (hasMore) {
const res = await fetch(
`${serverUrl}/sync/${appId}/pull?collection=${encodeURIComponent(syncName)}&since=${encodeURIComponent(cursor)}`,
{
headers: {
Authorization: `Bearer ${token}`,
'X-Client-Id': clientId,
},
}
);
if (!res.ok) break;
const data = await res.json();
hasMore = data.hasMore ?? false;
if (data.serverChanges && data.serverChanges.length > 0) {
await applyServerChanges(appId, data.serverChanges);
}
);
if (!res.ok) continue;
const data = await res.json();
if (!data.serverChanges || data.serverChanges.length === 0) continue;
// Apply changes to local DB
await applyServerChanges(appId, data.serverChanges);
// Update cursor
if (data.syncedUntil) {
await setSyncCursor(appId, tableName, data.syncedUntil);
if (data.syncedUntil) {
cursor = data.syncedUntil;
} else {
break;
}
}
// Update cursor after all pages fetched
await setSyncCursor(appId, tableName, cursor);
}
channel.lastError = null;