feat(notes): compact the operation log and purge old tombstones
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m17s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m17s
- Background job at start and every 6 h: log entries older than 30 days (private notes: 7) are removed, documents.log_floor marks the boundary; clients behind it get resync and keep their queued changes - Deleted blocks are purged 30 days after deleted_at via a "purge" log effect, so open editors drop them without reloading; a tombstone stays while a deleted child still points to it - Text for a purged block becomes a new block at the end; inserts under a purged parent go to the top level (server and browser model) - doc_op_results kept for 90 days; expired lease/session maps are swept - Tests: storage, API, engine resync, simulation with compactions Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aea4324c20
commit
f26f286fb8
17 files changed
+653
-48
No files matched your search
@@ -2,7 +2,7 @@ import 'fake-indexeddb/auto';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import type { DocumentBlock } from '../../types';
|
||||
import { DocSyncEngine, formBatch } from '../engine';
|
||||
import { applyOp, cloneDoc, Doc, emptyDoc, Op, Result } from '../model';
|
||||
import { applyEffect, applyOp, cloneDoc, Doc, Effect, emptyDoc, Op, Result } from '../model';
|
||||
import { IndexedDBStorage, MemoryStorage } from '../persist';
|
||||
import { PushChannel, PushKind } from '../push';
|
||||
import { LogEntry, SnapshotResponse, SyncRequest, SyncResponse, Transport, TransportError } from '../transport';
|
||||
@@ -14,6 +14,7 @@ class FakeServer {
|
||||
results = new Map<string, Result>();
|
||||
roles = new Map<string, string>(); // clientId -> role
|
||||
applyCount = new Map<string, number>();
|
||||
floor = 0; // log entries up to here were compacted away
|
||||
private waiters: (() => void)[] = [];
|
||||
|
||||
/** Resolves on the next change (used by the long-poll stand-in). */
|
||||
@@ -45,6 +46,14 @@ class FakeServer {
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Log compaction: purges the given tombstones (as a log entry) and drops all older log entries. */
|
||||
compact(purge: string[]) {
|
||||
this.floor = this.log.length;
|
||||
const effect: Effect = { kind: 'purge', ids: purge };
|
||||
applyEffect(this.doc, effect);
|
||||
this.log.push({ seq: this.log.length + 1, userId: '', clientId: '', opId: '', effect });
|
||||
}
|
||||
|
||||
snapshot(): SnapshotResponse {
|
||||
return {
|
||||
docId: 'doc',
|
||||
@@ -61,9 +70,11 @@ class FakeServer {
|
||||
const role = this.roles.get(body.clientId) || 'owner';
|
||||
if (body.ops.length > 0 && role === 'viewer') throw new TransportError('nur Lesen', 403);
|
||||
const results = body.ops.map((op) => this.apply(op, body.clientId));
|
||||
const compacted = body.lastSeq < this.floor;
|
||||
return {
|
||||
results,
|
||||
ops: this.log.filter((e) => e.seq > body.lastSeq),
|
||||
resync: compacted || undefined,
|
||||
ops: compacted ? [] : this.log.filter((e) => e.seq > body.lastSeq),
|
||||
headSeq: this.log.length,
|
||||
hasMore: false,
|
||||
role,
|
||||
@@ -206,6 +217,37 @@ describe('DocSyncEngine', () => {
|
||||
for (const count of server.applyCount.values()) expect(count).toBe(1);
|
||||
});
|
||||
|
||||
it('a client offline for weeks reloads after log compaction and keeps its unsent text', async () => {
|
||||
const server = new FakeServer([
|
||||
{ id: 'b1', content: 'Start' },
|
||||
{ id: 'b2', content: 'Wird gelöscht' },
|
||||
]);
|
||||
const netA = new FakeNet(server);
|
||||
const netB = new FakeNet(server);
|
||||
const a = await startEngine(netA);
|
||||
const b = await startEngine(netB);
|
||||
|
||||
netB.mode = 'down';
|
||||
edit(b, (blocks) => [{ ...blocks[0], content: 'Start (B)' }, { ...blocks[1], content: 'B tippt weiter' }]);
|
||||
await b.syncNow();
|
||||
|
||||
// Meanwhile A deletes b2, and weeks later the tombstone is purged and the log compacted
|
||||
edit(a, (blocks) => [blocks[0]]);
|
||||
await a.syncNow();
|
||||
server.compact(['b2']);
|
||||
await a.syncNow();
|
||||
expect(contents(a)).toEqual(['Start']);
|
||||
|
||||
netB.mode = 'ok';
|
||||
await b.syncNow();
|
||||
await b.syncNow();
|
||||
// Nothing typed offline is lost: the edit of the purged block comes back as a new block
|
||||
expect(serverContents(server)).toEqual(['Start (B)', 'B tippt weiter']);
|
||||
expect(contents(b)).toEqual(['Start (B)', 'B tippt weiter']);
|
||||
await a.syncNow();
|
||||
expect(contents(a)).toEqual(['Start (B)', 'B tippt weiter']);
|
||||
});
|
||||
|
||||
it('retries after a lost response without applying twice', async () => {
|
||||
const server = new FakeServer();
|
||||
const net = new FakeNet(server);
|
||||
|
||||
Reference in new issue
Block a user