feat(notes): compact the operation log and purge old tombstones
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m17s

- Background job at start and every 6 h: log entries older than 30 days
  (private notes: 7) are removed, documents.log_floor marks the boundary;
  clients behind it get resync and keep their queued changes
- Deleted blocks are purged 30 days after deleted_at via a "purge" log
  effect, so open editors drop them without reloading; a tombstone stays
  while a deleted child still points to it
- Text for a purged block becomes a new block at the end; inserts under a
  purged parent go to the top level (server and browser model)
- doc_op_results kept for 90 days; expired lease/session maps are swept
- Tests: storage, API, engine resync, simulation with compactions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-28 15:08:13 +01:00
1 parent aea4324c20
commit f26f286fb8
17 files changed
+653 -48

No files matched your search

@@ -2,7 +2,7 @@ import 'fake-indexeddb/auto';
import { describe, it, expect, vi } from 'vitest';
import type { DocumentBlock } from '../../types';
import { DocSyncEngine, formBatch } from '../engine';
import { applyOp, cloneDoc, Doc, emptyDoc, Op, Result } from '../model';
import { applyEffect, applyOp, cloneDoc, Doc, Effect, emptyDoc, Op, Result } from '../model';
import { IndexedDBStorage, MemoryStorage } from '../persist';
import { PushChannel, PushKind } from '../push';
import { LogEntry, SnapshotResponse, SyncRequest, SyncResponse, Transport, TransportError } from '../transport';
@@ -14,6 +14,7 @@ class FakeServer {
results = new Map<string, Result>();
roles = new Map<string, string>(); // clientId -> role
applyCount = new Map<string, number>();
floor = 0; // log entries up to here were compacted away
private waiters: (() => void)[] = [];
/** Resolves on the next change (used by the long-poll stand-in). */
@@ -45,6 +46,14 @@ class FakeServer {
return result;
}
/** Log compaction: purges the given tombstones (as a log entry) and drops all older log entries. */
compact(purge: string[]) {
this.floor = this.log.length;
const effect: Effect = { kind: 'purge', ids: purge };
applyEffect(this.doc, effect);
this.log.push({ seq: this.log.length + 1, userId: '', clientId: '', opId: '', effect });
}
snapshot(): SnapshotResponse {
return {
docId: 'doc',
@@ -61,9 +70,11 @@ class FakeServer {
const role = this.roles.get(body.clientId) || 'owner';
if (body.ops.length > 0 && role === 'viewer') throw new TransportError('nur Lesen', 403);
const results = body.ops.map((op) => this.apply(op, body.clientId));
const compacted = body.lastSeq < this.floor;
return {
results,
ops: this.log.filter((e) => e.seq > body.lastSeq),
resync: compacted || undefined,
ops: compacted ? [] : this.log.filter((e) => e.seq > body.lastSeq),
headSeq: this.log.length,
hasMore: false,
role,
@@ -206,6 +217,37 @@ describe('DocSyncEngine', () => {
for (const count of server.applyCount.values()) expect(count).toBe(1);
});
it('a client offline for weeks reloads after log compaction and keeps its unsent text', async () => {
const server = new FakeServer([
{ id: 'b1', content: 'Start' },
{ id: 'b2', content: 'Wird gelöscht' },
]);
const netA = new FakeNet(server);
const netB = new FakeNet(server);
const a = await startEngine(netA);
const b = await startEngine(netB);
netB.mode = 'down';
edit(b, (blocks) => [{ ...blocks[0], content: 'Start (B)' }, { ...blocks[1], content: 'B tippt weiter' }]);
await b.syncNow();
// Meanwhile A deletes b2, and weeks later the tombstone is purged and the log compacted
edit(a, (blocks) => [blocks[0]]);
await a.syncNow();
server.compact(['b2']);
await a.syncNow();
expect(contents(a)).toEqual(['Start']);
netB.mode = 'ok';
await b.syncNow();
await b.syncNow();
// Nothing typed offline is lost: the edit of the purged block comes back as a new block
expect(serverContents(server)).toEqual(['Start (B)', 'B tippt weiter']);
expect(contents(b)).toEqual(['Start (B)', 'B tippt weiter']);
await a.syncNow();
expect(contents(a)).toEqual(['Start (B)', 'B tippt weiter']);
});
it('retries after a lost response without applying twice', async () => {
const server = new FakeServer();
const net = new FakeNet(server);