feat(notes): compact the operation log and purge old tombstones

- Background job at start and every 6 h: log entries older than 30 days
  (private notes: 7) are removed, documents.log_floor marks the boundary;
  clients behind it get resync and keep their queued changes
- Deleted blocks are purged 30 days after deleted_at via a "purge" log
  effect, so open editors drop them without reloading; a tombstone stays
  while a deleted child still points to it
- Text for a purged block becomes a new block at the end; inserts under a
  purged parent go to the top level (server and browser model)
- doc_op_results kept for 90 days; expired lease/session maps are swept
- Tests: storage, API, engine resync, simulation with compactions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/storage/docsync_test.go

@@ -1,8 +1,10 @@
 package storage
 
 import (
+	"errors"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/sandrodach/mindsift/internal/collab"
 	"github.com/sandrodach/mindsift/internal/domain"
@@ -52,7 +54,7 @@ func sameOutline(t *testing.T, got []domain.DocumentBlock, want []domain.Documen
 
 func headSeq(t *testing.T, s *Store, docID string) int64 {
 	t.Helper()
-	_, head, err := s.GetDocOpsSince(docID, 0, 1)
+	_, head, err := s.GetDocOpsSince(docID, 0, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -234,3 +236,129 @@ func TestFixedBlockIDsWorkInSeveralNotes(t *testing.T) {
 		t.Fatal("block IDs must differ between notes")
 	}
 }
+
+func backdate(t *testing.T, s *Store, docID string, age time.Duration) {
+	t.Helper()
+	old := time.Now().Add(-age).UTC().Format(time.RFC3339)
+	for _, q := range []string{
+		`UPDATE doc_ops SET created_at = ? WHERE doc_id = ?`,
+		`UPDATE doc_op_results SET created_at = ? WHERE doc_id = ?`,
+		`UPDATE blocks SET deleted_at = ? WHERE document_id = ? AND deleted = 1`,
+	} {
+		if _, err := s.db.Exec(q, old, docID); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestCompactionKeepsStateAndForcesResync(t *testing.T) {
+	s, u := newSyncTestStore(t)
+	doc, _ := s.CreateDocument(u.ID, "", "Notiz", "📄", "", flatBlocks(
+		[3]interface{}{"a", 0, "Bleibt"},
+		[3]interface{}{"b", 0, "Gelöscht"},
+		[3]interface{}{"c", 1, "Kind von b"},
+	), nil, false)
+	a := collab.Actor{UserID: u.ID, ClientID: "tab1"}
+	if res, _, _ := s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "del-b", Kind: collab.OpDelete, BlockID: "b"}}, nil); res[0].Status != collab.StatusApplied {
+		t.Fatalf("delete: %+v", res)
+	}
+	backdate(t, s, doc.ID, 40*24*time.Hour)
+	// Idempotency results stay longer (90 days): only the very old one goes
+	s.db.Exec(`UPDATE doc_op_results SET created_at = ? WHERE op_id = 'del-b'`, time.Now().Add(-100*24*time.Hour).UTC().Format(time.RFC3339))
+
+	// A client that is up to date before compaction must only need the remaining log afterwards
+	snap, _ := s.GetDocSnapshot(doc.ID)
+	client := collab.NewDoc()
+	for i := range snap.Blocks {
+		b := snap.Blocks[i]
+		client.Blocks[b.ID] = &b
+	}
+	text := "Frisch"
+	if _, _, err := s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "fresh", Kind: collab.OpSetContent, BlockID: "a", BaseVersion: 1, Content: &text}}, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	st, err := s.CompactDocs(time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.Tombstones != 2 || st.LogEntries == 0 || st.OpResults != 1 || st.Purged[doc.ID] == 0 {
+		t.Fatalf("unexpected stats: %+v", st)
+	}
+	var rows int
+	_ = s.db.QueryRow(`SELECT COUNT(*) FROM blocks WHERE document_id = ? AND id IN ('b', 'c')`, doc.ID).Scan(&rows)
+	if rows != 0 {
+		t.Fatalf("tombstones must be removed from the table, %d left", rows)
+	}
+
+	// Behind the floor: reload needed
+	if _, _, err := s.GetDocOpsSince(doc.ID, 0, 100); !errors.Is(err, ErrLogCompacted) {
+		t.Fatalf("client behind the floor must be told to resync, got %v", err)
+	}
+	entries, head, err := s.GetDocOpsSince(doc.ID, snap.HeadSeq, 100)
+	if err != nil || len(entries) != 2 || entries[1].Effect.Kind != collab.OpPurge {
+		t.Fatalf("fresh edit and purge must remain: %+v %v", entries, err)
+	}
+	for _, e := range entries {
+		client.ApplyEffect(e.Effect)
+	}
+	final, _ := s.GetDocSnapshot(doc.ID)
+	if final.HeadSeq != head || len(final.Blocks) != len(client.Blocks) {
+		t.Fatalf("client after log (%d blocks) differs from snapshot (%d blocks)", len(client.Blocks), len(final.Blocks))
+	}
+	for _, b := range final.Blocks {
+		if cb := client.Blocks[b.ID]; cb == nil || *cb != b {
+			t.Fatalf("block %s differs: client %+v, server %+v", b.ID, cb, b)
+		}
+	}
+	got, _ := s.GetDocumentByID(u.ID, doc.ID)
+	sameOutline(t, got.Blocks, flatBlocks([3]interface{}{"a", 0, "Frisch"}))
+
+	// A second run has nothing left to do
+	if st, _ := s.CompactDocs(time.Now()); st.Docs != 0 || st.OpResults != 0 {
+		t.Fatalf("second run changed something: %+v", st)
+	}
+}
+
+func TestCompactionRetentionDependsOnSharing(t *testing.T) {
+	s, u := newSyncTestStore(t)
+	other, _ := s.CreateUser("ben@example.com", "hash", "Ben", "member")
+	private, _ := s.CreateDocument(u.ID, "", "Privat", "📄", "", flatBlocks([3]interface{}{"p", 0, "x"}), nil, false)
+	shared, _ := s.CreateDocument(u.ID, "", "Geteilt", "📄", "", flatBlocks([3]interface{}{"s", 0, "x"}), nil, false)
+	if _, err := s.CreateDocShare(u.ID, shared.ID, other.Email, "write"); err != nil {
+		t.Fatal(err)
+	}
+	backdate(t, s, private.ID, 10*24*time.Hour)
+	backdate(t, s, shared.ID, 10*24*time.Hour)
+	if _, err := s.CompactDocs(time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := s.GetDocOpsSince(private.ID, 0, 10); !errors.Is(err, ErrLogCompacted) {
+		t.Fatalf("private note: 10 days old log must be compacted, got %v", err)
+	}
+	if entries, _, err := s.GetDocOpsSince(shared.ID, 0, 10); err != nil || len(entries) == 0 {
+		t.Fatalf("shared note: log must be kept for 30 days: %d entries, %v", len(entries), err)
+	}
+}
+
+func TestDeletedAtFollowsDeleteAndRestore(t *testing.T) {
+	s, u := newSyncTestStore(t)
+	doc, _ := s.CreateDocument(u.ID, "", "Notiz", "📄", "", flatBlocks([3]interface{}{"a", 0, "x"}, [3]interface{}{"b", 0, "y"}), nil, false)
+	a := collab.Actor{UserID: u.ID, ClientID: "tab1"}
+	deletedAt := func() string {
+		var v string
+		_ = s.db.QueryRow(`SELECT deleted_at FROM blocks WHERE id = 'b'`).Scan(&v)
+		return v
+	}
+	s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "1", Kind: collab.OpDelete, BlockID: "b"}}, nil)
+	first := deletedAt()
+	if first == "" {
+		t.Fatal("deleting must set deleted_at")
+	}
+	// Other changes to the tombstone (none possible here) must not restart the clock; a restore clears it
+	text := "zurück"
+	s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "2", Kind: collab.OpSetContent, BlockID: "b", BaseVersion: 1, Content: &text}}, nil)
+	if deletedAt() != "" {
+		t.Fatal("restoring must clear deleted_at")
+	}
+}