feat(notes): compact the operation log and purge old tombstones

- Background job at start and every 6 h: log entries older than 30 days
  (private notes: 7) are removed, documents.log_floor marks the boundary;
  clients behind it get resync and keep their queued changes
- Deleted blocks are purged 30 days after deleted_at via a "purge" log
  effect, so open editors drop them without reloading; a tombstone stays
  while a deleted child still points to it
- Text for a purged block becomes a new block at the end; inserts under a
  purged parent go to the top level (server and browser model)
- doc_op_results kept for 90 days; expired lease/session maps are swept
- Tests: storage, API, engine resync, simulation with compactions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/storage/docsync.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/sandrodach/mindsift/internal/collab"
 	"github.com/sandrodach/mindsift/internal/domain"
@@ -20,6 +21,17 @@ import (
 // ErrDocNotFound is returned when a note does not exist or the user has no access to it.
 var ErrDocNotFound = errors.New("Notiz nicht gefunden oder kein Zugriff")
 
+// ErrLogCompacted means the requested log entries were removed by compaction; the client reloads the snapshot.
+var ErrLogCompacted = errors.New("Änderungsprotokoll verdichtet")
+
+// Retention of the operation log and of tombstones (see CompactDocs).
+const (
+	LogRetentionShared  = 30 * 24 * time.Hour
+	LogRetentionPrivate = 7 * 24 * time.Hour
+	TombstoneRetention  = 30 * 24 * time.Hour
+	OpResultRetention   = 90 * 24 * time.Hour
+)
+
 // Roles a user can have on a note.
 const (
 	DocRoleOwner  = "owner"
@@ -92,12 +104,20 @@ func (s *Store) migrateDocSync() {
 			created_at TEXT NOT NULL
 		);`,
 		"CREATE INDEX IF NOT EXISTS idx_doc_op_results_doc ON doc_op_results(doc_id);",
+		"ALTER TABLE blocks ADD COLUMN deleted_at TEXT NOT NULL DEFAULT '';",
+		"ALTER TABLE documents ADD COLUMN log_floor INTEGER NOT NULL DEFAULT 0;",
+		"CREATE INDEX IF NOT EXISTS idx_doc_ops_created ON doc_ops(created_at);",
+		"CREATE INDEX IF NOT EXISTS idx_doc_op_results_created ON doc_op_results(created_at);",
 	}
 	for _, q := range stmts {
 		if _, err := s.db.Exec(q); err != nil && !strings.Contains(err.Error(), "duplicate column") {
 			slog.Error("Migration der Notiz-Synchronisation fehlgeschlagen", "error", err, "statement", q)
 		}
 	}
+	// Tombstones from before deleted_at existed start their retention now
+	if _, err := s.db.Exec(`UPDATE blocks SET deleted_at = ? WHERE deleted = 1 AND deleted_at = ''`, domain.NowISO()); err != nil {
+		slog.Error("Migration der Notiz-Synchronisation fehlgeschlagen", "error", err)
+	}
 	s.backfillBlockTree()
 }
 
@@ -321,13 +341,14 @@ func (s *Store) applyDocChange(docID string, a collab.Actor, leases collab.Lease
 func persistDocState(tx *sql.Tx, docID string, before, after *collab.Doc, head int64, now string) error {
 	stmt, err := tx.Prepare(`
 		INSERT INTO blocks (id, document_id, parent_id, order_key, type, content, checked, collapsed, lang,
-		                    conflict_of, conflict_author, version, deleted, updated_by, level, position)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0)
+		                    conflict_of, conflict_author, version, deleted, deleted_at, updated_by, level, position)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0)
 		ON CONFLICT(id) DO UPDATE SET
 			parent_id = excluded.parent_id, order_key = excluded.order_key, type = excluded.type,
 			content = excluded.content, checked = excluded.checked, collapsed = excluded.collapsed,
 			lang = excluded.lang, conflict_of = excluded.conflict_of, conflict_author = excluded.conflict_author,
-			version = excluded.version, deleted = excluded.deleted, updated_by = excluded.updated_by
+			version = excluded.version, deleted = excluded.deleted, updated_by = excluded.updated_by,
+			deleted_at = CASE WHEN excluded.deleted = 0 THEN '' WHEN blocks.deleted = 1 THEN blocks.deleted_at ELSE excluded.deleted_at END
 		WHERE blocks.document_id = excluded.document_id`)
 	if err != nil {
 		return err
@@ -339,12 +360,23 @@ func persistDocState(tx *sql.Tx, docID string, before, after *collab.Doc, head i
 			continue
 		}
 		tagsChanged = true
+		deletedAt := ""
+		if b.Deleted {
+			deletedAt = now
+		}
 		if _, err := stmt.Exec(b.ID, docID, b.ParentID, b.OrderKey, b.Type, b.Content, boolInt(b.Props.Checked),
 			boolInt(b.Props.Collapsed), b.Props.Lang, b.Props.ConflictOf, b.Props.ConflictAuthor, b.Version,
-			boolInt(b.Deleted), b.UpdatedBy); err != nil {
+			boolInt(b.Deleted), deletedAt, b.UpdatedBy); err != nil {
 			return err
 		}
 	}
+	for id := range before.Blocks {
+		if _, ok := after.Blocks[id]; !ok {
+			if _, err := tx.Exec(`DELETE FROM blocks WHERE id = ? AND document_id = ?`, id, docID); err != nil {
+				return err
+			}
+		}
+	}
 
 	flat := flatDocumentBlocks(docID, after)
 	var plain strings.Builder
@@ -454,13 +486,23 @@ func (s *Store) GetThoughtByIDUnchecked(thoughtID string) (*domain.Thought, erro
 		 FROM thoughts WHERE id = ?`, thoughtID))
 }
 
-// GetDocOpsSince returns up to limit log entries after seq, plus the current head.
+// GetDocOpsSince returns up to limit log entries after seq, plus the current head. If entries after seq
+// were already compacted away it returns ErrLogCompacted (with the head): the client must reload the snapshot.
 func (s *Store) GetDocOpsSince(docID string, since int64, limit int) ([]DocLogEntry, int64, error) {
-	var head int64
-	if err := s.db.QueryRow(`SELECT head_seq FROM documents WHERE id = ?`, docID).Scan(&head); err != nil {
+	// One read transaction, so a compaction running in between cannot leave an unnoticed gap
+	tx, err := s.db.Begin()
+	if err != nil {
 		return nil, 0, err
 	}
-	rows, err := s.db.Query(
+	defer tx.Rollback()
+	var head, floor int64
+	if err := tx.QueryRow(`SELECT head_seq, log_floor FROM documents WHERE id = ?`, docID).Scan(&head, &floor); err != nil {
+		return nil, 0, err
+	}
+	if limit > 0 && since < floor {
+		return nil, head, ErrLogCompacted
+	}
+	rows, err := tx.Query(
 		`SELECT seq, op_id, user_id, client_id, payload FROM doc_ops WHERE doc_id = ? AND seq > ? ORDER BY seq ASC LIMIT ?`,
 		docID, since, limit)
 	if err != nil {
@@ -776,3 +818,128 @@ func longestIncreasing(keys []string) []bool {
 	}
 	return keep
 }
+
+// CompactStats counts what one compaction run removed.
+type CompactStats struct {
+	Docs       int
+	LogEntries int64
+	Tombstones int
+	OpResults  int64
+	Purged     map[string]int64 // docID -> new head, for notes whose tombstones were purged
+}
+
+// CompactDocs keeps the collaboration tables small:
+//   - log entries older than the retention (30 days for shared notes, 7 for private ones) are removed and
+//     the note remembers the boundary in log_floor; clients that are further behind reload the snapshot,
+//     their unsent changes stay queued and are sent afterwards,
+//   - deleted blocks older than 30 days are purged; this is itself a log entry ("purge"), so connected
+//     clients drop them without a reload,
+//   - idempotency results older than 90 days are removed.
+func (s *Store) CompactDocs(now time.Time) (CompactStats, error) {
+	st := CompactStats{Purged: map[string]int64{}}
+	ts := func(d time.Duration) string { return now.Add(-d).UTC().Format(time.RFC3339) }
+	tombCutoff := ts(TombstoneRetention)
+	rows, err := s.db.Query(
+		`SELECT doc_id FROM doc_ops WHERE created_at < ?
+		 UNION SELECT document_id FROM blocks WHERE deleted = 1 AND deleted_at != '' AND deleted_at < ?`,
+		ts(LogRetentionPrivate), tombCutoff)
+	if err != nil {
+		return st, err
+	}
+	var docIDs []string
+	for rows.Next() {
+		var id string
+		if rows.Scan(&id) == nil {
+			docIDs = append(docIDs, id)
+		}
+	}
+	rows.Close()
+
+	for _, docID := range docIDs {
+		cutoff := ts(LogRetentionPrivate)
+		if s.IsDocShared(docID) {
+			cutoff = ts(LogRetentionShared)
+		}
+		removed, purged, head, err := s.compactDoc(docID, cutoff, tombCutoff, now.UTC().Format(time.RFC3339))
+		if err != nil {
+			return st, fmt.Errorf("compact %s: %w", docID, err)
+		}
+		if purged > 0 {
+			st.Purged[docID] = head
+		}
+		if removed > 0 || purged > 0 {
+			st.Docs++
+		}
+		st.LogEntries += removed
+		st.Tombstones += purged
+	}
+
+	res, err := s.db.Exec(`DELETE FROM doc_op_results WHERE created_at < ?`, ts(OpResultRetention))
+	if err != nil {
+		return st, err
+	}
+	st.OpResults, _ = res.RowsAffected()
+	return st, nil
+}
+
+func (s *Store) compactDoc(docID, logCutoff, tombCutoff, now string) (removed int64, purged int, head int64, err error) {
+	unlock := lockDoc(docID)
+	defer unlock()
+	tx, err := s.db.Begin()
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	defer tx.Rollback()
+
+	doc, head, err := loadCollabDoc(tx, docID)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	var candidates []string
+	rows, err := tx.Query(`SELECT id FROM blocks WHERE document_id = ? AND deleted = 1 AND deleted_at != '' AND deleted_at < ?`, docID, tombCutoff)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	for rows.Next() {
+		var id string
+		if rows.Scan(&id) == nil {
+			candidates = append(candidates, id)
+		}
+	}
+	rows.Close()
+	if e := collab.Purge(doc, candidates); e != nil {
+		head++
+		payload, _ := json.Marshal(e)
+		if _, err := tx.Exec(
+			`INSERT INTO doc_ops (doc_id, seq, op_id, user_id, client_id, kind, block_id, payload, created_at) VALUES (?, ?, '', '', '', ?, '', ?, ?)`,
+			docID, head, e.Kind, string(payload), now); err != nil {
+			return 0, 0, 0, err
+		}
+		for _, id := range e.IDs {
+			if _, err := tx.Exec(`DELETE FROM blocks WHERE id = ? AND document_id = ?`, id, docID); err != nil {
+				return 0, 0, 0, err
+			}
+		}
+		// Only the position changes; tombstones are not part of the note's text, so nothing derived changes
+		if _, err := tx.Exec(`UPDATE documents SET head_seq = ? WHERE id = ?`, head, docID); err != nil {
+			return 0, 0, 0, err
+		}
+		purged = len(e.IDs)
+	}
+
+	var floor int64
+	if err := tx.QueryRow(`SELECT COALESCE(MAX(seq), 0) FROM doc_ops WHERE doc_id = ? AND created_at < ?`, docID, logCutoff).Scan(&floor); err != nil {
+		return 0, 0, 0, err
+	}
+	if floor > 0 {
+		res, err := tx.Exec(`DELETE FROM doc_ops WHERE doc_id = ? AND seq <= ?`, docID, floor)
+		if err != nil {
+			return 0, 0, 0, err
+		}
+		removed, _ = res.RowsAffected()
+		if _, err := tx.Exec(`UPDATE documents SET log_floor = MAX(log_floor, ?) WHERE id = ?`, floor, docID); err != nil {
+			return 0, 0, 0, err
+		}
+	}
+	return removed, purged, head, tx.Commit()
+}