feat(notes): compact the operation log and purge old tombstones

- Background job at start and every 6 h: log entries older than 30 days
  (private notes: 7) are removed, documents.log_floor marks the boundary;
  clients behind it get resync and keep their queued changes
- Deleted blocks are purged 30 days after deleted_at via a "purge" log
  effect, so open editors drop them without reloading; a tombstone stays
  while a deleted child still points to it
- Text for a purged block becomes a new block at the end; inserts under a
  purged parent go to the top level (server and browser model)
- doc_op_results kept for 90 days; expired lease/session maps are swept
- Tests: storage, API, engine resync, simulation with compactions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/docsync.go

@@ -2,6 +2,8 @@ package api
 
 import (
 	"encoding/json"
+	"errors"
+	"log/slog"
 	"net/http"
 	"strconv"
 	"time"
@@ -31,7 +33,7 @@ type syncResponse struct {
 	Ops      []storage.DocLogEntry `json:"ops"`
 	HeadSeq  int64                 `json:"headSeq"`
 	HasMore  bool                  `json:"hasMore"`
-	Resync   bool                  `json:"resync,omitempty"` // client state is ahead of the server: reload the snapshot
+	Resync   bool                  `json:"resync,omitempty"` // reload the snapshot (state ahead of the server or log compacted)
 	Role     string                `json:"role"`
 	Shared   bool                  `json:"shared"`
 	Sessions int                   `json:"sessions"`
@@ -155,11 +157,13 @@ func (s *Server) handleDocSync(w http.ResponseWriter, r *http.Request) {
 		logLimit = 0
 	}
 	entries, head, err := s.store.GetDocOpsSince(docID, req.LastSeq, logLimit)
-	if err != nil {
+	compacted := errors.Is(err, storage.ErrLogCompacted)
+	if err != nil && !compacted {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
-	if req.LastSeq > head && !req.SkipLog {
+	// Client state is ahead of the server, or so far behind that the log entries are compacted away
+	if (req.LastSeq > head || compacted) && !req.SkipLog {
 		resp.Resync = true
 		entries = []storage.DocLogEntry{}
 	}
@@ -253,3 +257,28 @@ func (s *Server) handleDocLeave(w http.ResponseWriter, r *http.Request) {
 	s.notifyDocChanged(docID, head)
 	writeJSON(w, http.StatusOK, map[string]bool{"success": true})
 }
+
+// compactionInterval is how often the operation log and tombstones are compacted (also once at start).
+const compactionInterval = 6 * time.Hour
+
+func (s *Server) startCompaction() {
+	for {
+		s.compactOnce()
+		time.Sleep(compactionInterval)
+	}
+}
+
+func (s *Server) compactOnce() {
+	s.leases.Sweep()
+	st, err := s.store.CompactDocs(time.Now())
+	if err != nil {
+		slog.Error("Verdichtung des Änderungsprotokolls fehlgeschlagen", "error", err)
+	}
+	// Open editors drop the purged tombstones with their next sync
+	for docID, head := range st.Purged {
+		s.notifyDocChanged(docID, head)
+	}
+	if st.Docs > 0 || st.OpResults > 0 {
+		slog.Info("Änderungsprotokoll verdichtet", "notes", st.Docs, "logEntries", st.LogEntries, "tombstones", st.Tombstones, "opResults", st.OpResults)
+	}
+}

internal/api/docsync_test.go

@@ -196,3 +196,36 @@ func itoa(n int64) string {
 	b, _ := json.Marshal(n)
 	return string(b)
 }
+
+func TestDocSyncAfterCompaction(t *testing.T) {
+	srv, store, ts, owner, ownerTok := startRealtimeServer(t)
+	doc, err := store.CreateDocument(owner.ID, "", "Alt", "📄", "", []domain.DocumentBlock{
+		{ID: "b1", Type: "bullet", Content: "Bleibt"}, {ID: "b2", Type: "bullet", Content: "Weg"},
+	}, nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, live := syncDoc(t, ts, ownerTok, doc.ID, map[string]interface{}{
+		"clientId": "live", "lastSeq": 0, "ops": []collab.Op{{OpID: "d", Kind: collab.OpDelete, BlockID: "b2"}},
+	})
+
+	// 40 days later: log and tombstone are past their retention
+	st, err := srv.store.CompactDocs(time.Now().Add(40 * 24 * time.Hour))
+	if err != nil || st.Tombstones != 1 {
+		t.Fatalf("compaction: %+v %v", st, err)
+	}
+
+	// A client that was up to date only gets the purge entry
+	_, up := syncDoc(t, ts, ownerTok, doc.ID, map[string]interface{}{"clientId": "live", "lastSeq": live.HeadSeq})
+	if up.Resync || len(up.Ops) != 1 || up.Ops[0].Effect.Kind != collab.OpPurge {
+		t.Fatalf("up-to-date client: resync=%v ops=%+v", up.Resync, up.Ops)
+	}
+	// A client behind the compacted log must reload; its own changes are still applied
+	code, old := syncDoc(t, ts, ownerTok, doc.ID, map[string]interface{}{
+		"clientId": "old", "lastSeq": 1,
+		"ops": []collab.Op{{OpID: "late", Kind: collab.OpSetContent, BlockID: "b1", BaseVersion: 1, Content: strp("Spät")}},
+	})
+	if code != http.StatusOK || !old.Resync || len(old.Ops) != 0 || old.Results[0].Status != collab.StatusApplied {
+		t.Fatalf("stale client: %d resync=%v results=%+v", code, old.Resync, old.Results)
+	}
+}

internal/api/server.go

@@ -46,6 +46,7 @@ func NewServer(cfg *config.Config, store *storage.Store, wsHub *ws.Hub, staticFS
 	}
 	wsHub.SetRoomAuthorizer(srv.canJoinRoom)
 	go srv.startAlarmTicker()
+	go srv.startCompaction()
 	return srv
 }