feat(notes): compact the operation log and purge old tombstones
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m17s

- Background job at start and every 6 h: log entries older than 30 days
  (private notes: 7) are removed, documents.log_floor marks the boundary;
  clients behind it get resync and keep their queued changes
- Deleted blocks are purged 30 days after deleted_at via a "purge" log
  effect, so open editors drop them without reloading; a tombstone stays
  while a deleted child still points to it
- Text for a purged block becomes a new block at the end; inserts under a
  purged parent go to the top level (server and browser model)
- doc_op_results kept for 90 days; expired lease/session maps are swept
- Tests: storage, API, engine resync, simulation with compactions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-28 15:08:13 +01:00
1 parent aea4324c20
commit f26f286fb8
17 files changed
+653 -48

No files matched your search

+66 -4
View File
@@ -14,6 +14,8 @@ const (
OpDelete = "delete"
OpRestore = "restore"
OpSetTitle = "set_title"
// OpPurge is only produced by log compaction: deleted blocks older than the retention are removed for good
OpPurge = "purge"
StatusApplied = "applied"
StatusConflict = "conflict"
@@ -131,6 +133,10 @@ func (d *Doc) ApplyEffect(e Effect) {
}
case OpSetTitle:
d.Title, d.TitleVersion = e.Title, e.TitleVersion
case OpPurge:
for _, id := range e.IDs {
delete(d.Blocks, id)
}
}
}
@@ -189,10 +195,11 @@ func (ap *applier) apply(op Op) Result {
parent = *op.ParentID
}
if parent != "" {
if _, ok := d.Blocks[parent]; !ok {
return rejected(op, "parent_missing")
if _, ok := d.Blocks[parent]; ok {
parent = d.liveAncestor(parent)
} else {
parent = "" // parent was purged after the retention period: keep the block at the top level
}
parent = d.liveAncestor(parent)
}
if !ValidOrderKey(op.OrderKey) {
return rejected(op, "invalid_order_key")
@@ -221,7 +228,7 @@ func (ap *applier) apply(op Op) Result {
case OpSetContent:
b, ok := d.Blocks[op.BlockID]
if !ok {
return rejected(op, "missing")
return ap.reviveAsNewBlock(op)
}
if op.Content == nil {
return rejected(op, "no_content")
@@ -372,6 +379,61 @@ func (ap *applier) restore(ids []string) {
}
}
// reviveAsNewBlock handles text for a block that no longer exists (its tombstone was purged after the
// retention period, e.g. a device was offline for weeks): the text becomes a new block at the end.
func (ap *applier) reviveAsNewBlock(op Op) Result {
if op.Content == nil || *op.Content == "" || op.BlockID == "" || len(op.BlockID) > 64 {
return rejected(op, "missing")
}
if ap.env.IDTaken != nil && ap.env.IDTaken(op.BlockID) {
return rejected(op, "missing")
}
if len(*op.Content) > maxContentLength {
return rejected(op, "too_long")
}
last := ""
if roots := ap.d.Children(""); len(roots) > 0 {
last = roots[len(roots)-1].OrderKey
}
key, err := KeyBetween(last, "")
if err != nil {
return rejected(op, "missing")
}
ap.emit(Effect{Kind: OpInsert, BlockID: op.BlockID, OrderKey: key, Type: "bullet", Content: *op.Content, Props: &Props{}, Version: 1})
return Result{Status: StatusApplied, Version: 1, Reason: "revived"}
}
// Purge removes tombstones for good. Only blocks whose whole deleted subtree is purged too may go, so no
// remaining block refers to a missing parent. Returns the effect (nil if nothing to purge).
func Purge(d *Doc, candidates []string) *Effect {
set := make(map[string]bool, len(candidates))
for _, id := range candidates {
if b, ok := d.Blocks[id]; ok && b.Deleted {
set[id] = true
}
}
for changed := true; changed; {
changed = false
for _, b := range d.Blocks {
if !set[b.ID] && set[b.ParentID] {
delete(set, b.ParentID) // a remaining block still points to it
changed = true
}
}
}
if len(set) == 0 {
return nil
}
ids := make([]string, 0, len(set))
for id := range set {
ids = append(ids, id)
}
sort.Strings(ids)
e := Effect{Kind: OpPurge, IDs: ids}
d.ApplyEffect(e)
return &e
}
// conflictCopy keeps the losing text as a new block directly below the contested one.
func (ap *applier) conflictCopy(op Op, b *Block, content string) Result {
id := ConflictBlockID(op.OpID)