feat(notes): browser sync engine with offline queue; editor saves through the operation log
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m26s

Phase 2 of the collaboration plan:
- frontend/src/lib/docSync: TypeScript port of the block tree, operations,
  effects and order keys (kept identical to Go via shared test vectors),
  diff from the editor's flat list to operations, IndexedDB persistence,
  HTTP transport and DocSyncEngine (one request in flight, idempotent
  resends, backoff, 3 s/30 s polling, keepalive flush and leave on exit,
  block leases for focus, 20 s grace and new blocks).
- Text changes carry the version the user saw when typing (first change
  wins when coalescing; a stale focused field keeps its old base), so a
  concurrent edit becomes a conflict copy instead of a silent overwrite.
- DocumentEditor uses the engine instead of the 800 ms whole-document PUT;
  editors (not only owners) can edit, remote changes appear live.
- Server: task IDs of linked blocks are delivered again, linked tasks follow
  ticked/deleted blocks after sync, note tags are derived, title changes
  notify other users; block IDs already used in another note are renamed
  (starter content with fixed IDs was empty for every further user).
- Fix: unlinking tasks from blocks wrote to non-existent columns and never
  worked (error was swallowed).
- Tests: Go/TS order key vectors, diff round trips, engine races (lost
  responses, offline + reload, edits arriving between typing and sending,
  stale focused field, duplicate first block), plus a Playwright test with
  two browsers against the real image (e2e/run.sh).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-28 12:28:15 +01:00
1 parent 1ea0c78b41
commit 51f90eb7e4
31 files changed
+2468 -207

No files matched your search

+22
View File
@@ -145,6 +145,9 @@ func (s *Server) handleDocSync(w http.ResponseWriter, r *http.Request) {
resp.Leases.Granted = append(resp.Leases.Granted, granted...)
}
}
if changed {
s.afterDocChange(docID, req.Ops, resp.Results)
}
entries, head, err := s.store.GetDocOpsSince(docID, req.LastSeq, syncMaxLog)
if err != nil {
@@ -168,6 +171,25 @@ func (s *Server) handleDocSync(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, resp)
}
// afterDocChange runs side effects of applied operations: linked tasks follow their blocks, and a new
// title reaches the note lists of everyone with access.
func (s *Server) afterDocChange(docID string, ops []collab.Op, results []collab.Result) {
titleChanged := false
for i, res := range results {
if i < len(ops) && ops[i].Kind == collab.OpSetTitle && res.Seq > 0 {
titleChanged = true
}
}
for _, taskID := range s.store.ReconcileDocTaskLinks(docID) {
if th, err := s.store.GetThoughtByIDUnchecked(taskID); err == nil {
s.wsHub.BroadcastToUsers(s.taskAudience(th), "TASK_UPDATED", th)
}
}
if titleChanged {
s.broadcastDocToAuthorizedUsers(docID, "DOC_UPDATED")
}
}
// notifyDocChanged wakes long-poll requests and pokes WebSocket/SSE clients in the note's room.
func (s *Server) notifyDocChanged(docID string, head int64) {
s.notifier.Notify(docID)
+103 -2
View File
@@ -355,9 +355,21 @@ func persistDocState(tx *sql.Tx, docID string, before, after *collab.Doc, head i
}
}
blocksJSON, _ := json.Marshal(flat)
tagSet := map[string]bool{}
for _, b := range flat {
for _, tag := range domain.ExtractTags(b.Content) {
tagSet[tag] = true
}
}
tags := make([]string, 0, len(tagSet))
for tag := range tagSet {
tags = append(tags, tag)
}
sort.Strings(tags)
tagsJSON, _ := json.Marshal(tags)
if _, err := tx.Exec(
`UPDATE documents SET title = ?, title_version = ?, head_seq = ?, markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ? WHERE id = ?`,
after.Title, after.TitleVersion, head, domain.CompileBlocksToMarkdown(flat), plain.String(), string(blocksJSON), now, docID,
`UPDATE documents SET title = ?, title_version = ?, head_seq = ?, markdown_cache = ?, content = ?, blocks_json = ?, tags = ?, updated_at = ? WHERE id = ?`,
after.Title, after.TitleVersion, head, domain.CompileBlocksToMarkdown(flat), plain.String(), string(blocksJSON), string(tagsJSON), now, docID,
); err != nil {
return err
}
@@ -377,6 +389,71 @@ func persistDocState(tx *sql.Tx, docID string, before, after *collab.Doc, head i
return nil
}
// docTaskLinks maps block IDs of a note to the IDs of tasks created from them ("Erweitert").
func (s *Store) docTaskLinks(docID string) map[string]string {
links := map[string]string{}
rows, err := s.db.Query(
`SELECT id, converted_ref_id FROM thoughts WHERE converted_type = 'document_block' AND converted_ref_id LIKE ?`,
docID+":%")
if err != nil {
return links
}
defer rows.Close()
for rows.Next() {
var taskID, ref string
if rows.Scan(&taskID, &ref) == nil {
if _, blockID, ok := strings.Cut(ref, ":"); ok {
links[blockID] = taskID
}
}
}
return links
}
// ReconcileDocTaskLinks brings tasks linked to blocks in line with the note: a ticked block completes its
// task, a deleted block unlinks it (the task itself stays). Returns the IDs of changed tasks.
func (s *Store) ReconcileDocTaskLinks(docID string) []string {
links := s.docTaskLinks(docID)
if len(links) == 0 {
return nil
}
d, _, err := loadCollabDoc(s.db, docID)
if err != nil {
return nil
}
var changed []string
for blockID, taskID := range links {
b, ok := d.Blocks[blockID]
if !ok || b.Deleted {
if _, err := s.db.Exec(`UPDATE thoughts SET converted_type = '', converted_ref_id = '' WHERE id = ?`, taskID); err == nil {
changed = append(changed, taskID)
}
continue
}
kanban := "todo"
if b.Props.Checked {
kanban = "done"
}
res, err := s.db.Exec(
`UPDATE thoughts SET is_completed = ?, kanban_status = ? WHERE id = ? AND is_completed != ?`,
boolInt(b.Props.Checked), kanban, taskID, boolInt(b.Props.Checked))
if err == nil {
if n, _ := res.RowsAffected(); n > 0 {
changed = append(changed, taskID)
}
}
}
sort.Strings(changed)
return changed
}
// GetThoughtByIDUnchecked loads a task without access check, for server-internal notifications.
func (s *Store) GetThoughtByIDUnchecked(thoughtID string) (*domain.Thought, error) {
return scanThought(s.db.QueryRow(
`SELECT id, user_id, content, tags, priority, source, status, is_completed, converted_type, converted_ref_id, due_date, task_list_id, kanban_status, notes, alarm_sent, created_at, triaged_at, subtasks, sort_order
FROM thoughts WHERE id = ?`, thoughtID))
}
// GetDocOpsSince returns up to limit log entries after seq, plus the current head.
func (s *Store) GetDocOpsSince(docID string, since int64, limit int) ([]DocLogEntry, int64, error) {
var head int64
@@ -495,12 +572,36 @@ const legacyClientID = "legacy"
// replaceDocBlocksInDB stores a complete flat block list by diffing it against the current tree and
// applying the difference as regular operations, so it shows up in the log like any other change.
func (s *Store) replaceDocBlocksInDB(docID, userID string, blocks []domain.DocumentBlock, _ string) {
blocks = s.renameForeignBlockIDs(docID, blocks)
a := collab.Actor{UserID: userID, ClientID: legacyClientID}
if _, _, err := s.applyDocChange(docID, a, nil, func(d *collab.Doc) []collab.Op { return legacyOps(d, blocks) }); err != nil {
slog.Error("Speichern der Notiz fehlgeschlagen", "doc_id", docID, "error", err)
}
}
// renameForeignBlockIDs gives new IDs to blocks whose ID already belongs to another note (e.g. fixed IDs of
// starter content or copied blocks). Block IDs are global, so keeping them would make the insert fail.
func (s *Store) renameForeignBlockIDs(docID string, blocks []domain.DocumentBlock) []domain.DocumentBlock {
out := blocks
copied := false
for i, b := range blocks {
id := strings.TrimSpace(b.ID)
if id == "" {
continue
}
var owner string
if s.db.QueryRow(`SELECT document_id FROM blocks WHERE id = ?`, id).Scan(&owner) != nil || owner == docID {
continue
}
if !copied {
out = append([]domain.DocumentBlock(nil), blocks...)
copied = true
}
out[i].ID = domain.NewID("blk")
}
return out
}
// applyLegacyTitle records a title change from the whole-document API in the log.
func (s *Store) applyLegacyTitle(docID, userID, title string) {
a := collab.Actor{UserID: userID, ClientID: legacyClientID}
+15 -9
View File
@@ -77,7 +77,8 @@ func (w *simWorld) edit(c *simClient) {
switch n := w.r.Intn(100); {
case n < 45 && target != nil: // typing
content := target.Content + fmt.Sprintf(" %s%d", c.actor.ClientID, c.counter)
c.pending = append(c.pending, collab.Op{OpID: w.newOpID(c), Kind: collab.OpSetContent, BlockID: target.ID, Content: &content})
// Base = the version the user sees while typing (like the browser engine)
c.pending = append(c.pending, collab.Op{OpID: w.newOpID(c), Kind: collab.OpSetContent, BlockID: target.ID, BaseVersion: target.Version, Content: &content})
c.holds[target.ID] = true
case n < 65: // Enter: new block after target (or first block)
parent, after := "", ""
@@ -133,7 +134,7 @@ func (w *simWorld) edit(c *simClient) {
c.pending = append(c.pending, collab.Op{OpID: w.newOpID(c), Kind: collab.OpSetProps, BlockID: target.ID, Patch: &collab.PropsPatch{Checked: &checked}})
case n < 96: // rename note
title := fmt.Sprintf("Titel %s%d", c.actor.ClientID, c.counter)
c.pending = append(c.pending, collab.Op{OpID: w.newOpID(c), Kind: collab.OpSetTitle, Title: &title})
c.pending = append(c.pending, collab.Op{OpID: w.newOpID(c), Kind: collab.OpSetTitle, BaseVersion: v.TitleVersion, Title: &title})
default: // stop editing some block
for id := range c.holds {
delete(c.holds, id)
@@ -142,15 +143,23 @@ func (w *simWorld) edit(c *simClient) {
}
}
// formBatch takes pending operations, keeps only the last text change per block and fills in base versions.
// formBatch takes pending operations and keeps only the last text change per block; its base version is
// the one of the first change, i.e. what the user saw when starting to type.
func (c *simClient) formBatch() []collab.Op {
lastText := map[string]int{}
lastTitle := -1
firstBase := map[string]int{}
lastTitle, firstTitleBase := -1, 0
for i, op := range c.pending {
switch op.Kind {
case collab.OpSetContent:
if _, ok := lastText[op.BlockID]; !ok {
firstBase[op.BlockID] = op.BaseVersion
}
lastText[op.BlockID] = i
case collab.OpSetTitle:
if lastTitle < 0 {
firstTitleBase = op.BaseVersion
}
lastTitle = i
}
}
@@ -161,15 +170,12 @@ func (c *simClient) formBatch() []collab.Op {
if lastText[op.BlockID] != i {
continue
}
op.BaseVersion = 1
if b, ok := c.confirmed.Blocks[op.BlockID]; ok {
op.BaseVersion = b.Version
}
op.BaseVersion = firstBase[op.BlockID]
case collab.OpSetTitle:
if i != lastTitle {
continue
}
op.BaseVersion = c.confirmed.TitleVersion
op.BaseVersion = firstTitleBase
}
batch = append(batch, op)
}
+54
View File
@@ -184,3 +184,57 @@ func TestBackfillConvertsFlatBlocksToTree(t *testing.T) {
want := flatBlocks([3]interface{}{"x", 0, "x"}, [3]interface{}{"y", 1, "y"}, [3]interface{}{"z", 1, "z"}, [3]interface{}{"w", 0, "w"})
sameOutline(t, got.Blocks, want)
}
func TestTaskLinksFollowTheirBlock(t *testing.T) {
s, u := newSyncTestStore(t)
doc, _ := s.CreateDocument(u.ID, "", "Notiz", "📄", "", []domain.DocumentBlock{{ID: "todo1", Type: "todo", Content: "Anrufen"}}, nil, false)
task, err := s.CreateTaskFromDocBlock(u.ID, doc.ID, "todo1", "Anrufen", false)
if err != nil {
t.Fatal(err)
}
got, _ := s.GetDocumentByID(u.ID, doc.ID)
if got.Blocks[0].TaskID != task.ID {
t.Fatalf("block must carry its task ID after loading, got %q", got.Blocks[0].TaskID)
}
a := collab.Actor{UserID: u.ID, ClientID: "tab"}
yes := true
if _, _, err := s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "chk", Kind: collab.OpSetProps, BlockID: "todo1", Patch: &collab.PropsPatch{Checked: &yes}}}, nil); err != nil {
t.Fatal(err)
}
if changed := s.ReconcileDocTaskLinks(doc.ID); len(changed) != 1 {
t.Fatalf("ticking the block must complete the task, changed=%v", changed)
}
if th, _ := s.GetThoughtByID(u.ID, task.ID); !th.IsCompleted {
t.Fatal("task not completed")
}
if _, _, err := s.ApplyDocOps(doc.ID, a, []collab.Op{{OpID: "del", Kind: collab.OpDelete, BlockID: "todo1"}}, nil); err != nil {
t.Fatal(err)
}
s.ReconcileDocTaskLinks(doc.ID)
th, err := s.GetThoughtByID(u.ID, task.ID)
if err != nil {
t.Fatal("deleting the block must not delete the task")
}
if th.ConvertedType == "document_block" {
t.Fatal("deleting the block must unlink the task")
}
}
func TestFixedBlockIDsWorkInSeveralNotes(t *testing.T) {
s, u := newSyncTestStore(t)
other, _ := s.CreateUser("second@example.com", "hash", "Second", "member")
blocks := flatBlocks([3]interface{}{"blk_1", 0, "Willkommen"}, [3]interface{}{"blk_2", 1, "Unterpunkt"})
first, _ := s.CreateDocument(u.ID, "", "Start", "📄", "", blocks, nil, false)
second, _ := s.CreateDocument(other.ID, "", "Start", "📄", "", blocks, nil, false)
a, _ := s.GetDocumentByID(u.ID, first.ID)
b, _ := s.GetDocumentByID(other.ID, second.ID)
if len(a.Blocks) != 2 || len(b.Blocks) != 2 || b.Blocks[0].Content != "Willkommen" || b.Blocks[1].Level != 1 {
t.Fatalf("second note must get its own copies of the blocks: %+v", b.Blocks)
}
if a.Blocks[0].ID == b.Blocks[0].ID {
t.Fatal("block IDs must differ between notes")
}
}
+14 -8
View File
@@ -1448,7 +1448,13 @@ func (s *Store) migrateWorkspaceToPureDB() {
func (s *Store) loadDocBlocks(docID string, fallbackJSON string) []domain.DocumentBlock {
if d, _, err := loadCollabDoc(s.db, docID); err == nil && len(d.Blocks) > 0 {
return flatDocumentBlocks(docID, d)
blocks := flatDocumentBlocks(docID, d)
if links := s.docTaskLinks(docID); len(links) > 0 {
for i := range blocks {
blocks[i].TaskID = links[blocks[i].ID]
}
}
return blocks
}
// Very old notes only stored blocks_json; convert them once
@@ -2228,18 +2234,18 @@ func (s *Store) UpdateDocument(userID, docID string, title, icon, folderPath *st
for _, b := range doc.Blocks {
remainingBlockIDs[b.ID] = true
}
rows, err := s.db.Query(`SELECT id, converted_ref_id, block_id FROM thoughts WHERE (doc_id = ? OR converted_ref_id LIKE ?) AND converted_type = 'document_block'`, docID, docID+":%")
rows, err := s.db.Query(`SELECT id, converted_ref_id FROM thoughts WHERE converted_ref_id LIKE ? AND converted_type = 'document_block'`, docID+":%")
if err == nil {
for rows.Next() {
var tid, refID, bID string
if err := rows.Scan(&tid, &refID, &bID); err == nil {
bid := bID
if bid == "" && strings.Contains(refID, ":") {
var tid, refID string
if err := rows.Scan(&tid, &refID); err == nil {
bid := ""
if strings.Contains(refID, ":") {
parts := strings.SplitN(refID, ":", 2)
bid = parts[1]
}
if !remainingBlockIDs[bid] {
_, _ = s.db.Exec(`UPDATE thoughts SET converted_type = '', converted_ref_id = '', doc_id = '', block_id = '' WHERE id = ?`, tid)
_, _ = s.db.Exec(`UPDATE thoughts SET converted_type = '', converted_ref_id = '' WHERE id = ?`, tid)
}
}
}
@@ -4111,7 +4117,7 @@ func (s *Store) UnlinkTaskFromDocBlock(userID, taskID string) {
}
}
}
_, _ = s.db.Exec(`UPDATE thoughts SET converted_type = '', converted_ref_id = '', doc_id = '', block_id = '' WHERE id = ?`, taskID)
_, _ = s.db.Exec(`UPDATE thoughts SET converted_type = '', converted_ref_id = '' WHERE id = ?`, taskID)
}
func (s *Store) SyncTaskFromBlockCheck(userID, taskID string, checked bool) {