sandrodach/MindSift
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(notes): operation log, block leases and sync API for collaborative editing
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 50s
Details

Browse Source 
Phase 1 of the collaboration plan (server side):
- internal/collab: block tree with parent IDs and fractional order keys,
  operations with conflict rules (version check, block leases, conflict
  copies instead of lost text, restore on late edits, cycle checks),
  in-memory leases/sessions and long-poll notifier.
- Storage: per-note lock + transaction, append-only effect log (doc_ops),
  idempotent results per opId (doc_op_results), snapshot, log paging,
  central DocRole. Old flat block data is converted to the tree.
- The existing whole-document save (PUT/PATCH) runs as an adapter through
  the same engine, so the current editor keeps working and writes to the log.
- API: GET /docs/{id}/snapshot, POST /docs/{id}/sync, GET /docs/{id}/changes
  (long poll), POST /docs/{id}/leave; DOC_SEQ poke for WebSocket/SSE rooms.
- Backend accepts editor (write) shares for notes and folders; the forced
  read-only migration is gone. The share dialog still offers read only.
- Tests: conflict matrix, order keys, leases, storage adapter round trip,
  idempotency, API protocol incl. long poll, and a randomized simulation
  (4 clients, lost/duplicated requests and responses, offline phases,
  lease expiry) checking convergence, no lost text and exactly-once results.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-28 11:47:56 +01:00
1 parent bcc06da3ee
commit 1ea0c78b41
23 files changed
+3266 -287
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/storage/sqlite.go
+74 -270
View File
@@ -415,11 +415,6 @@ func (s *Store) migrate() error {
// Backfill username for existing accounts where username is empty
_, _ = s.db.Exec(`UPDATE users SET username = LOWER(SUBSTR(email, 1, INSTR(email, '@') - 1)) WHERE (username IS NULL OR username = '') AND INSTR(email, '@') > 0`)
// Ensure all existing document & folder shares are strictly read-only
_, _ = s.db.Exec(`UPDATE doc_shares SET permission = 'read' WHERE permission != 'read'`)
_, _ = s.db.Exec(`UPDATE doc_team_shares SET permission = 'read' WHERE permission != 'read'`)
_, _ = s.db.Exec(`UPDATE workspace_folder_shares SET permission = 'read' WHERE permission != 'read'`)
// Default system settings (public registration disabled by default; invite or admin creation required)
_, _ = s.db.Exec(`INSERT OR IGNORE INTO system_settings (key, value) VALUES
('sharing_mode', 'teams_only'),
@@ -463,6 +458,9 @@ func (s *Store) migrate() error {
_, _ = s.db.Exec(`UPDATE team_members SET role = 'member' WHERE role NOT IN ('admin', 'member')`)
_, _ = s.db.Exec(`UPDATE users SET role = 'member' WHERE role = 'team_admin'`)
// Block tree, operation log and idempotency tables for collaborative editing
s.migrateDocSync()
// Migrate workspace documents, blocks, folders, and strip legacy folder:* tags into pure DB tables
s.migrateWorkspaceToPureDB()
@@ -1449,86 +1447,22 @@ func (s *Store) migrateWorkspaceToPureDB() {
}
func (s *Store) loadDocBlocks(docID string, fallbackJSON string) []domain.DocumentBlock {
rows, err := s.db.Query(`
SELECT id, document_id, type, content, level, position, checked, collapsed, lang
FROM blocks
WHERE document_id = ?
ORDER BY position ASC, rowid ASC
`, docID)
if err == nil {
blocks := make([]domain.DocumentBlock, 0)
for rows.Next() {
var b domain.DocumentBlock
var chk, col int
if scanErr := rows.Scan(&b.ID, &b.DocumentID, &b.Type, &b.Content, &b.Level, &b.Position, &chk, &col, &b.Lang); scanErr == nil {
b.Checked = chk != 0
b.Collapsed = col != 0
blocks = append(blocks, b)
}
}
_ = rows.Close()
if len(blocks) > 0 {
return blocks
}
if d, _, err := loadCollabDoc(s.db, docID); err == nil && len(d.Blocks) > 0 {
return flatDocumentBlocks(docID, d)
}
// Very old notes only stored blocks_json; convert them once
blocks := make([]domain.DocumentBlock, 0)
if fallbackJSON != "" && fallbackJSON != "[]" {
_ = json.Unmarshal([]byte(fallbackJSON), &blocks)
if len(blocks) > 0 {
s.replaceDocBlocksInDB(docID, "", blocks, domain.NowISO())
}
}
return blocks
}
func (s *Store) replaceDocBlocksInDB(docID, _ string, blocks []domain.DocumentBlock, now string) {
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ?`, docID)
if len(blocks) > 0 {
stmt, err := s.db.Prepare(`
INSERT INTO blocks (id, document_id, content, level, position, type, checked, collapsed, lang)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
document_id = excluded.document_id,
content = excluded.content,
level = excluded.level,
position = excluded.position,
type = excluded.type,
checked = excluded.checked,
collapsed = excluded.collapsed,
lang = excluded.lang
`)
if err == nil {
defer stmt.Close()
for i, b := range blocks {
bID := strings.TrimSpace(b.ID)
if bID == "" {
bID = domain.NewID("blk")
}
bType := b.Type
if bType == "" {
bType = "bullet"
}
chk := 0
if b.Checked {
chk = 1
}
col := 0
if b.Collapsed {
col = 1
}
pos := b.Position
if pos <= 0 {
pos = float64((i + 1) * 1000)
}
_, _ = stmt.Exec(bID, docID, b.Content, b.Level, pos, bType, chk, col, b.Lang)
if d, _, err := loadCollabDoc(s.db, docID); err == nil {
return flatDocumentBlocks(docID, d)
}
}
}
// Update markdown_cache on documents
mdCache := domain.CompileBlocksToMarkdown(blocks)
_, _ = s.db.Exec(`UPDATE documents SET markdown_cache = ?, updated_at = ? WHERE id = ?`, mdCache, now, docID)
return blocks
}
func (s *Store) LoadDocumentState(docID string) (string, []domain.DocumentBlock, error) {
@@ -1541,222 +1475,85 @@ func (s *Store) LoadDocumentState(docID string) (string, []domain.DocumentBlock,
return title, blocks, nil
}
// FlushDocumentState persists the in-memory blocks into `blocks`, extracts all #tags into `block_tags`,
// and compiles the hierarchical Markdown into `documents.markdown_cache`.
// FlushDocumentState stores a complete block list (and optionally the title) through the operation log.
func (s *Store) FlushDocumentState(docID string, title string, blocks []domain.DocumentBlock) error {
now := domain.NowISO()
// Normalize positions sequentially
normalized := make([]domain.DocumentBlock, len(blocks))
var plainBuilder strings.Builder
for i, b := range blocks {
if strings.TrimSpace(b.ID) == "" {
b.ID = domain.NewID("blk")
}
if b.Type == "" {
b.Type = "bullet"
}
b.DocumentID = docID
b.Position = float64((i + 1) * 1000)
normalized[i] = b
if b.Content != "" {
plainBuilder.WriteString(b.Content)
plainBuilder.WriteString("\n")
}
}
s.replaceDocBlocksInDB(docID, "", normalized, now)
_ = s.indexDocBlockTags(docID, "", normalized)
mdCache := domain.CompileBlocksToMarkdown(normalized)
plainContent := plainBuilder.String()
blocksJSON, _ := json.Marshal(normalized)
s.replaceDocBlocksInDB(docID, "", blocks, domain.NowISO())
if strings.TrimSpace(title) != "" {
_, err := s.db.Exec(`
UPDATE documents
SET title = ?, markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ?
WHERE id = ?
`, title, mdCache, plainContent, string(blocksJSON), now, docID)
return err
s.applyLegacyTitle(docID, "", title)
}
_, err := s.db.Exec(`
UPDATE documents
SET markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ?
WHERE id = ?
`, mdCache, plainContent, string(blocksJSON), now, docID)
return err
return nil
}
// PatchDocumentBlocks applies atomic block-level upserts, deletions, and order reconciliation without overwriting concurrent edits from other sessions.
// PatchDocumentBlocks applies block upserts, deletions and an optional order to a note through the
// operation log. Blocks not mentioned keep their position relative to each other.
func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPatchRequest) (*domain.Document, []string, error) {
if !CanEditDoc(s.DocRole(userID, docID)) {
return nil, nil, fmt.Errorf("keine Schreibberechtigung für dieses Dokument (nur Lesezugriff)")
}
doc, err := s.GetDocumentByID(userID, docID)
if err != nil {
return nil, nil, err
}
if !doc.IsOwner {
return nil, nil, fmt.Errorf("keine Schreibberechtigung für dieses Dokument (nur Lesezugriff)")
deleted := make(map[string]bool, len(req.DeletedIDs))
for _, id := range req.DeletedIDs {
deleted[strings.TrimSpace(id)] = true
}
now := domain.NowISO()
// Ensure blocks table is seeded if previously only in blocks_json
var existingCount int
_ = s.db.QueryRow(`SELECT COUNT(*) FROM blocks WHERE document_id = ?`, docID).Scan(&existingCount)
if existingCount == 0 && len(doc.Blocks) > 0 {
s.replaceDocBlocksInDB(docID, doc.UserID, doc.Blocks, doc.UpdatedAt)
}
// Read current ordered block IDs from DB before patch
var existingOrder []string
ordRows, err := s.db.Query(`SELECT id FROM blocks WHERE document_id = ? ORDER BY position ASC, rowid ASC`, docID)
if err == nil {
for ordRows.Next() {
var bid string
if err := ordRows.Scan(&bid); err == nil {
existingOrder = append(existingOrder, bid)
}
upserts := make(map[string]domain.DocumentBlock, len(req.Upserted))
for _, b := range req.Upserted {
if id := strings.TrimSpace(b.ID); id != "" && !deleted[id] {
upserts[id] = b
}
_ = ordRows.Close()
}
// 1. Delete explicitly deleted block IDs
deletedSet := make(map[string]bool, len(req.DeletedIDs))
for _, delID := range req.DeletedIDs {
delID = strings.TrimSpace(delID)
if delID == "" {
merged := make([]domain.DocumentBlock, 0, len(doc.Blocks)+len(upserts))
used := make(map[string]bool, len(upserts))
for _, b := range doc.Blocks {
if deleted[b.ID] {
continue
}
deletedSet[delID] = true
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ? AND id = ?`, docID, delID)
}
// Filter existingOrder to exclude deleted blocks
survivingExisting := make([]string, 0, len(existingOrder))
survivingSet := make(map[string]bool, len(existingOrder))
for _, bid := range existingOrder {
if !deletedSet[bid] {
survivingExisting = append(survivingExisting, bid)
survivingSet[bid] = true
}
}
// 2. Upsert modified or newly created blocks
upsertedIDs := make([]string, 0, len(req.Upserted))
for idx, b := range req.Upserted {
bID := strings.TrimSpace(b.ID)
if bID == "" || deletedSet[bID] {
if up, ok := upserts[b.ID]; ok {
merged = append(merged, up)
used[b.ID] = true
continue
}
bType := b.Type
if bType == "" {
bType = "bullet"
merged = append(merged, b)
}
for _, b := range req.Upserted {
if id := strings.TrimSpace(b.ID); id != "" && !used[id] && !deleted[id] {
merged = append(merged, b)
used[id] = true
}
chk := 0
if b.Checked {
chk = 1
}
col := 0
if b.Collapsed {
col = 1
}
defaultSort := float64((len(survivingExisting) + idx + 1) * 1000)
_, _ = s.db.Exec(`
INSERT INTO blocks (id, document_id, content, level, position, type, checked, collapsed, lang)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
type = excluded.type,
content = excluded.content,
level = excluded.level,
checked = excluded.checked,
collapsed = excluded.collapsed,
lang = excluded.lang
`, bID, docID, b.Content, b.Level, defaultSort, bType, chk, col, b.Lang)
upsertedIDs = append(upsertedIDs, bID)
survivingSet[bID] = true
}
// 3. Reconcile block ordering without dropping blocks concurrently created by other users
var mergedOrder []string
inMerged := make(map[string]bool)
if len(req.BlockOrder) > 0 {
for _, bid := range req.BlockOrder {
if !deletedSet[bid] && survivingSet[bid] && !inMerged[bid] {
mergedOrder = append(mergedOrder, bid)
inMerged[bid] = true
}
rank := make(map[string]int, len(req.BlockOrder))
for i, id := range req.BlockOrder {
rank[id] = i
}
for i, bid := range survivingExisting {
if inMerged[bid] {
continue
sort.SliceStable(merged, func(i, j int) bool {
ri, okI := rank[merged[i].ID]
rj, okJ := rank[merged[j].ID]
if okI && okJ {
return ri < rj
}
insertPos := 0
for j := i - 1; j >= 0; j-- {
predID := survivingExisting[j]
if inMerged[predID] {
for k, mID := range mergedOrder {
if mID == predID {
insertPos = k + 1
break
}
}
break
}
}
mergedOrder = append(mergedOrder[:insertPos], append([]string{bid}, mergedOrder[insertPos:]...)...)
inMerged[bid] = true
}
for _, bid := range upsertedIDs {
if !inMerged[bid] {
mergedOrder = append(mergedOrder, bid)
inMerged[bid] = true
}
}
} else {
for _, bid := range survivingExisting {
if !inMerged[bid] {
mergedOrder = append(mergedOrder, bid)
inMerged[bid] = true
}
}
for _, bid := range upsertedIDs {
if !inMerged[bid] {
mergedOrder = append(mergedOrder, bid)
inMerged[bid] = true
}
}
return okI && !okJ
})
}
// Update position in blocks
for idx, bid := range mergedOrder {
_, _ = s.db.Exec(`UPDATE blocks SET position = ? WHERE document_id = ? AND id = ?`, float64((idx+1)*1000), docID, bid)
s.replaceDocBlocksInDB(docID, userID, merged, domain.NowISO())
if req.Title != nil {
s.applyLegacyTitle(docID, userID, strings.TrimSpace(*req.Title))
}
// Reload authoritative block list and update document snapshot
finalBlocks := s.loadDocBlocks(docID, "")
var plainBuilder strings.Builder
for _, b := range finalBlocks {
if b.Content != "" {
plainBuilder.WriteString(b.Content)
plainBuilder.WriteString("\n")
}
updated, err := s.GetDocumentByID(userID, docID)
if err != nil {
return nil, nil, err
}
plainContent := plainBuilder.String()
mdCache := domain.CompileBlocksToMarkdown(finalBlocks)
blocksJSON, _ := json.Marshal(finalBlocks)
_, _ = s.db.Exec(`UPDATE documents SET markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ? WHERE id = ?`,
mdCache, plainContent, string(blocksJSON), now, docID)
doc.Blocks = finalBlocks
doc.MarkdownCache = mdCache
doc.Content = plainContent
doc.UpdatedAt = now
_ = s.indexDocBlockTags(doc.ID, doc.UserID, finalBlocks)
return doc, mergedOrder, nil
order := make([]string, len(updated.Blocks))
for i, b := range updated.Blocks {
order[i] = b.ID
}
return updated, order, nil
}
// GetAuthorizedDocUsers returns all user IDs who have read/write/owner access to docID.
@@ -2383,6 +2180,7 @@ func (s *Store) UpdateDocument(userID, docID string, title, icon, folderPath *st
if title != nil {
doc.Title = strings.TrimSpace(*title)
s.applyLegacyTitle(docID, userID, doc.Title)
}
if icon != nil {
doc.Icon = *icon
@@ -3445,8 +3243,7 @@ func (s *Store) CreateFolderShare(currentUserID, folderID, targetType, targetIde
}
}
// Geteilte Ordner können ausschließlich mit Leserberechtigung geteilt werden (wie Dokumente)
permission = "read"
permission = normalizeSharePermission(permission)
var targetID, targetName, email, avatarURL string
if cleanType == "team" {
@@ -3671,8 +3468,7 @@ func (s *Store) CreateDocTeamShare(currentUserID, docID, teamID, permission stri
return nil, fmt.Errorf("team nicht gefunden")
}
// Document sharing is strictly read-only
permission = "read"
permission = normalizeSharePermission(permission)
now := domain.NowISO()
shareID := domain.NewID("share")
@@ -3699,6 +3495,15 @@ func (s *Store) CreateDocTeamShare(currentUserID, docID, teamID, permission stri
}, nil
}
// normalizeSharePermission maps share roles from the API to the stored values: "write" (editor) or "read" (viewer).
func normalizeSharePermission(p string) string {
switch strings.ToLower(strings.TrimSpace(p)) {
case "write", "editor":
return "write"
}
return "read"
}
func (s *Store) CreateDocShare(currentUserID, docID, targetUserIdentifier, permission string) (*domain.DocShare, error) {
doc, err := s.GetDocumentByID(currentUserID, docID)
if err != nil {
@@ -3712,11 +3517,10 @@ func (s *Store) CreateDocShare(currentUserID, docID, targetUserIdentifier, permi
// If identifier looks like a team or is prefixed with team_, route to CreateDocTeamShare
if strings.HasPrefix(cleanTarget, "team_") {
return s.CreateDocTeamShare(currentUserID, docID, cleanTarget, "read")
return s.CreateDocTeamShare(currentUserID, docID, cleanTarget, permission)
}
// Document sharing is strictly read-only
permission = "read"
permission = normalizeSharePermission(permission)
// Resolve target user by ID, username, or Email
var targetUser *domain.User