sandrodach/MindSift
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(workspace): tabula-rasa outliner architecture, SQLite blocks/block_tags schema & echo-protected live room sync
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
sandrodach committed 2026-10-26 21:14:06 +01:00
1 parent f3b3f07e04
commit 86867b0d6f
17 files changed
+3409 -7178
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/api/server.go
+25 -225
View File
@@ -3,7 +3,6 @@ package api
import (
"bytes"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -32,7 +31,7 @@ type Server struct {
func NewServer(cfg *config.Config, store *storage.Store, wsHub *ws.Hub, staticFS fs.FS) *Server {
if wsHub != nil && store != nil {
wsHub.SetCRDTStore(store)
wsHub.SetDocStore(store)
}
srv := &Server{
cfg: cfg,
@@ -70,8 +69,6 @@ func (s *Server) Router() http.Handler {
r.Get("/api/v1/ws/spaces/{spaceId}", s.handleWebSocket)
r.Get("/api/v1/ws/lists/{listId}", s.handleWebSocket)
r.Get("/api/v1/ws/docs/{id}", s.handleDocWebSocket)
r.Get("/api/v1/sync/docs/{id}/stream", s.handleDocSyncStream)
r.Post("/api/v1/sync/docs/{id}/update", s.handleDocSyncUpdate)
r.Get("/api/v1/events", s.handleSSE)
r.Post("/api/v1/events/rooms", s.handleSSERooms)
@@ -144,18 +141,6 @@ func (s *Server) Router() http.Handler {
r.Post("/api/v1/documents/{id}/shares", s.handleCreateDocShare)
r.Delete("/api/v1/documents/{id}/shares/{userId}", s.handleDeleteDocShare)
// Document Locking
r.Get("/api/v1/documents/{id}/lock", s.handleGetDocLock)
r.Post("/api/v1/documents/{id}/lock", s.handleLockDoc)
r.Post("/api/v1/documents/{id}/lock/heartbeat", s.handleHeartbeatDocLock)
r.Delete("/api/v1/documents/{id}/lock", s.handleUnlockDoc)
// Document CRDT Realtime Sync
r.Get("/api/v1/documents/{id}/crdt", s.handleGetDocCRDT)
r.Put("/api/v1/documents/{id}/crdt", s.handleSaveDocCRDT)
r.Post("/api/v1/documents/{id}/awareness", s.handlePostDocAwareness)
r.Delete("/api/v1/documents/{id}/awareness", s.handleDeleteDocAwareness)
// Shared Lists & Realtime Collaboration
r.Get("/api/v1/lists", s.handleGetAllLists)
r.Post("/api/v1/lists", s.handleCreateList)
@@ -864,6 +849,18 @@ func (s *Server) handleGetDocument(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
if s.wsHub != nil {
if liveTitle, liveBlocks, ok := s.wsHub.GetActiveDocState(docID); ok {
if liveTitle != "" {
doc.Title = liveTitle
}
if len(liveBlocks) > 0 {
doc.Blocks = liveBlocks
doc.MarkdownCache = domain.CompileBlocksToMarkdown(liveBlocks)
doc.Content = doc.MarkdownCache
}
}
}
writeJSON(w, http.StatusOK, doc)
}
@@ -889,6 +886,9 @@ func (s *Server) handleUpdateDocument(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if s.wsHub != nil && (req.Title != nil || req.Blocks != nil) {
s.wsHub.SyncActiveDocRoom(docID, doc.Title, doc.Blocks)
}
if req.Title != nil || req.Icon != nil || req.FolderPath != nil || req.Blocks != nil || req.Tags != nil {
s.wsHub.BroadcastDocEvent(docID, "DOC_UPDATED", doc)
s.broadcastDocToAuthorizedUsers(docID, "DOC_UPDATED")
@@ -911,6 +911,9 @@ func (s *Server) handlePatchDocumentBlocks(w http.ResponseWriter, r *http.Reques
writeError(w, http.StatusBadRequest, err.Error())
return
}
if s.wsHub != nil {
s.wsHub.SyncActiveDocRoom(docID, doc.Title, doc.Blocks)
}
patchEvent := domain.DocBlocksPatchEvent{
DocID: docID,
@@ -933,6 +936,9 @@ func (s *Server) handlePatchDocumentBlocks(w http.ResponseWriter, r *http.Reques
func (s *Server) handleDeleteDocument(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
if s.wsHub != nil {
s.wsHub.CloseDocRoom(docID)
}
if err := s.store.DeleteDocument(userID, docID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
@@ -1307,30 +1313,7 @@ func (s *Server) handleGetCollaborators(w http.ResponseWriter, r *http.Request)
writeJSON(w, http.StatusOK, users)
}
// --- DOCUMENT LOCKING (Neutralized - Yjs CRDT Pessimistic Locks Removed) ---
func (s *Server) handleGetDocLock(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]interface{}{"locked": false})
}
func (s *Server) handleLockDoc(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
writeJSON(w, http.StatusOK, map[string]interface{}{
"locked": false,
"acquired": true,
"docId": docID,
})
}
func (s *Server) handleHeartbeatDocLock(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]bool{"extended": true})
}
func (s *Server) handleUnlockDoc(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]bool{"unlocked": true})
}
// --- DOCUMENT CRDT REALTIME SYNC & DUAL TRANSPORT ---
// --- DOCUMENT REALTIME WEBSOCKET ROOM ---
func (s *Server) authenticateUserFromReq(r *http.Request) string {
token := r.URL.Query().Get("token")
@@ -1379,192 +1362,9 @@ func (s *Server) handleDocWebSocket(w http.ResponseWriter, r *http.Request) {
}
}
clientID := r.URL.Query().Get("clientId")
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ServeDocWS(w, r, userID)
}
// GET /api/v1/sync/docs/{id}/stream
func (s *Server) handleDocSyncStream(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
if docID == "" {
writeError(w, http.StatusBadRequest, "docId erforderlich")
return
}
userID := s.authenticateUserFromReq(r)
if userID == "" {
userID = auth.GetUserID(r.Context())
}
if userID == "" {
userID = "anonymous"
}
if userID != "anonymous" {
if _, err := s.store.GetDocumentByID(userID, docID); err != nil {
writeError(w, http.StatusForbidden, "Kein Zugriff auf dieses Dokument")
return
}
}
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ServeDocSSE(w, r, userID)
}
// POST /api/v1/sync/docs/{id}/update
func (s *Server) handleDocSyncUpdate(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
if docID == "" {
writeError(w, http.StatusBadRequest, "docId erforderlich")
return
}
userID := s.authenticateUserFromReq(r)
if userID == "" {
userID = auth.GetUserID(r.Context())
}
if userID == "" {
writeError(w, http.StatusUnauthorized, "Nicht autorisiert")
return
}
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
if !doc.IsOwner && doc.Permission != "write" {
writeError(w, http.StatusForbidden, "Keine Schreibberechtigung")
return
}
contentType := r.Header.Get("Content-Type")
var updateBytes []byte
if strings.Contains(contentType, "application/octet-stream") {
body, err := io.ReadAll(io.LimitReader(r.Body, 8*1024*1024))
if err != nil {
writeError(w, http.StatusBadRequest, "Fehler beim Lesen des Payloads")
return
}
updateBytes = body
} else {
var req struct {
Update string `json:"update"`
State string `json:"state"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
if req.State != "" {
if raw, err := base64.StdEncoding.DecodeString(req.State); err == nil && len(raw) > 0 {
room := s.wsHub.GetOrCreateDocRoom(docID)
room.CompactState(raw, userID)
}
}
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 für Update")
return
}
updateBytes = raw
}
}
if len(updateBytes) > 0 {
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ApplyUpdate(updateBytes, userID, nil, nil)
}
writeJSON(w, http.StatusOK, map[string]interface{}{"success": true, "docId": docID})
}
func (s *Server) handleGetDocCRDT(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
if _, err := s.store.GetDocumentByID(userID, docID); err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
room := s.wsHub.GetOrCreateDocRoom(docID)
b64State, b64Updates := room.GetInitialSnapshot()
writeJSON(w, http.StatusOK, map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
})
}
func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
if !doc.IsOwner && doc.Permission != "write" {
writeError(w, http.StatusForbidden, "Keine Schreibberechtigung")
return
}
var req struct {
State string `json:"state"`
Update string `json:"update"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiger Request-Body")
return
}
room := s.wsHub.GetOrCreateDocRoom(docID)
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format für Update")
return
}
room.ApplyUpdate(raw, userID, nil, nil)
}
if req.State != "" {
raw, err := base64.StdEncoding.DecodeString(req.State)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format")
return
}
room.CompactState(raw, userID)
}
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handlePostDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
_ = doc
var payload json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS", payload, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handleDeleteDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": userID,
}, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
room.ServeDocWS(w, r, userID, clientID)
}
// --- REALTIME LISTS & SHARING ---
internal/api/server_test.go
+50 -155
View File
@@ -3,7 +3,6 @@ package api
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -842,9 +841,9 @@ func TestWebhookTriageBadgeAndIngestion(t *testing.T) {
}
}
func TestDocumentCRDTEndpoints(t *testing.T) {
func TestDocRoomLiveStateAndDebouncedFlush(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "crdt_api_test.db")
dbPath := filepath.Join(tmpDir, "docroom_test.db")
store, err := storage.NewStore(dbPath)
if err != nil {
@@ -860,59 +859,76 @@ func TestDocumentCRDTEndpoints(t *testing.T) {
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
user, err := store.CreateUser("crdt_api@example.com", "hash", "CRDT API User", "member")
user, err := store.CreateUser("outliner_ws@example.com", "hash", "WS Outliner User", "member")
if err != nil {
t.Fatalf("CreateUser failed: %v", err)
}
token, _ := auth.GenerateToken(cfg.JWTSecret, user.ID, user.Email, user.Role)
doc, err := store.CreateDocument(user.ID, "", "API CRDT Doc", "📄", "", nil, nil, false)
doc, err := store.CreateDocument(user.ID, "", "Initial Title", "📄", "", nil, nil, false)
if err != nil {
t.Fatalf("CreateDocument failed: %v", err)
}
// 1. Initial GET /api/v1/documents/{id}/crdt
req := httptest.NewRequest(http.MethodGet, "/api/v1/documents/"+doc.ID+"/crdt", nil)
// 1. Mutate in-memory DocRoom before 1.5s flush fires
room := hub.GetOrCreateDocRoom(doc.ID)
newContent := "Live RAM Block #ramtag"
newLevel := 1
room.HandleEvent(ws.DocSyncEvent{
Type: "title_updated",
ClientID: "cli_1",
Title: "RAM Updated Title",
}, nil)
room.HandleEvent(ws.DocSyncEvent{
Type: "block_created",
ClientID: "cli_1",
AfterBlockID: doc.Blocks[0].ID,
Block: &domain.DocumentBlock{
ID: "blk_ram_2",
Type: "text",
Content: newContent,
Level: newLevel,
},
}, nil)
// 2. Verify GET /api/v1/documents/{id} immediately returns the un-flushed in-memory state
req := httptest.NewRequest(http.MethodGet, "/api/v1/documents/"+doc.ID, nil)
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from GET crdt, got %d: %s", rec.Code, rec.Body.String())
t.Fatalf("expected 200 OK from GET document, got %d: %s", rec.Code, rec.Body.String())
}
var liveDoc domain.Document
_ = json.Unmarshal(rec.Body.Bytes(), &liveDoc)
if liveDoc.Title != "RAM Updated Title" {
t.Fatalf("expected live RAM title 'RAM Updated Title', got '%s'", liveDoc.Title)
}
if len(liveDoc.Blocks) != 2 || liveDoc.Blocks[1].Content != newContent {
t.Fatalf("expected 2 live blocks with RAM content, got %+v", liveDoc.Blocks)
}
// 2. PUT /api/v1/documents/{id}/crdt with base64 state
savePayload := []byte(`{"state":"AQIDBA=="}`)
req = httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID+"/crdt", bytes.NewReader(savePayload))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
// 3. Flush room to SQLite and verify persistence + tag extraction + markdown_cache
room.Flush()
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from PUT crdt, got %d: %s", rec.Code, rec.Body.String())
persistedDoc, err := store.GetDocumentByID(user.ID, doc.ID)
if err != nil {
t.Fatalf("GetDocumentByID failed: %v", err)
}
// 3. Verify state persisted
req = httptest.NewRequest(http.MethodGet, "/api/v1/documents/"+doc.ID+"/crdt", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from GET crdt, got %d", rec.Code)
if persistedDoc.Title != "RAM Updated Title" {
t.Fatalf("expected persisted title 'RAM Updated Title', got '%s'", persistedDoc.Title)
}
var crdtResp struct {
DocID string `json:"docId"`
State string `json:"state"`
if !strings.Contains(persistedDoc.MarkdownCache, " - Live RAM Block #ramtag") {
t.Fatalf("expected compiled markdown_cache to contain indented bullet, got:\n%s", persistedDoc.MarkdownCache)
}
_ = json.Unmarshal(rec.Body.Bytes(), &crdtResp)
if crdtResp.State != "AQIDBA==" {
t.Fatalf("expected state 'AQIDBA==', got '%s'", crdtResp.State)
tags, _ := store.GetWorkspaceTags(user.ID)
if len(tags) != 1 || tags[0].Tag != "ramtag" {
t.Fatalf("expected extracted #ramtag in block_tags, got %+v", tags)
}
}
func TestSSEAndAwarenessEndpoints(t *testing.T) {
func TestSSEGlobalEndpoint(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test_sse.db")
store, err := storage.NewStore(dbPath)
if err != nil {
@@ -921,7 +937,6 @@ func TestSSEAndAwarenessEndpoints(t *testing.T) {
defer store.Close()
hub := ws.NewHub()
hub.SetCRDTStore(store)
cfg := &config.Config{
JWTSecret: "test-jwt-secret-key",
AllowRegistration: false,
@@ -929,7 +944,6 @@ func TestSSEAndAwarenessEndpoints(t *testing.T) {
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
// 1. Setup user
setupBody := []byte(`{
"email": "sse_user@example.com",
"password": "secretpassword",
@@ -952,51 +966,13 @@ func TestSSEAndAwarenessEndpoints(t *testing.T) {
_ = json.Unmarshal(rec.Body.Bytes(), &authResp)
token := authResp.Token
// 2. Create Document
doc, err := store.CreateDocument(authResp.User.ID, "", "SSE Collaborative Note", "📄", "", nil, nil, false)
if err != nil {
t.Fatalf("failed to create document: %v", err)
}
// 3. Test Awareness POST and DELETE
awarenessBody := []byte(`{"userId":"` + authResp.User.ID + `","activeBlockId":"blk_123"}`)
req = httptest.NewRequest(http.MethodPost, "/api/v1/documents/"+doc.ID+"/awareness", bytes.NewReader(awarenessBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from POST awareness, got %d: %s", rec.Code, rec.Body.String())
}
req = httptest.NewRequest(http.MethodDelete, "/api/v1/documents/"+doc.ID+"/awareness", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from DELETE awareness, got %d", rec.Code)
}
// 4. Test Delta CRDT Update via PUT /api/v1/documents/{id}/crdt
updateBody := []byte(`{"update":"AQIDBA=="}`)
req = httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID+"/crdt", bytes.NewReader(updateBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from PUT crdt update, got %d: %s", rec.Code, rec.Body.String())
}
// 5. Test SSE Connection GET /api/v1/events
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req = httptest.NewRequest(http.MethodGet, "/api/v1/events?token="+token+"&docId="+doc.ID, nil).WithContext(ctx)
req = httptest.NewRequest(http.MethodGet, "/api/v1/events?token="+token, nil).WithContext(ctx)
rec = httptest.NewRecorder()
go func() {
// Cancel context after short moment to terminate stream cleanly
time.Sleep(50 * time.Millisecond)
cancel()
}()
@@ -1007,92 +983,11 @@ func TestSSEAndAwarenessEndpoints(t *testing.T) {
if !strings.Contains(contentType, "text/event-stream") {
t.Fatalf("expected Content-Type text/event-stream, got %s", contentType)
}
if rec.Header().Get("X-Accel-Buffering") != "no" {
t.Fatalf("expected X-Accel-Buffering: no, got %s", rec.Header().Get("X-Accel-Buffering"))
}
bodyStr := rec.Body.String()
if !strings.Contains(bodyStr, "SSE_CONNECTED") {
t.Fatalf("expected SSE_CONNECTED event in body, got: %s", bodyStr)
}
}
func TestDocRoomDualTransportAndDebouncedPersistence(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test_dual_transport.db")
store, err := storage.NewStore(dbPath)
if err != nil {
t.Fatalf("failed to init store: %v", err)
}
defer store.Close()
cfg := &config.Config{
JWTSecret: "test-secret-dual-transport",
AllowRegistration: true,
}
hub := ws.NewHub()
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
user, _ := store.CreateUser("dual@test.local", "hash", "Dual User", "member")
token, _ := auth.GenerateToken(cfg.JWTSecret, user.ID, user.Email, user.Role)
doc, _ := store.CreateDocument(user.ID, "", "Dual Transport Doc", "📄", "", nil, nil, false)
// 1. Test POST /api/v1/sync/docs/{id}/update with JSON base64
upd1 := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03})
req := httptest.NewRequest(http.MethodPost, "/api/v1/sync/docs/"+doc.ID+"/update", bytes.NewReader([]byte(`{"update":"`+upd1+`"}`)))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from sync update, got %d: %s", rec.Code, rec.Body.String())
}
// 2. Test POST /api/v1/sync/docs/{id}/update with octet-stream binary
binaryUpd := []byte{0x04, 0x05, 0x06}
req = httptest.NewRequest(http.MethodPost, "/api/v1/sync/docs/"+doc.ID+"/update", bytes.NewReader(binaryUpd))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/octet-stream")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from binary sync update, got %d: %s", rec.Code, rec.Body.String())
}
// 3. Test GET /api/v1/sync/docs/{id}/stream SSE connection
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req = httptest.NewRequest(http.MethodGet, "/api/v1/sync/docs/"+doc.ID+"/stream?token="+token, nil).WithContext(ctx)
rec = httptest.NewRecorder()
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
router.ServeHTTP(rec, req)
if !strings.Contains(rec.Header().Get("Content-Type"), "text/event-stream") {
t.Fatalf("expected text/event-stream, got %s", rec.Header().Get("Content-Type"))
}
bodyStr := rec.Body.String()
if !strings.Contains(bodyStr, "CRDT_INIT") {
t.Fatalf("expected CRDT_INIT in SSE stream, got %s", bodyStr)
}
// 4. Test Lock Neutrality: document update and block patch work without 423 Locked
updateReq := httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID, bytes.NewReader([]byte(`{"title":"Updated Without Locks"}`)))
updateReq.Header.Set("Authorization", "Bearer "+token)
updateReq.Header.Set("Content-Type", "application/json")
updateRec := httptest.NewRecorder()
router.ServeHTTP(updateRec, updateReq)
if updateRec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from document update, got %d: %s", updateRec.Code, updateRec.Body.String())
}
}
internal/domain/models.go
+56 -20
View File
@@ -260,13 +260,47 @@ type SystemSettings struct {
}
type DocumentBlock struct {
ID string `json:"id"`
Type string `json:"type"` // h1 | h2 | h3 | paragraph | todo | bullet | code | callout | divider | table | progress
Content string `json:"content"`
Checked bool `json:"checked,omitempty"`
Lang string `json:"lang,omitempty"`
Level int `json:"level,omitempty"`
Collapsed bool `json:"collapsed,omitempty"`
ID string `json:"id"`
DocumentID string `json:"documentId,omitempty"`
Type string `json:"type"` // h1 | h2 | h3 | paragraph | todo | bullet | code | callout | divider | table | progress
Content string `json:"content"`
Checked bool `json:"checked,omitempty"`
Lang string `json:"lang,omitempty"`
Level int `json:"level"`
Position float64 `json:"position"`
Collapsed bool `json:"collapsed,omitempty"`
}
func CompileBlocksToMarkdown(blocks []DocumentBlock) string {
var sb strings.Builder
for _, b := range blocks {
lvl := b.Level
if lvl < 0 {
lvl = 0
}
indent := strings.Repeat(" ", lvl)
switch b.Type {
case "h1":
sb.WriteString(indent + "- # " + b.Content + "\n")
case "h2":
sb.WriteString(indent + "- ## " + b.Content + "\n")
case "h3":
sb.WriteString(indent + "- ### " + b.Content + "\n")
case "todo":
mark := "[ ]"
if b.Checked {
mark = "[x]"
}
sb.WriteString(indent + "- " + mark + " " + b.Content + "\n")
case "code":
sb.WriteString(indent + "- ```" + b.Lang + "\n" + b.Content + "\n" + indent + " ```\n")
case "divider":
sb.WriteString(indent + "- ---\n")
default:
sb.WriteString(indent + "- " + b.Content + "\n")
}
}
return sb.String()
}
type DocBlocksPatchRequest struct {
@@ -310,19 +344,21 @@ type DocLockInfo struct {
}
type Document struct {
ID string `json:"id"`
UserID string `json:"userId"`
SpaceID string `json:"spaceId,omitempty"`
Title string `json:"title"`
Icon string `json:"icon"`
Content string `json:"content"`
Blocks []DocumentBlock `json:"blocks"`
Tags []string `json:"tags"`
FolderPath string `json:"folderPath,omitempty"`
IsPinned bool `json:"isPinned"`
IsArchived bool `json:"isArchived"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
ID string `json:"id"`
UserID string `json:"userId"`
SpaceID string `json:"spaceId,omitempty"`
Title string `json:"title"`
Icon string `json:"icon"`
Content string `json:"content"`
MarkdownCache string `json:"markdownCache,omitempty"`
Blocks []DocumentBlock `json:"blocks"`
Tags []string `json:"tags"`
FolderID string `json:"folderId,omitempty"`
FolderPath string `json:"folderPath,omitempty"`
IsPinned bool `json:"isPinned"`
IsArchived bool `json:"isArchived"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
// Collaboration & Permissions
Permission string `json:"permission"` // owner | write | read
internal/storage/sqlite.go
+185 -175
View File
@@ -162,9 +162,11 @@ func (s *Store) migrate() error {
CREATE TABLE IF NOT EXISTS documents (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
user_id TEXT NOT NULL DEFAULT '',
space_id TEXT DEFAULT '',
title TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
folder_id TEXT NOT NULL DEFAULT '',
markdown_cache TEXT NOT NULL DEFAULT '',
icon TEXT NOT NULL DEFAULT '📄',
content TEXT NOT NULL DEFAULT '',
blocks_json TEXT NOT NULL DEFAULT '[]',
@@ -172,13 +174,40 @@ func (s *Store) migrate() error {
folder_path TEXT NOT NULL DEFAULT '',
is_pinned INTEGER NOT NULL DEFAULT 0,
is_archived INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_documents_user ON documents(user_id, is_archived, updated_at DESC);
CREATE TABLE IF NOT EXISTS blocks (
id TEXT PRIMARY KEY,
document_id TEXT NOT NULL,
content TEXT NOT NULL DEFAULT '',
level INT NOT NULL DEFAULT 0,
position REAL NOT NULL DEFAULT 0,
type TEXT NOT NULL DEFAULT 'bullet',
checked INTEGER NOT NULL DEFAULT 0,
collapsed INTEGER NOT NULL DEFAULT 0,
lang TEXT NOT NULL DEFAULT '',
FOREIGN KEY(document_id) REFERENCES documents(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_blocks_doc_pos ON blocks(document_id, position ASC);
CREATE TABLE IF NOT EXISTS block_tags (
tag TEXT NOT NULL,
document_id TEXT NOT NULL,
block_id TEXT NOT NULL,
PRIMARY KEY(tag, block_id),
FOREIGN KEY(document_id) REFERENCES documents(id) ON DELETE CASCADE,
FOREIGN KEY(block_id) REFERENCES blocks(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_block_tags_tag ON block_tags(tag);
CREATE INDEX IF NOT EXISTS idx_block_tags_doc ON block_tags(document_id);
CREATE TABLE IF NOT EXISTS doc_shares (
id TEXT PRIMARY KEY,
doc_id TEXT NOT NULL,
@@ -208,22 +237,6 @@ func (s *Store) migrate() error {
INSERT OR IGNORE INTO doc_pins (user_id, doc_id, created_at)
SELECT user_id, id, updated_at FROM documents WHERE is_pinned = 1;
CREATE TABLE IF NOT EXISTS doc_block_tags (
id TEXT PRIMARY KEY,
tag TEXT NOT NULL,
user_id TEXT NOT NULL,
doc_id TEXT NOT NULL,
block_id TEXT NOT NULL,
created_at TEXT NOT NULL,
UNIQUE(doc_id, block_id, tag),
FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_doc_block_tags_tag ON doc_block_tags(tag);
CREATE INDEX IF NOT EXISTS idx_doc_block_tags_user_tag ON doc_block_tags(user_id, tag);
CREATE INDEX IF NOT EXISTS idx_doc_block_tags_doc ON doc_block_tags(doc_id);
CREATE TABLE IF NOT EXISTS tag_pages (
user_id TEXT NOT NULL,
tag TEXT NOT NULL,
@@ -232,25 +245,6 @@ func (s *Store) migrate() error {
PRIMARY KEY(user_id, tag)
);
CREATE TABLE IF NOT EXISTS document_crdt_state (
doc_id TEXT PRIMARY KEY,
state_blob BLOB NOT NULL,
updated_at TEXT NOT NULL,
updated_by TEXT NOT NULL DEFAULT '',
FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS document_crdt_updates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
doc_id TEXT NOT NULL,
update_blob BLOB NOT NULL,
created_at TEXT NOT NULL,
created_by TEXT NOT NULL DEFAULT '',
FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_doc_crdt_updates_doc ON document_crdt_updates(doc_id);
CREATE TABLE IF NOT EXISTS lists (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
@@ -382,17 +376,28 @@ func (s *Store) migrate() error {
"CREATE INDEX IF NOT EXISTS idx_workspace_folders_user ON workspace_folders(user_id);",
"CREATE INDEX IF NOT EXISTS idx_workspace_folders_sf ON workspace_folders(shared_folder_id);",
"CREATE TABLE IF NOT EXISTS document_blocks (id TEXT PRIMARY KEY, doc_id TEXT NOT NULL, sort_order REAL NOT NULL DEFAULT 0, type TEXT NOT NULL DEFAULT 'bullet', content TEXT NOT NULL DEFAULT '', level INTEGER NOT NULL DEFAULT 0, checked INTEGER NOT NULL DEFAULT 0, collapsed INTEGER NOT NULL DEFAULT 0, lang TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL, updated_by TEXT NOT NULL DEFAULT '', FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE);",
"CREATE INDEX IF NOT EXISTS idx_document_blocks_doc_order ON document_blocks(doc_id, sort_order);",
"ALTER TABLE documents ADD COLUMN folder_id TEXT NOT NULL DEFAULT '';",
"ALTER TABLE documents ADD COLUMN markdown_cache TEXT NOT NULL DEFAULT '';",
"CREATE TABLE IF NOT EXISTS blocks (id TEXT PRIMARY KEY, document_id TEXT NOT NULL, content TEXT NOT NULL DEFAULT '', level INT NOT NULL DEFAULT 0, position REAL NOT NULL DEFAULT 0, type TEXT NOT NULL DEFAULT 'bullet', checked INTEGER NOT NULL DEFAULT 0, collapsed INTEGER NOT NULL DEFAULT 0, lang TEXT NOT NULL DEFAULT '', FOREIGN KEY(document_id) REFERENCES documents(id) ON DELETE CASCADE);",
"CREATE INDEX IF NOT EXISTS idx_blocks_doc_pos ON blocks(document_id, position ASC);",
"CREATE TABLE IF NOT EXISTS block_tags (tag TEXT NOT NULL, document_id TEXT NOT NULL, block_id TEXT NOT NULL, PRIMARY KEY(tag, block_id), FOREIGN KEY(document_id) REFERENCES documents(id) ON DELETE CASCADE, FOREIGN KEY(block_id) REFERENCES blocks(id) ON DELETE CASCADE);",
"CREATE INDEX IF NOT EXISTS idx_block_tags_tag ON block_tags(tag);",
"CREATE INDEX IF NOT EXISTS idx_block_tags_doc ON block_tags(document_id);",
}
for _, q := range columnMigrations {
_, _ = s.db.Exec(q)
}
// Clean up legacy space_id from documents and task_lists if empty or orphaned
// Clean up legacy space and old document block/crdt tables
_, _ = s.db.Exec(`DROP TABLE IF EXISTS spaces`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS space_members`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS space_team_shares`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS document_crdt_state`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS document_crdt_updates`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS document_blocks`)
_, _ = s.db.Exec(`DROP TABLE IF EXISTS doc_block_tags`)
// Backfill username for existing accounts where username is empty
_, _ = s.db.Exec(`UPDATE users SET username = LOWER(SUBSTR(email, 1, INSTR(email, '@') - 1)) WHERE (username IS NULL OR username = '') AND INSTR(email, '@') > 0`)
@@ -445,7 +450,7 @@ func (s *Store) migrate() error {
// Reindex block tags if table is newly added or empty
var tagCount int
_ = s.db.QueryRow(`SELECT COUNT(*) FROM doc_block_tags`).Scan(&tagCount)
_ = s.db.QueryRow(`SELECT COUNT(*) FROM block_tags`).Scan(&tagCount)
if tagCount == 0 {
_ = s.reindexAllDocBlockTags()
}
@@ -1269,7 +1274,7 @@ func (s *Store) migrateWorkspaceToPureDB() {
for _, md := range docsToMigrate {
var count int
_ = s.db.QueryRow(`SELECT COUNT(*) FROM document_blocks WHERE doc_id = ?`, md.id).Scan(&count)
_ = s.db.QueryRow(`SELECT COUNT(*) FROM blocks WHERE document_id = ?`, md.id).Scan(&count)
if count == 0 && md.blocksJSON != "" && md.blocksJSON != "[]" {
var blocks []domain.DocumentBlock
if err := json.Unmarshal([]byte(md.blocksJSON), &blocks); err == nil && len(blocks) > 0 {
@@ -1327,17 +1332,17 @@ func (s *Store) migrateWorkspaceToPureDB() {
func (s *Store) loadDocBlocks(docID string, fallbackJSON string) []domain.DocumentBlock {
rows, err := s.db.Query(`
SELECT id, type, content, level, checked, collapsed, lang
FROM document_blocks
WHERE doc_id = ?
ORDER BY sort_order ASC, rowid ASC
SELECT id, document_id, type, content, level, position, checked, collapsed, lang
FROM blocks
WHERE document_id = ?
ORDER BY position ASC, rowid ASC
`, docID)
if err == nil {
blocks := make([]domain.DocumentBlock, 0)
for rows.Next() {
var b domain.DocumentBlock
var chk, col int
if scanErr := rows.Scan(&b.ID, &b.Type, &b.Content, &b.Level, &chk, &col, &b.Lang); scanErr == nil {
if scanErr := rows.Scan(&b.ID, &b.DocumentID, &b.Type, &b.Content, &b.Level, &b.Position, &chk, &col, &b.Lang); scanErr == nil {
b.Checked = chk != 0
b.Collapsed = col != 0
blocks = append(blocks, b)
@@ -1359,50 +1364,111 @@ func (s *Store) loadDocBlocks(docID string, fallbackJSON string) []domain.Docume
return blocks
}
func (s *Store) replaceDocBlocksInDB(docID, userID string, blocks []domain.DocumentBlock, now string) {
_, _ = s.db.Exec(`DELETE FROM document_blocks WHERE doc_id = ?`, docID)
if len(blocks) == 0 {
return
func (s *Store) replaceDocBlocksInDB(docID, _ string, blocks []domain.DocumentBlock, now string) {
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ?`, docID)
if len(blocks) > 0 {
stmt, err := s.db.Prepare(`
INSERT INTO blocks (id, document_id, content, level, position, type, checked, collapsed, lang)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
document_id = excluded.document_id,
content = excluded.content,
level = excluded.level,
position = excluded.position,
type = excluded.type,
checked = excluded.checked,
collapsed = excluded.collapsed,
lang = excluded.lang
`)
if err == nil {
defer stmt.Close()
for i, b := range blocks {
bID := strings.TrimSpace(b.ID)
if bID == "" {
bID = domain.NewID("blk")
}
bType := b.Type
if bType == "" {
bType = "bullet"
}
chk := 0
if b.Checked {
chk = 1
}
col := 0
if b.Collapsed {
col = 1
}
pos := b.Position
if pos <= 0 {
pos = float64((i + 1) * 1000)
}
_, _ = stmt.Exec(bID, docID, b.Content, b.Level, pos, bType, chk, col, b.Lang)
}
}
}
stmt, err := s.db.Prepare(`
INSERT INTO document_blocks (id, doc_id, sort_order, type, content, level, checked, collapsed, lang, updated_at, updated_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
doc_id = excluded.doc_id,
sort_order = excluded.sort_order,
type = excluded.type,
content = excluded.content,
level = excluded.level,
checked = excluded.checked,
collapsed = excluded.collapsed,
lang = excluded.lang,
updated_at = excluded.updated_at,
updated_by = excluded.updated_by
`)
if err != nil {
return
}
defer stmt.Close()
for i, b := range blocks {
bID := strings.TrimSpace(b.ID)
if bID == "" {
bID = domain.NewID("blk")
}
bType := b.Type
if bType == "" {
bType = "bullet"
}
chk := 0
if b.Checked {
chk = 1
}
col := 0
if b.Collapsed {
col = 1
}
_, _ = stmt.Exec(bID, docID, float64((i+1)*1000), bType, b.Content, b.Level, chk, col, b.Lang, now, userID)
// Update markdown_cache on documents
mdCache := domain.CompileBlocksToMarkdown(blocks)
_, _ = s.db.Exec(`UPDATE documents SET markdown_cache = ?, updated_at = ? WHERE id = ?`, mdCache, now, docID)
}
func (s *Store) LoadDocumentState(docID string) (string, []domain.DocumentBlock, error) {
var title, blocksJSON string
err := s.db.QueryRow(`SELECT title, blocks_json FROM documents WHERE id = ?`, docID).Scan(&title, &blocksJSON)
if err != nil {
return "", nil, err
}
blocks := s.loadDocBlocks(docID, blocksJSON)
return title, blocks, nil
}
// FlushDocumentState persists the in-memory blocks into `blocks`, extracts all #tags into `block_tags`,
// and compiles the hierarchical Markdown into `documents.markdown_cache`.
func (s *Store) FlushDocumentState(docID string, title string, blocks []domain.DocumentBlock) error {
now := domain.NowISO()
// Normalize positions sequentially
normalized := make([]domain.DocumentBlock, len(blocks))
var plainBuilder strings.Builder
for i, b := range blocks {
if strings.TrimSpace(b.ID) == "" {
b.ID = domain.NewID("blk")
}
if b.Type == "" {
b.Type = "bullet"
}
b.DocumentID = docID
b.Position = float64((i + 1) * 1000)
normalized[i] = b
if b.Content != "" {
plainBuilder.WriteString(b.Content)
plainBuilder.WriteString("\n")
}
}
s.replaceDocBlocksInDB(docID, "", normalized, now)
_ = s.indexDocBlockTags(docID, "", normalized)
mdCache := domain.CompileBlocksToMarkdown(normalized)
plainContent := plainBuilder.String()
blocksJSON, _ := json.Marshal(normalized)
if strings.TrimSpace(title) != "" {
_, err := s.db.Exec(`
UPDATE documents
SET title = ?, markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ?
WHERE id = ?
`, title, mdCache, plainContent, string(blocksJSON), now, docID)
return err
}
_, err := s.db.Exec(`
UPDATE documents
SET markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ?
WHERE id = ?
`, mdCache, plainContent, string(blocksJSON), now, docID)
return err
}
// PatchDocumentBlocks applies atomic block-level upserts, deletions, and order reconciliation without overwriting concurrent edits from other sessions.
@@ -1417,16 +1483,16 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
now := domain.NowISO()
// Ensure document_blocks is seeded if previously only in blocks_json
// Ensure blocks table is seeded if previously only in blocks_json
var existingCount int
_ = s.db.QueryRow(`SELECT COUNT(*) FROM document_blocks WHERE doc_id = ?`, docID).Scan(&existingCount)
_ = s.db.QueryRow(`SELECT COUNT(*) FROM blocks WHERE document_id = ?`, docID).Scan(&existingCount)
if existingCount == 0 && len(doc.Blocks) > 0 {
s.replaceDocBlocksInDB(docID, doc.UserID, doc.Blocks, doc.UpdatedAt)
}
// Read current ordered block IDs from DB before patch
var existingOrder []string
ordRows, err := s.db.Query(`SELECT id FROM document_blocks WHERE doc_id = ? ORDER BY sort_order ASC, rowid ASC`, docID)
ordRows, err := s.db.Query(`SELECT id FROM blocks WHERE document_id = ? ORDER BY position ASC, rowid ASC`, docID)
if err == nil {
for ordRows.Next() {
var bid string
@@ -1445,7 +1511,7 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
continue
}
deletedSet[delID] = true
_, _ = s.db.Exec(`DELETE FROM document_blocks WHERE doc_id = ? AND id = ?`, docID, delID)
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ? AND id = ?`, docID, delID)
}
// Filter existingOrder to exclude deleted blocks
@@ -1479,18 +1545,16 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
}
defaultSort := float64((len(survivingExisting) + idx + 1) * 1000)
_, _ = s.db.Exec(`
INSERT INTO document_blocks (id, doc_id, sort_order, type, content, level, checked, collapsed, lang, updated_at, updated_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO blocks (id, document_id, content, level, position, type, checked, collapsed, lang)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
type = excluded.type,
content = excluded.content,
level = excluded.level,
checked = excluded.checked,
collapsed = excluded.collapsed,
lang = excluded.lang,
updated_at = excluded.updated_at,
updated_by = excluded.updated_by
`, bID, docID, defaultSort, bType, b.Content, b.Level, chk, col, b.Lang, now, userID)
lang = excluded.lang
`, bID, docID, b.Content, b.Level, defaultSort, bType, chk, col, b.Lang)
upsertedIDs = append(upsertedIDs, bID)
survivingSet[bID] = true
}
@@ -1506,8 +1570,6 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
inMerged[bid] = true
}
}
// Any surviving block in DB that was not in req.BlockOrder (e.g. concurrently added by another user)
// is inserted right after its predecessor from survivingExisting!
for i, bid := range survivingExisting {
if inMerged[bid] {
continue
@@ -1528,7 +1590,6 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
mergedOrder = append(mergedOrder[:insertPos], append([]string{bid}, mergedOrder[insertPos:]...)...)
inMerged[bid] = true
}
// Ensure any upserted block not in BlockOrder is also appended
for _, bid := range upsertedIDs {
if !inMerged[bid] {
mergedOrder = append(mergedOrder, bid)
@@ -1550,9 +1611,9 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
}
}
// Update sort_order in document_blocks
// Update position in blocks
for idx, bid := range mergedOrder {
_, _ = s.db.Exec(`UPDATE document_blocks SET sort_order = ? WHERE doc_id = ? AND id = ?`, float64((idx+1)*1000), docID, bid)
_, _ = s.db.Exec(`UPDATE blocks SET position = ? WHERE document_id = ? AND id = ?`, float64((idx+1)*1000), docID, bid)
}
// Reload authoritative block list and update document snapshot
@@ -1565,12 +1626,14 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
}
}
plainContent := plainBuilder.String()
mdCache := domain.CompileBlocksToMarkdown(finalBlocks)
blocksJSON, _ := json.Marshal(finalBlocks)
_, _ = s.db.Exec(`UPDATE documents SET content = ?, blocks_json = ?, updated_at = ? WHERE id = ?`,
plainContent, string(blocksJSON), now, docID)
_, _ = s.db.Exec(`UPDATE documents SET markdown_cache = ?, content = ?, blocks_json = ?, updated_at = ? WHERE id = ?`,
mdCache, plainContent, string(blocksJSON), now, docID)
doc.Blocks = finalBlocks
doc.MarkdownCache = mdCache
doc.Content = plainContent
doc.UpdatedAt = now
_ = s.indexDocBlockTags(doc.ID, doc.UserID, finalBlocks)
@@ -1578,59 +1641,6 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
return doc, mergedOrder, nil
}
// GetDocumentCRDTState returns the compacted CRDT state blob and any pending updates for a document.
func (s *Store) GetDocumentCRDTState(docID string) ([]byte, [][]byte, error) {
var stateBlob []byte
err := s.db.QueryRow(`SELECT state_blob FROM document_crdt_state WHERE doc_id = ?`, docID).Scan(&stateBlob)
if err != nil && err != sql.ErrNoRows {
return nil, nil, err
}
rows, err := s.db.Query(`SELECT update_blob FROM document_crdt_updates WHERE doc_id = ? ORDER BY id ASC`, docID)
if err != nil {
return stateBlob, nil, nil
}
defer rows.Close()
var updates [][]byte
for rows.Next() {
var u []byte
if err := rows.Scan(&u); err == nil && len(u) > 0 {
updates = append(updates, u)
}
}
return stateBlob, updates, nil
}
// AppendDocumentCRDTUpdate appends a new CRDT update delta for a document.
func (s *Store) AppendDocumentCRDTUpdate(docID string, update []byte, userID string) error {
now := domain.NowISO()
_, err := s.db.Exec(`
INSERT INTO document_crdt_updates (doc_id, update_blob, created_at, created_by)
VALUES (?, ?, ?, ?)
`, docID, update, now, userID)
return err
}
// SaveDocumentCRDTState saves a compacted snapshot of the document CRDT state and prunes older incremental updates.
func (s *Store) SaveDocumentCRDTState(docID string, stateBlob []byte, userID string) error {
now := domain.NowISO()
_, err := s.db.Exec(`
INSERT INTO document_crdt_state (doc_id, state_blob, updated_at, updated_by)
VALUES (?, ?, ?, ?)
ON CONFLICT(doc_id) DO UPDATE SET
state_blob = excluded.state_blob,
updated_at = excluded.updated_at,
updated_by = excluded.updated_by
`, docID, stateBlob, now, userID)
if err != nil {
return err
}
// Prune incremental updates older than the snapshot
_, _ = s.db.Exec(`DELETE FROM document_crdt_updates WHERE doc_id = ?`, docID)
return nil
}
// GetAuthorizedDocUsers returns all user IDs who have read/write/owner access to docID.
func (s *Store) GetAuthorizedDocUsers(docID string) ([]string, error) {
userSet := make(map[string]bool)
@@ -1964,6 +1974,8 @@ func (s *Store) scanDocumentWithBlocks(rows interface {
_ = json.Unmarshal([]byte(tagsStr), &d.Tags)
d.Tags = stripFolderTags(d.Tags)
d.Blocks = s.loadDocBlocks(d.ID, blocksStr)
d.MarkdownCache = domain.CompileBlocksToMarkdown(d.Blocks)
d.FolderID = d.FolderPath
return d, nil
}
@@ -2303,8 +2315,8 @@ func (s *Store) DeleteDocument(userID, docID string) error {
_, _ = s.db.Exec(`DELETE FROM doc_user_folders WHERE user_id = ? AND doc_id = ?`, userID, docID)
return s.DeleteDocShare(userID, docID, userID)
}
_, _ = s.db.Exec(`DELETE FROM document_blocks WHERE doc_id = ?`, docID)
_, _ = s.db.Exec(`DELETE FROM doc_block_tags WHERE doc_id = ?`, docID)
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ?`, docID)
_, _ = s.db.Exec(`DELETE FROM block_tags WHERE document_id = ?`, docID)
_, _ = s.db.Exec(`DELETE FROM doc_user_folders WHERE doc_id = ?`, docID)
_, err = s.db.Exec(`DELETE FROM documents WHERE id = ?`, docID)
return err
@@ -2338,27 +2350,25 @@ func (s *Store) reindexAllDocBlockTags() error {
return nil
}
func (s *Store) indexDocBlockTags(docID, userID string, blocks []domain.DocumentBlock) error {
_, _ = s.db.Exec(`DELETE FROM doc_block_tags WHERE doc_id = ?`, docID)
func (s *Store) indexDocBlockTags(docID, _ string, blocks []domain.DocumentBlock) error {
_, _ = s.db.Exec(`DELETE FROM block_tags WHERE document_id = ?`, docID)
if len(blocks) == 0 {
return nil
}
stmt, err := s.db.Prepare(`INSERT OR IGNORE INTO doc_block_tags (id, tag, user_id, doc_id, block_id, created_at) VALUES (?, ?, ?, ?, ?, ?)`)
stmt, err := s.db.Prepare(`INSERT OR IGNORE INTO block_tags (tag, document_id, block_id) VALUES (?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
now := domain.NowISO()
for _, b := range blocks {
if b.Content == "" {
continue
}
tags := domain.ExtractTags(b.Content)
for _, tag := range tags {
tagID := domain.NewID("dbtag")
_, _ = stmt.Exec(tagID, tag, userID, docID, b.ID, now)
_, _ = stmt.Exec(tag, docID, b.ID)
}
}
return nil
@@ -2384,9 +2394,9 @@ func (s *Store) UpdateTagPageNote(userID, tag, content string) (string, error) {
func (s *Store) GetWorkspaceTags(userID string) ([]domain.TagSummary, error) {
rows, err := s.db.Query(
`SELECT t.tag, COUNT(DISTINCT t.doc_id || ':' || t.block_id) as count
FROM doc_block_tags t
JOIN documents d ON t.doc_id = d.id
`SELECT t.tag, COUNT(DISTINCT t.document_id || ':' || t.block_id) as count
FROM block_tags t
JOIN documents d ON t.document_id = d.id
LEFT JOIN doc_shares s ON s.doc_id = d.id AND s.user_id = ?
LEFT JOIN (
SELECT dts2.doc_id
@@ -2451,8 +2461,8 @@ func (s *Store) GetTagDetails(userID, tag string) (*domain.TagDetails, error) {
`SELECT t.block_id, d.id, d.title, d.icon, d.folder_path, d.blocks_json, d.updated_at, d.user_id,
COALESCE(u.display_name, '') as owner_name,
COALESCE(s.permission, dts.permission, '') as shared_permission
FROM doc_block_tags t
JOIN documents d ON t.doc_id = d.id
FROM block_tags t
JOIN documents d ON t.document_id = d.id
LEFT JOIN users u ON d.user_id = u.id
LEFT JOIN doc_shares s ON s.doc_id = d.id AND s.user_id = ?
LEFT JOIN (
@@ -2768,8 +2778,8 @@ func (s *Store) DeleteFolder(userID, folderPath string) ([]string, error) {
// Delete documents from SQLite, blocks, pins, shares, tags, user_folders
for _, id := range deletedDocIDs {
_, _ = s.db.Exec(`DELETE FROM document_blocks WHERE doc_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM doc_block_tags WHERE doc_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM blocks WHERE document_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM block_tags WHERE document_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM doc_pins WHERE doc_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM doc_shares WHERE doc_id = ?`, id)
_, _ = s.db.Exec(`DELETE FROM doc_team_shares WHERE doc_id = ?`, id)
internal/storage/sqlite_test.go
+42 -34
View File
@@ -683,9 +683,9 @@ func TestDatabaseFirstWorkspaceArchitecture(t *testing.T) {
}
}
func TestDocumentCRDTState(t *testing.T) {
func TestFlushDocumentStateAndSchema(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "crdt_test.db")
dbPath := filepath.Join(tmpDir, "tabula_rasa_test.db")
store, err := NewStore(dbPath)
if err != nil {
@@ -693,52 +693,60 @@ func TestDocumentCRDTState(t *testing.T) {
}
defer store.Close()
user, err := store.CreateUser("crdt@example.com", "hash", "CRDT User", "member")
user, err := store.CreateUser("outliner@example.com", "hash", "Outliner User", "member")
if err != nil {
t.Fatalf("CreateUser failed: %v", err)
}
doc, err := store.CreateDocument(user.ID, "", "CRDT Test Doc", "📝", "", nil, nil, false)
doc, err := store.CreateDocument(user.ID, "", "Initial Note", "📝", "Projekte", nil, nil, false)
if err != nil {
t.Fatalf("CreateDocument failed: %v", err)
}
// 1. Initial CRDT state is empty
state, updates, err := store.GetDocumentCRDTState(doc.ID)
blocks := []domain.DocumentBlock{
{
ID: "blk_root_1",
Type: "text",
Content: "Architektur Übersicht #backend #go",
Level: 0,
},
{
ID: "blk_child_1",
Type: "todo",
Content: "SQLite WAL Modus aktivieren #sqlite",
Level: 1,
Checked: true,
},
}
if err := store.FlushDocumentState(doc.ID, "Architektur Notiz", blocks); err != nil {
t.Fatalf("FlushDocumentState failed: %v", err)
}
updatedDoc, err := store.GetDocumentByID(user.ID, doc.ID)
if err != nil {
t.Fatalf("GetDocumentCRDTState failed: %v", err)
t.Fatalf("GetDocumentByID failed: %v", err)
}
if len(state) != 0 || len(updates) != 0 {
t.Fatalf("expected empty CRDT state, got state=%d updates=%d", len(state), len(updates))
if updatedDoc.Title != "Architektur Notiz" {
t.Fatalf("expected title 'Architektur Notiz', got '%s'", updatedDoc.Title)
}
if len(updatedDoc.Blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(updatedDoc.Blocks))
}
if updatedDoc.Blocks[1].Level != 1 || updatedDoc.Blocks[1].Position != 2000 {
t.Fatalf("expected block[1] level=1 position=2000, got level=%d position=%v", updatedDoc.Blocks[1].Level, updatedDoc.Blocks[1].Position)
}
if updatedDoc.MarkdownCache == "" {
t.Fatalf("expected markdown_cache to be populated")
}
// 2. Append incremental updates
u1 := []byte{1, 2, 3, 4}
u2 := []byte{5, 6, 7, 8}
if err := store.AppendDocumentCRDTUpdate(doc.ID, u1, user.ID); err != nil {
t.Fatalf("AppendDocumentCRDTUpdate 1 failed: %v", err)
}
if err := store.AppendDocumentCRDTUpdate(doc.ID, u2, user.ID); err != nil {
t.Fatalf("AppendDocumentCRDTUpdate 2 failed: %v", err)
}
_, updatesAfter, err := store.GetDocumentCRDTState(doc.ID)
if err != nil || len(updatesAfter) != 2 {
t.Fatalf("expected 2 updates, got %d (err: %v)", len(updatesAfter), err)
}
// 3. Compact snapshot state
snapshot := []byte{99, 100, 101}
if err := store.SaveDocumentCRDTState(doc.ID, snapshot, user.ID); err != nil {
t.Fatalf("SaveDocumentCRDTState failed: %v", err)
}
stateAfter, updatesAfterCompact, err := store.GetDocumentCRDTState(doc.ID)
tags, err := store.GetWorkspaceTags(user.ID)
if err != nil {
t.Fatalf("GetDocumentCRDTState after compact failed: %v", err)
t.Fatalf("GetWorkspaceTags failed: %v", err)
}
if len(stateAfter) != 3 || len(updatesAfterCompact) != 0 {
t.Fatalf("expected compacted state of length 3 and 0 updates, got state=%d updates=%d", len(stateAfter), len(updatesAfterCompact))
if len(tags) != 3 {
t.Fatalf("expected 3 extracted tags in block_tags (#backend, #go, #sqlite), got %d: %+v", len(tags), tags)
}
}
internal/ws/docroom.go
+386 -317
View File
@@ -1,9 +1,7 @@
package ws
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
@@ -15,386 +13,488 @@ import (
// DocWSClient represents an active WebSocket connection to a specific document room.
type DocWSClient struct {
hub *Hub
room *DocRoom
conn *websocket.Conn
userID string
send chan []byte
hub *Hub
room *DocRoom
conn *websocket.Conn
userID string
clientID string
send chan []byte
}
// DocSSEClient represents an active SSE connection to a specific document room.
type DocSSEClient struct {
hub *Hub
room *DocRoom
id string
userID string
send chan []byte
// DocSyncEvent is the lightweight JSON payload exchanged over /api/v1/ws/docs/{id}.
type DocSyncEvent struct {
Type string `json:"type"`
DocID string `json:"docId,omitempty"`
ClientID string `json:"clientId,omitempty"`
UserID string `json:"userId,omitempty"`
Title string `json:"title,omitempty"`
Block *domain.DocumentBlock `json:"block,omitempty"`
BlockID string `json:"blockId,omitempty"`
AfterBlockID string `json:"afterBlockId,omitempty"`
NewIndex *int `json:"newIndex,omitempty"`
Content *string `json:"content,omitempty"`
Level *int `json:"level,omitempty"`
Checked *bool `json:"checked,omitempty"`
Collapsed *bool `json:"collapsed,omitempty"`
BlockType *string `json:"blockType,omitempty"`
Blocks []domain.DocumentBlock `json:"blocks,omitempty"`
}
// DocRoom coordinates in-memory real-time updates and debounced persistence for one document.
// DocRoom coordinates in-memory real-time block state and 1.5s debounced SQLite persistence for one document.
type DocRoom struct {
mu sync.Mutex
docID string
hub *Hub
stateBlob []byte
pendingUpdates [][]byte
persistedUpdatesCount int
wsClients map[*DocWSClient]bool
sseClients map[*DocSSEClient]bool
debounceTimer *time.Timer
dirty bool
lastUserID string
mu sync.Mutex
docID string
hub *Hub
title string
blocks []domain.DocumentBlock
loaded bool
wsClients map[*DocWSClient]bool
debounceTimer *time.Timer
dirty bool
}
// GetOrCreateDocRoom returns the active in-memory room for docID or initializes one from SQLite.
func (h *Hub) GetOrCreateDocRoom(docID string) *DocRoom {
h.mu.Lock()
defer h.mu.Unlock()
if h.docRooms == nil {
h.docRooms = make(map[string]*DocRoom)
}
if room, exists := h.docRooms[docID]; exists {
return room
room, exists := h.docRooms[docID]
if !exists {
room = &DocRoom{
docID: docID,
hub: h,
wsClients: make(map[*DocWSClient]bool),
}
h.docRooms[docID] = room
}
store := h.docStore
h.mu.Unlock()
room := &DocRoom{
docID: docID,
hub: h,
wsClients: make(map[*DocWSClient]bool),
sseClients: make(map[*DocSSEClient]bool),
}
// Pre-load current state from CRDT store if available
if h.crdtStore != nil {
stateBlob, updates, err := h.crdtStore.GetDocumentCRDTState(docID)
room.mu.Lock()
if !room.loaded && store != nil {
title, blocks, err := store.LoadDocumentState(docID)
if err == nil {
room.stateBlob = stateBlob
room.pendingUpdates = updates
room.persistedUpdatesCount = len(updates)
room.title = title
room.blocks = normalizeRoomBlocks(docID, blocks)
room.loaded = true
}
}
room.mu.Unlock()
h.docRooms[docID] = room
return room
}
// ApplyUpdate processes an incremental Yjs binary update in-memory, broadcasts it immediately
// to all other connected clients (WS & SSE), and schedules an asynchronous debounced flush to SQLite.
func (r *DocRoom) ApplyUpdate(update []byte, fromUserID string, senderWS *DocWSClient, senderSSE *DocSSEClient) {
if len(update) == 0 {
// GetActiveDocState returns the live in-memory title and blocks if an active DocRoom exists in RAM.
func (h *Hub) GetActiveDocState(docID string) (string, []domain.DocumentBlock, bool) {
h.mu.RLock()
room, exists := h.docRooms[docID]
h.mu.RUnlock()
if !exists || room == nil {
return "", nil, false
}
room.mu.Lock()
defer room.mu.Unlock()
if !room.loaded {
return "", nil, false
}
blocksCopy := make([]domain.DocumentBlock, len(room.blocks))
copy(blocksCopy, room.blocks)
return room.title, blocksCopy, true
}
// SyncActiveDocRoom updates an in-memory room if one is currently loaded (e.g. after REST PUT/PATCH).
func (h *Hub) SyncActiveDocRoom(docID string, title string, blocks []domain.DocumentBlock) {
h.mu.RLock()
room, exists := h.docRooms[docID]
h.mu.RUnlock()
if !exists || room == nil {
return
}
r.mu.Lock()
r.pendingUpdates = append(r.pendingUpdates, update)
r.dirty = true
if fromUserID != "" {
r.lastUserID = fromUserID
room.mu.Lock()
if title != "" {
room.title = title
}
b64Update := base64.StdEncoding.EncodeToString(update)
msg := domain.WSMessage{
Event: "CRDT_UPDATE",
Payload: map[string]interface{}{
"docId": r.docID,
"update": b64Update,
"fromUserId": fromUserID,
},
Timestamp: domain.NowISO(),
if blocks != nil {
room.blocks = normalizeRoomBlocks(docID, blocks)
}
msgBytes, _ := json.Marshal(msg)
// Broadcast to all WS clients except sender
for client := range r.wsClients {
if senderWS != nil && client == senderWS {
continue
}
select {
case client.send <- msgBytes:
default:
}
}
// Broadcast to all SSE clients except sender
for client := range r.sseClients {
if senderSSE != nil && client == senderSSE {
continue
}
select {
case client.send <- msgBytes:
default:
}
}
// Schedule 2.5s debounced persistence
r.scheduleDebouncedFlushLocked()
r.mu.Unlock()
// Also inform legacy room subscribers if any
r.hub.BroadcastRoom("doc:"+r.docID, "CRDT_UPDATE", map[string]interface{}{
"docId": r.docID,
"update": b64Update,
"fromUserId": fromUserID,
}, nil)
room.loaded = true
room.mu.Unlock()
}
// CompactState replaces the incremental pending updates with a full compacted state blob.
func (r *DocRoom) CompactState(stateBlob []byte, fromUserID string) {
if len(stateBlob) == 0 {
return
// CloseDocRoom removes a document room from memory when a document is deleted.
func (h *Hub) CloseDocRoom(docID string) {
h.mu.Lock()
room, exists := h.docRooms[docID]
if exists {
delete(h.docRooms, docID)
}
h.mu.Unlock()
r.mu.Lock()
r.stateBlob = stateBlob
r.pendingUpdates = nil
r.persistedUpdatesCount = 0
r.dirty = true
if fromUserID != "" {
r.lastUserID = fromUserID
if exists && room != nil {
room.mu.Lock()
if room.debounceTimer != nil {
room.debounceTimer.Stop()
}
room.dirty = false
for c := range room.wsClients {
_ = c.conn.Close()
}
room.wsClients = make(map[*DocWSClient]bool)
room.mu.Unlock()
}
r.scheduleDebouncedFlushLocked()
r.mu.Unlock()
}
// BroadcastAwareness forwards user awareness state to all other participants in the room.
func (r *DocRoom) BroadcastAwareness(payload json.RawMessage, fromUserID string, senderWS *DocWSClient, senderSSE *DocSSEClient) {
msg := domain.WSMessage{
Event: "CRDT_AWARENESS",
Payload: payload,
Timestamp: domain.NowISO(),
func normalizeRoomBlocks(docID string, blocks []domain.DocumentBlock) []domain.DocumentBlock {
if len(blocks) == 0 {
return []domain.DocumentBlock{
{
ID: docID + "-b0",
DocumentID: docID,
Type: "text",
Content: "",
Level: 0,
Position: 1000,
},
}
}
msgBytes, _ := json.Marshal(msg)
out := make([]domain.DocumentBlock, len(blocks))
for i, b := range blocks {
if b.ID == "" {
b.ID = domain.NewID("blk")
}
b.DocumentID = docID
if b.Type == "" {
b.Type = "text"
}
if b.Level < 0 {
b.Level = 0
}
b.Position = float64((i + 1) * 1000)
out[i] = b
}
return out
}
// GetStateSnapshot returns a copy of the room's current live in-memory title and blocks.
func (r *DocRoom) GetStateSnapshot() (string, []domain.DocumentBlock) {
r.mu.Lock()
defer r.mu.Unlock()
blocksCopy := make([]domain.DocumentBlock, len(r.blocks))
copy(blocksCopy, r.blocks)
return r.title, blocksCopy
}
for client := range r.wsClients {
if senderWS != nil && client == senderWS {
continue
// HandleEvent applies a granular block or title mutation to the in-memory room,
// broadcasts it to all OTHER clients in the room (excluding senderClientID for echo protection),
// and resets the 1.5-second inactivity debounce timer for SQLite persistence.
func (r *DocRoom) HandleEvent(evt DocSyncEvent, sender *DocWSClient) {
r.mu.Lock()
evt.DocID = r.docID
if sender != nil {
if evt.ClientID == "" {
evt.ClientID = sender.clientID
}
select {
case client.send <- msgBytes:
default:
evt.UserID = sender.userID
}
mutated := false
switch evt.Type {
case "title_updated":
if evt.Title != "" || r.title != evt.Title {
r.title = evt.Title
mutated = true
}
case "block_created":
if evt.Block != nil {
newBlock := *evt.Block
if newBlock.ID == "" {
newBlock.ID = domain.NewID("blk")
}
newBlock.DocumentID = r.docID
if newBlock.Type == "" {
newBlock.Type = "text"
}
if newBlock.Level < 0 {
newBlock.Level = 0
}
// Avoid duplicate ID if already created
existsIdx := -1
for i, b := range r.blocks {
if b.ID == newBlock.ID {
existsIdx = i
break
}
}
if existsIdx >= 0 {
r.blocks[existsIdx] = newBlock
} else {
insertIdx := len(r.blocks)
if evt.AfterBlockID != "" {
for i, b := range r.blocks {
if b.ID == evt.AfterBlockID {
insertIdx = i + 1
break
}
}
} else if evt.NewIndex != nil && *evt.NewIndex >= 0 && *evt.NewIndex <= len(r.blocks) {
insertIdx = *evt.NewIndex
}
updated := make([]domain.DocumentBlock, 0, len(r.blocks)+1)
updated = append(updated, r.blocks[:insertIdx]...)
updated = append(updated, newBlock)
updated = append(updated, r.blocks[insertIdx:]...)
r.blocks = normalizeRoomBlocks(r.docID, updated)
}
evt.Block = &newBlock
mutated = true
}
case "block_updated":
targetID := evt.BlockID
if targetID == "" && evt.Block != nil {
targetID = evt.Block.ID
}
if targetID != "" {
for i := range r.blocks {
if r.blocks[i].ID == targetID {
if evt.Block != nil {
r.blocks[i].Content = evt.Block.Content
r.blocks[i].Level = evt.Block.Level
if evt.Block.Type != "" {
r.blocks[i].Type = evt.Block.Type
}
r.blocks[i].Checked = evt.Block.Checked
r.blocks[i].Collapsed = evt.Block.Collapsed
r.blocks[i].Lang = evt.Block.Lang
} else {
if evt.Content != nil {
r.blocks[i].Content = *evt.Content
}
if evt.Level != nil {
lvl := *evt.Level
if lvl < 0 {
lvl = 0
}
r.blocks[i].Level = lvl
}
if evt.Checked != nil {
r.blocks[i].Checked = *evt.Checked
}
if evt.Collapsed != nil {
r.blocks[i].Collapsed = *evt.Collapsed
}
if evt.BlockType != nil && *evt.BlockType != "" {
r.blocks[i].Type = *evt.BlockType
}
}
bCopy := r.blocks[i]
evt.Block = &bCopy
evt.BlockID = targetID
mutated = true
break
}
}
}
case "block_deleted":
if evt.BlockID != "" {
filtered := make([]domain.DocumentBlock, 0, len(r.blocks))
for _, b := range r.blocks {
if b.ID != evt.BlockID {
filtered = append(filtered, b)
}
}
r.blocks = normalizeRoomBlocks(r.docID, filtered)
mutated = true
}
case "block_moved":
if evt.BlockID != "" && evt.NewIndex != nil {
fromIdx := -1
for i, b := range r.blocks {
if b.ID == evt.BlockID {
fromIdx = i
break
}
}
if fromIdx >= 0 {
moving := r.blocks[fromIdx]
if evt.Level != nil {
lvl := *evt.Level
if lvl < 0 {
lvl = 0
}
moving.Level = lvl
}
without := make([]domain.DocumentBlock, 0, len(r.blocks)-1)
without = append(without, r.blocks[:fromIdx]...)
without = append(without, r.blocks[fromIdx+1:]...)
toIdx := *evt.NewIndex
if toIdx < 0 {
toIdx = 0
}
if toIdx > len(without) {
toIdx = len(without)
}
reordered := make([]domain.DocumentBlock, 0, len(r.blocks))
reordered = append(reordered, without[:toIdx]...)
reordered = append(reordered, moving)
reordered = append(reordered, without[toIdx:]...)
r.blocks = normalizeRoomBlocks(r.docID, reordered)
mutated = true
}
}
case "blocks_replaced":
if evt.Blocks != nil {
r.blocks = normalizeRoomBlocks(r.docID, evt.Blocks)
evt.Blocks = r.blocks
mutated = true
}
}
for client := range r.sseClients {
if senderSSE != nil && client == senderSSE {
continue
}
select {
case client.send <- msgBytes:
default:
if !mutated {
r.mu.Unlock()
return
}
r.dirty = true
r.loaded = true
msgBytes, err := json.Marshal(evt)
if err == nil {
for client := range r.wsClients {
// Strict Echo Protection: never send back to the originating connection or clientId
if sender != nil && (client == sender || (sender.clientID != "" && client.clientID == sender.clientID)) {
continue
}
if evt.ClientID != "" && client.clientID == evt.ClientID {
continue
}
select {
case client.send <- msgBytes:
default:
}
}
}
r.scheduleDebouncedFlushLocked()
r.mu.Unlock()
}
func (r *DocRoom) scheduleDebouncedFlushLocked() {
if r.debounceTimer != nil {
r.debounceTimer.Stop()
}
r.debounceTimer = time.AfterFunc(2500*time.Millisecond, func() {
r.debounceTimer = time.AfterFunc(1500*time.Millisecond, func() {
r.Flush()
})
}
// Flush persists in-memory pending updates to SQLite asynchronously in a background goroutine.
// Flush persists the current in-memory document state (blocks, #tags in block_tags, and markdown_cache) to SQLite.
func (r *DocRoom) Flush() {
r.mu.Lock()
if !r.dirty {
r.mu.Unlock()
return
}
if r.debounceTimer != nil {
r.debounceTimer.Stop()
r.debounceTimer = nil
}
// Prepare data to flush
docID := r.docID
userID := r.lastUserID
stateBlob := r.stateBlob
var updatesToFlush [][]byte
if len(stateBlob) > 0 && len(r.pendingUpdates) == 0 {
// Compacted state snapshot
r.dirty = false
r.mu.Unlock()
go func() {
if r.hub.crdtStore != nil {
_ = r.hub.crdtStore.SaveDocumentCRDTState(docID, stateBlob, userID)
}
}()
return
}
if r.persistedUpdatesCount < len(r.pendingUpdates) {
updatesToFlush = make([][]byte, len(r.pendingUpdates)-r.persistedUpdatesCount)
copy(updatesToFlush, r.pendingUpdates[r.persistedUpdatesCount:])
r.persistedUpdatesCount = len(r.pendingUpdates)
}
title := r.title
blocksCopy := make([]domain.DocumentBlock, len(r.blocks))
copy(blocksCopy, r.blocks)
r.dirty = false
r.mu.Unlock()
if len(updatesToFlush) > 0 {
go func() {
if r.hub.crdtStore != nil {
for _, u := range updatesToFlush {
_ = r.hub.crdtStore.AppendDocumentCRDTUpdate(docID, u, userID)
}
}
}()
}
}
r.hub.mu.RLock()
store := r.hub.docStore
r.hub.mu.RUnlock()
// GetInitialSnapshot returns the base64-encoded state blob and array of incremental updates.
func (r *DocRoom) GetInitialSnapshot() (string, []string) {
r.mu.Lock()
defer r.mu.Unlock()
b64State := ""
if len(r.stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(r.stateBlob)
}
var b64Updates []string
for _, u := range r.pendingUpdates {
if len(u) > 0 {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
if store != nil {
if err := store.FlushDocumentState(docID, title, blocksCopy); err != nil {
log.Printf("[doc-room] flush error for doc %s: %v", docID, err)
}
}
return b64State, b64Updates
}
// ServeDocWS handles WebSocket connection for GET /api/v1/ws/docs/{id}
func (r *DocRoom) ServeDocWS(w http.ResponseWriter, req *http.Request, userID string) {
// ServeDocWS handles WebSocket connections for GET /api/v1/ws/docs/{id}
func (r *DocRoom) ServeDocWS(w http.ResponseWriter, req *http.Request, userID string, clientID string) {
conn, err := upgrader.Upgrade(w, req, nil)
if err != nil {
log.Printf("[doc-ws] upgrade error for doc %s: %v", r.docID, err)
return
}
if clientID == "" {
clientID = req.URL.Query().Get("clientId")
}
if clientID == "" {
clientID = domain.NewID("cli")
}
client := &DocWSClient{
hub: r.hub,
room: r,
conn: conn,
userID: userID,
send: make(chan []byte, 256),
hub: r.hub,
room: r,
conn: conn,
userID: userID,
clientID: clientID,
send: make(chan []byte, 256),
}
r.mu.Lock()
r.wsClients[client] = true
title := r.title
blocksCopy := make([]domain.DocumentBlock, len(r.blocks))
copy(blocksCopy, r.blocks)
r.mu.Unlock()
// Send initial snapshot
state, updates := r.GetInitialSnapshot()
initMsg := domain.WSMessage{
Event: "CRDT_INIT",
Payload: map[string]interface{}{
"docId": r.docID,
"state": state,
"updates": updates,
},
Timestamp: domain.NowISO(),
// Immediately send live in-memory state to the newly connected client
initEvt := DocSyncEvent{
Type: "doc_state",
DocID: r.docID,
ClientID: clientID,
Title: title,
Blocks: blocksCopy,
}
if initBytes, err := json.Marshal(initEvt); err == nil {
client.send <- initBytes
}
initBytes, _ := json.Marshal(initMsg)
client.send <- initBytes
go client.writePump()
go client.readPump()
}
// ServeDocSSE handles SSE stream fallback for GET /api/v1/sync/docs/{id}/stream
func (r *DocRoom) ServeDocSSE(w http.ResponseWriter, req *http.Request, userID string) {
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "Streaming unsupported", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache, no-transform")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.Header().Set("Access-Control-Allow-Origin", "*")
client := &DocSSEClient{
hub: r.hub,
room: r,
id: domain.NewID("sse-doc"),
userID: userID,
send: make(chan []byte, 256),
}
r.mu.Lock()
r.sseClients[client] = true
r.mu.Unlock()
defer func() {
r.mu.Lock()
delete(r.sseClients, client)
clientCount := len(r.wsClients) + len(r.sseClients)
r.mu.Unlock()
if clientCount == 0 {
r.Flush()
}
}()
// Send initial connected comment and CRDT_INIT
fmt.Fprintf(w, ": connected (transport: sse, docId: %s)\n\n", r.docID)
state, updates := r.GetInitialSnapshot()
initMsg, _ := json.Marshal(domain.WSMessage{
Event: "CRDT_INIT",
Payload: map[string]interface{}{
"docId": r.docID,
"state": state,
"updates": updates,
},
Timestamp: domain.NowISO(),
})
fmt.Fprintf(w, "data: %s\n\n", initMsg)
flusher.Flush()
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
notify := req.Context().Done()
for {
select {
case <-notify:
return
case msg, ok := <-client.send:
if !ok {
return
}
fmt.Fprintf(w, "data: %s\n\n", msg)
flusher.Flush()
case <-ticker.C:
// Ping to keep proxy connection alive
fmt.Fprintf(w, ": ping\n\n")
flusher.Flush()
}
}
}
func (c *DocWSClient) readPump() {
defer func() {
c.room.mu.Lock()
delete(c.room.wsClients, c)
clientCount := len(c.room.wsClients) + len(c.room.sseClients)
remaining := len(c.room.wsClients)
c.room.mu.Unlock()
_ = c.conn.Close()
// Broadcast awareness leave
c.room.BroadcastAwareness([]byte(fmt.Sprintf(`{"userId":"%s","activeBlockId":null}`, c.userID)), c.userID, nil, nil)
if clientCount == 0 {
// Flush immediately when the last client leaves the document room
if remaining == 0 {
c.room.Flush()
}
}()
c.conn.SetReadLimit(8 * 1024 * 1024) // 8MB limit for binary/text updates
c.conn.SetReadLimit(4 * 1024 * 1024)
_ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second))
c.conn.SetPongHandler(func(string) error {
_ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second))
@@ -402,51 +502,20 @@ func (c *DocWSClient) readPump() {
})
for {
msgType, message, err := c.conn.ReadMessage()
_, message, err := c.conn.ReadMessage()
if err != nil {
break
}
if msgType == websocket.BinaryMessage {
// Direct binary Yjs update
c.room.ApplyUpdate(message, c.userID, c, nil)
continue
}
// Text JSON message
var incoming struct {
Event string `json:"event"`
Payload json.RawMessage `json:"payload"`
}
if json.Unmarshal(message, &incoming) == nil {
switch incoming.Event {
case "CRDT_UPDATE":
var p struct {
Update string `json:"update"`
}
if json.Unmarshal(incoming.Payload, &p) == nil && p.Update != "" {
if raw, err := base64.StdEncoding.DecodeString(p.Update); err == nil && len(raw) > 0 {
c.room.ApplyUpdate(raw, c.userID, c, nil)
}
}
case "CRDT_COMPACT":
var p struct {
State string `json:"state"`
}
if json.Unmarshal(incoming.Payload, &p) == nil && p.State != "" {
if raw, err := base64.StdEncoding.DecodeString(p.State); err == nil && len(raw) > 0 {
c.room.CompactState(raw, c.userID)
}
}
case "CRDT_AWARENESS":
c.room.BroadcastAwareness(incoming.Payload, c.userID, c, nil)
}
var evt DocSyncEvent
if err := json.Unmarshal(message, &evt); err == nil && evt.Type != "" {
c.room.HandleEvent(evt, c)
}
}
}
func (c *DocWSClient) writePump() {
ticker := time.NewTicker(30 * time.Second)
ticker := time.NewTicker(25 * time.Second)
defer func() {
ticker.Stop()
_ = c.conn.Close()
internal/ws/hub.go
+6 -84
View File
@@ -1,7 +1,6 @@
package ws
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
@@ -47,10 +46,9 @@ type docLockItem struct {
expiresAt time.Time
}
type CRDTStore interface {
GetDocumentCRDTState(docID string) ([]byte, [][]byte, error)
AppendDocumentCRDTUpdate(docID string, update []byte, userID string) error
SaveDocumentCRDTState(docID string, stateBlob []byte, userID string) error
type DocStore interface {
LoadDocumentState(docID string) (string, []domain.DocumentBlock, error)
FlushDocumentState(docID string, title string, blocks []domain.DocumentBlock) error
}
type Hub struct {
@@ -63,7 +61,7 @@ type Hub struct {
sseRooms map[string]map[*SSEClient]bool
locks map[string]*docLockItem
docRooms map[string]*DocRoom
crdtStore CRDTStore
docStore DocStore
}
func NewHub() *Hub {
@@ -81,10 +79,10 @@ func NewHub() *Hub {
return h
}
func (h *Hub) SetCRDTStore(store CRDTStore) {
func (h *Hub) SetDocStore(store DocStore) {
h.mu.Lock()
defer h.mu.Unlock()
h.crdtStore = store
h.docStore = store
}
func (h *Hub) Register(c *Client) {
@@ -688,39 +686,6 @@ func (h *Hub) ServeSSE(w http.ResponseWriter, r *http.Request, roomID, userID st
fmt.Fprintf(w, "data: %s\n\n", connectedMsg)
flusher.Flush()
// If initial rooms had doc rooms, send CRDT_INIT
if h.crdtStore != nil {
for rID := range client.subscribedRooms {
if strings.HasPrefix(rID, "doc:") {
docID := strings.TrimPrefix(rID, "doc:")
stateBlob, updates, err := h.crdtStore.GetDocumentCRDTState(docID)
if err == nil && (len(stateBlob) > 0 || len(updates) > 0) {
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
initMsg := domain.WSMessage{
Event: "CRDT_INIT",
Payload: map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
},
Timestamp: domain.NowISO(),
}
if initBytes, err := json.Marshal(initMsg); err == nil {
fmt.Fprintf(w, "data: %s\n\n", initBytes)
flusher.Flush()
}
}
}
}
}
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
@@ -811,49 +776,6 @@ func (c *Client) readPump() {
if c.hub.ReleaseDocLock(docID, c.userID, false) {
c.hub.BroadcastDocEvent(docID, "DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
}
case "CRDT_JOIN":
c.hub.JoinRoom(c, "doc:"+docID)
room := c.hub.GetOrCreateDocRoom(docID)
b64State, b64Updates := room.GetInitialSnapshot()
if len(b64State) > 0 || len(b64Updates) > 0 {
c.SendEvent("CRDT_INIT", map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
})
}
case "CRDT_UPDATE":
var p struct {
Update string `json:"update"`
}
_ = json.Unmarshal(incoming.Payload, &p)
if p.Update != "" {
if raw, err := base64.StdEncoding.DecodeString(p.Update); err == nil && len(raw) > 0 {
room := c.hub.GetOrCreateDocRoom(docID)
room.ApplyUpdate(raw, c.userID, nil, nil)
}
}
case "CRDT_COMPACT":
var p struct {
State string `json:"state"`
}
_ = json.Unmarshal(incoming.Payload, &p)
if p.State != "" {
if raw, err := base64.StdEncoding.DecodeString(p.State); err == nil && len(raw) > 0 {
room := c.hub.GetOrCreateDocRoom(docID)
room.CompactState(raw, c.userID)
}
}
case "CRDT_AWARENESS":
c.hub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS", incoming.Payload, c)
room := c.hub.GetOrCreateDocRoom(docID)
room.BroadcastAwareness(incoming.Payload, c.userID, nil, nil)
case "CRDT_LEAVE":
c.hub.LeaveRoom(c, "doc:"+docID)
c.hub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": c.userID,
}, c)
}
}
}