sandrodach/MindSift
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(crdt): implement proxy-resilient Yjs dual-transport sync with indexeddb offline-first and debounced sqlite persistence
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m11s
Details

Browse Source
This commit is contained in:
sandrodach committed 2026-10-26 19:54:51 +01:00
1 parent 297f3ed111
commit f3b3f07e04
10 files changed
+1248 -484
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/api/server.go
+163 -91
View File
@@ -69,6 +69,9 @@ func (s *Server) Router() http.Handler {
r.Get("/api/v1/ws", s.handleWebSocket)
r.Get("/api/v1/ws/spaces/{spaceId}", s.handleWebSocket)
r.Get("/api/v1/ws/lists/{listId}", s.handleWebSocket)
r.Get("/api/v1/ws/docs/{id}", s.handleDocWebSocket)
r.Get("/api/v1/sync/docs/{id}/stream", s.handleDocSyncStream)
r.Post("/api/v1/sync/docs/{id}/update", s.handleDocSyncUpdate)
r.Get("/api/v1/events", s.handleSSE)
r.Post("/api/v1/events/rooms", s.handleSSERooms)
@@ -868,16 +871,6 @@ func (s *Server) handleUpdateDocument(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
// Check document lock
if lock := s.wsHub.GetDocLock(docID); lock != nil && lock.UserID != userID {
w.WriteHeader(http.StatusLocked)
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"error": "Dokument wird aktuell bearbeitet von " + lock.UserName,
"currentLock": lock,
})
return
}
var req struct {
Title *string `json:"title"`
Icon *string `json:"icon"`
@@ -907,16 +900,6 @@ func (s *Server) handlePatchDocumentBlocks(w http.ResponseWriter, r *http.Reques
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
// Check document lock
if lock := s.wsHub.GetDocLock(docID); lock != nil && lock.UserID != userID {
w.WriteHeader(http.StatusLocked)
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"error": "Dokument wird aktuell bearbeitet von " + lock.UserName,
"currentLock": lock,
})
return
}
var req domain.DocBlocksPatchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
@@ -1324,68 +1307,178 @@ func (s *Server) handleGetCollaborators(w http.ResponseWriter, r *http.Request)
writeJSON(w, http.StatusOK, users)
}
// --- DOCUMENT LOCKING ---
// --- DOCUMENT LOCKING (Neutralized - Yjs CRDT Pessimistic Locks Removed) ---
func (s *Server) handleGetDocLock(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
lock := s.wsHub.GetDocLock(docID)
if lock == nil {
writeJSON(w, http.StatusOK, map[string]interface{}{"locked": false})
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"locked": true, "lock": lock})
writeJSON(w, http.StatusOK, map[string]interface{}{"locked": false})
}
func (s *Server) handleLockDoc(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
user, err := s.store.GetUserByID(userID)
userName := "Ein Benutzer"
if err == nil && user != nil && user.DisplayName != "" {
userName = user.DisplayName
}
info, acquired, currentLock := s.wsHub.AcquireDocLock(docID, userID, userName, 20*time.Second)
if !acquired {
w.WriteHeader(http.StatusLocked)
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"locked": true,
"acquired": false,
"currentLock": currentLock,
"message": "Wird aktuell bearbeitet von " + currentLock.UserName,
})
return
}
s.wsHub.BroadcastDocEvent(docID, "DOC_LOCKED", info)
writeJSON(w, http.StatusOK, map[string]interface{}{
"locked": true,
"locked": false,
"acquired": true,
"lock": info,
"docId": docID,
})
}
func (s *Server) handleHeartbeatDocLock(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
ok := s.wsHub.HeartbeatDocLock(docID, userID, 20*time.Second)
writeJSON(w, http.StatusOK, map[string]bool{"extended": ok})
writeJSON(w, http.StatusOK, map[string]bool{"extended": true})
}
func (s *Server) handleUnlockDoc(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
user, _ := s.store.GetUserByID(userID)
isAdmin := user != nil && user.Role == "admin"
released := s.wsHub.ReleaseDocLock(docID, userID, isAdmin)
if released {
s.wsHub.BroadcastDocEvent(docID, "DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
}
writeJSON(w, http.StatusOK, map[string]bool{"unlocked": released})
writeJSON(w, http.StatusOK, map[string]bool{"unlocked": true})
}
// --- DOCUMENT CRDT REALTIME SYNC ---
// --- DOCUMENT CRDT REALTIME SYNC & DUAL TRANSPORT ---
func (s *Server) authenticateUserFromReq(r *http.Request) string {
token := r.URL.Query().Get("token")
if token == "" {
if authHdr := r.Header.Get("Authorization"); strings.HasPrefix(authHdr, "Bearer ") {
token = strings.TrimPrefix(authHdr, "Bearer ")
}
}
if token != "" {
if claims, err := auth.ValidateToken(s.cfg.JWTSecret, token); err == nil {
return claims.UserID
}
}
return ""
}
// GET /api/v1/ws/docs/{id}
func (s *Server) handleDocWebSocket(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Sec-Websocket-Key") != "" {
if !strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade") {
r.Header.Set("Connection", "Upgrade")
}
if !strings.Contains(strings.ToLower(r.Header.Get("Upgrade")), "websocket") {
r.Header.Set("Upgrade", "websocket")
}
}
docID := chi.URLParam(r, "id")
if docID == "" {
writeError(w, http.StatusBadRequest, "docId erforderlich")
return
}
userID := s.authenticateUserFromReq(r)
if userID == "" {
userID = auth.GetUserID(r.Context())
}
if userID == "" {
userID = "anonymous"
}
if userID != "anonymous" {
if _, err := s.store.GetDocumentByID(userID, docID); err != nil {
writeError(w, http.StatusForbidden, "Kein Zugriff auf dieses Dokument")
return
}
}
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ServeDocWS(w, r, userID)
}
// GET /api/v1/sync/docs/{id}/stream
func (s *Server) handleDocSyncStream(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
if docID == "" {
writeError(w, http.StatusBadRequest, "docId erforderlich")
return
}
userID := s.authenticateUserFromReq(r)
if userID == "" {
userID = auth.GetUserID(r.Context())
}
if userID == "" {
userID = "anonymous"
}
if userID != "anonymous" {
if _, err := s.store.GetDocumentByID(userID, docID); err != nil {
writeError(w, http.StatusForbidden, "Kein Zugriff auf dieses Dokument")
return
}
}
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ServeDocSSE(w, r, userID)
}
// POST /api/v1/sync/docs/{id}/update
func (s *Server) handleDocSyncUpdate(w http.ResponseWriter, r *http.Request) {
docID := chi.URLParam(r, "id")
if docID == "" {
writeError(w, http.StatusBadRequest, "docId erforderlich")
return
}
userID := s.authenticateUserFromReq(r)
if userID == "" {
userID = auth.GetUserID(r.Context())
}
if userID == "" {
writeError(w, http.StatusUnauthorized, "Nicht autorisiert")
return
}
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
if !doc.IsOwner && doc.Permission != "write" {
writeError(w, http.StatusForbidden, "Keine Schreibberechtigung")
return
}
contentType := r.Header.Get("Content-Type")
var updateBytes []byte
if strings.Contains(contentType, "application/octet-stream") {
body, err := io.ReadAll(io.LimitReader(r.Body, 8*1024*1024))
if err != nil {
writeError(w, http.StatusBadRequest, "Fehler beim Lesen des Payloads")
return
}
updateBytes = body
} else {
var req struct {
Update string `json:"update"`
State string `json:"state"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
if req.State != "" {
if raw, err := base64.StdEncoding.DecodeString(req.State); err == nil && len(raw) > 0 {
room := s.wsHub.GetOrCreateDocRoom(docID)
room.CompactState(raw, userID)
}
}
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 für Update")
return
}
updateBytes = raw
}
}
if len(updateBytes) > 0 {
room := s.wsHub.GetOrCreateDocRoom(docID)
room.ApplyUpdate(updateBytes, userID, nil, nil)
}
writeJSON(w, http.StatusOK, map[string]interface{}{"success": true, "docId": docID})
}
func (s *Server) handleGetDocCRDT(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
@@ -1394,19 +1487,8 @@ func (s *Server) handleGetDocCRDT(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
stateBlob, updates, err := s.store.GetDocumentCRDTState(docID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
room := s.wsHub.GetOrCreateDocRoom(docID)
b64State, b64Updates := room.GetInitialSnapshot()
writeJSON(w, http.StatusOK, map[string]interface{}{
"docId": docID,
"state": b64State,
@@ -1434,21 +1516,14 @@ func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "Ungültiger Request-Body")
return
}
room := s.wsHub.GetOrCreateDocRoom(docID)
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format für Update")
return
}
if err := s.store.AppendDocumentCRDTUpdate(docID, raw, userID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_UPDATE", map[string]interface{}{
"docId": docID,
"update": req.Update,
"fromUserId": userID,
}, nil)
room.ApplyUpdate(raw, userID, nil, nil)
}
if req.State != "" {
raw, err := base64.StdEncoding.DecodeString(req.State)
@@ -1456,10 +1531,7 @@ func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format")
return
}
if err := s.store.SaveDocumentCRDTState(docID, raw, userID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
room.CompactState(raw, userID)
}
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
internal/api/server_test.go
+81
View File
@@ -3,6 +3,7 @@ package api
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -1015,3 +1016,83 @@ func TestSSEAndAwarenessEndpoints(t *testing.T) {
}
}
func TestDocRoomDualTransportAndDebouncedPersistence(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test_dual_transport.db")
store, err := storage.NewStore(dbPath)
if err != nil {
t.Fatalf("failed to init store: %v", err)
}
defer store.Close()
cfg := &config.Config{
JWTSecret: "test-secret-dual-transport",
AllowRegistration: true,
}
hub := ws.NewHub()
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
user, _ := store.CreateUser("dual@test.local", "hash", "Dual User", "member")
token, _ := auth.GenerateToken(cfg.JWTSecret, user.ID, user.Email, user.Role)
doc, _ := store.CreateDocument(user.ID, "", "Dual Transport Doc", "📄", "", nil, nil, false)
// 1. Test POST /api/v1/sync/docs/{id}/update with JSON base64
upd1 := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03})
req := httptest.NewRequest(http.MethodPost, "/api/v1/sync/docs/"+doc.ID+"/update", bytes.NewReader([]byte(`{"update":"`+upd1+`"}`)))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from sync update, got %d: %s", rec.Code, rec.Body.String())
}
// 2. Test POST /api/v1/sync/docs/{id}/update with octet-stream binary
binaryUpd := []byte{0x04, 0x05, 0x06}
req = httptest.NewRequest(http.MethodPost, "/api/v1/sync/docs/"+doc.ID+"/update", bytes.NewReader(binaryUpd))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/octet-stream")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from binary sync update, got %d: %s", rec.Code, rec.Body.String())
}
// 3. Test GET /api/v1/sync/docs/{id}/stream SSE connection
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req = httptest.NewRequest(http.MethodGet, "/api/v1/sync/docs/"+doc.ID+"/stream?token="+token, nil).WithContext(ctx)
rec = httptest.NewRecorder()
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
router.ServeHTTP(rec, req)
if !strings.Contains(rec.Header().Get("Content-Type"), "text/event-stream") {
t.Fatalf("expected text/event-stream, got %s", rec.Header().Get("Content-Type"))
}
bodyStr := rec.Body.String()
if !strings.Contains(bodyStr, "CRDT_INIT") {
t.Fatalf("expected CRDT_INIT in SSE stream, got %s", bodyStr)
}
// 4. Test Lock Neutrality: document update and block patch work without 423 Locked
updateReq := httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID, bytes.NewReader([]byte(`{"title":"Updated Without Locks"}`)))
updateReq.Header.Set("Authorization", "Bearer "+token)
updateReq.Header.Set("Content-Type", "application/json")
updateRec := httptest.NewRecorder()
router.ServeHTTP(updateRec, updateReq)
if updateRec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from document update, got %d: %s", updateRec.Code, updateRec.Body.String())
}
}