feat(realtime): implement SSE fallback, dual-transport hub & proxy resilience (resolves FLAW-03)
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 59s

This commit is contained in:
sandrodach committed 2026-10-26 16:55:11 +01:00
1 parent 80bb74b5b2
commit 6343d62bab
10 files changed
+1402 -240

No files matched your search

+115 -1
View File
@@ -69,6 +69,8 @@ func (s *Server) Router() http.Handler {
r.Get("/api/v1/ws", s.handleWebSocket)
r.Get("/api/v1/ws/spaces/{spaceId}", s.handleWebSocket)
r.Get("/api/v1/ws/lists/{listId}", s.handleWebSocket)
r.Get("/api/v1/events", s.handleSSE)
r.Post("/api/v1/events/rooms", s.handleSSERooms)
// Authenticated API Routes
r.Group(func(r chi.Router) {
@@ -148,6 +150,8 @@ func (s *Server) Router() http.Handler {
// Document CRDT Realtime Sync
r.Get("/api/v1/documents/{id}/crdt", s.handleGetDocCRDT)
r.Put("/api/v1/documents/{id}/crdt", s.handleSaveDocCRDT)
r.Post("/api/v1/documents/{id}/awareness", s.handlePostDocAwareness)
r.Delete("/api/v1/documents/{id}/awareness", s.handleDeleteDocAwareness)
// Shared Lists & Realtime Collaboration
r.Get("/api/v1/lists", s.handleGetAllLists)
@@ -1423,12 +1427,29 @@ func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
return
}
var req struct {
State string `json:"state"`
State string `json:"state"`
Update string `json:"update"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiger Request-Body")
return
}
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format für Update")
return
}
if err := s.store.AppendDocumentCRDTUpdate(docID, raw, userID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_UPDATE", map[string]interface{}{
"docId": docID,
"update": req.Update,
"fromUserId": userID,
}, nil)
}
if req.State != "" {
raw, err := base64.StdEncoding.DecodeString(req.State)
if err != nil {
@@ -1443,6 +1464,37 @@ func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handlePostDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
_ = doc
var payload json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS", payload, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handleDeleteDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": userID,
}, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
// --- REALTIME LISTS & SHARING ---
func (s *Server) handleGetAllLists(w http.ResponseWriter, r *http.Request) {
@@ -1712,6 +1764,68 @@ func (s *Server) handleWebSocket(w http.ResponseWriter, r *http.Request) {
s.wsHub.ServeWS(w, r, roomID, userID)
}
func (s *Server) handleSSE(w http.ResponseWriter, r *http.Request) {
roomID := r.URL.Query().Get("listId")
if roomID == "" {
roomID = r.URL.Query().Get("spaceId")
}
token := r.URL.Query().Get("token")
if token == "" {
if authHdr := r.Header.Get("Authorization"); strings.HasPrefix(authHdr, "Bearer ") {
token = strings.TrimPrefix(authHdr, "Bearer ")
}
}
userID := "anonymous"
if token != "" {
if claims, err := auth.ValidateToken(s.cfg.JWTSecret, token); err == nil {
userID = claims.UserID
}
}
var initialRooms []string
if roomsParam := r.URL.Query().Get("rooms"); roomsParam != "" {
for _, rm := range strings.Split(roomsParam, ",") {
rm = strings.TrimSpace(rm)
if rm != "" {
initialRooms = append(initialRooms, rm)
}
}
}
if docID := r.URL.Query().Get("docId"); docID != "" {
initialRooms = append(initialRooms, "doc:"+strings.TrimSpace(docID))
}
s.wsHub.ServeSSE(w, r, roomID, userID, initialRooms)
}
func (s *Server) handleSSERooms(w http.ResponseWriter, r *http.Request) {
var req struct {
SSEID string `json:"sseId"`
Room string `json:"room"`
DocID string `json:"docId"`
Action string `json:"action"` // "join" or "leave"
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
if req.Room == "" && req.DocID != "" {
req.Room = "doc:" + req.DocID
}
if req.SSEID == "" || req.Room == "" {
writeError(w, http.StatusBadRequest, "sseId und room erforderlich")
return
}
if req.Action == "leave" {
s.wsHub.LeaveSSERoomByID(req.SSEID, req.Room)
} else {
s.wsHub.JoinSSERoomByID(req.SSEID, req.Room)
}
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
// --- API KEYS & WEBHOOK INGESTION ---
func (s *Server) handleGetAPIKeys(w http.ResponseWriter, r *http.Request) {
+107
View File
@@ -2,6 +2,7 @@ package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -9,6 +10,7 @@ import (
"strings"
"sync"
"testing"
"time"
"github.com/sandrodach/mindsift/internal/auth"
"github.com/sandrodach/mindsift/internal/config"
@@ -908,3 +910,108 @@ func TestDocumentCRDTEndpoints(t *testing.T) {
t.Fatalf("expected state 'AQIDBA==', got '%s'", crdtResp.State)
}
}
func TestSSEAndAwarenessEndpoints(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test_sse.db")
store, err := storage.NewStore(dbPath)
if err != nil {
t.Fatalf("failed to init store: %v", err)
}
defer store.Close()
hub := ws.NewHub()
hub.SetCRDTStore(store)
cfg := &config.Config{
JWTSecret: "test-jwt-secret-key",
AllowRegistration: false,
}
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
// 1. Setup user
setupBody := []byte(`{
"email": "sse_user@example.com",
"password": "secretpassword",
"displayName": "SSE User",
"instanceName": "MindSift SSE",
"allowRegistration": false,
"seedSampleData": false
}`)
req := httptest.NewRequest(http.MethodPost, "/api/v1/auth/setup", bytes.NewReader(setupBody))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("setup failed: %s", rec.Body.String())
}
var authResp struct {
Token string `json:"token"`
User *domain.User `json:"user"`
}
_ = json.Unmarshal(rec.Body.Bytes(), &authResp)
token := authResp.Token
// 2. Create Document
doc, err := store.CreateDocument(authResp.User.ID, "", "SSE Collaborative Note", "📄", "", nil, nil, false)
if err != nil {
t.Fatalf("failed to create document: %v", err)
}
// 3. Test Awareness POST and DELETE
awarenessBody := []byte(`{"userId":"` + authResp.User.ID + `","activeBlockId":"blk_123"}`)
req = httptest.NewRequest(http.MethodPost, "/api/v1/documents/"+doc.ID+"/awareness", bytes.NewReader(awarenessBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from POST awareness, got %d: %s", rec.Code, rec.Body.String())
}
req = httptest.NewRequest(http.MethodDelete, "/api/v1/documents/"+doc.ID+"/awareness", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from DELETE awareness, got %d", rec.Code)
}
// 4. Test Delta CRDT Update via PUT /api/v1/documents/{id}/crdt
updateBody := []byte(`{"update":"AQIDBA=="}`)
req = httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID+"/crdt", bytes.NewReader(updateBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from PUT crdt update, got %d: %s", rec.Code, rec.Body.String())
}
// 5. Test SSE Connection GET /api/v1/events
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req = httptest.NewRequest(http.MethodGet, "/api/v1/events?token="+token+"&docId="+doc.ID, nil).WithContext(ctx)
rec = httptest.NewRecorder()
go func() {
// Cancel context after short moment to terminate stream cleanly
time.Sleep(50 * time.Millisecond)
cancel()
}()
router.ServeHTTP(rec, req)
contentType := rec.Header().Get("Content-Type")
if !strings.Contains(contentType, "text/event-stream") {
t.Fatalf("expected Content-Type text/event-stream, got %s", contentType)
}
if rec.Header().Get("X-Accel-Buffering") != "no" {
t.Fatalf("expected X-Accel-Buffering: no, got %s", rec.Header().Get("X-Accel-Buffering"))
}
bodyStr := rec.Body.String()
if !strings.Contains(bodyStr, "SSE_CONNECTED") {
t.Fatalf("expected SSE_CONNECTED event in body, got: %s", bodyStr)
}
}
+297 -13
View File
@@ -3,6 +3,7 @@ package ws
import (
"encoding/base64"
"encoding/json"
"fmt"
"log"
"net/http"
"strings"
@@ -30,6 +31,14 @@ type Client struct {
subscribedRooms map[string]bool
}
type SSEClient struct {
id string
userID string
roomID string
send chan []byte
subscribedRooms map[string]bool
}
type docLockItem struct {
docID string
userID string
@@ -45,20 +54,26 @@ type CRDTStore interface {
}
type Hub struct {
mu sync.RWMutex
rooms map[string]map[*Client]bool
userRooms map[string]map[*Client]bool
allClients map[*Client]bool
locks map[string]*docLockItem
crdtStore CRDTStore
mu sync.RWMutex
rooms map[string]map[*Client]bool
userRooms map[string]map[*Client]bool
allClients map[*Client]bool
sseClients map[*SSEClient]bool
sseUserRooms map[string]map[*SSEClient]bool
sseRooms map[string]map[*SSEClient]bool
locks map[string]*docLockItem
crdtStore CRDTStore
}
func NewHub() *Hub {
h := &Hub{
rooms: make(map[string]map[*Client]bool),
userRooms: make(map[string]map[*Client]bool),
allClients: make(map[*Client]bool),
locks: make(map[string]*docLockItem),
rooms: make(map[string]map[*Client]bool),
userRooms: make(map[string]map[*Client]bool),
allClients: make(map[*Client]bool),
sseClients: make(map[*SSEClient]bool),
sseUserRooms: make(map[string]map[*SSEClient]bool),
sseRooms: make(map[string]map[*SSEClient]bool),
locks: make(map[string]*docLockItem),
}
go h.cleanupLocksLoop()
return h
@@ -85,7 +100,7 @@ func (h *Hub) Register(c *Client) {
}
h.userRooms[c.userID][c] = true
}
onlineCount := len(h.allClients)
onlineCount := len(h.allClients) + len(h.sseClients)
h.mu.Unlock()
if c.roomID != "" {
@@ -126,11 +141,14 @@ func (h *Hub) Unregister(c *Client) {
if c.userID != "" && c.userID != "anonymous" {
if uRoom, ok := h.userRooms[c.userID]; ok {
delete(uRoom, c)
userRemaining = len(uRoom)
if userRemaining == 0 {
userRemaining += len(uRoom)
if len(uRoom) == 0 {
delete(h.userRooms, c.userID)
}
}
if sseRoom, ok := h.sseUserRooms[c.userID]; ok {
userRemaining += len(sseRoom)
}
}
close(c.send)
@@ -161,6 +179,142 @@ func (h *Hub) Unregister(c *Client) {
}
}
func (h *Hub) RegisterSSE(c *SSEClient) {
h.mu.Lock()
h.sseClients[c] = true
if c.roomID != "" {
if h.sseRooms[c.roomID] == nil {
h.sseRooms[c.roomID] = make(map[*SSEClient]bool)
}
h.sseRooms[c.roomID][c] = true
}
if c.userID != "" && c.userID != "anonymous" {
if h.sseUserRooms[c.userID] == nil {
h.sseUserRooms[c.userID] = make(map[*SSEClient]bool)
}
h.sseUserRooms[c.userID][c] = true
}
if c.subscribedRooms != nil {
for rID := range c.subscribedRooms {
if h.sseRooms[rID] == nil {
h.sseRooms[rID] = make(map[*SSEClient]bool)
}
h.sseRooms[rID][c] = true
}
}
onlineCount := len(h.allClients) + len(h.sseClients)
h.mu.Unlock()
if c.roomID != "" {
h.Broadcast(c.roomID, "", "USER_PRESENCE", map[string]interface{}{
"onlineCount": onlineCount,
})
}
}
func (h *Hub) UnregisterSSE(c *SSEClient) {
h.mu.Lock()
delete(h.sseClients, c)
if c.roomID != "" {
if room, ok := h.sseRooms[c.roomID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.sseRooms, c.roomID)
}
}
}
var docRoomsLeft []string
if c.subscribedRooms != nil {
for rID := range c.subscribedRooms {
if room, ok := h.sseRooms[rID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.sseRooms, rID)
}
}
if strings.HasPrefix(rID, "doc:") {
docRoomsLeft = append(docRoomsLeft, strings.TrimPrefix(rID, "doc:"))
}
}
}
userRemaining := 0
if c.userID != "" && c.userID != "anonymous" {
if uRoom, ok := h.sseUserRooms[c.userID]; ok {
delete(uRoom, c)
userRemaining += len(uRoom)
if len(uRoom) == 0 {
delete(h.sseUserRooms, c.userID)
}
}
if wsRoom, ok := h.userRooms[c.userID]; ok {
userRemaining += len(wsRoom)
}
}
close(c.send)
var releasedDocIDs []string
if userRemaining == 0 && c.userID != "" {
now := time.Now()
for docID, l := range h.locks {
if l.userID == c.userID || l.expiresAt.Before(now) {
releasedDocIDs = append(releasedDocIDs, docID)
delete(h.locks, docID)
}
}
}
h.mu.Unlock()
for _, docID := range releasedDocIDs {
h.BroadcastToAll("DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
}
for _, docID := range docRoomsLeft {
h.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": c.userID,
}, nil)
}
}
func (h *Hub) JoinSSERoomByID(sseID, roomID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
for client := range h.sseClients {
if client.id == sseID {
if h.sseRooms[roomID] == nil {
h.sseRooms[roomID] = make(map[*SSEClient]bool)
}
h.sseRooms[roomID][client] = true
if client.subscribedRooms == nil {
client.subscribedRooms = make(map[string]bool)
}
client.subscribedRooms[roomID] = true
return true
}
}
return false
}
func (h *Hub) LeaveSSERoomByID(sseID, roomID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
for client := range h.sseClients {
if client.id == sseID {
if room, ok := h.sseRooms[roomID]; ok {
delete(room, client)
if len(room) == 0 {
delete(h.sseRooms, roomID)
}
}
if client.subscribedRooms != nil {
delete(client.subscribedRooms, roomID)
}
return true
}
}
return false
}
// Document Locking Methods
func (h *Hub) AcquireDocLock(docID, userID, userName string, ttl time.Duration) (*domain.DocLockInfo, bool, *domain.DocLockInfo) {
@@ -300,6 +454,12 @@ func (h *Hub) BroadcastToAll(event string, payload interface{}) {
default:
}
}
for sseClient := range h.sseClients {
select {
case sseClient.send <- data:
default:
}
}
}
func (h *Hub) BroadcastToUser(userID, event string, payload interface{}) {
@@ -324,6 +484,14 @@ func (h *Hub) BroadcastToUser(userID, event string, payload interface{}) {
}
}
}
if sseRoom, ok := h.sseUserRooms[userID]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
}
func (h *Hub) BroadcastTaskEvent(userID, event string, payload interface{}) {
@@ -363,6 +531,14 @@ func (h *Hub) Broadcast(spaceID, listID, event string, payload interface{}) {
}
}
}
if sseRoom, ok := h.sseRooms[targetRoom]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
}
func (h *Hub) BroadcastListEvent(listID, event string, payload interface{}) {
@@ -421,6 +597,14 @@ func (h *Hub) BroadcastRoom(roomID, event string, payload interface{}, excludeCl
}
}
}
if sseRoom, ok := h.sseRooms[roomID]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
}
func (c *Client) SendEvent(event string, payload interface{}) {
@@ -458,6 +642,106 @@ func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request, roomID, userID str
go client.readPump()
}
func (h *Hub) ServeSSE(w http.ResponseWriter, r *http.Request, roomID, userID string, initialRooms []string) {
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache, no-transform")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.Header().Set("Access-Control-Allow-Origin", "*")
client := &SSEClient{
id: domain.NewID("sse"),
userID: userID,
roomID: roomID,
send: make(chan []byte, 128),
subscribedRooms: make(map[string]bool),
}
for _, room := range initialRooms {
if strings.TrimSpace(room) != "" {
client.subscribedRooms[strings.TrimSpace(room)] = true
}
}
h.RegisterSSE(client)
defer h.UnregisterSSE(client)
// Send initial connected comment and event
fmt.Fprintf(w, ": connected (transport: sse, id: %s)\n\n", client.id)
connectedMsg, _ := json.Marshal(domain.WSMessage{
Event: "SSE_CONNECTED",
Payload: map[string]interface{}{
"sseId": client.id,
"transport": "sse",
"userId": userID,
},
Timestamp: domain.NowISO(),
})
fmt.Fprintf(w, "data: %s\n\n", connectedMsg)
flusher.Flush()
// If initial rooms had doc rooms, send CRDT_INIT
if h.crdtStore != nil {
for rID := range client.subscribedRooms {
if strings.HasPrefix(rID, "doc:") {
docID := strings.TrimPrefix(rID, "doc:")
stateBlob, updates, err := h.crdtStore.GetDocumentCRDTState(docID)
if err == nil && (len(stateBlob) > 0 || len(updates) > 0) {
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
initMsg := domain.WSMessage{
Event: "CRDT_INIT",
Payload: map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
},
Timestamp: domain.NowISO(),
}
if initBytes, err := json.Marshal(initMsg); err == nil {
fmt.Fprintf(w, "data: %s\n\n", initBytes)
flusher.Flush()
}
}
}
}
}
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
notify := r.Context().Done()
for {
select {
case <-notify:
return
case msg, ok := <-client.send:
if !ok {
return
}
fmt.Fprintf(w, "data: %s\n\n", msg)
flusher.Flush()
case <-ticker.C:
// Periodic ping comment to keep HTTP connection alive and prevent proxy timeouts
fmt.Fprintf(w, ": ping\n\n")
flusher.Flush()
}
}
}
func (c *Client) readPump() {
defer func() {
c.hub.Unregister(c)