Dev #6

Merged
sandrodach merged 6 commits from dev into main 2026-10-26 18:54:23 +01:00
19 changed files with 2558 additions and 575 deletions

No files matched your search

+12 -11
View File
@@ -17,9 +17,10 @@ Doku/
├── modules/ ├── modules/
│ ├── triage.md # Modul 1: Triage & Quick-Dump (State Machine & Audio) │ ├── triage.md # Modul 1: Triage & Quick-Dump (State Machine & Audio)
│ ├── tasks.md # Modul 2: Aufgaben & Kanban-System (Alarme & Scheduler) │ ├── tasks.md # Modul 2: Aufgaben & Kanban-System (Alarme & Scheduler)
│ ├── workspace.md # Modul 3: Workspace Outliner-Engine, Wiki-Links & Disk-Sync │ ├── workspace.md # Modul 3: Workspace Outliner-Engine, Wiki-Links & Pure-DB Storage
│ └── lists.md # Modul 4: Geteilte Listen (Live-Kollaboration & D&D) │ └── lists.md # Modul 4: Geteilte Listen (Live-Kollaboration & D&D)
├── sharing-and-permissions.md # Synchrones Teilen, Rollen, Isolation & Mount-Entkopplung ├── sharing-and-permissions.md # Synchrones Teilen, Rollen, Isolation & Mount-Entkopplung
├── reverse-proxy.md # Nginx, Traefik, Caddy, Cloudflare Konfiguration & SSE
├── theming.md # Single-File CSS Theme-Engine & Design-System ├── theming.md # Single-File CSS Theme-Engine & Design-System
└── flaws.md # Technische Schwachstellen, Risiken & zukünftige Optimierungen └── flaws.md # Technische Schwachstellen, Risiken & zukünftige Optimierungen
``` ```
@@ -32,21 +33,23 @@ Doku/
3. **[REST-API Referenz](api/rest.md)** 3. **[REST-API Referenz](api/rest.md)**
Spezifikation aller Endpunkte für Authentifizierung, Gedanken, Aufgaben, Dokumente, Listen, Teams und Webhooks. Spezifikation aller Endpunkte für Authentifizierung, Gedanken, Aufgaben, Dokumente, Listen, Teams und Webhooks.
4. **[WebSocket & Echtzeit-Synchronisation](api/websocket.md)** 4. **[WebSocket & Echtzeit-Synchronisation](api/websocket.md)**
Gorilla WebSocket Hub, Rooms, Presence-Tracking, Document Locking Mechanismen und Polling-Fallback. Gorilla WebSocket Hub, Rooms, Presence-Tracking, Yjs CRDT Kollaboration und SSE Fallback.
5. **[Kernmodul: Triage & Quick-Dump](modules/triage.md)** 5. **[Kernmodul: Triage & Quick-Dump](modules/triage.md)**
Instant-Capture-Pipeline, Finite State Machine, Gestensteuerung, Spracheingabe und Webhook-Ingestion. Instant-Capture-Pipeline, Finite State Machine, Gestensteuerung, Spracheingabe und Webhook-Ingestion.
6. **[Kernmodul: Aufgaben & Kanban-System](modules/tasks.md)** 6. **[Kernmodul: Aufgaben & Kanban-System](modules/tasks.md)**
Multi-Listen-Konzept, 3-Spalten Kanban-Board, Subtasks, Fälligkeiten und Multi-Channel Alarm-Ticker (SMTP, Discord, Webhooks). Multi-Listen-Konzept, 3-Spalten Kanban-Board, Subtasks, Fälligkeiten und Multi-Channel Alarm-Ticker (SMTP, Discord, Webhooks).
7. **[Kernmodul: Workspace & Outliner-Engine](modules/workspace.md)** 7. **[Kernmodul: Workspace & Outliner-Engine](modules/workspace.md)**
Logseq-inspirierte hierarchische Block-Engine, Zoom-Hoisting, Wiki-Links, Disk-Mirroring (`.md`) und Globales To-Do Rollup. Logseq-inspirierte hierarchische Block-Engine, Zoom-Hoisting, Wiki-Links, Pure-Database Storage und Globales To-Do Rollup.
8. **[Kernmodul: Geteilte Listen & Kollaboration](modules/lists.md)** 8. **[Kernmodul: Geteilte Listen & Kollaboration](modules/lists.md)**
Listen-Typen, Mengen, Kategorien, Live-Checkoff und Drag & Drop Sortierung. Listen-Typen, Mengen, Kategorien, Live-Checkoff und Drag & Drop Sortierung.
9. **[Synchrones Teilen & Berechtigungskonzept](sharing-and-permissions.md)** 9. **[Synchrones Teilen & Berechtigungskonzept](sharing-and-permissions.md)**
Rollenmodell (Admin/Member), System-Isolations-Modi (`strict_private`, `teams_only`, `open`), entkoppelte Ordner-Mounts und Notizen-Freigaben. Rollenmodell (Admin/Member), System-Isolations-Modi (`strict_private`, `teams_only`, `open`), entkoppelte Ordner-Mounts und Notizen-Freigaben.
10. **[Single-File Theming & Design-System](theming.md)** 10. **[Reverse-Proxy- & Ingress-Handbuch](reverse-proxy.md)**
Produktionsfertige Konfigurationen für Nginx, Traefik, Caddy, Cloudflare und Apache mit WebSocket-Upgrades und SSE-Streaming.
11. **[Single-File Theming & Design-System](theming.md)**
1-Datei = 1-Theme Philosophie, die 6 CSS-Variablen-Blöcke, Semantische CSS-Klassen und Geometrie-Steuerung. 1-Datei = 1-Theme Philosophie, die 6 CSS-Variablen-Blöcke, Semantische CSS-Klassen und Geometrie-Steuerung.
11. **[Technische Schwachstellen & Optimierungspotenziale (Flaws)](flaws.md)** 12. **[Technische Schwachstellen & Optimierungspotenziale (Flaws)](flaws.md)**
Kritische Analyse von Concurrency-Grenzen, Dual-Storage-Risiken, Undo/Redo, Volltextsuche und Skalierungs-Engpässen. Kritische Analyse von Concurrency-Grenzen, Volltextsuche, Viewport-Anpassung und Skalierungs-Engpässen.
--- ---
@@ -68,12 +71,11 @@ graph TB
ChiRouter["Chi Router (REST API)"] ChiRouter["Chi Router (REST API)"]
WSHub["Gorilla WebSocket Hub & Lock Manager"] WSHub["Gorilla WebSocket Hub & Lock Manager"]
AlarmTicker["Alarm Scheduler (Background Ticker)"] AlarmTicker["Alarm Scheduler (Background Ticker)"]
DiskSync["Disk Mirror Engine (/data/workspace)"] StorageEngine["Storage Layer (Domain & Repositories)"]
end end
subgraph Storage ["Storage Layer"] subgraph Storage ["Storage Layer"]
SQLite[("SQLite WAL Database (/data/mindsift.db)")] SQLite[("SQLite WAL Database (/data/mindsift.db)")]
DiskFiles[("Physische Markdown Files (/data/workspace)")]
end end
subgraph External ["Externe Schnittstellen"] subgraph External ["Externe Schnittstellen"]
@@ -86,9 +88,8 @@ graph TB
WSClient -->|WSS Realtime| Traefik WSClient -->|WSS Realtime| Traefik
Traefik --> ChiRouter Traefik --> ChiRouter
Traefik --> WSHub Traefik --> WSHub
ChiRouter --> SQLite ChiRouter --> StorageEngine
ChiRouter --> DiskSync StorageEngine --> SQLite
DiskSync --> DiskFiles
AlarmTicker --> Discord AlarmTicker --> Discord
AlarmTicker --> SMTPServer AlarmTicker --> SMTPServer
HomeAssistant -->|POST /api/v1/inbox| Traefik HomeAssistant -->|POST /api/v1/inbox| Traefik
+88 -31
View File
@@ -64,50 +64,107 @@ Alle WebSocket-Nachrichten werden als JSON mit einheitlicher Hülle übertragen:
--- ---
## 🔒 4. Dokument-Sperrsystem (Document Locking) ---
Um Schreibkonflikte bei Notizen im Workspace zu vermeiden, verwendet MindSift ein **serverseitiges Lease-Locking mit Heartbeat**: ## ⚡ 4. CRDT-Echtzeitprotokoll (Yjs Collaborative Editing)
Seit Version 2.1 unterstützt MindSift **echte, konfliktfreie Mehrbenutzer-Kollaboration** für Notizen via Yjs CRDT. Clients tauschen binäre Delta-Updates und Live-Awareness-Informationen über dedizierte WebSocket-Nachrichten aus:
### CRDT-Nachrichtentypen
| Event | Richtung | Zweck / Payload |
|---|---|---|
| `CRDT_JOIN` | Client ➔ Server | Tritt dem Dokumentenraum (`doc:{docId}`) bei. |
| `CRDT_INIT` | Server ➔ Client | Sendet den aktuellen Basiszustand (`stateBlob`) und offene Delta-Updates (`updates[]`) an den Client. |
| `CRDT_UPDATE` | Bidirektional | Überträgt inkrementelle binäre Yjs-Updates (Base64-kodiert). Server speichert in `document_crdt_updates` und verteilt an alle anderen Raumnutzer. |
| `CRDT_COMPACT` | Client ➔ Server | Komprimiert mehrere Updates zu einem konsolidierten Snapshot in `document_crdt_state`. |
| `CRDT_AWARENESS` | Bidirektional | Überträgt Live-Präsenz (Fokus auf `blockId`, Benutzername, Farbe). |
| `CRDT_AWARENESS_LEAVE` | Bidirektional | Signalisiert, dass ein Nutzer den Fokus verloren hat oder das Dokument verlassen hat. |
| `CRDT_LEAVE` | Client ➔ Server | Verlässt den Dokumentenraum explizit. |
```mermaid ```mermaid
sequenceDiagram sequenceDiagram
autonumber autonumber
actor UserA as Benutzer A actor UserA as Benutzer A
participant WS as WebSocket Hub participant WS as WebSocket Hub & SQLite
actor UserB as Benutzer B actor UserB as Benutzer B
UserA->>WS: LOCK_DOC (docId: "doc_123") UserA->>WS: CRDT_JOIN (docId: "doc_123")
WS-->>UserA: Lock bestätigt (30s TTL) WS-->>UserA: CRDT_INIT (stateBlob, updates)
WS->>UserB: DOC_LOCKED (docId: "doc_123", lockedBy: "Benutzer A") UserB->>WS: CRDT_JOIN (docId: "doc_123")
Note over UserB: Editor gesperrt (Schreibschutz aktiv) WS-->>UserB: CRDT_INIT (stateBlob, updates)
loop Alle 15 Sekunden UserA->>WS: CRDT_UPDATE (docId, updateBlob)
UserA->>WS: HEARTBEAT_DOC (docId: "doc_123") WS->>WS: Append in document_crdt_updates
WS-->>UserA: TTL um 30s verlängert WS->>UserB: CRDT_UPDATE (docId, updateBlob)
end Note over UserB: Yjs wendet Delta atomar an (0 Konflikte)
alt Reguläres Schließen UserA->>WS: CRDT_AWARENESS (docId, focusedBlockId)
UserA->>WS: UNLOCK_DOC (docId: "doc_123") WS->>UserB: CRDT_AWARENESS (docId, user, focusedBlockId)
WS->>UserB: DOC_UNLOCKED (docId: "doc_123") Note over UserB: Avatar-Badge erscheint am Block
Note over UserB: Editor wieder freigegeben
else Verbindungsabbruch / Tab geschlossen
Note over WS: Heartbeat bleibt aus (TTL läuft ab)
WS->>UserB: DOC_UNLOCKED (Automatisch nach Ablauf)
end
``` ```
### Locking-Events
- **`LOCK_DOC` (Client ➔ Server):** Fordert eine 30-Sekunden-Sperre für ein Dokument an.
- **`HEARTBEAT_DOC` (Client ➔ Server):** Verlängert die Sperre alle 15 Sekunden, solange der Benutzer das Dokument geöffnet hat.
- **`UNLOCK_DOC` (Client ➔ Server):** Gibt die Sperre beim Verlassen des Dokuments frei.
- **`DOC_LOCKED` (Server ➔ Clients):** Informiert andere Benutzer, dass das Dokument schreibgeschützt ist.
- **`DOC_UNLOCKED` (Server ➔ Clients):** Hebt den Schreibschutz auf.
--- ---
## 🔄 5. Fallback-Strategie & Reconnect ## 🔒 5. Abwärtskompatibles Document-Lease-Locking
In strikten Firmennetzwerken oder bei Proxy-Problemen (z.B. fehlende `Upgrade`-Header bei HTTP/1.1) kann eine WebSocket-Verbindung fehlschlagen: Vor Version 2.1 nutzte MindSift ein serverseitiges Lease-Locking mit 30-Sekunden-Heartbeats. Durch die CRDT-Engine wurde die blockierende Bearbeitungssperre (`isLockedByOther`) abgelöst:
- **Aktuelles Verhalten:** Mehrere Benutzer mit Bearbeitungsrechten können zeitgleich dasselbe Dokument bearbeiten.
- **Presence & Locks:** Die Events `LOCK_DOC` und `HEARTBEAT_DOC` bleiben abwärtskompatibel erhalten und dienen nun der reinen Präsenzanzeige (z.B. gelber Statusindikator „wird gerade von X angesehen").
- **Schreibschutz:** Ein echter Schreibschutz (`isReadOnly`) greift ausschließlich noch dann, wenn der Nutzer nur über Leserechte (`permission === 'read'`) für die Notiz oder den Ordner verfügt.
---
## 🔄 5. Fallback-Strategie & Proxy-Resilienz (Dual-Transport Architektur)
In restriktiven Firmennetzwerken, hinter Unternehmens-Proxies (z.B. Zscaler, BlueCoat) oder fehlerhaft konfigurierten Reverse-Proxies wird der `Upgrade: websocket`-Header häufig verworfen (`NS_ERROR_CONNECTION_REFUSED`, Code 1006 / 1002 / 1008).
MindSift löst dieses Problem durch eine **vollwertige Dual-Transport-Architektur** ([FLAW-03](../flaws.md)):
```
┌───────────────────────────────┐
│ MindSift Frontend │
└───────────────┬───────────────┘
│
┌─────────────────────┴─────────────────────┐
▼ ▼
Standard: WebSockets Fallback: SSE + REST
(wss://.../api/v1/ws) (https://.../api/v1/events)
│ │
│ [Proxy Upgrade Block (Code 1006/Timeout)]│
│ ────────────────────────────────────────> │
▼ ▼
┌────────────────────────┐ ┌────────────────────────┐
│ Gorilla WS Conn │ │ Server-Sent Events │
│ Bidirektionaler │ │ (text/event-stream) │
│ TCP-WebSocket │ │ Standard HTTPS Port │
└────────────┬───────────┘ └────────────┬───────────┘
│ │
└─────────────────────┬─────────────────────┘
▼
┌─────────────────────────────┐
│ Realtime Hub │
│ (Multiplexing WS & SSE) │
└─────────────────────────────┘
```
### 1. Automatischer SSE-Fallback (`GET /api/v1/events`)
- Scheitert der WebSocket-Handshake oder blockiert ein Proxy das Upgrade, wechselt MindSift innerhalb von 3,5 Sekunden automatisch auf einen **Server-Sent Events (SSE) Stream**.
- Da SSE ein standardmäßiger HTTP/1.1- bzw. HTTP/2-GET-Request über Standard-Port 443/80 ist (`Content-Type: text/event-stream`), wird er von jedem Proxy und jeder Firewall ungehindert durchgeleitet.
- Header `X-Accel-Buffering: no` und `Cache-Control: no-cache` verhindern Pufferung im Proxy; periodische Ping-Kommentare (`: ping\n\n`) alle 15 Sekunden halten Timeouts fern.
- Der Client bleibt dadurch **vollständig echtzeitfähig** (`Live (SSE)`).
### 2. REST-Endpunkte für Upstream-Aktionen im SSE-Modus
Da SSE unidirektional (Server ➔ Client) ist, stellt MindSift für Schreibvorgänge im SSE-Modus REST-Äquivalente bereit:
- `PUT /api/v1/documents/:id/crdt`: Unterstützt sowohl volles `state` (Compaction) als auch inkrementelles `update` (Delta). Der Hub persistiert das Update und broadcastet es sofort an alle anderen WS- und SSE-Clients im Dokumentenraum.
- `POST /api/v1/documents/:id/awareness`: Überträgt Cursor- und Blockfokus-Präsenzen.
- `DELETE /api/v1/documents/:id/awareness`: Informiert Peers beim Verlassen des Dokuments.
- `POST /api/v1/events/rooms`: Erlaubt SSE-Clients, Dokument- oder Listenräumen dynamisch beizutreten (`join`) oder sie zu verlassen (`leave`).
### 3. Verbindungs-Indikator
- **Grün pulsierend:** Live verbunden über WebSockets (`Live (WebSocket)`).
- **Türkis pulsierend:** Live verbunden über Server-Sent Events (`Live (Server-Sent Events / Proxy-Stream)`).
- **Gelb/Amber:** Temporäre Unterbrechung mit automatischem HTTP-Polling-Schutz.
Ausführliche Konfigurationsbeispiele für Nginx, Traefik, Caddy, Cloudflare und Apache finden sich im separaten Handbuch:
👉 **[Reverse-Proxy-Konfigurationshandbuch](../reverse-proxy.md)**
1. **Exponentielles Backoff:** Das Frontend ([`frontend/src/lib/api.ts`](../../frontend/src/lib/api.ts)) versucht automatisch die Wiederverbindung nach 1s, 2s, 4s, 8s, bis maximal 30s.
2. **HTTP-Polling Fallback:** Geteilte Listen und Notizen aktivieren im Hintergrund ein periodisches Polling (5s Intervall), falls der WebSocket-Status nicht `CONNECTED` ist.
3. **Status-Indikator:** Ein dezenter grüner bzw. gelber Indikator oben rechts im Header zeigt dem Benutzer jederzeit den aktuellen Live-Verbindungsstatus an.
+2 -2
View File
@@ -138,5 +138,5 @@ MindSift wird als schlanker, autarker Docker-Container ausgeliefert:
3. **Stage 3 (Production Runtime):** `alpine:3.20` mit `ca-certificates` und `tzdata`. Das finale Image ist ca. 45 MB groß und benötigt keine externen Datenbankserver oder Abhängigkeiten. 3. **Stage 3 (Production Runtime):** `alpine:3.20` mit `ca-certificates` und `tzdata`. Das finale Image ist ca. 45 MB groß und benötigt keine externen Datenbankserver oder Abhängigkeiten.
Alle persistenten Daten befinden sich in einem einzigen Verzeichnis `/data`: Alle persistenten Daten befinden sich in einem einzigen Verzeichnis `/data`:
- `/data/mindsift.db` (SQLite Hauptdatenbank) - `/data/mindsift.db` (SQLite Hauptdatenbank inklusive WAL-Journal und Shared-Memory-Dateien)
- `/data/workspace/` (Physische Markdown-Dateien der Workspace-Notizen) - Notizen und Workspace-Blöcke werden zu 100% in der SQLite-Datenbank persistiert (keine losen Markdown-Dateien auf der Festplatte).
+2 -2
View File
@@ -208,7 +208,7 @@ CREATE TABLE IF NOT EXISTS task_list_shares (
### 2.4 Workspace: Dokumente, Blöcke & Geteilte Ordner ### 2.4 Workspace: Dokumente, Blöcke & Geteilte Ordner
#### `documents` & `document_blocks` (Database-First Outliner) #### `documents` & `document_blocks` (Database-First Outliner)
Dokumente werden als Kopfdatensatz in `documents` gespeichert, während der Inhalt als feingranulare Blöcke in `document_blocks` verwaltet wird. Zusätzlich spiegelt MindSift die Daten synchron als physische `.md`-Datei auf Festplatte. Dokumente werden als Kopfdatensatz in `documents` gespeichert, während der Inhalt als feingranulare Blöcke in `document_blocks` verwaltet wird. MindSift speichert alle Notizen zu 100% in SQLite (keine physischen `.md`-Dateien auf Festplatte).
```sql ```sql
CREATE TABLE IF NOT EXISTS documents ( CREATE TABLE IF NOT EXISTS documents (
@@ -216,7 +216,7 @@ CREATE TABLE IF NOT EXISTS documents (
user_id TEXT NOT NULL, -- Eigentümer user_id TEXT NOT NULL, -- Eigentümer
title TEXT NOT NULL, -- Dokumenttitel title TEXT NOT NULL, -- Dokumenttitel
icon TEXT NOT NULL DEFAULT '📄', -- Emoji-Icon icon TEXT NOT NULL DEFAULT '📄', -- Emoji-Icon
content TEXT NOT NULL DEFAULT '', -- Markdown-Volltext (Spiegelung) content TEXT NOT NULL DEFAULT '', -- Plaintext / Text-Repräsentation (Cache)
blocks_json TEXT NOT NULL DEFAULT '[]', -- Legacy JSON-Cache blocks_json TEXT NOT NULL DEFAULT '[]', -- Legacy JSON-Cache
tags TEXT NOT NULL DEFAULT '[]', -- JSON Array von Tags tags TEXT NOT NULL DEFAULT '[]', -- JSON Array von Tags
folder_path TEXT NOT NULL DEFAULT '', -- Relativer Ordnerpfad des Eigentümers folder_path TEXT NOT NULL DEFAULT '', -- Relativer Ordnerpfad des Eigentümers
+40 -25
View File
@@ -8,8 +8,8 @@ Diese Datei dokumentiert eine ehrliche und schonungslose technische Analyse alle
| ID | Bereich | Problem / Schwachstelle | Schweregrad | Risiko im Produktivbetrieb | | ID | Bereich | Problem / Schwachstelle | Schweregrad | Risiko im Produktivbetrieb |
|---|---|---|---|---| |---|---|---|---|---|
| **FLAW-01** | Workspace Sync | Kein echtes CRDT/OT – Lock-Heartbeat-Modell mit Last-Write-Wins | **Kritisch** | Zeilenverlust bei gleichzeitigen Bearbeitungen oder Netzschwankungen | | **FLAW-01** | Workspace Sync | Kein echtes CRDT/OT – Lock-Heartbeat-Modell mit Last-Write-Wins | **Gelöst (v2.1)** | Behoben durch Yjs CRDT-Engine & blockbasierte kollaborative Bearbeitung |
| **FLAW-02** | Storage | Dual-Storage Drift (SQLite `document_blocks` vs. Dateisystem `.md`) | **Hoch** | Externe Änderungen an `.md` werden nicht automatisch in SQLite gespiegelt | | **FLAW-02** | Storage | Dual-Storage Drift (SQLite vs. Dateisystem) | **Obsolet / Gelöst** | Kein Risiko: MindSift nutzt Pure-DB (keine `.md`-Dateien auf Platte) |
| **FLAW-03** | Netzwerk / WS | Proxy-Inkompatibilität bei WebSocket-Upgrades & Polling-Fallback | **Mittel** | Verzögerte Synchronisierung (1–3s Polling) in Firmennetzwerken | | **FLAW-03** | Netzwerk / WS | Proxy-Inkompatibilität bei WebSocket-Upgrades & Polling-Fallback | **Mittel** | Verzögerte Synchronisierung (1–3s Polling) in Firmennetzwerken |
| **FLAW-04** | Datenbank / Suche | Volltextsuche via `LIKE '%query%'` statt SQLite FTS5 | **Mittel** | Performance-Einbrüche bei wachsenden Notizbeständen (>10.000 Blöcke) | | **FLAW-04** | Datenbank / Suche | Volltextsuche via `LIKE '%query%'` statt SQLite FTS5 | **Mittel** | Performance-Einbrüche bei wachsenden Notizbeständen (>10.000 Blöcke) |
| **FLAW-05** | Frontend DOM | Fehlende Virtualisierung im Outliner-Baum | **Mittel** | Frame-Drops und Lags bei Dokumenten mit >500 Blöcken auf Mobilgeräten | | **FLAW-05** | Frontend DOM | Fehlende Virtualisierung im Outliner-Baum | **Mittel** | Frame-Drops und Lags bei Dokumenten mit >500 Blöcken auf Mobilgeräten |
@@ -23,35 +23,50 @@ Diese Datei dokumentiert eine ehrliche und schonungslose technische Analyse alle
## 🔍 Detaillierte Analyse & Maßnahmen ## 🔍 Detaillierte Analyse & Maßnahmen
### FLAW-01: Kein echtes CRDT/OT – Lock-Heartbeat & Last-Write-Wins ### FLAW-01: [GELÖST / IMPLEMENTIERT] Yjs Outliner CRDT & Multi-User Kollaboration
- **Betroffene Komponenten:** [`api/websocket.md`](./api/websocket.md), [`modules/workspace.md`](./modules/workspace.md), `internal/ws/hub.go`, `frontend/src/lib/components/Outliner.svelte` - **Status:** **Gelöst via Yjs CRDT Engine (Lösung i)**
- **Ursache:** MindSift nutzt zur Vermeidung von Konflikten eine kooperative Bearbeitungssperre (*Document Lease Lock*, 30 Sekunden TTL mit Heartbeat). Wenn zwei Nutzer dasselbe Dokument geöffnet haben, blockiert der aktive Schreiber das Bearbeiten für andere. Verliert ein Nutzer jedoch kurzzeitig die WebSocket-Verbindung oder tippt während des Übergangs zum Polling-Fallback, werden Änderungen per REST via Last-Write-Wins (`PUT /api/v1/documents/:id`) gespeichert. - **Betroffene Komponenten:** [`api/websocket.md`](./api/websocket.md), [`modules/workspace.md`](./modules/workspace.md), `internal/ws/hub.go`, `internal/storage/sqlite.go`, `frontend/src/lib/crdt.ts`, `frontend/src/features/workspace/WorkspaceView.tsx`
- **Folgen:** Wie in früheren Versionen beobachtet, können bei parallelem Schreiben Zeilen überschrieben werden oder Strukturhierarchien (Parent-IDs) verloren gehen. - **Ursache (Historisch):** MindSift nutzte ein kooperatives Dokumenten-Lease-Locking mit Last-Write-Wins. Bei parallelem Zugriff oder Netztrennung konnten Änderungen überschrieben werden.
- **Empfohlene Lösung:** - **Implementierte Lösung (Yjs CRDT):**
1. Langfristige Migration auf ein echtes blockweises CRDT-Framework (wie [Yjs](https://yjs.dev/) oder [Automerge](https://automerge.org/)). 1. **Block-Level Yjs CRDT:** Implementierung eines vollwertigen CRDT-Modells mit `Y.Doc`. Die Blockreihenfolge wird über ein `Y.Array<string>` (Block-IDs) konfliktfrei mit dem YATA-Algorithmus aufgelöst. Inhalte und Metadaten (Text, Level, Checked, Collapsed) sind in `Y.Map` gekapselt.
2. Kurzfristig: Optimistisches Block-Locking (Sperre nur auf Block-Ebene `block_id` statt des gesamten Dokuments) und atomare Versionsprüfung (*Optimistic Concurrency Control* mit `version_seq`). 2. **Wegfall von Ganzdokument-Sperren:** Keine blockierenden Sperren mehr (`isLockedByOther` entfernt). Mehrere Nutzer können zeitgleich am selben Dokument schreiben.
3. **Dual-Persistence:** CRDT-Zustände und Delta-Updates werden binär/Base64 in SQLite (`document_crdt_state`, `document_crdt_updates`) persistiert. Zur Aufrechterhaltung relationaler Indizes (Global To-Dos, Tag-Indizierung, Suche) werden Blöcke parallel in `document_blocks` materialisiert.
4. **Live-Awareness & Presence:** Block-Fokus und Cursor-Zustand werden via `CRDT_AWARENESS` übertragen. Nutzer sehen ein Live-Badge im Header (`Live (N)`) sowie farbige Kollaborator-Pillen an aktiven Blöcken.
5. **REST- & Proxy-Sicherheit:** Ergänzend zu WebSockets stehen Endpunkte `GET /api/v1/documents/:id/crdt` und `PUT /api/v1/documents/:id/crdt` bereit.
--- ---
### FLAW-02: Dual-Storage Drift (SQLite vs. Dateisystem `.md`) ### FLAW-02: [OBSOLET / GELÖST] Dual-Storage Drift (SQLite vs. Dateisystem)
- **Betroffene Komponenten:** [`database.md`](./database.md), `internal/storage/sqlite.go`, `internal/storage/markdown_sync.go` - **Status:** **Entkräftet & Behoben (Pure Database Architecture)**
- **Ursache:** MindSift pflegt Notizen primär in SQLite (`documents`, `document_blocks`), exportiert bzw. spiegelt sie jedoch als reine `.md`-Dateien auf das Dateisystem (`data/notes/`), um Exportierbarkeit und lokale Lesbarkeit zu gewährleisten. - **Klarstellung:** In früheren Entwürfen wurde angedacht, Notizen zusätzlich als Markdown-Dateien auf dem Dateisystem zu spiegeln. MindSift speichert Notizen und Blöcke jedoch **zu 100% relational in SQLite** (`documents` und `document_blocks`).
- **Problem:** Es existiert kein systemweiter File-Watcher ([fsnotify](https://github.com/fsnotify/fsnotify)). Bearbeitet der Nutzer eine Notizdatei direkt auf dem Host (z. B. via Obsidian, VS Code, Git-Sync oder SSH), bekommt die SQLite-Datenbank dies nicht mit. Beim nächsten Web-Edit überschreibt SQLite die Datei auf der Festplatte. - **Verifikation:**
- **Empfohlene Lösung:** - Es existieren **keine** Markdown-Dateien auf der Festplatte.
1. Integrieren eines Background-File-Watchers mit `fsnotify` für das Datenverzeichnis. - Das Go-Backend schreibt keine Dateien in ein Notizverzeichnis.
2. Implementieren einer Hash-Prüfung (`sha256` der Datei vs. DB-Metadaten) vor dem Speichern. Bei Abweichung: Nutzerkonflikt-Dialog anzeigen. - Unnötige Verzeichniserstellungen (`docDir`/`workspace`) wurden aus `internal/storage/sqlite.go` vollständig entfernt.
- **Ergebnis:** Es gibt keinen Dual-Storage und somit keinen File-Drift oder externe Dateikonflikte. Notizen verhalten sich vollständig transaktional und konsistent innerhalb der SQLite-Datenbank.
--- ---
### FLAW-03: Proxy-Inkompatibilität bei WebSocket-Upgrades ### FLAW-03: [GELÖST / IMPLEMENTIERT] Proxy-Inkompatibilität bei WebSocket-Upgrades & Polling-Fallback
- **Betroffene Komponenten:** [`api/websocket.md`](./api/websocket.md), `frontend/src/lib/stores/websocket.js` - **Status:** **Gelöst & Vollständig Implementiert (Dual-Transport SSE Fallback & Proxy-Resilienz)**
- **Ursache:** In restriktiven Firmennetzwerken oder über bestimmte Reverse-Proxies (Corporate SSL-Termination, Firewalls) schlägt der HTTP-Upgrade-Header `Upgrade: websocket` fehl (`NS_ERROR_CONNECTION_REFUSED` oder `400 Bad Request`). - **Betroffene Komponenten:** [`api/websocket.md`](./api/websocket.md), [`reverse-proxy.md`](./reverse-proxy.md), [`internal/ws/hub.go`](../internal/ws/hub.go), [`internal/api/server.go`](../internal/api/server.go), [`frontend/src/lib/api.ts`](../frontend/src/lib/api.ts)
- **Folgen:** Der Client schaltet in den HTTP-Polling-Fallback um (alle 2 Sekunden). Dadurch: - **Problemursache:** Corporate Proxies (z.B. Zscaler, BlueCoat, strikte Firewalls) oder unvollständig konfigurierte Reverse-Proxies verwerfen den HTTP-Upgrade-Header `Upgrade: websocket` (`NS_ERROR_CONNECTION_REFUSED`, Status 1006 / 1002). Früher fiel der Client dabei auf 2-Sekunden-Polling zurück, was zu Verzögerungen und Last führte.
- Werden Sperren (*Document Locks*) und Online-Statusanzeigen verzögert übertragen. - **Implementierte Lösung:**
- Entsteht unnötige Serverlast durch kontinuierliche HTTP-Requests. 1. **Unidirektionaler Server-Sent Events (SSE) Fallback:**
- **Empfohlene Lösung:** - Neuer Streaming-Endpunkt `GET /api/v1/events` im Go-Backend mit `Content-Type: text/event-stream`, `X-Accel-Buffering: no` und 15s Heartbeat-Pings (`: ping\n\n`).
1. Sicherstellen von standardmäßigen Nginx/Traefik-Snippets in der Dokumentation (`proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade";`). - Standard HTTPS Ports (443/80), wodurch kein Proxy den Datenstrom blockieren kann.
2. Optionale Implementierung von Server-Sent Events (SSE) als unidirektionale Echtzeit-Alternative für Lesevorgänge bei blockiertem WebSocket. 2. **Hub-Multiplexing:**
- `ws.Hub` verwaltet WebSocket- und SSE-Clients parallel in gemeinsamen Räumen (`rooms`, `userRooms`, `allClients`).
- Sämtliche Events (`BroadcastToAll`, `BroadcastToUser`, `BroadcastRoom`) werden transparent an beide Transportarten verteilt.
3. **REST-Parität für Outbound CRDT & Awareness:**
- `PUT /api/v1/documents/:id/crdt`: Akzeptiert Delta-Updates (`update`) und broadcastet sie an alle Peers.
- `POST /api/v1/documents/:id/awareness` und `DELETE /api/v1/documents/:id/awareness`: Übertragen Blockfokus und Cursor ohne WebSocket.
- `POST /api/v1/events/rooms`: Erlaubt SSE-Clients dynamisches Room-Joining.
4. **Frontend Realtime Transport Manager (`frontend/src/lib/api.ts`):**
- Automatischer 3,5s-Watchdog: Schlägt das WebSocket-Upgrade fehl oder wird Code 1006/1002 gemeldet, wechselt MindSift nahtlos auf `EventSource('/api/v1/events')`.
- Grüner/türkiser Statusindikator im Header zeigt `Live (WebSocket)` bzw. `Live (Server-Sent Events / Proxy-Stream)`.
5. **Reverse-Proxy-Handbuch:**
- Vollständige Produktions-Konfigurationen für Nginx, Traefik, Caddy, Cloudflare und Apache unter [`reverse-proxy.md`](./reverse-proxy.md).
--- ---
+31 -21
View File
@@ -1,6 +1,6 @@
# 📝 Kernmodul 3: Workspace Outliner-Engine, Wiki-Links & Disk-Sync # 📝 Kernmodul 3: Workspace Outliner-Engine, Wiki-Links & Pure-DB Storage
Der MindSift Workspace ([`WorkspaceView.tsx`](../../frontend/src/features/workspace/WorkspaceView.tsx)) kombiniert ein **hierarchisches Bullet-Point Notizsystem (inspiriert von Logseq)** mit einem **persönlichen Markdown-Wiki** und einem **globalen To-Do Rollup Dashboard**. Der MindSift Workspace ([`WorkspaceView.tsx`](../../frontend/src/features/workspace/WorkspaceView.tsx)) kombiniert ein **hierarchisches Bullet-Point Notizsystem (inspiriert von Logseq)** mit einem **persönlichen Wiki** und einem **globalen To-Do Rollup Dashboard**.
--- ---
@@ -46,30 +46,40 @@ Der Bullet-Dot dient gleichzeitig als Drag-Anfasser. Wird ein Block verschoben,
--- ---
## 💾 3. Dual-Storage: Database-First & Disk-Mirroring ---
MindSift löst das Dilemma zwischen schneller relationaler Abfrage und portabler Dateispeicherung durch ein **synchrones Dual-Storage-Modell**: ## ⚡ 3. Yjs CRDT Real-Time Engine & Dual-Persistence
Ab MindSift v2.1 setzt der Workspace auf ein **konfliktfreies Replicated Data Type (CRDT) Modell** auf Basis von [Yjs](https://yjs.dev/):
### Funktionsweise
1. **CRDT-Struktur im Client:**
- Ein `Y.Doc` pro geöffnetem Dokument.
- `Y.Array<string>` (`blocks`): Führt die Block-IDs in korrekter Reihenfolge. Der YATA-Algorithmus garantiert, dass simultanes Einfügen, Verschieben oder Löschen von Blöcken durch mehrere Nutzer deterministisch und konfliktfrei konvergiert.
- `Y.Map` (`block:<id>`): Speichert Inhalt (`content`), Einrückung (`level`), Erledigungsstatus (`checked`) und Faltungsstatus (`collapsed`).
2. **Kollaborative Bearbeitung ohne Sperren:**
- Nutzer können simultan dieselbe Notiz bearbeiten.
- Fokus- und Cursor-Positionen werden via `CRDT_AWARENESS` übertragen und als farbige Benutzer-Pillen direkt am jeweiligen Block visualisiert.
### Dual-Persistence in SQLite
``` ```
Client (Block-Editor) Client (Y.Doc Mutation)
│ PATCH /api/v1/documents/:id/blocks │
├──▶ WebSocket (CRDT_UPDATE Base64 Delta)
│ │
│ ▼
│ SQLite: document_crdt_updates / document_crdt_state
│ (Mathematisch exakte CRDT-Konvergenz)
│
└──▶ Periodischer Snapshot / Flush
│
▼ ▼
Backend Engine SQLite: document_blocks
┌──────────────────────────────────────────────┐ (Relationale Indizes für Global To-Dos, Tag-Explorer & FTS)
│ 1. Transaktion in SQLite (document_blocks) │
│ -> Sort-Order, Content, Level, Checked │
│ 2. Volltext-Generierung (Markdown-String) │
│ 3. Schreiben auf Disk (/data/workspace/...) │
└──────────────────────────────────────────────┘
``` ```
1. **Präzision & Geschwindigkeit (SQLite):** 1. **Mathematische CRDT-Garantie:** Binäre Yjs-Updates werden in `document_crdt_updates` und periodisch als Snapshot in `document_crdt_state` gesichert. Auch nach einem Server-Neustart oder Offline-Phasen konvergieren alle Clients verlustfrei.
- Jeder Block besitzt eine eindeutige ID (`blk_...`) und eine Gleitkomma-Reihenfolge (`sort_order REAL`). 2. **Relationale Query-Fähigkeit:** Um schnelle SQL-Abfragen für das globale To-Do Rollup Dashboard und Hashtags zu ermöglichen, werden die Blöcke parallel in `document_blocks` materialisiert. Es existieren **keine Markdown-Dateien auf der Festplatte**.
- Schnelle Status-Updates (z.B. Checkbox abhaken) schreiben nur genau eine Zeile in die Datenbank.
2. **Datensouveränität & Portabilität (Disk-Mirror):**
- Zeitgleich spiegelt der Server das Dokument als Standard-`.md`-Datei unter `/data/workspace/<user_id>/<ordner>/<dateiname>.md`.
- Einrückungen werden nach dem 2-Leerzeichen-Standard (` - [ ] `) formatiert.
- Administratoren können das Verzeichnis jederzeit mit externen Werkzeugen (Obsidian, Logseq, Git, rsync) sichern.
--- ---
@@ -79,4 +89,4 @@ Client (Block-Editor)
- Scannt in Echtzeit alle unvollendeten `- [ ] ` Checkboxen über alle Ordner und Notizen hinweg. - Scannt in Echtzeit alle unvollendeten `- [ ] ` Checkboxen über alle Ordner und Notizen hinweg.
- Übersichtliche Gruppierung nach **Ordner ➔ Dokument ➔ Aufgabe**. - Übersichtliche Gruppierung nach **Ordner ➔ Dokument ➔ Aufgabe**.
- **Ordner-Filter:** Klick auf einen Ordner isoliert nur dessen Aufgaben für fokussiertes Arbeiten. - **Ordner-Filter:** Klick auf einen Ordner isoliert nur dessen Aufgaben für fokussiertes Arbeiten.
- **2-Wege-Live-Sync:** Wird eine Checkbox im Rollup abgehakt, wird der Block in SQLite sofort auf `checked = 1` aktualisiert und die physische `.md`-Datei synchronisiert. - **2-Wege-Live-Sync:** Wird eine Checkbox im Rollup abgehakt, wird der Block in SQLite sofort auf `checked = 1` aktualisiert und via WebSocket an alle aktiven Clients gebroadcastet.
+229
View File
@@ -0,0 +1,229 @@
# 🛡️ Reverse-Proxy & WebServer-Konfigurationshandbuch
Dieses Dokument beschreibt die empfohlene Konfiguration von Reverse-Proxies und Ingress-Controllern für MindSift. MindSift nutzt eine **hybride Realtime-Architektur** bestehend aus bidirektionalen **WebSockets** und einem automatischen **Server-Sent Events (SSE) Fallback**, um selbst in restriktivsten Firmennetzwerken, hinter strikten Firewalls oder bei Deep-Packet-Inspection unterbrechungsfreie Live-Synchronisation zu gewährleisten.
---
## ⚡ Wichtige Anforderungen auf einen Blick
| Protokoll | Standard-Port | Benötigte Header & Einstellungen | Zweck |
| :--- | :--- | :--- | :--- |
| **WebSockets** | 443 (WSS) / 80 (WS) | `Upgrade $http_upgrade`<br>`Connection "Upgrade"`<br>`proxy_read_timeout 3600s` | Bidirektionale Yjs-CRDT-Synchronisation & Awareness |
| **Server-Sent Events (SSE)** | 443 (HTTPS) / 80 (HTTP) | `X-Accel-Buffering: no`<br>`proxy_buffering off`<br>`proxy_cache off` | Fallback bei blockiertem WebSocket-Upgrade |
| **REST API & Uploads** | 443 (HTTPS) / 80 (HTTP) | `client_max_body_size 64M` | Notizen, Anhänge & CRDT-Zustandsblöcke |
---
## 1. Nginx
Eine typische `nginx.conf` bzw. Site-Konfiguration unter `/etc/nginx/sites-available/mindsift`:
```nginx
# Map für sauberes Connection-Handling bei WebSocket-Upgrades
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen 80;
server_name mindsift.example.com;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
server_name mindsift.example.com;
ssl_certificate /etc/letsencrypt/live/mindsift.example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/mindsift.example.com/privkey.pem;
# Maximale Payload-Größe für Backups und große CRDT-Zustände
client_max_body_size 64M;
# MindSift Frontend & REST API
location / {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# 1. WebSocket Endpunkte (/api/v1/ws)
location /api/v1/ws {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
# Upgrade-Header zwingend erforderlich
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Verhindert vorzeitigen Verbindungsabbruch durch Proxy-Inaktivitäts-Timeout
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
# 2. Server-Sent Events Fallback (/api/v1/events)
location /api/v1/events {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
# Pufferung zwingend deaktivieren für sofortiges Event-Streaming
proxy_buffering off;
proxy_cache off;
proxy_set_header X-Accel-Buffering no;
proxy_set_header Connection '';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 86400s;
}
}
```
---
## 2. Traefik (Docker Compose / Ingress)
Bei Verwendung von Traefik Version 2/3 müssen keine expliziten WebSocket-Upgrades deklariert werden, da Traefik diese transparent durchleitet. Für Server-Sent Events und Timeouts empfehlen sich Middleware-Definitionen:
```yaml
version: '3.8'
services:
mindsift:
image: ghcr.io/sandrodach/mindsift:latest
restart: unless-stopped
environment:
- PORT=8080
- DB_PATH=/app/data/mindsift.db
volumes:
- ./data:/app/data
labels:
- "traefik.enable=true"
- "traefik.http.routers.mindsift.rule=Host(`mindsift.example.com`)"
- "traefik.http.routers.mindsift.entrypoints=websecure"
- "traefik.http.routers.mindsift.tls.certresolver=letsencrypt"
- "traefik.http.services.mindsift.loadbalancer.server.port=8080"
# Verhindert Pufferung und hält persistente SSE-Streams am Leben
- "traefik.http.middlewares.mindsift-buffering.buffering.maxRequestBodyBytes=67108864"
- "traefik.http.routers.mindsift.middlewares=mindsift-buffering"
```
---
## 3. Caddy Server (Caddyfile)
Caddy unterstützt WebSockets und HTTP/2 Server-Sent Events von Haus aus vollständig ("out-of-the-box"):
```caddyfile
mindsift.example.com {
encode gzip zstd
# Notizen, CRDT & Event-Stream ohne Pufferung weiterleiten
reverse_proxy 127.0.0.1:8080 {
# Streaming für SSE optimieren
flush_interval -1
}
}
```
---
## 4. Cloudflare Proxy (Orange Cloud)
Wird MindSift hinter dem Cloudflare CDN betrieben:
1. **WebSockets aktivieren:**
- Navigiere in das Cloudflare Dashboard: **Network** > **WebSockets** > Schalter auf **ON** stellen.
2. **Cloudflare Rocket Loader / Auto Minify:**
- Deaktiviere "Rocket Loader" für MindSift, da asynchrone Skripteingriffe die Yjs-CRDT-Initialisierung stören können.
3. **Cloudflare WAF / Bot Fight Mode:**
- Stelle sicher, dass `GET /api/v1/events` und `GET /api/v1/ws*` nicht von "Challenge"-Seiten blockiert werden.
4. **SSE-Unterstützung:**
- Cloudflare unterstützt Server-Sent Events nativ über HTTP/2 und HTTP/3. Dank der vom MindSift-Backend gesetzten Header `X-Accel-Buffering: no` und `Cache-Control: no-cache` streamt Cloudflare Echtzeit-Events verzögerungsfrei an die Clients.
---
## 5. Apache 2.4 (`mod_proxy_wstunnel`)
Unter Debian/Ubuntu:
```bash
sudo a2enmod proxy proxy_http proxy_wstunnel headers
sudo systemctl restart apache2
```
VirtualHost-Konfiguration (`/etc/apache2/sites-available/mindsift.conf`):
```apache
<VirtualHost *:443>
ServerName mindsift.example.com
SSLEngine on
SSLCertificateFile /etc/letsencrypt/live/mindsift.example.com/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/mindsift.example.com/privkey.pem
# 1. WebSocket Tunneling
RewriteEngine On
RewriteCond %{HTTP:Upgrade} =websocket [NC]
RewriteRule /(.*) ws://127.0.0.1:8080/$1 [P,L]
RewriteCond %{HTTP:Upgrade} !=websocket [NC]
RewriteRule /(.*) http://127.0.0.1:8080/$1 [P,L]
# 2. SSE Response Flushing
<Location "/api/v1/events">
ProxyPass "http://127.0.0.1:8080/api/v1/events" flushpackets=on
ProxyPassReverse "http://127.0.0.1:8080/api/v1/events"
</Location>
ProxyPass / http://127.0.0.1:8080/
ProxyPassReverse / http://127.0.0.1:8080/
</VirtualHost>
```
---
## 🔍 Diagnose & Verifikation
Mit den folgenden Befehlen lässt sich die Erreichbarkeit und Streaming-Funktion prüfen:
### 1. Server-Sent Events Test (Standard-HTTPS)
```bash
curl -N -H "Accept: text/event-stream" -H "Authorization: Bearer <DEIN_TOKEN>" https://mindsift.example.com/api/v1/events
```
*Erwartete Ausgabe:*
```text
: connected (transport: sse, id: sse_...)
data: {"event":"SSE_CONNECTED","payload":{"sseId":"sse_...","transport":"sse"},"timestamp":"..."}
: ping
```
### 2. WebSocket Handshake Test
```bash
curl -i -N \
-H "Connection: Upgrade" \
-H "Upgrade: websocket" \
-H "Host: mindsift.example.com" \
-H "Origin: https://mindsift.example.com" \
-H "Sec-WebSocket-Key: SGVsbG8sIHdvcmxkIQ==" \
-H "Sec-WebSocket-Version: 13" \
https://mindsift.example.com/api/v1/ws?token=<DEIN_TOKEN>
```
*Erwartete Antwort:* `101 Switching Protocols`
---
## 🧭 Verwandte Dokumente
- 🔌 [WebSocket & Live-Protokoll](api/websocket.md)
- 🏗️ [Gesamtarchitektur](architecture.md)
- 🚨 [Bekannte Schwachstellen & Flaws](flaws.md)
+50 -1
View File
@@ -13,7 +13,8 @@
"lucide-react": "^0.468.0", "lucide-react": "^0.468.0",
"prismjs": "^1.30.0", "prismjs": "^1.30.0",
"react": "^18.3.1", "react": "^18.3.1",
"react-dom": "^18.3.1" "react-dom": "^18.3.1",
"yjs": "^13.6.33"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^18.3.12", "@types/react": "^18.3.12",
@@ -1840,6 +1841,16 @@
"node": ">=0.12.0" "node": ">=0.12.0"
} }
}, },
"node_modules/isomorphic.js": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz",
"integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==",
"license": "MIT",
"funding": {
"type": "GitHub Sponsors ❤",
"url": "https://github.com/sponsors/dmonad"
}
},
"node_modules/jiti": { "node_modules/jiti": {
"version": "1.21.7", "version": "1.21.7",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz",
@@ -1882,6 +1893,27 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/lib0": {
"version": "0.2.119",
"resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.119.tgz",
"integrity": "sha512-vh+TiejVy8Xm+hCdnRfn9BthGNvNUZdEXltNjVZL3TzRIKP3s1lnQh8pWd/Q0wCgY0FPAGVVjFF4ZiHUV3LFgA==",
"license": "MIT",
"dependencies": {
"isomorphic.js": "^0.2.4"
},
"bin": {
"0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js",
"0gentesthtml": "bin/gentesthtml.js",
"0serve": "bin/0serve.js"
},
"engines": {
"node": ">=16"
},
"funding": {
"type": "GitHub Sponsors ❤",
"url": "https://github.com/sponsors/dmonad"
}
},
"node_modules/lilconfig": { "node_modules/lilconfig": {
"version": "3.1.4", "version": "3.1.4",
"resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.4.tgz", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.4.tgz",
@@ -2756,6 +2788,23 @@
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
"dev": true, "dev": true,
"license": "ISC" "license": "ISC"
},
"node_modules/yjs": {
"version": "13.6.33",
"resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.33.tgz",
"integrity": "sha512-q/UYvr1gOk4XII+Cu1CW2dINjYqJJFifTzogEaIARznrvX7uOlHsdTEXcnOBMuIsrCGEEKdNxi74kyMLZlKVfw==",
"license": "MIT",
"dependencies": {
"lib0": "^0.2.99"
},
"engines": {
"node": ">=16.0.0",
"npm": ">=8.0.0"
},
"funding": {
"type": "GitHub Sponsors ❤",
"url": "https://github.com/sponsors/dmonad"
}
} }
} }
} }
+2 -1
View File
@@ -16,7 +16,8 @@
"lucide-react": "^0.468.0", "lucide-react": "^0.468.0",
"prismjs": "^1.30.0", "prismjs": "^1.30.0",
"react": "^18.3.1", "react": "^18.3.1",
"react-dom": "^18.3.1" "react-dom": "^18.3.1",
"yjs": "^13.6.33"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^18.3.12", "@types/react": "^18.3.12",
+47 -33
View File
@@ -10,7 +10,7 @@ import {
Hash, Hash,
} from 'lucide-react'; } from 'lucide-react';
import { ListItem, SharedList, ListShare, Team, User, WSMessage } from '../../lib/types'; import { ListItem, SharedList, ListShare, Team, User, WSMessage } from '../../lib/types';
import { api, buildWebSocketURL, triggerHaptic } from '../../lib/api'; import { api, buildWebSocketURL, triggerHaptic, createRealtimeConnection, RealtimeTransport } from '../../lib/api';
import { UnifiedShareModal } from '../../components/UnifiedShareModal'; import { UnifiedShareModal } from '../../components/UnifiedShareModal';
import { useMobileActions } from '../../context/MobileActionContext'; import { useMobileActions } from '../../context/MobileActionContext';
@@ -86,6 +86,7 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
const [isSubmittingItem, setIsSubmittingItem] = useState(false); const [isSubmittingItem, setIsSubmittingItem] = useState(false);
const [wsConnected, setWsConnected] = useState(false); const [wsConnected, setWsConnected] = useState(false);
const [realtimeTransport, setRealtimeTransport] = useState<RealtimeTransport>('none');
const [showNewListModal, setShowNewListModal] = useState(false); const [showNewListModal, setShowNewListModal] = useState(false);
const [newListTitle, setNewListTitle] = useState(''); const [newListTitle, setNewListTitle] = useState('');
const [newListIcon, setNewListIcon] = useState('🛒'); const [newListIcon, setNewListIcon] = useState('🛒');
@@ -105,6 +106,7 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
const [dragOverCategoryTarget, setDragOverCategoryTarget] = useState<string | null>(null); const [dragOverCategoryTarget, setDragOverCategoryTarget] = useState<string | null>(null);
const wsRef = useRef<WebSocket | null>(null); const wsRef = useRef<WebSocket | null>(null);
const realtimeConnRef = useRef<any>(null);
const loadLists = async () => { const loadLists = async () => {
try { try {
@@ -147,26 +149,20 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
useEffect(() => { useEffect(() => {
if (!activeList?.id) return; if (!activeList?.id) return;
if (wsRef.current) { if (realtimeConnRef.current) {
wsRef.current.close(); try {
realtimeConnRef.current.close();
} catch {}
realtimeConnRef.current = null;
} }
let ws: WebSocket | null = null; const conn = createRealtimeConnection({
try { listId: activeList.id,
ws = new WebSocket(buildWebSocketURL(activeList.id)); onStatusChange: (status) => {
wsRef.current = ws; setWsConnected(status.connected);
setRealtimeTransport(status.transport);
ws.onopen = () => { },
setWsConnected(true); onMessage: (msg: WSMessage) => {
};
ws.onclose = () => {
setWsConnected(false);
};
ws.onmessage = (evt) => {
try {
const msg: WSMessage = JSON.parse(evt.data);
const msgListId = msg.listId || (msg as any).list_id; const msgListId = msg.listId || (msg as any).list_id;
if (msg.event === 'ITEM_CREATED' && msgListId) { if (msg.event === 'ITEM_CREATED' && msgListId) {
const newItem: ListItem = msg.payload; const newItem: ListItem = msg.payload;
@@ -214,16 +210,18 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
) { ) {
loadLists(); loadLists();
} }
} catch { },
// ignore malformed frames });
}
}; realtimeConnRef.current = conn;
} catch {
setWsConnected(false);
}
return () => { return () => {
if (ws) ws.close(); if (realtimeConnRef.current) {
try {
realtimeConnRef.current.close();
} catch {}
realtimeConnRef.current = null;
}
}; };
}, [activeList?.id]); }, [activeList?.id]);
@@ -821,12 +819,20 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
</button> </button>
)} )}
<div <div
className={`h-9 w-9 flex items-center justify-center rounded-btn border ${ className={`h-9 w-9 flex items-center justify-center rounded-btn border transition-colors ${
wsConnected wsConnected
? 'bg-emerald-500/10 border-emerald-500/30 text-emerald-500' ? realtimeTransport === 'sse'
? 'bg-teal-500/10 border-teal-500/30 text-teal-400'
: 'bg-emerald-500/10 border-emerald-500/30 text-emerald-500'
: 'bg-canvas border-border text-muted' : 'bg-canvas border-border text-muted'
}`} }`}
title={wsConnected ? 'Live synchronisiert (Online)' : 'Offline'} title={
wsConnected
? realtimeTransport === 'sse'
? 'Live synchronisiert (Server-Sent Events / Proxy-Stream)'
: 'Live synchronisiert (WebSocket online)'
: 'Offline'
}
> >
<Radio className={`w-3.5 h-3.5 ${wsConnected ? 'animate-pulse' : ''}`} /> <Radio className={`w-3.5 h-3.5 ${wsConnected ? 'animate-pulse' : ''}`} />
</div> </div>
@@ -927,12 +933,20 @@ export const SharedListsView: React.FC<SharedListsViewProps> = ({ onRefreshGloba
{/* Live Sync Status */} {/* Live Sync Status */}
<div <div
className={`h-8 w-8 flex items-center justify-center rounded-btn border ${ className={`h-8 w-8 flex items-center justify-center rounded-btn border transition-colors ${
wsConnected wsConnected
? 'bg-emerald-500/10 border-emerald-500/30 text-emerald-500' ? realtimeTransport === 'sse'
? 'bg-teal-500/10 border-teal-500/30 text-teal-400'
: 'bg-emerald-500/10 border-emerald-500/30 text-emerald-500'
: 'bg-canvas border-border text-muted' : 'bg-canvas border-border text-muted'
}`} }`}
title={wsConnected ? 'Live synchronisiert (Online)' : 'Offline'} title={
wsConnected
? realtimeTransport === 'sse'
? 'Live synchronisiert (Server-Sent Events / Proxy-Stream)'
: 'Live synchronisiert (WebSocket online)'
: 'Offline'
}
> >
<Radio className={`w-3.5 h-3.5 ${wsConnected ? 'animate-pulse' : ''}`} /> <Radio className={`w-3.5 h-3.5 ${wsConnected ? 'animate-pulse' : ''}`} />
</div> </div>
File diff suppressed because it is too large. Load diff
+408 -21
View File
@@ -602,6 +602,38 @@ export const api = {
}; };
}, },
async getDocumentCRDT(id: string): Promise<{ docId: string; state?: string; updates?: string[] }> {
return request<{ docId: string; state?: string; updates?: string[] }>(`/api/v1/documents/${id}/crdt`);
},
async saveDocumentCRDT(id: string, payload: string | { state?: string; update?: string }): Promise<{ success: boolean }> {
const body = typeof payload === 'string' ? { state: payload } : payload;
return request<{ success: boolean }>(`/api/v1/documents/${id}/crdt`, {
method: 'PUT',
body: JSON.stringify(body),
});
},
async sendDocumentAwareness(id: string, payload: any): Promise<{ success: boolean }> {
return request<{ success: boolean }>(`/api/v1/documents/${id}/awareness`, {
method: 'POST',
body: JSON.stringify(payload),
});
},
async clearDocumentAwareness(id: string): Promise<{ success: boolean }> {
return request<{ success: boolean }>(`/api/v1/documents/${id}/awareness`, {
method: 'DELETE',
});
},
async joinSSERoom(sseId: string, room: string, action: 'join' | 'leave' = 'join'): Promise<{ success: boolean }> {
return request<{ success: boolean }>('/api/v1/events/rooms', {
method: 'POST',
body: JSON.stringify({ sseId, room, action }),
});
},
async deleteDocument(id: string): Promise<void> { async deleteDocument(id: string): Promise<void> {
await request(`/api/v1/documents/${id}`, { method: 'DELETE' }); await request(`/api/v1/documents/${id}`, { method: 'DELETE' });
}, },
@@ -1042,11 +1074,17 @@ export function getOfflineQueue(): Thought[] {
} }
} }
export type RealtimeTransport = 'ws' | 'sse' | 'none';
let activeTransport: RealtimeTransport = 'none';
let activeGlobalWS: WebSocket | null = null; let activeGlobalWS: WebSocket | null = null;
let activeGlobalSSE: EventSource | null = null;
let activeSSEId: string | null = null;
let wsReconnectTimer: any = null; let wsReconnectTimer: any = null;
let wsConnectionWatchdog: any = null;
let wsListeners: ((msg: WSMessage) => void)[] = []; let wsListeners: ((msg: WSMessage) => void)[] = [];
let wsStatusListeners: ((connected: boolean) => void)[] = []; let realtimeStatusListeners: ((connected: boolean, transport: RealtimeTransport) => void)[] = [];
let isWSConnected = false; let isRealtimeConnectedState = false;
let manualDisconnect = false; let manualDisconnect = false;
let wsReconnectDelay = 2000; let wsReconnectDelay = 2000;
@@ -1059,28 +1097,116 @@ export function buildWebSocketURL(listId?: string): string {
return `${protocol}//${window.location.host}/api/v1/ws?token=${encodeURIComponent(token)}`; return `${protocol}//${window.location.host}/api/v1/ws?token=${encodeURIComponent(token)}`;
} }
export function buildSSEURL(listId?: string, docId?: string, rooms?: string[]): string {
const token = getToken() || '';
const params = new URLSearchParams();
if (token) params.set('token', token);
if (listId) params.set('listId', listId);
if (docId) params.set('docId', docId);
if (rooms && rooms.length > 0) params.set('rooms', rooms.join(','));
return `/api/v1/events?${params.toString()}`;
}
export function isWebSocketConnected(): boolean { export function isWebSocketConnected(): boolean {
return isWSConnected && activeGlobalWS !== null && activeGlobalWS.readyState === WebSocket.OPEN; return activeTransport === 'ws' && activeGlobalWS !== null && activeGlobalWS.readyState === WebSocket.OPEN;
}
export function isRealtimeConnected(): boolean {
return isRealtimeConnectedState;
}
export function getRealtimeTransport(): RealtimeTransport {
return activeTransport;
}
export function getSSEId(): string | null {
return activeSSEId;
} }
export function onWebSocketStatus(callback: (connected: boolean) => void): () => void { export function onWebSocketStatus(callback: (connected: boolean) => void): () => void {
wsStatusListeners.push(callback); const wrapper = (connected: boolean) => callback(connected);
callback(isWebSocketConnected()); realtimeStatusListeners.push(wrapper);
callback(isRealtimeConnected());
return () => { return () => {
wsStatusListeners = wsStatusListeners.filter((fn) => fn !== callback); realtimeStatusListeners = realtimeStatusListeners.filter((fn) => fn !== wrapper);
}; };
} }
function notifyStatusChange(connected: boolean) { export function onRealtimeStatus(callback: (connected: boolean, transport: RealtimeTransport) => void): () => void {
isWSConnected = connected; realtimeStatusListeners.push(callback);
wsStatusListeners.forEach((fn) => { callback(isRealtimeConnected(), activeTransport);
return () => {
realtimeStatusListeners = realtimeStatusListeners.filter((fn) => fn !== callback);
};
}
function notifyRealtimeStatusChange(connected: boolean, transport: RealtimeTransport) {
isRealtimeConnectedState = connected;
activeTransport = transport;
realtimeStatusListeners.forEach((fn) => {
try { try {
fn(connected); fn(connected, transport);
} catch (e) { } catch (e) {
console.error('WS status listener error', e); console.error('Realtime status listener error', e);
} }
}); });
window.dispatchEvent(new CustomEvent('mindsift:ws:status', { detail: { connected } })); window.dispatchEvent(new CustomEvent('mindsift:ws:status', { detail: { connected, transport } }));
}
export function connectSSE(rooms?: string[]): void {
if (activeGlobalSSE) {
try { activeGlobalSSE.close(); } catch {}
activeGlobalSSE = null;
}
try {
const url = buildSSEURL(undefined, undefined, rooms);
const es = new EventSource(url);
activeGlobalSSE = es;
es.onopen = () => {
console.info('[MindSift Realtime] Server-Sent Events (SSE) verbunden (Proxy-Fallback aktiv).');
wsReconnectDelay = 2000;
notifyRealtimeStatusChange(true, 'sse');
};
es.onmessage = (event) => {
try {
const msg = JSON.parse(event.data) as WSMessage;
if (msg.event === 'SSE_CONNECTED') {
activeSSEId = msg.payload?.sseId || null;
}
wsListeners.forEach((fn) => {
try {
fn(msg);
} catch (e) {
console.error('SSE listener error', e);
}
});
window.dispatchEvent(new CustomEvent('mindsift:ws', { detail: msg }));
} catch (err) {
console.error('Failed to parse SSE message', err);
}
};
es.onerror = (e) => {
console.warn('[MindSift Realtime] SSE getrennt oder Fehler aufgetreten:', e);
if (activeTransport === 'sse') {
notifyRealtimeStatusChange(false, 'none');
}
try { es.close(); } catch {}
activeGlobalSSE = null;
if (!manualDisconnect && getToken()) {
clearTimeout(wsReconnectTimer);
wsReconnectTimer = setTimeout(() => {
connectWebSocket();
}, 5000);
}
};
} catch (err) {
console.error('Failed to initialize EventSource SSE', err);
notifyRealtimeStatusChange(false, 'none');
}
} }
export function connectWebSocket(): void { export function connectWebSocket(): void {
@@ -1092,14 +1218,31 @@ export function connectWebSocket(): void {
return; return;
} }
clearTimeout(wsConnectionWatchdog);
try { try {
const url = buildWebSocketURL(); const url = buildWebSocketURL();
const ws = new WebSocket(url); const ws = new WebSocket(url);
activeGlobalWS = ws; activeGlobalWS = ws;
// 3.5s watchdog for corporate proxies silently stripping Upgrade header
wsConnectionWatchdog = setTimeout(() => {
if (ws && ws.readyState !== WebSocket.OPEN) {
console.warn('[MindSift Realtime] WebSocket connection timeout (Proxy Upgrade Block). Wechsle automatisch zu SSE...');
try { ws.close(); } catch {}
activeGlobalWS = null;
connectSSE();
}
}, 3500);
ws.onopen = () => { ws.onopen = () => {
clearTimeout(wsConnectionWatchdog);
if (activeGlobalSSE) {
try { activeGlobalSSE.close(); } catch {}
activeGlobalSSE = null;
}
wsReconnectDelay = 2000; wsReconnectDelay = 2000;
notifyStatusChange(true); notifyRealtimeStatusChange(true, 'ws');
}; };
ws.onmessage = (event) => { ws.onmessage = (event) => {
@@ -1119,10 +1262,22 @@ export function connectWebSocket(): void {
}; };
ws.onclose = (evt: CloseEvent) => { ws.onclose = (evt: CloseEvent) => {
clearTimeout(wsConnectionWatchdog);
activeGlobalWS = null; activeGlobalWS = null;
console.warn('[MindSift Global WS] Closed with code:', evt?.code, 'reason:', evt?.reason); console.warn('[MindSift Global WS] Closed with code:', evt?.code, 'reason:', evt?.reason);
notifyStatusChange(false);
if (!manualDisconnect && getToken()) { const isProxyIssue = evt?.code === 1006 || evt?.code === 1002 || evt?.code === 1008;
if (isProxyIssue && !manualDisconnect && getToken()) {
console.warn('[MindSift Realtime] Proxy/Firewall Inkompatibilität erkannt (Code ' + evt.code + '). Wechsle zu SSE...');
connectSSE();
return;
}
if (activeTransport === 'ws') {
notifyRealtimeStatusChange(false, 'none');
}
if (!manualDisconnect && getToken() && activeTransport !== 'sse') {
clearTimeout(wsReconnectTimer); clearTimeout(wsReconnectTimer);
const delay = wsReconnectDelay; const delay = wsReconnectDelay;
wsReconnectDelay = Math.min(30000, Math.round(wsReconnectDelay * 1.5)); wsReconnectDelay = Math.min(30000, Math.round(wsReconnectDelay * 1.5));
@@ -1133,27 +1288,28 @@ export function connectWebSocket(): void {
}; };
ws.onerror = (e) => { ws.onerror = (e) => {
clearTimeout(wsConnectionWatchdog);
console.warn('[MindSift Global WS] Error:', e); console.warn('[MindSift Global WS] Error:', e);
notifyStatusChange(false);
try { try {
ws.close(); ws.close();
} catch {} } catch {}
}; };
} catch (e) { } catch (e) {
console.error('Failed to connect WebSocket', e); clearTimeout(wsConnectionWatchdog);
notifyStatusChange(false); console.warn('Failed to connect WebSocket, switching to SSE:', e);
connectSSE();
} }
} }
// Ensure reconnect when browser recovers from sleep or tab visibility // Ensure reconnect when browser recovers from sleep or tab visibility
if (typeof window !== 'undefined') { if (typeof window !== 'undefined') {
window.addEventListener('online', () => { window.addEventListener('online', () => {
if (getToken() && !isWebSocketConnected()) { if (getToken() && !isRealtimeConnected()) {
connectWebSocket(); connectWebSocket();
} }
}); });
document.addEventListener('visibilitychange', () => { document.addEventListener('visibilitychange', () => {
if (document.visibilityState === 'visible' && getToken() && !isWebSocketConnected()) { if (document.visibilityState === 'visible' && getToken() && !isRealtimeConnected()) {
connectWebSocket(); connectWebSocket();
} }
}); });
@@ -1162,18 +1318,47 @@ if (typeof window !== 'undefined') {
export function disconnectWebSocket(): void { export function disconnectWebSocket(): void {
manualDisconnect = true; manualDisconnect = true;
clearTimeout(wsReconnectTimer); clearTimeout(wsReconnectTimer);
clearTimeout(wsConnectionWatchdog);
if (activeGlobalWS) { if (activeGlobalWS) {
try { try {
activeGlobalWS.close(); activeGlobalWS.close();
} catch {} } catch {}
activeGlobalWS = null; activeGlobalWS = null;
} }
notifyStatusChange(false); if (activeGlobalSSE) {
try {
activeGlobalSSE.close();
} catch {}
activeGlobalSSE = null;
}
notifyRealtimeStatusChange(false, 'none');
} }
export function sendWebSocketMessage(msg: any): void { export function sendWebSocketMessage(msg: any): void {
if (activeGlobalWS && activeGlobalWS.readyState === WebSocket.OPEN) { if (activeGlobalWS && activeGlobalWS.readyState === WebSocket.OPEN) {
activeGlobalWS.send(typeof msg === 'string' ? msg : JSON.stringify(msg)); activeGlobalWS.send(typeof msg === 'string' ? msg : JSON.stringify(msg));
return;
}
// SSE Fallback: dispatch via REST endpoints
try {
const parsed = typeof msg === 'string' ? JSON.parse(msg) : msg;
if (parsed.event === 'CRDT_UPDATE' && parsed.docId) {
api.saveDocumentCRDT(parsed.docId, { update: parsed.payload?.update });
} else if (parsed.event === 'CRDT_AWARENESS' && parsed.docId) {
api.sendDocumentAwareness(parsed.docId, parsed.payload);
} else if (parsed.event === 'CRDT_LEAVE' && parsed.docId) {
api.clearDocumentAwareness(parsed.docId);
if (activeSSEId) api.joinSSERoom(activeSSEId, `doc:${parsed.docId}`, 'leave');
} else if (parsed.event === 'CRDT_JOIN' && parsed.docId) {
if (activeSSEId) api.joinSSERoom(activeSSEId, `doc:${parsed.docId}`, 'join');
} else if (parsed.event === 'LOCK_DOC' && parsed.docId) {
api.lockDocument(parsed.docId);
} else if (parsed.event === 'UNLOCK_DOC' && parsed.docId) {
api.unlockDocument(parsed.docId);
}
} catch (err) {
console.error('Error dispatching fallback REST message for SSE:', err);
} }
} }
@@ -1183,3 +1368,205 @@ export function onWebSocketMessage(callback: (msg: WSMessage) => void): () => vo
wsListeners = wsListeners.filter((fn) => fn !== callback); wsListeners = wsListeners.filter((fn) => fn !== callback);
}; };
} }
export function createRealtimeConnection(options: {
listId?: string;
docId?: string;
rooms?: string[];
onMessage?: (msg: WSMessage) => void;
onStatusChange?: (status: { connected: boolean; transport: RealtimeTransport; reason?: string }) => void;
}) {
let isClosed = false;
let currentTransport: RealtimeTransport = 'none';
let ws: WebSocket | null = null;
let es: EventSource | null = null;
let watchdogTimer: any = null;
let reconnectTimer: any = null;
let reconnectDelay = 2000;
let sseId: string | null = null;
const notify = (connected: boolean, transport: RealtimeTransport, reason?: string) => {
currentTransport = transport;
options.onStatusChange?.({ connected, transport, reason });
};
const connectSSE = () => {
if (isClosed) return;
if (es) {
try { es.close(); } catch {}
es = null;
}
try {
const allRooms = [...(options.rooms || [])];
if (options.docId) allRooms.push(`doc:${options.docId}`);
if (options.listId) allRooms.push(options.listId);
const url = buildSSEURL(options.listId, options.docId, allRooms);
es = new EventSource(url);
es.onopen = () => {
if (isClosed) return;
reconnectDelay = 2000;
notify(true, 'sse');
};
es.onmessage = (evt) => {
if (isClosed) return;
try {
const msg = JSON.parse(evt.data) as WSMessage;
if (msg.event === 'SSE_CONNECTED') {
sseId = msg.payload?.sseId || null;
}
options.onMessage?.(msg);
} catch (e) {
console.error('SSE JSON error', e);
}
};
es.onerror = (e) => {
if (isClosed) return;
notify(false, 'none', 'SSE-Verbindungsabbruch');
try { es?.close(); } catch {}
es = null;
clearTimeout(reconnectTimer);
reconnectTimer = setTimeout(() => {
if (!isClosed) connectWS();
}, 5000);
};
} catch (e: any) {
notify(false, 'none', e?.message || 'SSE Initialisierungsfehler');
}
};
const connectWS = () => {
if (isClosed) return;
const token = getToken();
if (!token) {
notify(false, 'none', 'Kein Authentifizierungs-Token vorhanden');
return;
}
if (ws) {
try { ws.close(); } catch {}
ws = null;
}
clearTimeout(watchdogTimer);
try {
const url = buildWebSocketURL(options.listId);
ws = new WebSocket(url);
// 3.5s watchdog for corporate proxies silently discarding Upgrade header
watchdogTimer = setTimeout(() => {
if (ws && ws.readyState !== WebSocket.OPEN && !isClosed) {
console.warn('[MindSift Realtime] WebSocket connection timeout (Proxy Upgrade block). Falling back to SSE...');
try { ws.close(); } catch {}
ws = null;
connectSSE();
}
}, 3500);
ws.onopen = () => {
if (isClosed) return;
clearTimeout(watchdogTimer);
reconnectDelay = 2000;
notify(true, 'ws');
if (options.docId) {
try {
ws?.send(JSON.stringify({ event: 'CRDT_JOIN', docId: options.docId }));
} catch {}
}
};
ws.onmessage = (evt) => {
if (isClosed) return;
try {
const msg = JSON.parse(evt.data) as WSMessage;
options.onMessage?.(msg);
} catch (e) {
console.error('WS JSON error', e);
}
};
ws.onclose = (evt: CloseEvent) => {
if (isClosed) return;
clearTimeout(watchdogTimer);
ws = null;
const isProxyError = evt?.code === 1006 || evt?.code === 1002 || evt?.code === 1008;
if (isProxyError) {
console.warn(`[MindSift Realtime] Proxy WebSocket rejection detected (Code ${evt.code}). Falling back to SSE...`);
connectSSE();
return;
}
const reason = evt.reason || (evt.code === 1000 ? 'Normal geschlossen' : `Code ${evt.code}`);
notify(false, 'none', reason);
clearTimeout(reconnectTimer);
const delay = reconnectDelay;
reconnectDelay = Math.min(30000, Math.round(reconnectDelay * 1.5));
reconnectTimer = setTimeout(connectWS, delay);
};
ws.onerror = (e) => {
if (isClosed) return;
clearTimeout(watchdogTimer);
try { ws?.close(); } catch {}
};
} catch (e: any) {
clearTimeout(watchdogTimer);
console.warn('[MindSift Realtime] WebSocket constructor failed, trying SSE fallback:', e);
connectSSE();
}
};
connectWS();
return {
getTransport: () => currentTransport,
isConnected: () => currentTransport !== 'none',
send: (msg: any) => {
const str = typeof msg === 'string' ? msg : JSON.stringify(msg);
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(str);
return;
}
// SSE Fallback: dispatch via REST endpoints
try {
const parsed = typeof msg === 'string' ? JSON.parse(msg) : msg;
if (parsed.event === 'CRDT_UPDATE' && parsed.docId) {
api.saveDocumentCRDT(parsed.docId, { update: parsed.payload?.update });
} else if (parsed.event === 'CRDT_AWARENESS' && parsed.docId) {
api.sendDocumentAwareness(parsed.docId, parsed.payload);
} else if (parsed.event === 'CRDT_LEAVE' && parsed.docId) {
api.clearDocumentAwareness(parsed.docId);
if (sseId) api.joinSSERoom(sseId, `doc:${parsed.docId}`, 'leave');
} else if (parsed.event === 'CRDT_JOIN' && parsed.docId) {
if (sseId) api.joinSSERoom(sseId, `doc:${parsed.docId}`, 'join');
} else if (parsed.event === 'LOCK_DOC' && parsed.docId) {
api.lockDocument(parsed.docId);
} else if (parsed.event === 'UNLOCK_DOC' && parsed.docId) {
api.unlockDocument(parsed.docId);
}
} catch (err) {
console.error('Error dispatching fallback REST message for connection:', err);
}
},
close: () => {
isClosed = true;
clearTimeout(watchdogTimer);
clearTimeout(reconnectTimer);
if (ws) {
try { ws.close(); } catch {}
ws = null;
}
if (es) {
try { es.close(); } catch {}
es = null;
}
notify(false, 'none');
},
};
}
+271
View File
@@ -0,0 +1,271 @@
import * as Y from 'yjs';
import { BlockType, DocumentBlock } from './types';
// Convert Uint8Array to Base64
export function uint8ArrayToBase64(bytes: Uint8Array): string {
let binary = '';
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
// Convert Base64 to Uint8Array
export function base64ToUint8Array(base64: string): Uint8Array {
const binaryString = atob(base64);
const len = binaryString.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes;
}
const COLLAB_COLORS = [
'#3B82F6', // Blue
'#10B981', // Emerald
'#F59E0B', // Amber
'#8B5CF6', // Purple
'#EC4899', // Pink
'#06B6D4', // Cyan
'#F97316', // Orange
'#14B8A6', // Teal
];
export function getCollaboratorColor(userId: string): string {
if (!userId) return COLLAB_COLORS[0];
let hash = 0;
for (let i = 0; i < userId.length; i++) {
hash = (hash << 5) - hash + userId.charCodeAt(i);
hash |= 0;
}
const idx = Math.abs(hash) % COLLAB_COLORS.length;
return COLLAB_COLORS[idx];
}
export interface CollaboratorAwareness {
userId: string;
userName: string;
activeBlockId: string | null;
color: string;
lastActive: number;
}
export class DocumentCRDTSession {
public docId: string;
public doc: Y.Doc;
private order: Y.Array<string>;
private blocks: Y.Map<Y.Map<any>>;
private destroyed: boolean = false;
public onLocalUpdate?: (base64Update: string) => void;
public onRemoteChange?: (blocks: DocumentBlock[]) => void;
constructor(docId: string) {
this.docId = docId;
this.doc = new Y.Doc({ guid: docId });
this.order = this.doc.getArray<string>('blockOrder');
this.blocks = this.doc.getMap<Y.Map<any>>('blocks');
this.doc.on('update', (update: Uint8Array, origin: any) => {
if (this.destroyed) return;
if (origin !== 'remote') {
const b64 = uint8ArrayToBase64(update);
if (this.onLocalUpdate) {
this.onLocalUpdate(b64);
}
} else {
if (this.onRemoteChange) {
this.onRemoteChange(this.getBlocks());
}
}
});
}
public destroy(): void {
this.destroyed = true;
this.doc.destroy();
}
public getBlocks(): DocumentBlock[] {
const orderList = this.order.toArray();
const result: DocumentBlock[] = [];
for (const id of orderList) {
const bMap = this.blocks.get(id);
if (bMap) {
result.push({
id,
docId: this.docId,
type: (bMap.get('type') || 'bullet') as BlockType,
content: bMap.get('content') || '',
level: typeof bMap.get('level') === 'number' ? bMap.get('level') : 0,
checked: !!bMap.get('checked'),
collapsed: !!bMap.get('collapsed'),
lang: bMap.get('lang') || '',
sortOrder: typeof bMap.get('sortOrder') === 'number' ? bMap.get('sortOrder') : 0,
updatedAt: bMap.get('updatedAt') || undefined,
});
}
}
return result;
}
public initFromBlocks(initialBlocks: DocumentBlock[]): void {
this.doc.transact(() => {
if (this.order.length === 0 && initialBlocks.length > 0) {
const ids: string[] = [];
for (const b of initialBlocks) {
ids.push(b.id);
const bMap = new Y.Map<any>();
bMap.set('id', b.id);
bMap.set('type', b.type || 'bullet');
bMap.set('content', b.content || '');
bMap.set('level', b.level || 0);
bMap.set('checked', !!b.checked);
bMap.set('collapsed', !!b.collapsed);
bMap.set('lang', b.lang || '');
bMap.set('sortOrder', b.sortOrder || 0);
bMap.set('updatedAt', b.updatedAt || new Date().toISOString());
this.blocks.set(b.id, bMap);
}
this.order.push(ids);
}
}, 'init');
}
public updateBlock(id: string, updates: Partial<DocumentBlock>): void {
this.doc.transact(() => {
const bMap = this.blocks.get(id);
if (bMap) {
if (updates.type !== undefined) bMap.set('type', updates.type);
if (updates.content !== undefined) bMap.set('content', updates.content);
if (updates.level !== undefined) bMap.set('level', updates.level);
if (updates.checked !== undefined) bMap.set('checked', updates.checked);
if (updates.collapsed !== undefined) bMap.set('collapsed', updates.collapsed);
if (updates.lang !== undefined) bMap.set('lang', updates.lang);
if (updates.sortOrder !== undefined) bMap.set('sortOrder', updates.sortOrder);
bMap.set('updatedAt', new Date().toISOString());
}
}, 'local');
}
public insertBlock(index: number, block: DocumentBlock): void {
this.doc.transact(() => {
const bMap = new Y.Map<any>();
bMap.set('id', block.id);
bMap.set('type', block.type || 'bullet');
bMap.set('content', block.content || '');
bMap.set('level', block.level || 0);
bMap.set('checked', !!block.checked);
bMap.set('collapsed', !!block.collapsed);
bMap.set('lang', block.lang || '');
bMap.set('sortOrder', block.sortOrder || 0);
bMap.set('updatedAt', new Date().toISOString());
this.blocks.set(block.id, bMap);
if (index >= 0 && index <= this.order.length) {
this.order.insert(index, [block.id]);
} else {
this.order.push([block.id]);
}
}, 'local');
}
public deleteBlock(id: string): void {
this.doc.transact(() => {
const currentOrder = this.order.toArray();
const idx = currentOrder.indexOf(id);
if (idx !== -1) {
this.order.delete(idx, 1);
}
this.blocks.delete(id);
}, 'local');
}
public deleteMultipleBlocks(ids: string[]): void {
if (ids.length === 0) return;
const toDeleteSet = new Set(ids);
this.doc.transact(() => {
const currentOrder = this.order.toArray();
// Delete from order backwards to preserve indices
for (let i = currentOrder.length - 1; i >= 0; i--) {
if (toDeleteSet.has(currentOrder[i])) {
this.order.delete(i, 1);
}
}
for (const id of ids) {
this.blocks.delete(id);
}
}, 'local');
}
public reorderBlocks(newOrder: string[]): void {
this.doc.transact(() => {
this.order.delete(0, this.order.length);
this.order.push(newOrder);
}, 'local');
}
public syncFullBlocks(blocks: DocumentBlock[]): void {
this.doc.transact(() => {
const nextIds = blocks.map((b) => b.id);
const currentIds = new Set(this.order.toArray());
// Upsert all blocks
for (const b of blocks) {
let bMap = this.blocks.get(b.id);
if (!bMap) {
bMap = new Y.Map<any>();
this.blocks.set(b.id, bMap);
}
bMap.set('id', b.id);
bMap.set('type', b.type || 'bullet');
bMap.set('content', b.content || '');
bMap.set('level', b.level || 0);
bMap.set('checked', !!b.checked);
bMap.set('collapsed', !!b.collapsed);
bMap.set('lang', b.lang || '');
bMap.set('sortOrder', b.sortOrder || 0);
bMap.set('updatedAt', b.updatedAt || new Date().toISOString());
}
// Delete blocks no longer present
const nextIdSet = new Set(nextIds);
currentIds.forEach((id) => {
if (!nextIdSet.has(id)) {
this.blocks.delete(id);
}
});
// Update order if different
const currentOrder = this.order.toArray();
const orderChanged =
currentOrder.length !== nextIds.length || currentOrder.some((id, idx) => id !== nextIds[idx]);
if (orderChanged) {
this.order.delete(0, this.order.length);
this.order.push(nextIds);
}
}, 'local');
}
public applyRemoteUpdate(base64Update: string): void {
try {
const bytes = base64ToUint8Array(base64Update);
Y.applyUpdate(this.doc, bytes, 'remote');
} catch (err) {
console.error('[CRDT] Failed to apply remote update:', err);
}
}
public applyRemoteUpdates(base64Updates: string[]): void {
for (const b64 of base64Updates) {
this.applyRemoteUpdate(b64);
}
}
public encodeState(): string {
const bytes = Y.encodeStateAsUpdate(this.doc);
return uint8ArrayToBase64(bytes);
}
}
+3
View File
@@ -157,12 +157,15 @@ export type BlockType = 'h1' | 'h2' | 'h3' | 'paragraph' | 'todo' | 'bullet' | '
export interface DocumentBlock { export interface DocumentBlock {
id: string; id: string;
docId?: string;
type: BlockType; type: BlockType;
content: string; content: string;
checked?: boolean; checked?: boolean;
lang?: string; lang?: string;
level?: number; level?: number;
collapsed?: boolean; collapsed?: boolean;
sortOrder?: number;
updatedAt?: string;
} }
export interface DocBlocksPatchRequest { export interface DocBlocksPatchRequest {
+184
View File
@@ -3,6 +3,7 @@ package api
import ( import (
"bytes" "bytes"
"crypto/tls" "crypto/tls"
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@@ -30,6 +31,9 @@ type Server struct {
} }
func NewServer(cfg *config.Config, store *storage.Store, wsHub *ws.Hub, staticFS fs.FS) *Server { func NewServer(cfg *config.Config, store *storage.Store, wsHub *ws.Hub, staticFS fs.FS) *Server {
if wsHub != nil && store != nil {
wsHub.SetCRDTStore(store)
}
srv := &Server{ srv := &Server{
cfg: cfg, cfg: cfg,
store: store, store: store,
@@ -65,6 +69,8 @@ func (s *Server) Router() http.Handler {
r.Get("/api/v1/ws", s.handleWebSocket) r.Get("/api/v1/ws", s.handleWebSocket)
r.Get("/api/v1/ws/spaces/{spaceId}", s.handleWebSocket) r.Get("/api/v1/ws/spaces/{spaceId}", s.handleWebSocket)
r.Get("/api/v1/ws/lists/{listId}", s.handleWebSocket) r.Get("/api/v1/ws/lists/{listId}", s.handleWebSocket)
r.Get("/api/v1/events", s.handleSSE)
r.Post("/api/v1/events/rooms", s.handleSSERooms)
// Authenticated API Routes // Authenticated API Routes
r.Group(func(r chi.Router) { r.Group(func(r chi.Router) {
@@ -141,6 +147,12 @@ func (s *Server) Router() http.Handler {
r.Post("/api/v1/documents/{id}/lock/heartbeat", s.handleHeartbeatDocLock) r.Post("/api/v1/documents/{id}/lock/heartbeat", s.handleHeartbeatDocLock)
r.Delete("/api/v1/documents/{id}/lock", s.handleUnlockDoc) r.Delete("/api/v1/documents/{id}/lock", s.handleUnlockDoc)
// Document CRDT Realtime Sync
r.Get("/api/v1/documents/{id}/crdt", s.handleGetDocCRDT)
r.Put("/api/v1/documents/{id}/crdt", s.handleSaveDocCRDT)
r.Post("/api/v1/documents/{id}/awareness", s.handlePostDocAwareness)
r.Delete("/api/v1/documents/{id}/awareness", s.handleDeleteDocAwareness)
// Shared Lists & Realtime Collaboration // Shared Lists & Realtime Collaboration
r.Get("/api/v1/lists", s.handleGetAllLists) r.Get("/api/v1/lists", s.handleGetAllLists)
r.Post("/api/v1/lists", s.handleCreateList) r.Post("/api/v1/lists", s.handleCreateList)
@@ -1373,6 +1385,116 @@ func (s *Server) handleUnlockDoc(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]bool{"unlocked": released}) writeJSON(w, http.StatusOK, map[string]bool{"unlocked": released})
} }
// --- DOCUMENT CRDT REALTIME SYNC ---
func (s *Server) handleGetDocCRDT(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
if _, err := s.store.GetDocumentByID(userID, docID); err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
stateBlob, updates, err := s.store.GetDocumentCRDTState(docID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
})
}
func (s *Server) handleSaveDocCRDT(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
if !doc.IsOwner && doc.Permission != "write" {
writeError(w, http.StatusForbidden, "Keine Schreibberechtigung")
return
}
var req struct {
State string `json:"state"`
Update string `json:"update"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiger Request-Body")
return
}
if req.Update != "" {
raw, err := base64.StdEncoding.DecodeString(req.Update)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format für Update")
return
}
if err := s.store.AppendDocumentCRDTUpdate(docID, raw, userID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_UPDATE", map[string]interface{}{
"docId": docID,
"update": req.Update,
"fromUserId": userID,
}, nil)
}
if req.State != "" {
raw, err := base64.StdEncoding.DecodeString(req.State)
if err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges Base64 Format")
return
}
if err := s.store.SaveDocumentCRDTState(docID, raw, userID); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
}
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handlePostDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
doc, err := s.store.GetDocumentByID(userID, docID)
if err != nil {
writeError(w, http.StatusNotFound, "Dokument nicht gefunden")
return
}
_ = doc
var payload json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS", payload, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
func (s *Server) handleDeleteDocAwareness(w http.ResponseWriter, r *http.Request) {
userID := auth.GetUserID(r.Context())
docID := chi.URLParam(r, "id")
s.wsHub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": userID,
}, nil)
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
// --- REALTIME LISTS & SHARING --- // --- REALTIME LISTS & SHARING ---
func (s *Server) handleGetAllLists(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGetAllLists(w http.ResponseWriter, r *http.Request) {
@@ -1642,6 +1764,68 @@ func (s *Server) handleWebSocket(w http.ResponseWriter, r *http.Request) {
s.wsHub.ServeWS(w, r, roomID, userID) s.wsHub.ServeWS(w, r, roomID, userID)
} }
func (s *Server) handleSSE(w http.ResponseWriter, r *http.Request) {
roomID := r.URL.Query().Get("listId")
if roomID == "" {
roomID = r.URL.Query().Get("spaceId")
}
token := r.URL.Query().Get("token")
if token == "" {
if authHdr := r.Header.Get("Authorization"); strings.HasPrefix(authHdr, "Bearer ") {
token = strings.TrimPrefix(authHdr, "Bearer ")
}
}
userID := "anonymous"
if token != "" {
if claims, err := auth.ValidateToken(s.cfg.JWTSecret, token); err == nil {
userID = claims.UserID
}
}
var initialRooms []string
if roomsParam := r.URL.Query().Get("rooms"); roomsParam != "" {
for _, rm := range strings.Split(roomsParam, ",") {
rm = strings.TrimSpace(rm)
if rm != "" {
initialRooms = append(initialRooms, rm)
}
}
}
if docID := r.URL.Query().Get("docId"); docID != "" {
initialRooms = append(initialRooms, "doc:"+strings.TrimSpace(docID))
}
s.wsHub.ServeSSE(w, r, roomID, userID, initialRooms)
}
func (s *Server) handleSSERooms(w http.ResponseWriter, r *http.Request) {
var req struct {
SSEID string `json:"sseId"`
Room string `json:"room"`
DocID string `json:"docId"`
Action string `json:"action"` // "join" or "leave"
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "Ungültiges JSON")
return
}
if req.Room == "" && req.DocID != "" {
req.Room = "doc:" + req.DocID
}
if req.SSEID == "" || req.Room == "" {
writeError(w, http.StatusBadRequest, "sseId und room erforderlich")
return
}
if req.Action == "leave" {
s.wsHub.LeaveSSERoomByID(req.SSEID, req.Room)
} else {
s.wsHub.JoinSSERoomByID(req.SSEID, req.Room)
}
writeJSON(w, http.StatusOK, map[string]bool{"success": true})
}
// --- API KEYS & WEBHOOK INGESTION --- // --- API KEYS & WEBHOOK INGESTION ---
func (s *Server) handleGetAPIKeys(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGetAPIKeys(w http.ResponseWriter, r *http.Request) {
+177
View File
@@ -2,6 +2,7 @@ package api
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@@ -9,6 +10,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time"
"github.com/sandrodach/mindsift/internal/auth" "github.com/sandrodach/mindsift/internal/auth"
"github.com/sandrodach/mindsift/internal/config" "github.com/sandrodach/mindsift/internal/config"
@@ -838,3 +840,178 @@ func TestWebhookTriageBadgeAndIngestion(t *testing.T) {
t.Fatalf("expected thought content to contain 'Waschmaschine', got '%s'", ingestResp.Thought.Content) t.Fatalf("expected thought content to contain 'Waschmaschine', got '%s'", ingestResp.Thought.Content)
} }
} }
func TestDocumentCRDTEndpoints(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "crdt_api_test.db")
store, err := storage.NewStore(dbPath)
if err != nil {
t.Fatalf("NewStore failed: %v", err)
}
defer store.Close()
cfg := &config.Config{
JWTSecret: "test-jwt-secret-key",
AllowRegistration: true,
}
hub := ws.NewHub()
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
user, err := store.CreateUser("crdt_api@example.com", "hash", "CRDT API User", "member")
if err != nil {
t.Fatalf("CreateUser failed: %v", err)
}
token, _ := auth.GenerateToken(cfg.JWTSecret, user.ID, user.Email, user.Role)
doc, err := store.CreateDocument(user.ID, "", "API CRDT Doc", "📄", "", nil, nil, false)
if err != nil {
t.Fatalf("CreateDocument failed: %v", err)
}
// 1. Initial GET /api/v1/documents/{id}/crdt
req := httptest.NewRequest(http.MethodGet, "/api/v1/documents/"+doc.ID+"/crdt", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from GET crdt, got %d: %s", rec.Code, rec.Body.String())
}
// 2. PUT /api/v1/documents/{id}/crdt with base64 state
savePayload := []byte(`{"state":"AQIDBA=="}`)
req = httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID+"/crdt", bytes.NewReader(savePayload))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from PUT crdt, got %d: %s", rec.Code, rec.Body.String())
}
// 3. Verify state persisted
req = httptest.NewRequest(http.MethodGet, "/api/v1/documents/"+doc.ID+"/crdt", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from GET crdt, got %d", rec.Code)
}
var crdtResp struct {
DocID string `json:"docId"`
State string `json:"state"`
}
_ = json.Unmarshal(rec.Body.Bytes(), &crdtResp)
if crdtResp.State != "AQIDBA==" {
t.Fatalf("expected state 'AQIDBA==', got '%s'", crdtResp.State)
}
}
func TestSSEAndAwarenessEndpoints(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test_sse.db")
store, err := storage.NewStore(dbPath)
if err != nil {
t.Fatalf("failed to init store: %v", err)
}
defer store.Close()
hub := ws.NewHub()
hub.SetCRDTStore(store)
cfg := &config.Config{
JWTSecret: "test-jwt-secret-key",
AllowRegistration: false,
}
srv := NewServer(cfg, store, hub, nil)
router := srv.Router()
// 1. Setup user
setupBody := []byte(`{
"email": "sse_user@example.com",
"password": "secretpassword",
"displayName": "SSE User",
"instanceName": "MindSift SSE",
"allowRegistration": false,
"seedSampleData": false
}`)
req := httptest.NewRequest(http.MethodPost, "/api/v1/auth/setup", bytes.NewReader(setupBody))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("setup failed: %s", rec.Body.String())
}
var authResp struct {
Token string `json:"token"`
User *domain.User `json:"user"`
}
_ = json.Unmarshal(rec.Body.Bytes(), &authResp)
token := authResp.Token
// 2. Create Document
doc, err := store.CreateDocument(authResp.User.ID, "", "SSE Collaborative Note", "📄", "", nil, nil, false)
if err != nil {
t.Fatalf("failed to create document: %v", err)
}
// 3. Test Awareness POST and DELETE
awarenessBody := []byte(`{"userId":"` + authResp.User.ID + `","activeBlockId":"blk_123"}`)
req = httptest.NewRequest(http.MethodPost, "/api/v1/documents/"+doc.ID+"/awareness", bytes.NewReader(awarenessBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from POST awareness, got %d: %s", rec.Code, rec.Body.String())
}
req = httptest.NewRequest(http.MethodDelete, "/api/v1/documents/"+doc.ID+"/awareness", nil)
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from DELETE awareness, got %d", rec.Code)
}
// 4. Test Delta CRDT Update via PUT /api/v1/documents/{id}/crdt
updateBody := []byte(`{"update":"AQIDBA=="}`)
req = httptest.NewRequest(http.MethodPut, "/api/v1/documents/"+doc.ID+"/crdt", bytes.NewReader(updateBody))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from PUT crdt update, got %d: %s", rec.Code, rec.Body.String())
}
// 5. Test SSE Connection GET /api/v1/events
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req = httptest.NewRequest(http.MethodGet, "/api/v1/events?token="+token+"&docId="+doc.ID, nil).WithContext(ctx)
rec = httptest.NewRecorder()
go func() {
// Cancel context after short moment to terminate stream cleanly
time.Sleep(50 * time.Millisecond)
cancel()
}()
router.ServeHTTP(rec, req)
contentType := rec.Header().Get("Content-Type")
if !strings.Contains(contentType, "text/event-stream") {
t.Fatalf("expected Content-Type text/event-stream, got %s", contentType)
}
if rec.Header().Get("X-Accel-Buffering") != "no" {
t.Fatalf("expected X-Accel-Buffering: no, got %s", rec.Header().Get("X-Accel-Buffering"))
}
bodyStr := rec.Body.String()
if !strings.Contains(bodyStr, "SSE_CONNECTED") {
t.Fatalf("expected SSE_CONNECTED event in body, got: %s", bodyStr)
}
}
+73 -5
View File
@@ -21,19 +21,15 @@ import (
type Store struct { type Store struct {
db *sql.DB db *sql.DB
docDir string
initMu sync.Mutex initMu sync.Mutex
} }
func NewStore(dbPath string) (*Store, error) { func NewStore(dbPath string) (*Store, error) {
var docDir string
if dbPath != ":memory:" && !strings.HasPrefix(dbPath, "file:") { if dbPath != ":memory:" && !strings.HasPrefix(dbPath, "file:") {
dir := filepath.Dir(dbPath) dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil { if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create data directory %s: %w", dir, err) return nil, fmt.Errorf("failed to create data directory %s: %w", dir, err)
} }
docDir = filepath.Join(dir, "workspace")
_ = os.MkdirAll(docDir, 0755)
} }
dsn := dbPath dsn := dbPath
@@ -49,7 +45,7 @@ func NewStore(dbPath string) (*Store, error) {
db.SetMaxOpenConns(8) db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4) db.SetMaxIdleConns(4)
store := &Store{db: db, docDir: docDir} store := &Store{db: db}
if err := store.migrate(); err != nil { if err := store.migrate(); err != nil {
return nil, fmt.Errorf("failed to run migrations: %w", err) return nil, fmt.Errorf("failed to run migrations: %w", err)
} }
@@ -236,6 +232,25 @@ func (s *Store) migrate() error {
PRIMARY KEY(user_id, tag) PRIMARY KEY(user_id, tag)
); );
CREATE TABLE IF NOT EXISTS document_crdt_state (
doc_id TEXT PRIMARY KEY,
state_blob BLOB NOT NULL,
updated_at TEXT NOT NULL,
updated_by TEXT NOT NULL DEFAULT '',
FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS document_crdt_updates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
doc_id TEXT NOT NULL,
update_blob BLOB NOT NULL,
created_at TEXT NOT NULL,
created_by TEXT NOT NULL DEFAULT '',
FOREIGN KEY(doc_id) REFERENCES documents(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_doc_crdt_updates_doc ON document_crdt_updates(doc_id);
CREATE TABLE IF NOT EXISTS lists ( CREATE TABLE IF NOT EXISTS lists (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
@@ -1563,6 +1578,59 @@ func (s *Store) PatchDocumentBlocks(userID, docID string, req domain.DocBlocksPa
return doc, mergedOrder, nil return doc, mergedOrder, nil
} }
// GetDocumentCRDTState returns the compacted CRDT state blob and any pending updates for a document.
func (s *Store) GetDocumentCRDTState(docID string) ([]byte, [][]byte, error) {
var stateBlob []byte
err := s.db.QueryRow(`SELECT state_blob FROM document_crdt_state WHERE doc_id = ?`, docID).Scan(&stateBlob)
if err != nil && err != sql.ErrNoRows {
return nil, nil, err
}
rows, err := s.db.Query(`SELECT update_blob FROM document_crdt_updates WHERE doc_id = ? ORDER BY id ASC`, docID)
if err != nil {
return stateBlob, nil, nil
}
defer rows.Close()
var updates [][]byte
for rows.Next() {
var u []byte
if err := rows.Scan(&u); err == nil && len(u) > 0 {
updates = append(updates, u)
}
}
return stateBlob, updates, nil
}
// AppendDocumentCRDTUpdate appends a new CRDT update delta for a document.
func (s *Store) AppendDocumentCRDTUpdate(docID string, update []byte, userID string) error {
now := domain.NowISO()
_, err := s.db.Exec(`
INSERT INTO document_crdt_updates (doc_id, update_blob, created_at, created_by)
VALUES (?, ?, ?, ?)
`, docID, update, now, userID)
return err
}
// SaveDocumentCRDTState saves a compacted snapshot of the document CRDT state and prunes older incremental updates.
func (s *Store) SaveDocumentCRDTState(docID string, stateBlob []byte, userID string) error {
now := domain.NowISO()
_, err := s.db.Exec(`
INSERT INTO document_crdt_state (doc_id, state_blob, updated_at, updated_by)
VALUES (?, ?, ?, ?)
ON CONFLICT(doc_id) DO UPDATE SET
state_blob = excluded.state_blob,
updated_at = excluded.updated_at,
updated_by = excluded.updated_by
`, docID, stateBlob, now, userID)
if err != nil {
return err
}
// Prune incremental updates older than the snapshot
_, _ = s.db.Exec(`DELETE FROM document_crdt_updates WHERE doc_id = ?`, docID)
return nil
}
// GetAuthorizedDocUsers returns all user IDs who have read/write/owner access to docID. // GetAuthorizedDocUsers returns all user IDs who have read/write/owner access to docID.
func (s *Store) GetAuthorizedDocUsers(docID string) ([]string, error) { func (s *Store) GetAuthorizedDocUsers(docID string) ([]string, error) {
userSet := make(map[string]bool) userSet := make(map[string]bool)
+60 -9
View File
@@ -1,7 +1,6 @@
package storage package storage
import ( import (
"os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -332,7 +331,6 @@ func TestEnsureDemoAccountsAndLoginLookup(t *testing.T) {
func TestFolderManagementAndDeletion(t *testing.T) { func TestFolderManagementAndDeletion(t *testing.T) {
tmpDir := t.TempDir() tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "folder_test.db") dbPath := filepath.Join(tmpDir, "folder_test.db")
docDir := filepath.Join(tmpDir, "workspace")
store, err := NewStore(dbPath) store, err := NewStore(dbPath)
if err != nil { if err != nil {
@@ -345,7 +343,7 @@ func TestFolderManagementAndDeletion(t *testing.T) {
t.Fatalf("CreateUser failed: %v", err) t.Fatalf("CreateUser failed: %v", err)
} }
// 1. Create folders on disk // 1. Create folders
f1, err := store.CreateFolder(user.ID, "Projekte/MindSift") f1, err := store.CreateFolder(user.ID, "Projekte/MindSift")
if err != nil || f1 != "Projekte/MindSift" { if err != nil || f1 != "Projekte/MindSift" {
t.Fatalf("CreateFolder failed: %v, f1=%s", err, f1) t.Fatalf("CreateFolder failed: %v, f1=%s", err, f1)
@@ -401,12 +399,6 @@ func TestFolderManagementAndDeletion(t *testing.T) {
if err == nil { if err == nil {
t.Fatalf("expected doc2 to be deleted") t.Fatalf("expected doc2 to be deleted")
} }
// Verify disk directory was deleted
diskFolder := filepath.Join(docDir, user.ID, "Projekte/MindSift")
if _, err := os.Stat(diskFolder); !os.IsNotExist(err) {
t.Fatalf("expected directory %s to be deleted on disk", diskFolder)
}
} }
func TestWorkspaceBlockHashtagsAndDetails(t *testing.T) { func TestWorkspaceBlockHashtagsAndDetails(t *testing.T) {
@@ -691,3 +683,62 @@ func TestDatabaseFirstWorkspaceArchitecture(t *testing.T) {
} }
} }
func TestDocumentCRDTState(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "crdt_test.db")
store, err := NewStore(dbPath)
if err != nil {
t.Fatalf("NewStore failed: %v", err)
}
defer store.Close()
user, err := store.CreateUser("crdt@example.com", "hash", "CRDT User", "member")
if err != nil {
t.Fatalf("CreateUser failed: %v", err)
}
doc, err := store.CreateDocument(user.ID, "", "CRDT Test Doc", "📝", "", nil, nil, false)
if err != nil {
t.Fatalf("CreateDocument failed: %v", err)
}
// 1. Initial CRDT state is empty
state, updates, err := store.GetDocumentCRDTState(doc.ID)
if err != nil {
t.Fatalf("GetDocumentCRDTState failed: %v", err)
}
if len(state) != 0 || len(updates) != 0 {
t.Fatalf("expected empty CRDT state, got state=%d updates=%d", len(state), len(updates))
}
// 2. Append incremental updates
u1 := []byte{1, 2, 3, 4}
u2 := []byte{5, 6, 7, 8}
if err := store.AppendDocumentCRDTUpdate(doc.ID, u1, user.ID); err != nil {
t.Fatalf("AppendDocumentCRDTUpdate 1 failed: %v", err)
}
if err := store.AppendDocumentCRDTUpdate(doc.ID, u2, user.ID); err != nil {
t.Fatalf("AppendDocumentCRDTUpdate 2 failed: %v", err)
}
_, updatesAfter, err := store.GetDocumentCRDTState(doc.ID)
if err != nil || len(updatesAfter) != 2 {
t.Fatalf("expected 2 updates, got %d (err: %v)", len(updatesAfter), err)
}
// 3. Compact snapshot state
snapshot := []byte{99, 100, 101}
if err := store.SaveDocumentCRDTState(doc.ID, snapshot, user.ID); err != nil {
t.Fatalf("SaveDocumentCRDTState failed: %v", err)
}
stateAfter, updatesAfterCompact, err := store.GetDocumentCRDTState(doc.ID)
if err != nil {
t.Fatalf("GetDocumentCRDTState after compact failed: %v", err)
}
if len(stateAfter) != 3 || len(updatesAfterCompact) != 0 {
t.Fatalf("expected compacted state of length 3 and 0 updates, got state=%d updates=%d", len(stateAfter), len(updatesAfterCompact))
}
}
+455 -7
View File
@@ -1,9 +1,12 @@
package ws package ws
import ( import (
"encoding/base64"
"encoding/json" "encoding/json"
"fmt"
"log" "log"
"net/http" "net/http"
"strings"
"sync" "sync"
"time" "time"
@@ -12,8 +15,8 @@ import (
) )
var upgrader = websocket.Upgrader{ var upgrader = websocket.Upgrader{
ReadBufferSize: 2048, ReadBufferSize: 4096,
WriteBufferSize: 2048, WriteBufferSize: 4096,
CheckOrigin: func(r *http.Request) bool { CheckOrigin: func(r *http.Request) bool {
return true // Self-hosted reverse proxy compatible return true // Self-hosted reverse proxy compatible
}, },
@@ -25,6 +28,15 @@ type Client struct {
roomID string roomID string
userID string userID string
send chan []byte send chan []byte
subscribedRooms map[string]bool
}
type SSEClient struct {
id string
userID string
roomID string
send chan []byte
subscribedRooms map[string]bool
} }
type docLockItem struct { type docLockItem struct {
@@ -35,12 +47,22 @@ type docLockItem struct {
expiresAt time.Time expiresAt time.Time
} }
type CRDTStore interface {
GetDocumentCRDTState(docID string) ([]byte, [][]byte, error)
AppendDocumentCRDTUpdate(docID string, update []byte, userID string) error
SaveDocumentCRDTState(docID string, stateBlob []byte, userID string) error
}
type Hub struct { type Hub struct {
mu sync.RWMutex mu sync.RWMutex
rooms map[string]map[*Client]bool rooms map[string]map[*Client]bool
userRooms map[string]map[*Client]bool userRooms map[string]map[*Client]bool
allClients map[*Client]bool allClients map[*Client]bool
sseClients map[*SSEClient]bool
sseUserRooms map[string]map[*SSEClient]bool
sseRooms map[string]map[*SSEClient]bool
locks map[string]*docLockItem locks map[string]*docLockItem
crdtStore CRDTStore
} }
func NewHub() *Hub { func NewHub() *Hub {
@@ -48,12 +70,21 @@ func NewHub() *Hub {
rooms: make(map[string]map[*Client]bool), rooms: make(map[string]map[*Client]bool),
userRooms: make(map[string]map[*Client]bool), userRooms: make(map[string]map[*Client]bool),
allClients: make(map[*Client]bool), allClients: make(map[*Client]bool),
sseClients: make(map[*SSEClient]bool),
sseUserRooms: make(map[string]map[*SSEClient]bool),
sseRooms: make(map[string]map[*SSEClient]bool),
locks: make(map[string]*docLockItem), locks: make(map[string]*docLockItem),
} }
go h.cleanupLocksLoop() go h.cleanupLocksLoop()
return h return h
} }
func (h *Hub) SetCRDTStore(store CRDTStore) {
h.mu.Lock()
defer h.mu.Unlock()
h.crdtStore = store
}
func (h *Hub) Register(c *Client) { func (h *Hub) Register(c *Client) {
h.mu.Lock() h.mu.Lock()
h.allClients[c] = true h.allClients[c] = true
@@ -69,7 +100,7 @@ func (h *Hub) Register(c *Client) {
} }
h.userRooms[c.userID][c] = true h.userRooms[c.userID][c] = true
} }
onlineCount := len(h.allClients) onlineCount := len(h.allClients) + len(h.sseClients)
h.mu.Unlock() h.mu.Unlock()
if c.roomID != "" { if c.roomID != "" {
@@ -90,15 +121,34 @@ func (h *Hub) Unregister(c *Client) {
} }
} }
} }
var docRoomsLeft []string
if c.subscribedRooms != nil {
for rID := range c.subscribedRooms {
if room, ok := h.rooms[rID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.rooms, rID)
}
}
if strings.HasPrefix(rID, "doc:") {
docRoomsLeft = append(docRoomsLeft, strings.TrimPrefix(rID, "doc:"))
}
}
}
userRemaining := 0 userRemaining := 0
if c.userID != "" && c.userID != "anonymous" { if c.userID != "" && c.userID != "anonymous" {
if uRoom, ok := h.userRooms[c.userID]; ok { if uRoom, ok := h.userRooms[c.userID]; ok {
delete(uRoom, c) delete(uRoom, c)
userRemaining = len(uRoom) userRemaining += len(uRoom)
if userRemaining == 0 { if len(uRoom) == 0 {
delete(h.userRooms, c.userID) delete(h.userRooms, c.userID)
} }
} }
if sseRoom, ok := h.sseUserRooms[c.userID]; ok {
userRemaining += len(sseRoom)
}
} }
close(c.send) close(c.send)
@@ -119,6 +169,150 @@ func (h *Hub) Unregister(c *Client) {
for _, docID := range releasedDocIDs { for _, docID := range releasedDocIDs {
h.BroadcastToAll("DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID}) h.BroadcastToAll("DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
} }
// Broadcast awareness leave for doc rooms
for _, docID := range docRoomsLeft {
h.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": c.userID,
}, nil)
}
}
func (h *Hub) RegisterSSE(c *SSEClient) {
h.mu.Lock()
h.sseClients[c] = true
if c.roomID != "" {
if h.sseRooms[c.roomID] == nil {
h.sseRooms[c.roomID] = make(map[*SSEClient]bool)
}
h.sseRooms[c.roomID][c] = true
}
if c.userID != "" && c.userID != "anonymous" {
if h.sseUserRooms[c.userID] == nil {
h.sseUserRooms[c.userID] = make(map[*SSEClient]bool)
}
h.sseUserRooms[c.userID][c] = true
}
if c.subscribedRooms != nil {
for rID := range c.subscribedRooms {
if h.sseRooms[rID] == nil {
h.sseRooms[rID] = make(map[*SSEClient]bool)
}
h.sseRooms[rID][c] = true
}
}
onlineCount := len(h.allClients) + len(h.sseClients)
h.mu.Unlock()
if c.roomID != "" {
h.Broadcast(c.roomID, "", "USER_PRESENCE", map[string]interface{}{
"onlineCount": onlineCount,
})
}
}
func (h *Hub) UnregisterSSE(c *SSEClient) {
h.mu.Lock()
delete(h.sseClients, c)
if c.roomID != "" {
if room, ok := h.sseRooms[c.roomID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.sseRooms, c.roomID)
}
}
}
var docRoomsLeft []string
if c.subscribedRooms != nil {
for rID := range c.subscribedRooms {
if room, ok := h.sseRooms[rID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.sseRooms, rID)
}
}
if strings.HasPrefix(rID, "doc:") {
docRoomsLeft = append(docRoomsLeft, strings.TrimPrefix(rID, "doc:"))
}
}
}
userRemaining := 0
if c.userID != "" && c.userID != "anonymous" {
if uRoom, ok := h.sseUserRooms[c.userID]; ok {
delete(uRoom, c)
userRemaining += len(uRoom)
if len(uRoom) == 0 {
delete(h.sseUserRooms, c.userID)
}
}
if wsRoom, ok := h.userRooms[c.userID]; ok {
userRemaining += len(wsRoom)
}
}
close(c.send)
var releasedDocIDs []string
if userRemaining == 0 && c.userID != "" {
now := time.Now()
for docID, l := range h.locks {
if l.userID == c.userID || l.expiresAt.Before(now) {
releasedDocIDs = append(releasedDocIDs, docID)
delete(h.locks, docID)
}
}
}
h.mu.Unlock()
for _, docID := range releasedDocIDs {
h.BroadcastToAll("DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
}
for _, docID := range docRoomsLeft {
h.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": c.userID,
}, nil)
}
}
func (h *Hub) JoinSSERoomByID(sseID, roomID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
for client := range h.sseClients {
if client.id == sseID {
if h.sseRooms[roomID] == nil {
h.sseRooms[roomID] = make(map[*SSEClient]bool)
}
h.sseRooms[roomID][client] = true
if client.subscribedRooms == nil {
client.subscribedRooms = make(map[string]bool)
}
client.subscribedRooms[roomID] = true
return true
}
}
return false
}
func (h *Hub) LeaveSSERoomByID(sseID, roomID string) bool {
h.mu.Lock()
defer h.mu.Unlock()
for client := range h.sseClients {
if client.id == sseID {
if room, ok := h.sseRooms[roomID]; ok {
delete(room, client)
if len(room) == 0 {
delete(h.sseRooms, roomID)
}
}
if client.subscribedRooms != nil {
delete(client.subscribedRooms, roomID)
}
return true
}
}
return false
} }
// Document Locking Methods // Document Locking Methods
@@ -260,6 +454,12 @@ func (h *Hub) BroadcastToAll(event string, payload interface{}) {
default: default:
} }
} }
for sseClient := range h.sseClients {
select {
case sseClient.send <- data:
default:
}
}
} }
func (h *Hub) BroadcastToUser(userID, event string, payload interface{}) { func (h *Hub) BroadcastToUser(userID, event string, payload interface{}) {
@@ -284,6 +484,14 @@ func (h *Hub) BroadcastToUser(userID, event string, payload interface{}) {
} }
} }
} }
if sseRoom, ok := h.sseUserRooms[userID]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
} }
func (h *Hub) BroadcastTaskEvent(userID, event string, payload interface{}) { func (h *Hub) BroadcastTaskEvent(userID, event string, payload interface{}) {
@@ -323,12 +531,97 @@ func (h *Hub) Broadcast(spaceID, listID, event string, payload interface{}) {
} }
} }
} }
if sseRoom, ok := h.sseRooms[targetRoom]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
} }
func (h *Hub) BroadcastListEvent(listID, event string, payload interface{}) { func (h *Hub) BroadcastListEvent(listID, event string, payload interface{}) {
h.Broadcast("", listID, event, payload) h.Broadcast("", listID, event, payload)
} }
func (h *Hub) JoinRoom(c *Client, roomID string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.rooms[roomID] == nil {
h.rooms[roomID] = make(map[*Client]bool)
}
h.rooms[roomID][c] = true
if c.subscribedRooms == nil {
c.subscribedRooms = make(map[string]bool)
}
c.subscribedRooms[roomID] = true
}
func (h *Hub) LeaveRoom(c *Client, roomID string) {
h.mu.Lock()
defer h.mu.Unlock()
if room, ok := h.rooms[roomID]; ok {
delete(room, c)
if len(room) == 0 {
delete(h.rooms, roomID)
}
}
if c.subscribedRooms != nil {
delete(c.subscribedRooms, roomID)
}
}
func (h *Hub) BroadcastRoom(roomID, event string, payload interface{}, excludeClient *Client) {
msg := domain.WSMessage{
Event: event,
Payload: payload,
Timestamp: domain.NowISO(),
}
data, err := json.Marshal(msg)
if err != nil {
return
}
h.mu.RLock()
defer h.mu.RUnlock()
if room, ok := h.rooms[roomID]; ok {
for client := range room {
if excludeClient != nil && client == excludeClient {
continue
}
select {
case client.send <- data:
default:
}
}
}
if sseRoom, ok := h.sseRooms[roomID]; ok {
for sseClient := range sseRoom {
select {
case sseClient.send <- data:
default:
}
}
}
}
func (c *Client) SendEvent(event string, payload interface{}) {
msg := domain.WSMessage{
Event: event,
Payload: payload,
Timestamp: domain.NowISO(),
}
data, err := json.Marshal(msg)
if err == nil {
select {
case c.send <- data:
default:
}
}
}
func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request, roomID, userID string) { func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request, roomID, userID string) {
conn, err := upgrader.Upgrade(w, r, nil) conn, err := upgrader.Upgrade(w, r, nil)
if err != nil { if err != nil {
@@ -349,13 +642,113 @@ func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request, roomID, userID str
go client.readPump() go client.readPump()
} }
func (h *Hub) ServeSSE(w http.ResponseWriter, r *http.Request, roomID, userID string, initialRooms []string) {
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache, no-transform")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.Header().Set("Access-Control-Allow-Origin", "*")
client := &SSEClient{
id: domain.NewID("sse"),
userID: userID,
roomID: roomID,
send: make(chan []byte, 128),
subscribedRooms: make(map[string]bool),
}
for _, room := range initialRooms {
if strings.TrimSpace(room) != "" {
client.subscribedRooms[strings.TrimSpace(room)] = true
}
}
h.RegisterSSE(client)
defer h.UnregisterSSE(client)
// Send initial connected comment and event
fmt.Fprintf(w, ": connected (transport: sse, id: %s)\n\n", client.id)
connectedMsg, _ := json.Marshal(domain.WSMessage{
Event: "SSE_CONNECTED",
Payload: map[string]interface{}{
"sseId": client.id,
"transport": "sse",
"userId": userID,
},
Timestamp: domain.NowISO(),
})
fmt.Fprintf(w, "data: %s\n\n", connectedMsg)
flusher.Flush()
// If initial rooms had doc rooms, send CRDT_INIT
if h.crdtStore != nil {
for rID := range client.subscribedRooms {
if strings.HasPrefix(rID, "doc:") {
docID := strings.TrimPrefix(rID, "doc:")
stateBlob, updates, err := h.crdtStore.GetDocumentCRDTState(docID)
if err == nil && (len(stateBlob) > 0 || len(updates) > 0) {
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
initMsg := domain.WSMessage{
Event: "CRDT_INIT",
Payload: map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
},
Timestamp: domain.NowISO(),
}
if initBytes, err := json.Marshal(initMsg); err == nil {
fmt.Fprintf(w, "data: %s\n\n", initBytes)
flusher.Flush()
}
}
}
}
}
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
notify := r.Context().Done()
for {
select {
case <-notify:
return
case msg, ok := <-client.send:
if !ok {
return
}
fmt.Fprintf(w, "data: %s\n\n", msg)
flusher.Flush()
case <-ticker.C:
// Periodic ping comment to keep HTTP connection alive and prevent proxy timeouts
fmt.Fprintf(w, ": ping\n\n")
flusher.Flush()
}
}
}
func (c *Client) readPump() { func (c *Client) readPump() {
defer func() { defer func() {
c.hub.Unregister(c) c.hub.Unregister(c)
_ = c.conn.Close() _ = c.conn.Close()
}() }()
c.conn.SetReadLimit(16384) c.conn.SetReadLimit(4 * 1024 * 1024) // 4MB for large document CRDT updates
_ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second)) _ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second))
c.conn.SetPongHandler(func(string) error { c.conn.SetPongHandler(func(string) error {
_ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second)) _ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second))
@@ -368,7 +761,7 @@ func (c *Client) readPump() {
break break
} }
// Handle client-initiated lock and unlock messages // Handle client-initiated lock, CRDT, and presence messages
var incoming struct { var incoming struct {
Event string `json:"event"` Event string `json:"event"`
DocID string `json:"docId"` DocID string `json:"docId"`
@@ -396,6 +789,61 @@ func (c *Client) readPump() {
if c.hub.ReleaseDocLock(docID, c.userID, false) { if c.hub.ReleaseDocLock(docID, c.userID, false) {
c.hub.BroadcastDocEvent(docID, "DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID}) c.hub.BroadcastDocEvent(docID, "DOC_UNLOCKED", map[string]string{"docId": docID, "doc_id": docID})
} }
case "CRDT_JOIN":
c.hub.JoinRoom(c, "doc:"+docID)
if c.hub.crdtStore != nil {
stateBlob, updates, err := c.hub.crdtStore.GetDocumentCRDTState(docID)
if err == nil && (len(stateBlob) > 0 || len(updates) > 0) {
var b64Updates []string
for _, u := range updates {
b64Updates = append(b64Updates, base64.StdEncoding.EncodeToString(u))
}
b64State := ""
if len(stateBlob) > 0 {
b64State = base64.StdEncoding.EncodeToString(stateBlob)
}
c.SendEvent("CRDT_INIT", map[string]interface{}{
"docId": docID,
"state": b64State,
"updates": b64Updates,
})
}
}
case "CRDT_UPDATE":
var p struct {
Update string `json:"update"`
}
_ = json.Unmarshal(incoming.Payload, &p)
if p.Update != "" {
c.hub.BroadcastRoom("doc:"+docID, "CRDT_UPDATE", map[string]interface{}{
"docId": docID,
"update": p.Update,
"fromUserId": c.userID,
}, c)
if c.hub.crdtStore != nil {
if raw, err := base64.StdEncoding.DecodeString(p.Update); err == nil && len(raw) > 0 {
_ = c.hub.crdtStore.AppendDocumentCRDTUpdate(docID, raw, c.userID)
}
}
}
case "CRDT_COMPACT":
var p struct {
State string `json:"state"`
}
_ = json.Unmarshal(incoming.Payload, &p)
if p.State != "" && c.hub.crdtStore != nil {
if raw, err := base64.StdEncoding.DecodeString(p.State); err == nil && len(raw) > 0 {
_ = c.hub.crdtStore.SaveDocumentCRDTState(docID, raw, c.userID)
}
}
case "CRDT_AWARENESS":
c.hub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS", incoming.Payload, c)
case "CRDT_LEAVE":
c.hub.LeaveRoom(c, "doc:"+docID)
c.hub.BroadcastRoom("doc:"+docID, "CRDT_AWARENESS_LEAVE", map[string]interface{}{
"docId": docID,
"userId": c.userID,
}, c)
} }
} }
} }