refactor(crdt): implement proxy-resilient Yjs dual-transport sync with indexeddb offline-first and debounced sqlite persistence
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m11s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m11s
This commit is contained in:
sandrodach committed
1 parent
297f3ed111
commit
f3b3f07e04
10 files changed
+1248
-484
No files matched your search
@@ -0,0 +1,380 @@
|
||||
import * as Y from 'yjs';
|
||||
import { IndexeddbPersistence } from 'y-indexeddb';
|
||||
import { getToken } from './api';
|
||||
|
||||
export type SyncTransportStatus = 'ws' | 'sse' | 'offline';
|
||||
|
||||
export interface YjsHybridProviderOptions {
|
||||
docId: string;
|
||||
ydoc: Y.Doc;
|
||||
onStatusChange?: (status: SyncTransportStatus) => void;
|
||||
onAwareness?: (payload: any) => void;
|
||||
onSynced?: () => void;
|
||||
}
|
||||
|
||||
export class YjsHybridProvider {
|
||||
public readonly docId: string;
|
||||
public readonly ydoc: Y.Doc;
|
||||
private status: SyncTransportStatus = 'offline';
|
||||
private indexeddbProvider: IndexeddbPersistence;
|
||||
private ws: WebSocket | null = null;
|
||||
private eventSource: EventSource | null = null;
|
||||
private wsWatchdogTimer: any = null;
|
||||
private destroyed: boolean = false;
|
||||
private isConnectingWS: boolean = false;
|
||||
private isConnectingSSE: boolean = false;
|
||||
private onStatusChange?: (status: SyncTransportStatus) => void;
|
||||
private onAwareness?: (payload: any) => void;
|
||||
private onSynced?: () => void;
|
||||
|
||||
constructor(options: YjsHybridProviderOptions) {
|
||||
this.docId = options.docId;
|
||||
this.ydoc = options.ydoc;
|
||||
this.onStatusChange = options.onStatusChange;
|
||||
this.onAwareness = options.onAwareness;
|
||||
this.onSynced = options.onSynced;
|
||||
|
||||
// 1. Offline-First: Bind IndexedDB persistence
|
||||
this.indexeddbProvider = new IndexeddbPersistence(`mindsift-doc-${this.docId}`, this.ydoc);
|
||||
this.indexeddbProvider.on('synced', () => {
|
||||
if (this.destroyed) return;
|
||||
if (this.onSynced) {
|
||||
this.onSynced();
|
||||
}
|
||||
});
|
||||
|
||||
// 2. Listen to local document updates
|
||||
this.ydoc.on('update', this.handleDocUpdate);
|
||||
|
||||
// 3. Connect via Primary Transport (WebSocket) with 4-second watchdog fallback
|
||||
this.connect();
|
||||
|
||||
// 4. Online/offline window event listeners
|
||||
if (typeof window !== 'undefined') {
|
||||
window.addEventListener('online', this.handleOnline);
|
||||
window.addEventListener('offline', this.handleOffline);
|
||||
}
|
||||
}
|
||||
|
||||
private setStatus(newStatus: SyncTransportStatus) {
|
||||
if (this.status !== newStatus) {
|
||||
this.status = newStatus;
|
||||
if (this.onStatusChange) {
|
||||
this.onStatusChange(newStatus);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public getStatus(): SyncTransportStatus {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
private handleOnline = () => {
|
||||
if (this.destroyed) return;
|
||||
if (this.status === 'offline') {
|
||||
this.connect();
|
||||
}
|
||||
};
|
||||
|
||||
private handleOffline = () => {
|
||||
if (this.destroyed) return;
|
||||
this.setStatus('offline');
|
||||
};
|
||||
|
||||
public connect() {
|
||||
if (this.destroyed) return;
|
||||
this.connectWebSocket();
|
||||
}
|
||||
|
||||
private connectWebSocket() {
|
||||
if (this.destroyed || this.isConnectingWS) return;
|
||||
this.isConnectingWS = true;
|
||||
|
||||
this.cleanupWebSocket();
|
||||
|
||||
const token = getToken() || '';
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const wsUrl = `${protocol}//${window.location.host}/api/v1/ws/docs/${encodeURIComponent(this.docId)}?token=${encodeURIComponent(token)}`;
|
||||
|
||||
// 4-second watchdog timer: If WebSocket doesn't establish within 4 seconds, fallback to HTTP/SSE
|
||||
this.clearWatchdog();
|
||||
this.wsWatchdogTimer = setTimeout(() => {
|
||||
if (this.status !== 'ws') {
|
||||
console.warn(`[YjsProvider] WebSocket timeout (4s) for doc ${this.docId}. Switching to HTTP fallback (SSE + POST)...`);
|
||||
this.fallbackToHTTP();
|
||||
}
|
||||
}, 4000);
|
||||
|
||||
try {
|
||||
const ws = new WebSocket(wsUrl);
|
||||
ws.binaryType = 'arraybuffer';
|
||||
this.ws = ws;
|
||||
|
||||
ws.onopen = () => {
|
||||
this.isConnectingWS = false;
|
||||
this.clearWatchdog();
|
||||
this.cleanupSSE();
|
||||
this.setStatus('ws');
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
if (this.destroyed) return;
|
||||
|
||||
if (event.data instanceof ArrayBuffer) {
|
||||
const update = new Uint8Array(event.data);
|
||||
Y.applyUpdate(this.ydoc, update, 'remote');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.event === 'CRDT_INIT' && data.payload) {
|
||||
this.handleInit(data.payload);
|
||||
} else if (data.event === 'CRDT_UPDATE' && data.payload) {
|
||||
this.handleRemoteUpdate(data.payload);
|
||||
} else if (data.event === 'CRDT_AWARENESS' && data.payload) {
|
||||
if (this.onAwareness) {
|
||||
this.onAwareness(data.payload);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[YjsProvider] WS message parse error:', err);
|
||||
}
|
||||
};
|
||||
|
||||
ws.onerror = (err) => {
|
||||
console.warn(`[YjsProvider] WebSocket error on doc ${this.docId}:`, err);
|
||||
this.isConnectingWS = false;
|
||||
this.fallbackToHTTP();
|
||||
};
|
||||
|
||||
ws.onclose = (event) => {
|
||||
this.isConnectingWS = false;
|
||||
// If closed abnormally (proxy drop, 1006, etc.), fallback to HTTP
|
||||
if (!this.destroyed && this.status === 'ws') {
|
||||
console.warn(`[YjsProvider] WebSocket closed (code ${event.code}). Falling back to HTTP/SSE...`);
|
||||
this.fallbackToHTTP();
|
||||
}
|
||||
};
|
||||
} catch (e) {
|
||||
console.warn(`[YjsProvider] WebSocket creation failed:`, e);
|
||||
this.isConnectingWS = false;
|
||||
this.fallbackToHTTP();
|
||||
}
|
||||
}
|
||||
|
||||
private fallbackToHTTP() {
|
||||
if (this.destroyed) return;
|
||||
this.cleanupWebSocket();
|
||||
this.connectSSE();
|
||||
}
|
||||
|
||||
private connectSSE() {
|
||||
if (this.destroyed || this.isConnectingSSE) return;
|
||||
this.isConnectingSSE = true;
|
||||
|
||||
this.cleanupSSE();
|
||||
|
||||
const token = getToken() || '';
|
||||
const sseUrl = `/api/v1/sync/docs/${encodeURIComponent(this.docId)}/stream?token=${encodeURIComponent(token)}`;
|
||||
|
||||
try {
|
||||
const es = new EventSource(sseUrl);
|
||||
this.eventSource = es;
|
||||
|
||||
es.onopen = () => {
|
||||
this.isConnectingSSE = false;
|
||||
this.setStatus('sse');
|
||||
};
|
||||
|
||||
es.onmessage = (event) => {
|
||||
if (this.destroyed) return;
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.event === 'CRDT_INIT' && data.payload) {
|
||||
this.handleInit(data.payload);
|
||||
} else if (data.event === 'CRDT_UPDATE' && data.payload) {
|
||||
this.handleRemoteUpdate(data.payload);
|
||||
} else if (data.event === 'CRDT_AWARENESS' && data.payload) {
|
||||
if (this.onAwareness) {
|
||||
this.onAwareness(data.payload);
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
};
|
||||
|
||||
es.onerror = () => {
|
||||
this.isConnectingSSE = false;
|
||||
if (!this.destroyed) {
|
||||
this.setStatus('offline');
|
||||
}
|
||||
};
|
||||
} catch (e) {
|
||||
this.isConnectingSSE = false;
|
||||
this.setStatus('offline');
|
||||
}
|
||||
}
|
||||
|
||||
private handleInit(payload: { state?: string; updates?: string[] }) {
|
||||
if (payload.state) {
|
||||
try {
|
||||
const stateBytes = this.base64ToUint8(payload.state);
|
||||
Y.applyUpdate(this.ydoc, stateBytes, 'remote');
|
||||
} catch (e) {
|
||||
console.error('[YjsProvider] Failed to apply initial state:', e);
|
||||
}
|
||||
}
|
||||
if (Array.isArray(payload.updates)) {
|
||||
for (const u of payload.updates) {
|
||||
if (u) {
|
||||
try {
|
||||
const updateBytes = this.base64ToUint8(u);
|
||||
Y.applyUpdate(this.ydoc, updateBytes, 'remote');
|
||||
} catch (e) {
|
||||
console.error('[YjsProvider] Failed to apply initial update:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.onSynced) {
|
||||
this.onSynced();
|
||||
}
|
||||
}
|
||||
|
||||
private handleRemoteUpdate(payload: { update?: string }) {
|
||||
if (payload.update) {
|
||||
try {
|
||||
const bytes = this.base64ToUint8(payload.update);
|
||||
Y.applyUpdate(this.ydoc, bytes, 'remote');
|
||||
} catch (e) {
|
||||
console.error('[YjsProvider] Failed to apply remote update:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private handleDocUpdate = (update: Uint8Array, origin: any) => {
|
||||
// Only send updates originating locally (ignore 'remote' and 'indexeddb')
|
||||
if (this.destroyed || origin === 'remote' || origin === this.indexeddbProvider) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.status === 'ws' && this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.ws.send(update);
|
||||
return;
|
||||
} catch (e) {
|
||||
console.warn('[YjsProvider] Failed to send update over WS, falling back to HTTP POST:', e);
|
||||
this.fallbackToHTTP();
|
||||
}
|
||||
}
|
||||
|
||||
// Upstream via HTTP POST fallback
|
||||
this.sendUpdateViaHTTP(update);
|
||||
};
|
||||
|
||||
private async sendUpdateViaHTTP(update: Uint8Array) {
|
||||
const token = getToken() || '';
|
||||
const url = `/api/v1/sync/docs/${encodeURIComponent(this.docId)}/update?token=${encodeURIComponent(token)}`;
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
body: new Blob([update as any]),
|
||||
});
|
||||
if (!res.ok) {
|
||||
console.warn(`[YjsProvider] HTTP sync update status: ${res.status}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[YjsProvider] Network error during HTTP sync update (persisted locally in IndexedDB):', e);
|
||||
this.setStatus('offline');
|
||||
}
|
||||
}
|
||||
|
||||
public sendAwareness(payload: any) {
|
||||
if (this.destroyed) return;
|
||||
|
||||
if (this.status === 'ws' && this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.ws.send(JSON.stringify({
|
||||
event: 'CRDT_AWARENESS',
|
||||
docId: this.docId,
|
||||
payload,
|
||||
}));
|
||||
return;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// HTTP awareness fallback
|
||||
const token = getToken() || '';
|
||||
fetch(`/api/v1/documents/${encodeURIComponent(this.docId)}/awareness`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
private clearWatchdog() {
|
||||
if (this.wsWatchdogTimer) {
|
||||
clearTimeout(this.wsWatchdogTimer);
|
||||
this.wsWatchdogTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private cleanupWebSocket() {
|
||||
this.clearWatchdog();
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.onopen = null;
|
||||
this.ws.onmessage = null;
|
||||
this.ws.onerror = null;
|
||||
this.ws.onclose = null;
|
||||
this.ws.close();
|
||||
} catch {}
|
||||
this.ws = null;
|
||||
}
|
||||
}
|
||||
|
||||
private cleanupSSE() {
|
||||
if (this.eventSource) {
|
||||
try {
|
||||
this.eventSource.onopen = null;
|
||||
this.eventSource.onmessage = null;
|
||||
this.eventSource.onerror = null;
|
||||
this.eventSource.close();
|
||||
} catch {}
|
||||
this.eventSource = null;
|
||||
}
|
||||
}
|
||||
|
||||
private base64ToUint8(base64: string): Uint8Array {
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public destroy() {
|
||||
this.destroyed = true;
|
||||
this.clearWatchdog();
|
||||
this.cleanupWebSocket();
|
||||
this.cleanupSSE();
|
||||
this.ydoc.off('update', this.handleDocUpdate);
|
||||
if (this.indexeddbProvider) {
|
||||
try {
|
||||
this.indexeddbProvider.destroy();
|
||||
} catch {}
|
||||
}
|
||||
if (typeof window !== 'undefined') {
|
||||
window.removeEventListener('online', this.handleOnline);
|
||||
window.removeEventListener('offline', this.handleOffline);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user