// Voice input for all places with a microphone button. The server decides how it works (STT_* in the // environment, reported by /auth/status): // server record with MediaRecorder, the own Whisper server turns it into text after stopping // browser the browser's speech recognition (Chrome, Edge, Safari; text appears while speaking) // off no voice input import { api } from './api'; import type { VoiceMode } from './types'; export type EffectiveVoiceMode = VoiceMode | 'unsupported'; export interface VoiceConfig { mode: EffectiveVoiceMode; maxSeconds: number; } let configPromise: Promise | null = null; let loadedConfig: VoiceConfig | null = null; export function browserRecognition(): any { if (typeof window === 'undefined') return null; return (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition || null; } export function canRecord(): boolean { return typeof window !== 'undefined' && typeof (window as any).MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia; } /** What this browser can actually do with the server's setting. */ export function effectiveMode(mode: VoiceMode | undefined): EffectiveVoiceMode { if (mode === 'off') return 'off'; if (mode === 'server') return canRecord() ? 'server' : 'unsupported'; return browserRecognition() ? 'browser' : 'unsupported'; } /** Loaded once per page; the setting only changes with a server restart. */ export function loadVoiceConfig(): Promise { if (!configPromise) { configPromise = api .getAuthStatus() .then((st) => { loadedConfig = { mode: effectiveMode(st.voice?.mode), maxSeconds: st.voice?.maxSeconds || 600 }; return loadedConfig; }) .catch(() => { configPromise = null; // offline at start: ask again next time return { mode: effectiveMode('browser'), maxSeconds: 600 }; }); } return configPromise; } /** The setting if already loaded; until then the browser's recognition (a button must react at once). */ export function currentVoiceConfig(): VoiceConfig { return loadedConfig ?? { mode: effectiveMode('browser'), maxSeconds: 600 }; } /** For tests. */ export function resetVoiceConfig(): void { configPromise = null; loadedConfig = null; } // Containers in order of preference; Safari only records MP4/AAC const RECORDING_TYPES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus']; function recordingType(): string | undefined { const MR = (window as any).MediaRecorder; if (!MR?.isTypeSupported) return undefined; return RECORDING_TYPES.find((t) => MR.isTypeSupported(t)); } /** * One recording from the microphone, with a live input level for wave animations. stop() resolves * with the audio; cancel() throws it away. */ export class VoiceRecorder { private stream: MediaStream | null = null; private recorder: MediaRecorder | null = null; private chunks: Blob[] = []; private audioCtx: AudioContext | null = null; private analyser: AnalyserNode | null = null; private samples: Uint8Array | null = null; private done: ((blob: Blob | null) => void) | null = null; async start(): Promise { this.stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); this.startSegmentRecorder(); try { const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext; this.audioCtx = new Ctx(); const source = this.audioCtx!.createMediaStreamSource(this.stream); this.analyser = this.audioCtx!.createAnalyser(); this.analyser.fftSize = 512; source.connect(this.analyser); this.samples = new Uint8Array(this.analyser.fftSize); void this.audioCtx!.resume?.(); } catch { this.analyser = null; // no level, the recording still works } } private startSegmentRecorder(): void { if (!this.stream) return; const type = recordingType(); const rec = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined); const chunks: Blob[] = []; rec.ondataavailable = (e) => { if (e.data.size > 0) chunks.push(e.data); }; rec.onstop = () => { const blob = chunks.length ? new Blob(chunks, { type: rec.mimeType || type || 'audio/webm' }) : null; this.release(); this.done?.(blob); this.done = null; }; this.recorder = rec; this.chunks = chunks; rec.start(1000); } /** Input level 0..1 (roughly loudness), or null when unknown. */ level(): number | null { if (!this.analyser || !this.samples) return null; this.analyser.getByteTimeDomainData(this.samples as Uint8Array); let sum = 0; for (const v of this.samples) { const x = (v - 128) / 128; sum += x * x; } return Math.min(1, Math.sqrt(sum / this.samples.length) * 4); } stop(): Promise { return new Promise((resolve) => { if (!this.recorder || this.recorder.state === 'inactive') { this.release(); resolve(null); return; } this.done = resolve; this.recorder.stop(); }); } /** * Stops the current MediaRecorder segment and immediately starts a new one on the same MediaStream. * Resolves with the finished segment's audio Blob while recording continues seamlessly. */ cutSegment(): Promise { if (!this.stream || !this.recorder || this.recorder.state === 'inactive') { return Promise.resolve(null); } const oldRecorder = this.recorder; const oldChunks = this.chunks; this.startSegmentRecorder(); return new Promise((resolve) => { oldRecorder.onstop = () => { const blob = oldChunks.length ? new Blob(oldChunks, { type: oldRecorder.mimeType || 'audio/webm' }) : null; resolve(blob); }; oldRecorder.stop(); }); } cancel(): void { this.done = null; if (this.recorder && this.recorder.state !== 'inactive') { this.recorder.onstop = () => this.release(); this.recorder.stop(); } else { this.release(); } } private release(): void { this.stream?.getTracks().forEach((t) => t.stop()); this.stream = null; void this.audioCtx?.close?.().catch(() => {}); this.audioCtx = null; this.analyser = null; } }