feat(voice): all microphone buttons use the server's speech recognition
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s

- New useVoiceInput hook (lib/voice.ts): "server" records with MediaRecorder
  and sends to /voice/transcribe, "browser" keeps the live recognition,
  "off" or missing browser support hides the microphone.
- Quick dump bar, mobile capture sheet and hold-to-record use it; live
  waves from the real input level (VoiceWave) and "Wird erkannt…".
- Recordings stop after STT_MAX_SECONDS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-29 22:23:48 +01:00
1 parent 7801cd13b9
commit 7fa21e485b
9 files changed
+753 -248

No files matched your search

+157
View File
@@ -0,0 +1,157 @@
// Voice input for all places with a microphone button. The server decides how it works (STT_* in the
// environment, reported by /auth/status):
// server record with MediaRecorder, the own Whisper server turns it into text after stopping
// browser the browser's speech recognition (Chrome, Edge, Safari; text appears while speaking)
// off no voice input
import { api } from './api';
import type { VoiceMode } from './types';
export type EffectiveVoiceMode = VoiceMode | 'unsupported';
export interface VoiceConfig {
mode: EffectiveVoiceMode;
maxSeconds: number;
}
let configPromise: Promise<VoiceConfig> | null = null;
let loadedConfig: VoiceConfig | null = null;
export function browserRecognition(): any {
if (typeof window === 'undefined') return null;
return (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition || null;
}
export function canRecord(): boolean {
return typeof window !== 'undefined' && typeof (window as any).MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia;
}
/** What this browser can actually do with the server's setting. */
export function effectiveMode(mode: VoiceMode | undefined): EffectiveVoiceMode {
if (mode === 'off') return 'off';
if (mode === 'server') return canRecord() ? 'server' : 'unsupported';
return browserRecognition() ? 'browser' : 'unsupported';
}
/** Loaded once per page; the setting only changes with a server restart. */
export function loadVoiceConfig(): Promise<VoiceConfig> {
if (!configPromise) {
configPromise = api
.getAuthStatus()
.then((st) => {
loadedConfig = { mode: effectiveMode(st.voice?.mode), maxSeconds: st.voice?.maxSeconds || 600 };
return loadedConfig;
})
.catch(() => {
configPromise = null; // offline at start: ask again next time
return { mode: effectiveMode('browser'), maxSeconds: 600 };
});
}
return configPromise;
}
/** The setting if already loaded; until then the browser's recognition (a button must react at once). */
export function currentVoiceConfig(): VoiceConfig {
return loadedConfig ?? { mode: effectiveMode('browser'), maxSeconds: 600 };
}
/** For tests. */
export function resetVoiceConfig(): void {
configPromise = null;
loadedConfig = null;
}
// Containers in order of preference; Safari only records MP4/AAC
const RECORDING_TYPES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus'];
function recordingType(): string | undefined {
const MR = (window as any).MediaRecorder;
if (!MR?.isTypeSupported) return undefined;
return RECORDING_TYPES.find((t) => MR.isTypeSupported(t));
}
/**
* One recording from the microphone, with a live input level for wave animations. stop() resolves
* with the audio; cancel() throws it away.
*/
export class VoiceRecorder {
private stream: MediaStream | null = null;
private recorder: MediaRecorder | null = null;
private chunks: Blob[] = [];
private audioCtx: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private samples: Uint8Array | null = null;
private done: ((blob: Blob | null) => void) | null = null;
async start(): Promise<void> {
this.stream = await navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
const type = recordingType();
this.recorder = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
this.chunks = [];
this.recorder.ondataavailable = (e) => {
if (e.data.size > 0) this.chunks.push(e.data);
};
this.recorder.onstop = () => {
const blob = this.chunks.length ? new Blob(this.chunks, { type: this.recorder?.mimeType || type || 'audio/webm' }) : null;
this.release();
this.done?.(blob);
this.done = null;
};
this.recorder.start(1000);
try {
const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext;
this.audioCtx = new Ctx();
const source = this.audioCtx!.createMediaStreamSource(this.stream);
this.analyser = this.audioCtx!.createAnalyser();
this.analyser.fftSize = 512;
source.connect(this.analyser);
this.samples = new Uint8Array(this.analyser.fftSize);
void this.audioCtx!.resume?.();
} catch {
this.analyser = null; // no level, the recording still works
}
}
/** Input level 0..1 (roughly loudness), or null when unknown. */
level(): number | null {
if (!this.analyser || !this.samples) return null;
this.analyser.getByteTimeDomainData(this.samples as Uint8Array<ArrayBuffer>);
let sum = 0;
for (const v of this.samples) {
const x = (v - 128) / 128;
sum += x * x;
}
return Math.min(1, Math.sqrt(sum / this.samples.length) * 4);
}
stop(): Promise<Blob | null> {
return new Promise((resolve) => {
if (!this.recorder || this.recorder.state === 'inactive') {
this.release();
resolve(null);
return;
}
this.done = resolve;
this.recorder.stop();
});
}
cancel(): void {
this.done = null;
if (this.recorder && this.recorder.state !== 'inactive') {
this.recorder.onstop = () => this.release();
this.recorder.stop();
} else {
this.release();
}
}
private release(): void {
this.stream?.getTracks().forEach((t) => t.stop());
this.stream = null;
void this.audioCtx?.close?.().catch(() => {});
this.audioCtx = null;
this.analyser = null;
}
}