feat(voice): all microphone buttons use the server's speech recognition
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s
- New useVoiceInput hook (lib/voice.ts): "server" records with MediaRecorder and sends to /voice/transcribe, "browser" keeps the live recognition, "off" or missing browser support hides the microphone. - Quick dump bar, mobile capture sheet and hold-to-record use it; live waves from the real input level (VoiceWave) and "Wird erkannt…". - Recordings stop after STT_MAX_SECONDS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed
1 parent
7801cd13b9
commit
7fa21e485b
9 files changed
+753
-248
No files matched your search
@@ -0,0 +1,133 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { act, renderHook, waitFor } from '@testing-library/react';
|
||||
|
||||
const getAuthStatus = vi.fn();
|
||||
const transcribe = vi.fn();
|
||||
vi.mock('../api', () => ({ api: { getAuthStatus: (...a: any[]) => getAuthStatus(...a), transcribe: (...a: any[]) => transcribe(...a) } }));
|
||||
|
||||
import { useVoiceInput } from '../useVoiceInput';
|
||||
import { resetVoiceConfig } from '../voice';
|
||||
|
||||
class FakeMediaRecorder {
|
||||
static isTypeSupported = (t: string) => t === 'audio/webm;codecs=opus';
|
||||
state: 'inactive' | 'recording' = 'inactive';
|
||||
mimeType = 'audio/webm;codecs=opus';
|
||||
ondataavailable: ((e: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
constructor(public stream: any) {}
|
||||
start() {
|
||||
this.state = 'recording';
|
||||
}
|
||||
stop() {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio'], { type: this.mimeType }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
|
||||
const stopTrack = vi.fn();
|
||||
|
||||
beforeEach(() => {
|
||||
resetVoiceConfig();
|
||||
getAuthStatus.mockReset();
|
||||
transcribe.mockReset();
|
||||
stopTrack.mockReset();
|
||||
(window as any).MediaRecorder = FakeMediaRecorder;
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia: vi.fn(async () => ({ getTracks: () => [{ stop: stopTrack }] })) },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete (window as any).MediaRecorder;
|
||||
delete (window as any).SpeechRecognition;
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('useVoiceInput', () => {
|
||||
it('server mode: records, sends the audio to the own server and returns its text', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } });
|
||||
transcribe.mockResolvedValue('Milch kaufen');
|
||||
const onText = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('server'));
|
||||
expect(result.current.available).toBe(true);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
expect(result.current.state).toBe('listening');
|
||||
|
||||
await act(async () => {
|
||||
await result.current.stop();
|
||||
});
|
||||
expect(transcribe).toHaveBeenCalledTimes(1);
|
||||
expect((transcribe.mock.calls[0][0] as Blob).type).toBe('audio/webm;codecs=opus');
|
||||
expect(onText).toHaveBeenCalledWith('Milch kaufen', true);
|
||||
expect(result.current.state).toBe('idle');
|
||||
expect(stopTrack).toHaveBeenCalled(); // microphone released
|
||||
});
|
||||
|
||||
it('server mode: cancel throws the recording away', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } });
|
||||
const onText = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('server'));
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
act(() => result.current.cancel());
|
||||
expect(transcribe).not.toHaveBeenCalled();
|
||||
expect(onText).not.toHaveBeenCalled();
|
||||
expect(result.current.state).toBe('idle');
|
||||
});
|
||||
|
||||
it('server mode: a failing server reports an error instead of text', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } });
|
||||
transcribe.mockRejectedValue(new Error('Spracherkennung fehlgeschlagen'));
|
||||
const onText = vi.fn();
|
||||
const onError = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText, onError }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('server'));
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
await act(async () => {
|
||||
await result.current.stop();
|
||||
});
|
||||
expect(onError).toHaveBeenCalledWith('Spracherkennung fehlgeschlagen');
|
||||
expect(onText).not.toHaveBeenCalled();
|
||||
expect(result.current.state).toBe('idle');
|
||||
});
|
||||
|
||||
it('server mode: stops by itself after the maximum length', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 5 } });
|
||||
transcribe.mockResolvedValue('Zu lang');
|
||||
const onText = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('server'));
|
||||
vi.useFakeTimers();
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(5000);
|
||||
});
|
||||
vi.useRealTimers();
|
||||
await waitFor(() => expect(onText).toHaveBeenCalledWith('Zu lang', true));
|
||||
});
|
||||
|
||||
it('hides the microphone when switched off, and without browser recognition', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'off', maxSeconds: 600 } });
|
||||
const off = renderHook(() => useVoiceInput({ onText: vi.fn() }));
|
||||
await waitFor(() => expect(off.result.current.mode).toBe('off'));
|
||||
expect(off.result.current.available).toBe(false);
|
||||
|
||||
resetVoiceConfig();
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'browser', maxSeconds: 600 } });
|
||||
const none = renderHook(() => useVoiceInput({ onText: vi.fn() }));
|
||||
await waitFor(() => expect(none.result.current.mode).toBe('unsupported'));
|
||||
expect(none.result.current.available).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { api } from './api';
|
||||
import { browserRecognition, currentVoiceConfig, EffectiveVoiceMode, loadVoiceConfig, VoiceRecorder } from './voice';
|
||||
|
||||
export type VoiceState = 'idle' | 'listening' | 'transcribing';
|
||||
|
||||
export interface VoiceInputOptions {
|
||||
/** Text of the current recording; in browser mode repeatedly while speaking (final=false), always once at the end (final=true). */
|
||||
onText: (text: string, final: boolean) => void;
|
||||
onError?: (message: string) => void;
|
||||
/** Browser mode: keep listening through pauses (hold-to-record) instead of stopping after a sentence. */
|
||||
continuous?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Microphone input for one button: start, stop (text arrives), cancel (nothing arrives). How the text
|
||||
* is made depends on the server setting, see lib/voice.ts. mode is null until known, "off" or
|
||||
* "unsupported" means: do not show a microphone.
|
||||
*/
|
||||
export function useVoiceInput({ onText, onError, continuous = false }: VoiceInputOptions) {
|
||||
const [mode, setMode] = useState<EffectiveVoiceMode | null>(null);
|
||||
const [state, setState] = useState<VoiceState>('idle');
|
||||
const maxSecondsRef = useRef(600);
|
||||
const recorderRef = useRef<VoiceRecorder | null>(null);
|
||||
const recognitionRef = useRef<any>(null);
|
||||
const timerRef = useRef<number | null>(null);
|
||||
const cancelledRef = useRef(false);
|
||||
const transcriptRef = useRef('');
|
||||
const mountedRef = useRef(true);
|
||||
const onTextRef = useRef(onText);
|
||||
onTextRef.current = onText;
|
||||
const onErrorRef = useRef(onError);
|
||||
onErrorRef.current = onError;
|
||||
|
||||
useEffect(() => {
|
||||
mountedRef.current = true;
|
||||
void loadVoiceConfig().then((cfg) => {
|
||||
if (!mountedRef.current) return;
|
||||
setMode(cfg.mode);
|
||||
maxSecondsRef.current = cfg.maxSeconds;
|
||||
});
|
||||
return () => {
|
||||
mountedRef.current = false;
|
||||
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
|
||||
recorderRef.current?.cancel();
|
||||
try {
|
||||
recognitionRef.current?.abort?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const setStateSafe = (s: VoiceState) => {
|
||||
if (mountedRef.current) setState(s);
|
||||
};
|
||||
const fail = (message: string) => {
|
||||
setStateSafe('idle');
|
||||
onErrorRef.current?.(message);
|
||||
};
|
||||
const clearTimer = () => {
|
||||
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
|
||||
timerRef.current = null;
|
||||
};
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
clearTimer();
|
||||
if (recognitionRef.current) {
|
||||
try {
|
||||
recognitionRef.current.stop();
|
||||
} catch {
|
||||
// onend still follows
|
||||
}
|
||||
return;
|
||||
}
|
||||
const rec = recorderRef.current;
|
||||
if (!rec) return;
|
||||
recorderRef.current = null;
|
||||
const blob = await rec.stop();
|
||||
if (cancelledRef.current) return;
|
||||
if (!blob) {
|
||||
setStateSafe('idle');
|
||||
onTextRef.current('', true);
|
||||
return;
|
||||
}
|
||||
setStateSafe('transcribing');
|
||||
try {
|
||||
const text = await api.transcribe(blob);
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(text, true);
|
||||
} catch (err: any) {
|
||||
fail(err?.message || 'Spracherkennung fehlgeschlagen');
|
||||
}
|
||||
// fail only touches refs and state setters
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
const startServer = async () => {
|
||||
const rec = new VoiceRecorder();
|
||||
recorderRef.current = rec;
|
||||
setStateSafe('listening');
|
||||
try {
|
||||
await rec.start();
|
||||
} catch {
|
||||
recorderRef.current = null;
|
||||
fail('Kein Zugriff auf das Mikrofon (Berechtigung, oder die Seite läuft nicht über HTTPS)');
|
||||
return;
|
||||
}
|
||||
if (cancelledRef.current || recorderRef.current !== rec) {
|
||||
rec.cancel();
|
||||
return;
|
||||
}
|
||||
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
|
||||
};
|
||||
|
||||
const startBrowser = () => {
|
||||
const Recognition = browserRecognition();
|
||||
if (!Recognition) {
|
||||
fail('Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
return;
|
||||
}
|
||||
const recognition = new Recognition();
|
||||
recognition.lang = 'de-DE';
|
||||
recognition.interimResults = true;
|
||||
recognition.continuous = continuous;
|
||||
recognition.onresult = (event: any) => {
|
||||
let combined = '';
|
||||
for (let i = 0; i < event.results.length; i++) combined += event.results[i][0].transcript;
|
||||
transcriptRef.current = combined;
|
||||
onTextRef.current(combined, false);
|
||||
};
|
||||
recognition.onerror = (event: any) => {
|
||||
if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') {
|
||||
cancelledRef.current = true;
|
||||
recognitionRef.current = null;
|
||||
fail('Mikrofon nicht freigegeben');
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
clearTimer();
|
||||
recognitionRef.current = null;
|
||||
if (cancelledRef.current) return;
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(transcriptRef.current, true);
|
||||
};
|
||||
recognitionRef.current = recognition;
|
||||
setStateSafe('listening');
|
||||
try {
|
||||
recognition.start();
|
||||
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
|
||||
} catch {
|
||||
recognitionRef.current = null;
|
||||
fail('Spracheingabe konnte nicht gestartet werden');
|
||||
}
|
||||
};
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recorderRef.current || recognitionRef.current) return;
|
||||
cancelledRef.current = false;
|
||||
transcriptRef.current = '';
|
||||
const current = mode ?? currentVoiceConfig().mode;
|
||||
if (current === 'server') await startServer();
|
||||
else if (current === 'browser') startBrowser();
|
||||
else fail(current === 'off' ? 'Spracheingabe ist ausgeschaltet' : 'Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [mode, continuous]);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
clearTimer();
|
||||
recorderRef.current?.cancel();
|
||||
recorderRef.current = null;
|
||||
try {
|
||||
recognitionRef.current?.abort?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
recognitionRef.current = null;
|
||||
setStateSafe('idle');
|
||||
}, []);
|
||||
|
||||
const toggle = useCallback(() => {
|
||||
if (state === 'listening') void stop();
|
||||
else if (state === 'idle') void start();
|
||||
}, [state, start, stop]);
|
||||
|
||||
/** Current input level 0..1 for waves (server mode), null when unknown. */
|
||||
const level = useCallback(() => recorderRef.current?.level() ?? null, []);
|
||||
|
||||
return {
|
||||
mode,
|
||||
available: mode === 'server' || mode === 'browser',
|
||||
state,
|
||||
start,
|
||||
stop,
|
||||
cancel,
|
||||
toggle,
|
||||
level,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Voice input for all places with a microphone button. The server decides how it works (STT_* in the
|
||||
// environment, reported by /auth/status):
|
||||
// server record with MediaRecorder, the own Whisper server turns it into text after stopping
|
||||
// browser the browser's speech recognition (Chrome, Edge, Safari; text appears while speaking)
|
||||
// off no voice input
|
||||
import { api } from './api';
|
||||
import type { VoiceMode } from './types';
|
||||
|
||||
export type EffectiveVoiceMode = VoiceMode | 'unsupported';
|
||||
|
||||
export interface VoiceConfig {
|
||||
mode: EffectiveVoiceMode;
|
||||
maxSeconds: number;
|
||||
}
|
||||
|
||||
let configPromise: Promise<VoiceConfig> | null = null;
|
||||
let loadedConfig: VoiceConfig | null = null;
|
||||
|
||||
export function browserRecognition(): any {
|
||||
if (typeof window === 'undefined') return null;
|
||||
return (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition || null;
|
||||
}
|
||||
|
||||
export function canRecord(): boolean {
|
||||
return typeof window !== 'undefined' && typeof (window as any).MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia;
|
||||
}
|
||||
|
||||
/** What this browser can actually do with the server's setting. */
|
||||
export function effectiveMode(mode: VoiceMode | undefined): EffectiveVoiceMode {
|
||||
if (mode === 'off') return 'off';
|
||||
if (mode === 'server') return canRecord() ? 'server' : 'unsupported';
|
||||
return browserRecognition() ? 'browser' : 'unsupported';
|
||||
}
|
||||
|
||||
/** Loaded once per page; the setting only changes with a server restart. */
|
||||
export function loadVoiceConfig(): Promise<VoiceConfig> {
|
||||
if (!configPromise) {
|
||||
configPromise = api
|
||||
.getAuthStatus()
|
||||
.then((st) => {
|
||||
loadedConfig = { mode: effectiveMode(st.voice?.mode), maxSeconds: st.voice?.maxSeconds || 600 };
|
||||
return loadedConfig;
|
||||
})
|
||||
.catch(() => {
|
||||
configPromise = null; // offline at start: ask again next time
|
||||
return { mode: effectiveMode('browser'), maxSeconds: 600 };
|
||||
});
|
||||
}
|
||||
return configPromise;
|
||||
}
|
||||
|
||||
/** The setting if already loaded; until then the browser's recognition (a button must react at once). */
|
||||
export function currentVoiceConfig(): VoiceConfig {
|
||||
return loadedConfig ?? { mode: effectiveMode('browser'), maxSeconds: 600 };
|
||||
}
|
||||
|
||||
/** For tests. */
|
||||
export function resetVoiceConfig(): void {
|
||||
configPromise = null;
|
||||
loadedConfig = null;
|
||||
}
|
||||
|
||||
// Containers in order of preference; Safari only records MP4/AAC
|
||||
const RECORDING_TYPES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus'];
|
||||
|
||||
function recordingType(): string | undefined {
|
||||
const MR = (window as any).MediaRecorder;
|
||||
if (!MR?.isTypeSupported) return undefined;
|
||||
return RECORDING_TYPES.find((t) => MR.isTypeSupported(t));
|
||||
}
|
||||
|
||||
/**
|
||||
* One recording from the microphone, with a live input level for wave animations. stop() resolves
|
||||
* with the audio; cancel() throws it away.
|
||||
*/
|
||||
export class VoiceRecorder {
|
||||
private stream: MediaStream | null = null;
|
||||
private recorder: MediaRecorder | null = null;
|
||||
private chunks: Blob[] = [];
|
||||
private audioCtx: AudioContext | null = null;
|
||||
private analyser: AnalyserNode | null = null;
|
||||
private samples: Uint8Array | null = null;
|
||||
private done: ((blob: Blob | null) => void) | null = null;
|
||||
|
||||
async start(): Promise<void> {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
const type = recordingType();
|
||||
this.recorder = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
|
||||
this.chunks = [];
|
||||
this.recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) this.chunks.push(e.data);
|
||||
};
|
||||
this.recorder.onstop = () => {
|
||||
const blob = this.chunks.length ? new Blob(this.chunks, { type: this.recorder?.mimeType || type || 'audio/webm' }) : null;
|
||||
this.release();
|
||||
this.done?.(blob);
|
||||
this.done = null;
|
||||
};
|
||||
this.recorder.start(1000);
|
||||
try {
|
||||
const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext;
|
||||
this.audioCtx = new Ctx();
|
||||
const source = this.audioCtx!.createMediaStreamSource(this.stream);
|
||||
this.analyser = this.audioCtx!.createAnalyser();
|
||||
this.analyser.fftSize = 512;
|
||||
source.connect(this.analyser);
|
||||
this.samples = new Uint8Array(this.analyser.fftSize);
|
||||
void this.audioCtx!.resume?.();
|
||||
} catch {
|
||||
this.analyser = null; // no level, the recording still works
|
||||
}
|
||||
}
|
||||
|
||||
/** Input level 0..1 (roughly loudness), or null when unknown. */
|
||||
level(): number | null {
|
||||
if (!this.analyser || !this.samples) return null;
|
||||
this.analyser.getByteTimeDomainData(this.samples as Uint8Array<ArrayBuffer>);
|
||||
let sum = 0;
|
||||
for (const v of this.samples) {
|
||||
const x = (v - 128) / 128;
|
||||
sum += x * x;
|
||||
}
|
||||
return Math.min(1, Math.sqrt(sum / this.samples.length) * 4);
|
||||
}
|
||||
|
||||
stop(): Promise<Blob | null> {
|
||||
return new Promise((resolve) => {
|
||||
if (!this.recorder || this.recorder.state === 'inactive') {
|
||||
this.release();
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
this.done = resolve;
|
||||
this.recorder.stop();
|
||||
});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.done = null;
|
||||
if (this.recorder && this.recorder.state !== 'inactive') {
|
||||
this.recorder.onstop = () => this.release();
|
||||
this.recorder.stop();
|
||||
} else {
|
||||
this.release();
|
||||
}
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
this.stream?.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
void this.audioCtx?.close?.().catch(() => {});
|
||||
this.audioCtx = null;
|
||||
this.analyser = null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user