feat(voice): live dictation with browser pause restart and server audio chunking
This commit is contained in:
1 parent
446f8e4649
commit
6795edbf4b
5 files changed
+415
-77
No files matched your search
@@ -9,16 +9,24 @@ interface DictationBlockProps {
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
function formatDuration(sec: number): string {
|
||||
const m = String(Math.floor(sec / 60)).padStart(2, '0');
|
||||
const s = String(sec % 60).padStart(2, '0');
|
||||
return `${m}:${s}`;
|
||||
}
|
||||
|
||||
// What an empty bullet turns into after /diktat: listening starts right away, the whole bullet is one
|
||||
// button with the live wave and a translucent "Stopp" on top. A tap ends it; the text lands in the
|
||||
// bullet as plain text.
|
||||
// button with live text growing above the wave, mm:ss timer and a translucent "Stopp" on top. A tap ends
|
||||
// it; the text lands in the bullet as plain text.
|
||||
export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel }) => {
|
||||
const [interim, setInterim] = useState('');
|
||||
const [elapsedSec, setElapsedSec] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const doneRef = useRef(false);
|
||||
const buttonRef = useRef<HTMLButtonElement>(null);
|
||||
|
||||
const voice = useVoiceInput({
|
||||
live: true,
|
||||
continuous: true,
|
||||
onText: (text, final) => {
|
||||
if (!final) {
|
||||
@@ -45,11 +53,17 @@ export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel
|
||||
triggerHaptic([15, 60, 25]);
|
||||
void voice.start();
|
||||
buttonRef.current?.focus();
|
||||
const started = Date.now();
|
||||
const timer = window.setInterval(() => {
|
||||
setElapsedSec(Math.floor((Date.now() - started) / 1000));
|
||||
}, 500);
|
||||
return () => window.clearInterval(timer);
|
||||
// start once when the bullet turns into the dictation
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
const transcribing = voice.state === 'transcribing';
|
||||
const chunkBusy = voice.pendingChunks > 0 || transcribing;
|
||||
|
||||
return (
|
||||
<button
|
||||
@@ -71,27 +85,35 @@ export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel
|
||||
}}
|
||||
disabled={transcribing || !!error}
|
||||
aria-label="Diktat beenden"
|
||||
className="relative w-full min-h-[56px] sm:min-h-[44px] my-1 rounded-card border border-accent bg-accent-dim overflow-hidden flex items-center justify-center cursor-pointer disabled:cursor-default focus:outline-none"
|
||||
className="relative w-full min-h-[56px] sm:min-h-[44px] my-1 px-3 py-2 rounded-card border border-accent bg-accent-dim overflow-hidden flex flex-col gap-1.5 text-left cursor-pointer disabled:cursor-default focus:outline-none"
|
||||
>
|
||||
{/* Live wave across the whole bullet */}
|
||||
<VoiceWave getLevel={voice.level} bars={32} className="absolute inset-x-3 inset-y-2 justify-between h-auto text-accent opacity-60" />
|
||||
{interim && !error && (
|
||||
<span className="absolute inset-x-3 bottom-1 text-[11px] text-main truncate text-left pointer-events-none">{interim}</span>
|
||||
{(interim || chunkBusy) && !error && (
|
||||
<div className="w-full text-sm text-main whitespace-pre-wrap break-words leading-relaxed pointer-events-none">
|
||||
{interim}
|
||||
{chunkBusy && <span className="text-muted animate-pulse">{interim ? ' …' : '…'}</span>}
|
||||
</div>
|
||||
)}
|
||||
{/* Floating stop label on top */}
|
||||
<span className="relative z-10 flex items-center gap-1.5 px-3 py-1 rounded-pill bg-panel/70 text-main text-xs font-semibold backdrop-blur-sm">
|
||||
{error ? (
|
||||
<span className="text-status-err font-normal">{error}</span>
|
||||
) : transcribing ? (
|
||||
<>
|
||||
<Loader2 className="w-3.5 h-3.5 animate-spin" /> Wird erkannt…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Square className="w-3 h-3 fill-current" /> Stopp
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
<div className="relative w-full min-h-[32px] flex items-center justify-between gap-2">
|
||||
{/* Live wave across the bottom bar */}
|
||||
<VoiceWave getLevel={voice.level} bars={32} className="absolute inset-x-0 inset-y-0 justify-between h-auto text-accent opacity-60" />
|
||||
<span className="relative z-10 px-2 py-0.5 rounded-pill bg-panel/70 text-muted text-[11px] font-mono tabular-nums backdrop-blur-sm">
|
||||
{formatDuration(elapsedSec)}
|
||||
</span>
|
||||
{/* Floating stop label on top */}
|
||||
<span className="relative z-10 flex items-center gap-1.5 px-3 py-1 rounded-pill bg-panel/70 text-main text-xs font-semibold backdrop-blur-sm">
|
||||
{error ? (
|
||||
<span className="text-status-err font-normal">{error}</span>
|
||||
) : transcribing ? (
|
||||
<>
|
||||
<Loader2 className="w-3.5 h-3.5 animate-spin" /> Wird erkannt…
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Square className="w-3 h-3 fill-current" /> Stopp
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
};
|
||||
@@ -130,4 +130,121 @@ describe('useVoiceInput', () => {
|
||||
await waitFor(() => expect(none.result.current.mode).toBe('unsupported'));
|
||||
expect(none.result.current.available).toBe(false);
|
||||
});
|
||||
|
||||
it('browser mode with live=true: restarts on pause (onend), keeps base text, rate-limits restarts to 300ms, and sends final=true only on stop()', async () => {
|
||||
const instances: any[] = [];
|
||||
class FakeSpeechRecognition {
|
||||
lang = '';
|
||||
interimResults = false;
|
||||
continuous = false;
|
||||
onresult: ((e: any) => void) | null = null;
|
||||
onerror: ((e: any) => void) | null = null;
|
||||
onend: (() => void) | null = null;
|
||||
constructor() {
|
||||
instances.push(this);
|
||||
}
|
||||
start() {}
|
||||
stop() {
|
||||
this.onend?.();
|
||||
}
|
||||
abort() {}
|
||||
}
|
||||
(window as any).SpeechRecognition = FakeSpeechRecognition;
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'browser', maxSeconds: 600 } });
|
||||
|
||||
const onText = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText, live: true }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('browser'));
|
||||
|
||||
vi.useFakeTimers();
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
expect(instances).toHaveLength(1);
|
||||
|
||||
// First utterance before a pause
|
||||
act(() => {
|
||||
instances[0].onresult?.({ results: [[{ transcript: 'Erster Satz.' }]] });
|
||||
});
|
||||
expect(onText).toHaveBeenLastCalledWith('Erster Satz.', false);
|
||||
|
||||
// Browser ends recognition on speech pause -> immediate restart (#2)
|
||||
act(() => {
|
||||
instances[0].onend?.();
|
||||
});
|
||||
expect(instances).toHaveLength(2);
|
||||
expect(onText).not.toHaveBeenCalledWith(expect.anything(), true);
|
||||
|
||||
// Another immediate onend within <300ms waits until 300ms have passed
|
||||
act(() => {
|
||||
instances[1].onend?.();
|
||||
});
|
||||
expect(instances).toHaveLength(2);
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(300);
|
||||
});
|
||||
expect(instances).toHaveLength(3);
|
||||
|
||||
// Second utterance appends to base text
|
||||
act(() => {
|
||||
instances[2].onresult?.({ results: [[{ transcript: 'Zweiter Satz.' }]] });
|
||||
});
|
||||
expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', false);
|
||||
|
||||
// Explicit stop() finishes with final=true
|
||||
await act(async () => {
|
||||
await result.current.stop();
|
||||
});
|
||||
expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', true);
|
||||
expect(result.current.state).toBe('idle');
|
||||
});
|
||||
|
||||
it('server mode with live=true: cuts segments onto new MediaRecorders on the same stream, orders out-of-order responses by chunk index, and releases stream at the end', async () => {
|
||||
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } });
|
||||
|
||||
const resolvers: Array<(text: string) => void> = [];
|
||||
transcribe.mockImplementation(
|
||||
() =>
|
||||
new Promise<string>((resolve) => {
|
||||
resolvers.push(resolve);
|
||||
})
|
||||
);
|
||||
|
||||
const onText = vi.fn();
|
||||
const { result } = renderHook(() => useVoiceInput({ onText, live: true }));
|
||||
await waitFor(() => expect(result.current.mode).toBe('server'));
|
||||
|
||||
vi.useFakeTimers();
|
||||
await act(async () => {
|
||||
await result.current.start();
|
||||
});
|
||||
|
||||
// Advance 8s -> segment 0 is cut and sent to transcribe, stream is NOT released yet
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(8000);
|
||||
});
|
||||
expect(transcribe).toHaveBeenCalledTimes(1);
|
||||
expect(stopTrack).not.toHaveBeenCalled();
|
||||
|
||||
// Advance 4s and stop -> segment 1 (final) is sent to transcribe, stream IS released now
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(4000);
|
||||
await result.current.stop();
|
||||
});
|
||||
expect(transcribe).toHaveBeenCalledTimes(2);
|
||||
expect(stopTrack).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Resolve chunk 1 BEFORE chunk 0 to verify index-based ordering
|
||||
await act(async () => {
|
||||
resolvers[1]('zweiter Teil');
|
||||
});
|
||||
expect(onText).toHaveBeenLastCalledWith('zweiter Teil', false);
|
||||
|
||||
// Now resolve chunk 0 -> both chunks are done, ordered "Erster Teil zweiter Teil" with final=true
|
||||
await act(async () => {
|
||||
resolvers[0]('Erster Teil');
|
||||
});
|
||||
expect(onText).toHaveBeenLastCalledWith('Erster Teil zweiter Teil', true);
|
||||
expect(result.current.state).toBe('idle');
|
||||
});
|
||||
});
|
||||
@@ -10,28 +10,54 @@ export interface VoiceInputOptions {
|
||||
onError?: (message: string) => void;
|
||||
/** Browser mode: keep listening through pauses (hold-to-record) instead of stopping after a sentence. */
|
||||
continuous?: boolean;
|
||||
/** Live dictation mode (/diktat): auto-restart browser speech recognition on pause, and chunk server recordings every ~8s or ~0.8s of silence. */
|
||||
live?: boolean;
|
||||
}
|
||||
|
||||
const LIVE_CHUNK_MS = 8000;
|
||||
const LIVE_SILENCE_MS = 800;
|
||||
const LIVE_TICK_MS = 200;
|
||||
const LIVE_SPEECH_LEVEL = 0.02;
|
||||
const MIN_RESTART_INTERVAL_MS = 300;
|
||||
|
||||
/**
|
||||
* Microphone input for one button: start, stop (text arrives), cancel (nothing arrives). How the text
|
||||
* is made depends on the server setting, see lib/voice.ts. mode is null until known, "off" or
|
||||
* "unsupported" means: do not show a microphone.
|
||||
*/
|
||||
export function useVoiceInput({ onText, onError, continuous = false }: VoiceInputOptions) {
|
||||
export function useVoiceInput({ onText, onError, continuous = false, live = false }: VoiceInputOptions) {
|
||||
const [mode, setMode] = useState<EffectiveVoiceMode | null>(null);
|
||||
const [state, setState] = useState<VoiceState>('idle');
|
||||
const [pendingChunks, setPendingChunks] = useState(0);
|
||||
const maxSecondsRef = useRef(600);
|
||||
const recorderRef = useRef<VoiceRecorder | null>(null);
|
||||
const recognitionRef = useRef<any>(null);
|
||||
const timerRef = useRef<number | null>(null);
|
||||
const segmentTimerRef = useRef<number | null>(null);
|
||||
const restartTimerRef = useRef<number | null>(null);
|
||||
const cancelledRef = useRef(false);
|
||||
const stoppedRef = useRef(false);
|
||||
const transcriptRef = useRef('');
|
||||
const baseTranscriptRef = useRef('');
|
||||
const lastRestartAtRef = useRef(0);
|
||||
const chunkIndexRef = useRef(0);
|
||||
const chunkResultsRef = useRef<Map<number, string>>(new Map());
|
||||
const inFlightChunksRef = useRef(0);
|
||||
const mountedRef = useRef(true);
|
||||
const onTextRef = useRef(onText);
|
||||
onTextRef.current = onText;
|
||||
const onErrorRef = useRef(onError);
|
||||
onErrorRef.current = onError;
|
||||
|
||||
const clearAllTimers = () => {
|
||||
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
|
||||
timerRef.current = null;
|
||||
if (segmentTimerRef.current !== null) window.clearInterval(segmentTimerRef.current);
|
||||
segmentTimerRef.current = null;
|
||||
if (restartTimerRef.current !== null) window.clearTimeout(restartTimerRef.current);
|
||||
restartTimerRef.current = null;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
mountedRef.current = true;
|
||||
void loadVoiceConfig().then((cfg) => {
|
||||
@@ -41,7 +67,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
});
|
||||
return () => {
|
||||
mountedRef.current = false;
|
||||
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
|
||||
clearAllTimers();
|
||||
recorderRef.current?.cancel();
|
||||
try {
|
||||
recognitionRef.current?.abort?.();
|
||||
@@ -54,17 +80,65 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
const setStateSafe = (s: VoiceState) => {
|
||||
if (mountedRef.current) setState(s);
|
||||
};
|
||||
const setPendingChunksSafe = (n: number) => {
|
||||
if (mountedRef.current) setPendingChunks(n);
|
||||
};
|
||||
const fail = (message: string) => {
|
||||
setStateSafe('idle');
|
||||
setPendingChunksSafe(0);
|
||||
onErrorRef.current?.(message);
|
||||
};
|
||||
const clearTimer = () => {
|
||||
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
|
||||
timerRef.current = null;
|
||||
|
||||
const joinedChunkText = () =>
|
||||
[...chunkResultsRef.current.entries()]
|
||||
.sort((a, b) => a[0] - b[0])
|
||||
.map(([, t]) => t.trim())
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
|
||||
const processSegmentPromise = (idx: number, blobPromise: Promise<Blob | null>) => {
|
||||
inFlightChunksRef.current += 1;
|
||||
setPendingChunksSafe(inFlightChunksRef.current);
|
||||
void blobPromise
|
||||
.then(async (blob) => {
|
||||
if (cancelledRef.current) return;
|
||||
if (!blob || blob.size === 0) {
|
||||
inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1);
|
||||
setPendingChunksSafe(inFlightChunksRef.current);
|
||||
if (stoppedRef.current && inFlightChunksRef.current === 0) {
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(joinedChunkText(), true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const text = await api.transcribe(blob);
|
||||
if (cancelledRef.current) return;
|
||||
chunkResultsRef.current.set(idx, text || '');
|
||||
inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1);
|
||||
setPendingChunksSafe(inFlightChunksRef.current);
|
||||
const combined = joinedChunkText();
|
||||
transcriptRef.current = combined;
|
||||
if (stoppedRef.current && inFlightChunksRef.current === 0) {
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(combined, true);
|
||||
} else {
|
||||
onTextRef.current(combined, false);
|
||||
}
|
||||
})
|
||||
.catch((err: any) => {
|
||||
if (cancelledRef.current) return;
|
||||
cancelledRef.current = true;
|
||||
inFlightChunksRef.current = 0;
|
||||
clearAllTimers();
|
||||
recorderRef.current?.cancel();
|
||||
recorderRef.current = null;
|
||||
fail(err?.message || 'Spracherkennung fehlgeschlagen');
|
||||
});
|
||||
};
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
clearTimer();
|
||||
stoppedRef.current = true;
|
||||
clearAllTimers();
|
||||
if (recognitionRef.current) {
|
||||
try {
|
||||
recognitionRef.current.stop();
|
||||
@@ -74,8 +148,20 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
return;
|
||||
}
|
||||
const rec = recorderRef.current;
|
||||
if (!rec) return;
|
||||
if (!rec) {
|
||||
if (live && state === 'listening') {
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(transcriptRef.current, true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
recorderRef.current = null;
|
||||
if (live) {
|
||||
const idx = chunkIndexRef.current++;
|
||||
setStateSafe('transcribing');
|
||||
processSegmentPromise(idx, rec.stop());
|
||||
return;
|
||||
}
|
||||
const blob = await rec.stop();
|
||||
if (cancelledRef.current) return;
|
||||
if (!blob) {
|
||||
@@ -93,11 +179,15 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
}
|
||||
// fail only touches refs and state setters
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
}, [live, state]);
|
||||
|
||||
const startServer = async () => {
|
||||
const rec = new VoiceRecorder();
|
||||
recorderRef.current = rec;
|
||||
chunkIndexRef.current = 0;
|
||||
chunkResultsRef.current = new Map();
|
||||
inFlightChunksRef.current = 0;
|
||||
setPendingChunksSafe(0);
|
||||
setStateSafe('listening');
|
||||
try {
|
||||
await rec.start();
|
||||
@@ -111,6 +201,104 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
return;
|
||||
}
|
||||
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
|
||||
if (live) {
|
||||
let segmentMs = 0;
|
||||
let silentMs = 0;
|
||||
let hadSpeech = false;
|
||||
segmentTimerRef.current = window.setInterval(() => {
|
||||
if (cancelledRef.current || stoppedRef.current || recorderRef.current !== rec) return;
|
||||
const lvl = rec.level();
|
||||
if (lvl !== null) {
|
||||
if (lvl >= LIVE_SPEECH_LEVEL) {
|
||||
hadSpeech = true;
|
||||
silentMs = 0;
|
||||
} else if (hadSpeech) {
|
||||
silentMs += LIVE_TICK_MS;
|
||||
}
|
||||
}
|
||||
segmentMs += LIVE_TICK_MS;
|
||||
if (segmentMs >= LIVE_CHUNK_MS || (hadSpeech && silentMs >= LIVE_SILENCE_MS)) {
|
||||
segmentMs = 0;
|
||||
silentMs = 0;
|
||||
hadSpeech = false;
|
||||
const idx = chunkIndexRef.current++;
|
||||
processSegmentPromise(idx, rec.cutSegment());
|
||||
}
|
||||
}, LIVE_TICK_MS);
|
||||
}
|
||||
};
|
||||
|
||||
const spawnBrowserRecognition = (Recognition: any, isRestart: boolean) => {
|
||||
const recognition = new Recognition();
|
||||
recognition.lang = 'de-DE';
|
||||
recognition.interimResults = true;
|
||||
recognition.continuous = continuous || live;
|
||||
recognition.onresult = (event: any) => {
|
||||
let sessionText = '';
|
||||
for (let i = 0; i < event.results.length; i++) sessionText += event.results[i][0].transcript;
|
||||
if (live && baseTranscriptRef.current) {
|
||||
const base = baseTranscriptRef.current.trimEnd();
|
||||
const next = sessionText.trimStart();
|
||||
transcriptRef.current = next ? `${base} ${next}` : base;
|
||||
} else {
|
||||
transcriptRef.current = sessionText;
|
||||
}
|
||||
onTextRef.current(transcriptRef.current, false);
|
||||
};
|
||||
recognition.onerror = (event: any) => {
|
||||
if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') {
|
||||
cancelledRef.current = true;
|
||||
clearAllTimers();
|
||||
recognitionRef.current = null;
|
||||
fail('Mikrofon nicht freigegeben');
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
recognitionRef.current = null;
|
||||
if (cancelledRef.current) {
|
||||
clearAllTimers();
|
||||
return;
|
||||
}
|
||||
if (!live || stoppedRef.current) {
|
||||
clearAllTimers();
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(transcriptRef.current, true);
|
||||
return;
|
||||
}
|
||||
baseTranscriptRef.current = transcriptRef.current;
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastRestartAtRef.current;
|
||||
const doRestart = () => {
|
||||
if (cancelledRef.current || stoppedRef.current) return;
|
||||
lastRestartAtRef.current = Date.now();
|
||||
spawnBrowserRecognition(Recognition, true);
|
||||
};
|
||||
if (lastRestartAtRef.current === 0 || elapsed >= MIN_RESTART_INTERVAL_MS) {
|
||||
doRestart();
|
||||
} else {
|
||||
restartTimerRef.current = window.setTimeout(() => {
|
||||
restartTimerRef.current = null;
|
||||
doRestart();
|
||||
}, MIN_RESTART_INTERVAL_MS - elapsed);
|
||||
}
|
||||
};
|
||||
recognitionRef.current = recognition;
|
||||
if (!isRestart) setStateSafe('listening');
|
||||
try {
|
||||
recognition.start();
|
||||
if (!isRestart) {
|
||||
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
|
||||
}
|
||||
} catch {
|
||||
recognitionRef.current = null;
|
||||
if (!isRestart) {
|
||||
fail('Spracheingabe konnte nicht gestartet werden');
|
||||
} else {
|
||||
clearAllTimers();
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(transcriptRef.current, true);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const startBrowser = () => {
|
||||
@@ -119,55 +307,31 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
fail('Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
return;
|
||||
}
|
||||
const recognition = new Recognition();
|
||||
recognition.lang = 'de-DE';
|
||||
recognition.interimResults = true;
|
||||
recognition.continuous = continuous;
|
||||
recognition.onresult = (event: any) => {
|
||||
let combined = '';
|
||||
for (let i = 0; i < event.results.length; i++) combined += event.results[i][0].transcript;
|
||||
transcriptRef.current = combined;
|
||||
onTextRef.current(combined, false);
|
||||
};
|
||||
recognition.onerror = (event: any) => {
|
||||
if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') {
|
||||
cancelledRef.current = true;
|
||||
recognitionRef.current = null;
|
||||
fail('Mikrofon nicht freigegeben');
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
clearTimer();
|
||||
recognitionRef.current = null;
|
||||
if (cancelledRef.current) return;
|
||||
setStateSafe('idle');
|
||||
onTextRef.current(transcriptRef.current, true);
|
||||
};
|
||||
recognitionRef.current = recognition;
|
||||
setStateSafe('listening');
|
||||
try {
|
||||
recognition.start();
|
||||
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
|
||||
} catch {
|
||||
recognitionRef.current = null;
|
||||
fail('Spracheingabe konnte nicht gestartet werden');
|
||||
}
|
||||
baseTranscriptRef.current = '';
|
||||
lastRestartAtRef.current = 0;
|
||||
spawnBrowserRecognition(Recognition, false);
|
||||
};
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recorderRef.current || recognitionRef.current) return;
|
||||
cancelledRef.current = false;
|
||||
stoppedRef.current = false;
|
||||
transcriptRef.current = '';
|
||||
baseTranscriptRef.current = '';
|
||||
lastRestartAtRef.current = 0;
|
||||
const current = mode ?? currentVoiceConfig().mode;
|
||||
if (current === 'server') await startServer();
|
||||
else if (current === 'browser') startBrowser();
|
||||
else fail(current === 'off' ? 'Spracheingabe ist ausgeschaltet' : 'Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [mode, continuous]);
|
||||
}, [mode, continuous, live]);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
clearTimer();
|
||||
stoppedRef.current = true;
|
||||
inFlightChunksRef.current = 0;
|
||||
setPendingChunksSafe(0);
|
||||
clearAllTimers();
|
||||
recorderRef.current?.cancel();
|
||||
recorderRef.current = null;
|
||||
try {
|
||||
@@ -191,6 +355,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
|
||||
mode,
|
||||
available: mode === 'server' || mode === 'browser',
|
||||
state,
|
||||
pendingChunks,
|
||||
start,
|
||||
stop,
|
||||
cancel,
|
||||
|
||||
+44
-13
@@ -86,19 +86,7 @@ export class VoiceRecorder {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
const type = recordingType();
|
||||
this.recorder = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
|
||||
this.chunks = [];
|
||||
this.recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) this.chunks.push(e.data);
|
||||
};
|
||||
this.recorder.onstop = () => {
|
||||
const blob = this.chunks.length ? new Blob(this.chunks, { type: this.recorder?.mimeType || type || 'audio/webm' }) : null;
|
||||
this.release();
|
||||
this.done?.(blob);
|
||||
this.done = null;
|
||||
};
|
||||
this.recorder.start(1000);
|
||||
this.startSegmentRecorder();
|
||||
try {
|
||||
const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext;
|
||||
this.audioCtx = new Ctx();
|
||||
@@ -113,6 +101,25 @@ export class VoiceRecorder {
|
||||
}
|
||||
}
|
||||
|
||||
private startSegmentRecorder(): void {
|
||||
if (!this.stream) return;
|
||||
const type = recordingType();
|
||||
const rec = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
|
||||
const chunks: Blob[] = [];
|
||||
rec.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunks.push(e.data);
|
||||
};
|
||||
rec.onstop = () => {
|
||||
const blob = chunks.length ? new Blob(chunks, { type: rec.mimeType || type || 'audio/webm' }) : null;
|
||||
this.release();
|
||||
this.done?.(blob);
|
||||
this.done = null;
|
||||
};
|
||||
this.recorder = rec;
|
||||
this.chunks = chunks;
|
||||
rec.start(1000);
|
||||
}
|
||||
|
||||
/** Input level 0..1 (roughly loudness), or null when unknown. */
|
||||
level(): number | null {
|
||||
if (!this.analyser || !this.samples) return null;
|
||||
@@ -137,6 +144,30 @@ export class VoiceRecorder {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the current MediaRecorder segment and immediately starts a new one on the same MediaStream.
|
||||
* Resolves with the finished segment's audio Blob while recording continues seamlessly.
|
||||
*/
|
||||
cutSegment(): Promise<Blob | null> {
|
||||
if (!this.stream || !this.recorder || this.recorder.state === 'inactive') {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
const oldRecorder = this.recorder;
|
||||
const oldChunks = this.chunks;
|
||||
|
||||
this.startSegmentRecorder();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
oldRecorder.onstop = () => {
|
||||
const blob = oldChunks.length
|
||||
? new Blob(oldChunks, { type: oldRecorder.mimeType || 'audio/webm' })
|
||||
: null;
|
||||
resolve(blob);
|
||||
};
|
||||
oldRecorder.stop();
|
||||
});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.done = null;
|
||||
if (this.recorder && this.recorder.state !== 'inactive') {
|
||||
|
||||
Reference in new issue
Block a user