feat(voice): live dictation with browser pause restart and server audio chunking

This commit is contained in:
sandrodach committed 2026-10-30 09:27:30 +01:00
1 parent 446f8e4649
commit 6795edbf4b
5 files changed
+415 -77

No files matched your search

+3
View File
@@ -6,6 +6,9 @@ Das Format folgt [Keep a Changelog](https://keepachangelog.com/de/1.1.0/), die V
## [Unreleased]
### Changed
- `/diktat` in Notizen: schreibt live mit (mehrzeiliger Text über den Wellen mit Zeitanzeige `mm:ss`), bricht im Browser-Modus bei Sprechpausen nicht mehr ab und schickt im Server-Modus alle ~8 Sekunden bzw. nach kurzen Sprechpausen Abschnitte zur Erkennung vorab.
### Fixed
- Notizen: Link-Vorschau-Karten erscheinen nur noch, wenn der gesamte Punkt aus einer einzigen `https`-Adresse besteht (oder über `/link` in einem leeren Punkt eingefügt wurde), nicht mehr bei Links mitten im Fließtext.
- Notizen: Das Slash-Menü (`/`) öffnet sich nur noch am Textanfang oder nach einem Leerzeichen (nicht mehr beim Einfügen von Web-Adressen mit Schrägstrichen), schließt zuverlässig mit `Esc` oder `Backspace` und scrollt bei Pfeiltasten-Navigation automatisch mit.
@@ -9,16 +9,24 @@ interface DictationBlockProps {
onCancel: () => void;
}
function formatDuration(sec: number): string {
const m = String(Math.floor(sec / 60)).padStart(2, '0');
const s = String(sec % 60).padStart(2, '0');
return `${m}:${s}`;
}
// What an empty bullet turns into after /diktat: listening starts right away, the whole bullet is one
// button with the live wave and a translucent "Stopp" on top. A tap ends it; the text lands in the
// bullet as plain text.
// button with live text growing above the wave, mm:ss timer and a translucent "Stopp" on top. A tap ends
// it; the text lands in the bullet as plain text.
export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel }) => {
const [interim, setInterim] = useState('');
const [elapsedSec, setElapsedSec] = useState(0);
const [error, setError] = useState<string | null>(null);
const doneRef = useRef(false);
const buttonRef = useRef<HTMLButtonElement>(null);
const voice = useVoiceInput({
live: true,
continuous: true,
onText: (text, final) => {
if (!final) {
@@ -45,11 +53,17 @@ export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel
triggerHaptic([15, 60, 25]);
void voice.start();
buttonRef.current?.focus();
const started = Date.now();
const timer = window.setInterval(() => {
setElapsedSec(Math.floor((Date.now() - started) / 1000));
}, 500);
return () => window.clearInterval(timer);
// start once when the bullet turns into the dictation
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const transcribing = voice.state === 'transcribing';
const chunkBusy = voice.pendingChunks > 0 || transcribing;
return (
<button
@@ -71,27 +85,35 @@ export const DictationBlock: React.FC<DictationBlockProps> = ({ onDone, onCancel
}}
disabled={transcribing || !!error}
aria-label="Diktat beenden"
className="relative w-full min-h-[56px] sm:min-h-[44px] my-1 rounded-card border border-accent bg-accent-dim overflow-hidden flex items-center justify-center cursor-pointer disabled:cursor-default focus:outline-none"
className="relative w-full min-h-[56px] sm:min-h-[44px] my-1 px-3 py-2 rounded-card border border-accent bg-accent-dim overflow-hidden flex flex-col gap-1.5 text-left cursor-pointer disabled:cursor-default focus:outline-none"
>
{/* Live wave across the whole bullet */}
<VoiceWave getLevel={voice.level} bars={32} className="absolute inset-x-3 inset-y-2 justify-between h-auto text-accent opacity-60" />
{interim && !error && (
<span className="absolute inset-x-3 bottom-1 text-[11px] text-main truncate text-left pointer-events-none">{interim}</span>
{(interim || chunkBusy) && !error && (
<div className="w-full text-sm text-main whitespace-pre-wrap break-words leading-relaxed pointer-events-none">
{interim}
{chunkBusy && <span className="text-muted animate-pulse">{interim ? ' …' : '…'}</span>}
</div>
)}
{/* Floating stop label on top */}
<span className="relative z-10 flex items-center gap-1.5 px-3 py-1 rounded-pill bg-panel/70 text-main text-xs font-semibold backdrop-blur-sm">
{error ? (
<span className="text-status-err font-normal">{error}</span>
) : transcribing ? (
<>
<Loader2 className="w-3.5 h-3.5 animate-spin" /> Wird erkannt…
</>
) : (
<>
<Square className="w-3 h-3 fill-current" /> Stopp
</>
)}
</span>
<div className="relative w-full min-h-[32px] flex items-center justify-between gap-2">
{/* Live wave across the bottom bar */}
<VoiceWave getLevel={voice.level} bars={32} className="absolute inset-x-0 inset-y-0 justify-between h-auto text-accent opacity-60" />
<span className="relative z-10 px-2 py-0.5 rounded-pill bg-panel/70 text-muted text-[11px] font-mono tabular-nums backdrop-blur-sm">
{formatDuration(elapsedSec)}
</span>
{/* Floating stop label on top */}
<span className="relative z-10 flex items-center gap-1.5 px-3 py-1 rounded-pill bg-panel/70 text-main text-xs font-semibold backdrop-blur-sm">
{error ? (
<span className="text-status-err font-normal">{error}</span>
) : transcribing ? (
<>
<Loader2 className="w-3.5 h-3.5 animate-spin" /> Wird erkannt…
</>
) : (
<>
<Square className="w-3 h-3 fill-current" /> Stopp
</>
)}
</span>
</div>
</button>
);
};
@@ -130,4 +130,121 @@ describe('useVoiceInput', () => {
await waitFor(() => expect(none.result.current.mode).toBe('unsupported'));
expect(none.result.current.available).toBe(false);
});
it('browser mode with live=true: restarts on pause (onend), keeps base text, rate-limits restarts to 300ms, and sends final=true only on stop()', async () => {
const instances: any[] = [];
class FakeSpeechRecognition {
lang = '';
interimResults = false;
continuous = false;
onresult: ((e: any) => void) | null = null;
onerror: ((e: any) => void) | null = null;
onend: (() => void) | null = null;
constructor() {
instances.push(this);
}
start() {}
stop() {
this.onend?.();
}
abort() {}
}
(window as any).SpeechRecognition = FakeSpeechRecognition;
getAuthStatus.mockResolvedValue({ voice: { mode: 'browser', maxSeconds: 600 } });
const onText = vi.fn();
const { result } = renderHook(() => useVoiceInput({ onText, live: true }));
await waitFor(() => expect(result.current.mode).toBe('browser'));
vi.useFakeTimers();
await act(async () => {
await result.current.start();
});
expect(instances).toHaveLength(1);
// First utterance before a pause
act(() => {
instances[0].onresult?.({ results: [[{ transcript: 'Erster Satz.' }]] });
});
expect(onText).toHaveBeenLastCalledWith('Erster Satz.', false);
// Browser ends recognition on speech pause -> immediate restart (#2)
act(() => {
instances[0].onend?.();
});
expect(instances).toHaveLength(2);
expect(onText).not.toHaveBeenCalledWith(expect.anything(), true);
// Another immediate onend within <300ms waits until 300ms have passed
act(() => {
instances[1].onend?.();
});
expect(instances).toHaveLength(2);
act(() => {
vi.advanceTimersByTime(300);
});
expect(instances).toHaveLength(3);
// Second utterance appends to base text
act(() => {
instances[2].onresult?.({ results: [[{ transcript: 'Zweiter Satz.' }]] });
});
expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', false);
// Explicit stop() finishes with final=true
await act(async () => {
await result.current.stop();
});
expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', true);
expect(result.current.state).toBe('idle');
});
it('server mode with live=true: cuts segments onto new MediaRecorders on the same stream, orders out-of-order responses by chunk index, and releases stream at the end', async () => {
getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } });
const resolvers: Array<(text: string) => void> = [];
transcribe.mockImplementation(
() =>
new Promise<string>((resolve) => {
resolvers.push(resolve);
})
);
const onText = vi.fn();
const { result } = renderHook(() => useVoiceInput({ onText, live: true }));
await waitFor(() => expect(result.current.mode).toBe('server'));
vi.useFakeTimers();
await act(async () => {
await result.current.start();
});
// Advance 8s -> segment 0 is cut and sent to transcribe, stream is NOT released yet
await act(async () => {
await vi.advanceTimersByTimeAsync(8000);
});
expect(transcribe).toHaveBeenCalledTimes(1);
expect(stopTrack).not.toHaveBeenCalled();
// Advance 4s and stop -> segment 1 (final) is sent to transcribe, stream IS released now
await act(async () => {
await vi.advanceTimersByTimeAsync(4000);
await result.current.stop();
});
expect(transcribe).toHaveBeenCalledTimes(2);
expect(stopTrack).toHaveBeenCalledTimes(1);
// Resolve chunk 1 BEFORE chunk 0 to verify index-based ordering
await act(async () => {
resolvers[1]('zweiter Teil');
});
expect(onText).toHaveBeenLastCalledWith('zweiter Teil', false);
// Now resolve chunk 0 -> both chunks are done, ordered "Erster Teil zweiter Teil" with final=true
await act(async () => {
resolvers[0]('Erster Teil');
});
expect(onText).toHaveBeenLastCalledWith('Erster Teil zweiter Teil', true);
expect(result.current.state).toBe('idle');
});
});
+208 -43
View File
@@ -10,28 +10,54 @@ export interface VoiceInputOptions {
onError?: (message: string) => void;
/** Browser mode: keep listening through pauses (hold-to-record) instead of stopping after a sentence. */
continuous?: boolean;
/** Live dictation mode (/diktat): auto-restart browser speech recognition on pause, and chunk server recordings every ~8s or ~0.8s of silence. */
live?: boolean;
}
const LIVE_CHUNK_MS = 8000;
const LIVE_SILENCE_MS = 800;
const LIVE_TICK_MS = 200;
const LIVE_SPEECH_LEVEL = 0.02;
const MIN_RESTART_INTERVAL_MS = 300;
/**
* Microphone input for one button: start, stop (text arrives), cancel (nothing arrives). How the text
* is made depends on the server setting, see lib/voice.ts. mode is null until known, "off" or
* "unsupported" means: do not show a microphone.
*/
export function useVoiceInput({ onText, onError, continuous = false }: VoiceInputOptions) {
export function useVoiceInput({ onText, onError, continuous = false, live = false }: VoiceInputOptions) {
const [mode, setMode] = useState<EffectiveVoiceMode | null>(null);
const [state, setState] = useState<VoiceState>('idle');
const [pendingChunks, setPendingChunks] = useState(0);
const maxSecondsRef = useRef(600);
const recorderRef = useRef<VoiceRecorder | null>(null);
const recognitionRef = useRef<any>(null);
const timerRef = useRef<number | null>(null);
const segmentTimerRef = useRef<number | null>(null);
const restartTimerRef = useRef<number | null>(null);
const cancelledRef = useRef(false);
const stoppedRef = useRef(false);
const transcriptRef = useRef('');
const baseTranscriptRef = useRef('');
const lastRestartAtRef = useRef(0);
const chunkIndexRef = useRef(0);
const chunkResultsRef = useRef<Map<number, string>>(new Map());
const inFlightChunksRef = useRef(0);
const mountedRef = useRef(true);
const onTextRef = useRef(onText);
onTextRef.current = onText;
const onErrorRef = useRef(onError);
onErrorRef.current = onError;
const clearAllTimers = () => {
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
timerRef.current = null;
if (segmentTimerRef.current !== null) window.clearInterval(segmentTimerRef.current);
segmentTimerRef.current = null;
if (restartTimerRef.current !== null) window.clearTimeout(restartTimerRef.current);
restartTimerRef.current = null;
};
useEffect(() => {
mountedRef.current = true;
void loadVoiceConfig().then((cfg) => {
@@ -41,7 +67,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
});
return () => {
mountedRef.current = false;
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
clearAllTimers();
recorderRef.current?.cancel();
try {
recognitionRef.current?.abort?.();
@@ -54,17 +80,65 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
const setStateSafe = (s: VoiceState) => {
if (mountedRef.current) setState(s);
};
const setPendingChunksSafe = (n: number) => {
if (mountedRef.current) setPendingChunks(n);
};
const fail = (message: string) => {
setStateSafe('idle');
setPendingChunksSafe(0);
onErrorRef.current?.(message);
};
const clearTimer = () => {
if (timerRef.current !== null) window.clearTimeout(timerRef.current);
timerRef.current = null;
const joinedChunkText = () =>
[...chunkResultsRef.current.entries()]
.sort((a, b) => a[0] - b[0])
.map(([, t]) => t.trim())
.filter(Boolean)
.join(' ');
const processSegmentPromise = (idx: number, blobPromise: Promise<Blob | null>) => {
inFlightChunksRef.current += 1;
setPendingChunksSafe(inFlightChunksRef.current);
void blobPromise
.then(async (blob) => {
if (cancelledRef.current) return;
if (!blob || blob.size === 0) {
inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1);
setPendingChunksSafe(inFlightChunksRef.current);
if (stoppedRef.current && inFlightChunksRef.current === 0) {
setStateSafe('idle');
onTextRef.current(joinedChunkText(), true);
}
return;
}
const text = await api.transcribe(blob);
if (cancelledRef.current) return;
chunkResultsRef.current.set(idx, text || '');
inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1);
setPendingChunksSafe(inFlightChunksRef.current);
const combined = joinedChunkText();
transcriptRef.current = combined;
if (stoppedRef.current && inFlightChunksRef.current === 0) {
setStateSafe('idle');
onTextRef.current(combined, true);
} else {
onTextRef.current(combined, false);
}
})
.catch((err: any) => {
if (cancelledRef.current) return;
cancelledRef.current = true;
inFlightChunksRef.current = 0;
clearAllTimers();
recorderRef.current?.cancel();
recorderRef.current = null;
fail(err?.message || 'Spracherkennung fehlgeschlagen');
});
};
const stop = useCallback(async () => {
clearTimer();
stoppedRef.current = true;
clearAllTimers();
if (recognitionRef.current) {
try {
recognitionRef.current.stop();
@@ -74,8 +148,20 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
return;
}
const rec = recorderRef.current;
if (!rec) return;
if (!rec) {
if (live && state === 'listening') {
setStateSafe('idle');
onTextRef.current(transcriptRef.current, true);
}
return;
}
recorderRef.current = null;
if (live) {
const idx = chunkIndexRef.current++;
setStateSafe('transcribing');
processSegmentPromise(idx, rec.stop());
return;
}
const blob = await rec.stop();
if (cancelledRef.current) return;
if (!blob) {
@@ -93,11 +179,15 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
}
// fail only touches refs and state setters
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
}, [live, state]);
const startServer = async () => {
const rec = new VoiceRecorder();
recorderRef.current = rec;
chunkIndexRef.current = 0;
chunkResultsRef.current = new Map();
inFlightChunksRef.current = 0;
setPendingChunksSafe(0);
setStateSafe('listening');
try {
await rec.start();
@@ -111,6 +201,104 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
return;
}
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
if (live) {
let segmentMs = 0;
let silentMs = 0;
let hadSpeech = false;
segmentTimerRef.current = window.setInterval(() => {
if (cancelledRef.current || stoppedRef.current || recorderRef.current !== rec) return;
const lvl = rec.level();
if (lvl !== null) {
if (lvl >= LIVE_SPEECH_LEVEL) {
hadSpeech = true;
silentMs = 0;
} else if (hadSpeech) {
silentMs += LIVE_TICK_MS;
}
}
segmentMs += LIVE_TICK_MS;
if (segmentMs >= LIVE_CHUNK_MS || (hadSpeech && silentMs >= LIVE_SILENCE_MS)) {
segmentMs = 0;
silentMs = 0;
hadSpeech = false;
const idx = chunkIndexRef.current++;
processSegmentPromise(idx, rec.cutSegment());
}
}, LIVE_TICK_MS);
}
};
const spawnBrowserRecognition = (Recognition: any, isRestart: boolean) => {
const recognition = new Recognition();
recognition.lang = 'de-DE';
recognition.interimResults = true;
recognition.continuous = continuous || live;
recognition.onresult = (event: any) => {
let sessionText = '';
for (let i = 0; i < event.results.length; i++) sessionText += event.results[i][0].transcript;
if (live && baseTranscriptRef.current) {
const base = baseTranscriptRef.current.trimEnd();
const next = sessionText.trimStart();
transcriptRef.current = next ? `${base} ${next}` : base;
} else {
transcriptRef.current = sessionText;
}
onTextRef.current(transcriptRef.current, false);
};
recognition.onerror = (event: any) => {
if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') {
cancelledRef.current = true;
clearAllTimers();
recognitionRef.current = null;
fail('Mikrofon nicht freigegeben');
}
};
recognition.onend = () => {
recognitionRef.current = null;
if (cancelledRef.current) {
clearAllTimers();
return;
}
if (!live || stoppedRef.current) {
clearAllTimers();
setStateSafe('idle');
onTextRef.current(transcriptRef.current, true);
return;
}
baseTranscriptRef.current = transcriptRef.current;
const now = Date.now();
const elapsed = now - lastRestartAtRef.current;
const doRestart = () => {
if (cancelledRef.current || stoppedRef.current) return;
lastRestartAtRef.current = Date.now();
spawnBrowserRecognition(Recognition, true);
};
if (lastRestartAtRef.current === 0 || elapsed >= MIN_RESTART_INTERVAL_MS) {
doRestart();
} else {
restartTimerRef.current = window.setTimeout(() => {
restartTimerRef.current = null;
doRestart();
}, MIN_RESTART_INTERVAL_MS - elapsed);
}
};
recognitionRef.current = recognition;
if (!isRestart) setStateSafe('listening');
try {
recognition.start();
if (!isRestart) {
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
}
} catch {
recognitionRef.current = null;
if (!isRestart) {
fail('Spracheingabe konnte nicht gestartet werden');
} else {
clearAllTimers();
setStateSafe('idle');
onTextRef.current(transcriptRef.current, true);
}
}
};
const startBrowser = () => {
@@ -119,55 +307,31 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
fail('Spracheingabe wird von diesem Browser nicht unterstützt');
return;
}
const recognition = new Recognition();
recognition.lang = 'de-DE';
recognition.interimResults = true;
recognition.continuous = continuous;
recognition.onresult = (event: any) => {
let combined = '';
for (let i = 0; i < event.results.length; i++) combined += event.results[i][0].transcript;
transcriptRef.current = combined;
onTextRef.current(combined, false);
};
recognition.onerror = (event: any) => {
if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') {
cancelledRef.current = true;
recognitionRef.current = null;
fail('Mikrofon nicht freigegeben');
}
};
recognition.onend = () => {
clearTimer();
recognitionRef.current = null;
if (cancelledRef.current) return;
setStateSafe('idle');
onTextRef.current(transcriptRef.current, true);
};
recognitionRef.current = recognition;
setStateSafe('listening');
try {
recognition.start();
timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000);
} catch {
recognitionRef.current = null;
fail('Spracheingabe konnte nicht gestartet werden');
}
baseTranscriptRef.current = '';
lastRestartAtRef.current = 0;
spawnBrowserRecognition(Recognition, false);
};
const start = useCallback(async () => {
if (recorderRef.current || recognitionRef.current) return;
cancelledRef.current = false;
stoppedRef.current = false;
transcriptRef.current = '';
baseTranscriptRef.current = '';
lastRestartAtRef.current = 0;
const current = mode ?? currentVoiceConfig().mode;
if (current === 'server') await startServer();
else if (current === 'browser') startBrowser();
else fail(current === 'off' ? 'Spracheingabe ist ausgeschaltet' : 'Spracheingabe wird von diesem Browser nicht unterstützt');
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [mode, continuous]);
}, [mode, continuous, live]);
const cancel = useCallback(() => {
cancelledRef.current = true;
clearTimer();
stoppedRef.current = true;
inFlightChunksRef.current = 0;
setPendingChunksSafe(0);
clearAllTimers();
recorderRef.current?.cancel();
recorderRef.current = null;
try {
@@ -191,6 +355,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu
mode,
available: mode === 'server' || mode === 'browser',
state,
pendingChunks,
start,
stop,
cancel,
+44 -13
View File
@@ -86,19 +86,7 @@ export class VoiceRecorder {
this.stream = await navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
const type = recordingType();
this.recorder = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
this.chunks = [];
this.recorder.ondataavailable = (e) => {
if (e.data.size > 0) this.chunks.push(e.data);
};
this.recorder.onstop = () => {
const blob = this.chunks.length ? new Blob(this.chunks, { type: this.recorder?.mimeType || type || 'audio/webm' }) : null;
this.release();
this.done?.(blob);
this.done = null;
};
this.recorder.start(1000);
this.startSegmentRecorder();
try {
const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext;
this.audioCtx = new Ctx();
@@ -113,6 +101,25 @@ export class VoiceRecorder {
}
}
private startSegmentRecorder(): void {
if (!this.stream) return;
const type = recordingType();
const rec = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
const chunks: Blob[] = [];
rec.ondataavailable = (e) => {
if (e.data.size > 0) chunks.push(e.data);
};
rec.onstop = () => {
const blob = chunks.length ? new Blob(chunks, { type: rec.mimeType || type || 'audio/webm' }) : null;
this.release();
this.done?.(blob);
this.done = null;
};
this.recorder = rec;
this.chunks = chunks;
rec.start(1000);
}
/** Input level 0..1 (roughly loudness), or null when unknown. */
level(): number | null {
if (!this.analyser || !this.samples) return null;
@@ -137,6 +144,30 @@ export class VoiceRecorder {
});
}
/**
* Stops the current MediaRecorder segment and immediately starts a new one on the same MediaStream.
* Resolves with the finished segment's audio Blob while recording continues seamlessly.
*/
cutSegment(): Promise<Blob | null> {
if (!this.stream || !this.recorder || this.recorder.state === 'inactive') {
return Promise.resolve(null);
}
const oldRecorder = this.recorder;
const oldChunks = this.chunks;
this.startSegmentRecorder();
return new Promise((resolve) => {
oldRecorder.onstop = () => {
const blob = oldChunks.length
? new Blob(oldChunks, { type: oldRecorder.mimeType || 'audio/webm' })
: null;
resolve(blob);
};
oldRecorder.stop();
});
}
cancel(): void {
this.done = null;
if (this.recorder && this.recorder.state !== 'inactive') {