diff --git a/CHANGELOG.md b/CHANGELOG.md index 5537a78..1cf7027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Das Format folgt [Keep a Changelog](https://keepachangelog.com/de/1.1.0/), die V ## [Unreleased] +### Changed +- `/diktat` in Notizen: schreibt live mit (mehrzeiliger Text über den Wellen mit Zeitanzeige `mm:ss`), bricht im Browser-Modus bei Sprechpausen nicht mehr ab und schickt im Server-Modus alle ~8 Sekunden bzw. nach kurzen Sprechpausen Abschnitte zur Erkennung vorab. + ### Fixed - Notizen: Link-Vorschau-Karten erscheinen nur noch, wenn der gesamte Punkt aus einer einzigen `https`-Adresse besteht (oder über `/link` in einem leeren Punkt eingefügt wurde), nicht mehr bei Links mitten im Fließtext. - Notizen: Das Slash-Menü (`/`) öffnet sich nur noch am Textanfang oder nach einem Leerzeichen (nicht mehr beim Einfügen von Web-Adressen mit Schrägstrichen), schließt zuverlässig mit `Esc` oder `Backspace` und scrollt bei Pfeiltasten-Navigation automatisch mit. diff --git a/frontend/src/features/workspace/DictationBlock.tsx b/frontend/src/features/workspace/DictationBlock.tsx index 8322208..d8bf720 100644 --- a/frontend/src/features/workspace/DictationBlock.tsx +++ b/frontend/src/features/workspace/DictationBlock.tsx @@ -9,16 +9,24 @@ interface DictationBlockProps { onCancel: () => void; } +function formatDuration(sec: number): string { + const m = String(Math.floor(sec / 60)).padStart(2, '0'); + const s = String(sec % 60).padStart(2, '0'); + return `${m}:${s}`; +} + // What an empty bullet turns into after /diktat: listening starts right away, the whole bullet is one -// button with the live wave and a translucent "Stopp" on top. A tap ends it; the text lands in the -// bullet as plain text. +// button with live text growing above the wave, mm:ss timer and a translucent "Stopp" on top. A tap ends +// it; the text lands in the bullet as plain text. export const DictationBlock: React.FC = ({ onDone, onCancel }) => { const [interim, setInterim] = useState(''); + const [elapsedSec, setElapsedSec] = useState(0); const [error, setError] = useState(null); const doneRef = useRef(false); const buttonRef = useRef(null); const voice = useVoiceInput({ + live: true, continuous: true, onText: (text, final) => { if (!final) { @@ -45,11 +53,17 @@ export const DictationBlock: React.FC = ({ onDone, onCancel triggerHaptic([15, 60, 25]); void voice.start(); buttonRef.current?.focus(); + const started = Date.now(); + const timer = window.setInterval(() => { + setElapsedSec(Math.floor((Date.now() - started) / 1000)); + }, 500); + return () => window.clearInterval(timer); // start once when the bullet turns into the dictation // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const transcribing = voice.state === 'transcribing'; + const chunkBusy = voice.pendingChunks > 0 || transcribing; return ( ); }; diff --git a/frontend/src/lib/__tests__/useVoiceInput.test.tsx b/frontend/src/lib/__tests__/useVoiceInput.test.tsx index 76d9faa..615a000 100644 --- a/frontend/src/lib/__tests__/useVoiceInput.test.tsx +++ b/frontend/src/lib/__tests__/useVoiceInput.test.tsx @@ -130,4 +130,121 @@ describe('useVoiceInput', () => { await waitFor(() => expect(none.result.current.mode).toBe('unsupported')); expect(none.result.current.available).toBe(false); }); + + it('browser mode with live=true: restarts on pause (onend), keeps base text, rate-limits restarts to 300ms, and sends final=true only on stop()', async () => { + const instances: any[] = []; + class FakeSpeechRecognition { + lang = ''; + interimResults = false; + continuous = false; + onresult: ((e: any) => void) | null = null; + onerror: ((e: any) => void) | null = null; + onend: (() => void) | null = null; + constructor() { + instances.push(this); + } + start() {} + stop() { + this.onend?.(); + } + abort() {} + } + (window as any).SpeechRecognition = FakeSpeechRecognition; + getAuthStatus.mockResolvedValue({ voice: { mode: 'browser', maxSeconds: 600 } }); + + const onText = vi.fn(); + const { result } = renderHook(() => useVoiceInput({ onText, live: true })); + await waitFor(() => expect(result.current.mode).toBe('browser')); + + vi.useFakeTimers(); + await act(async () => { + await result.current.start(); + }); + expect(instances).toHaveLength(1); + + // First utterance before a pause + act(() => { + instances[0].onresult?.({ results: [[{ transcript: 'Erster Satz.' }]] }); + }); + expect(onText).toHaveBeenLastCalledWith('Erster Satz.', false); + + // Browser ends recognition on speech pause -> immediate restart (#2) + act(() => { + instances[0].onend?.(); + }); + expect(instances).toHaveLength(2); + expect(onText).not.toHaveBeenCalledWith(expect.anything(), true); + + // Another immediate onend within <300ms waits until 300ms have passed + act(() => { + instances[1].onend?.(); + }); + expect(instances).toHaveLength(2); + act(() => { + vi.advanceTimersByTime(300); + }); + expect(instances).toHaveLength(3); + + // Second utterance appends to base text + act(() => { + instances[2].onresult?.({ results: [[{ transcript: 'Zweiter Satz.' }]] }); + }); + expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', false); + + // Explicit stop() finishes with final=true + await act(async () => { + await result.current.stop(); + }); + expect(onText).toHaveBeenLastCalledWith('Erster Satz. Zweiter Satz.', true); + expect(result.current.state).toBe('idle'); + }); + + it('server mode with live=true: cuts segments onto new MediaRecorders on the same stream, orders out-of-order responses by chunk index, and releases stream at the end', async () => { + getAuthStatus.mockResolvedValue({ voice: { mode: 'server', maxSeconds: 600 } }); + + const resolvers: Array<(text: string) => void> = []; + transcribe.mockImplementation( + () => + new Promise((resolve) => { + resolvers.push(resolve); + }) + ); + + const onText = vi.fn(); + const { result } = renderHook(() => useVoiceInput({ onText, live: true })); + await waitFor(() => expect(result.current.mode).toBe('server')); + + vi.useFakeTimers(); + await act(async () => { + await result.current.start(); + }); + + // Advance 8s -> segment 0 is cut and sent to transcribe, stream is NOT released yet + await act(async () => { + await vi.advanceTimersByTimeAsync(8000); + }); + expect(transcribe).toHaveBeenCalledTimes(1); + expect(stopTrack).not.toHaveBeenCalled(); + + // Advance 4s and stop -> segment 1 (final) is sent to transcribe, stream IS released now + await act(async () => { + await vi.advanceTimersByTimeAsync(4000); + await result.current.stop(); + }); + expect(transcribe).toHaveBeenCalledTimes(2); + expect(stopTrack).toHaveBeenCalledTimes(1); + + // Resolve chunk 1 BEFORE chunk 0 to verify index-based ordering + await act(async () => { + resolvers[1]('zweiter Teil'); + }); + expect(onText).toHaveBeenLastCalledWith('zweiter Teil', false); + + // Now resolve chunk 0 -> both chunks are done, ordered "Erster Teil zweiter Teil" with final=true + await act(async () => { + resolvers[0]('Erster Teil'); + }); + expect(onText).toHaveBeenLastCalledWith('Erster Teil zweiter Teil', true); + expect(result.current.state).toBe('idle'); + }); }); diff --git a/frontend/src/lib/useVoiceInput.ts b/frontend/src/lib/useVoiceInput.ts index b139a24..783d683 100644 --- a/frontend/src/lib/useVoiceInput.ts +++ b/frontend/src/lib/useVoiceInput.ts @@ -10,28 +10,54 @@ export interface VoiceInputOptions { onError?: (message: string) => void; /** Browser mode: keep listening through pauses (hold-to-record) instead of stopping after a sentence. */ continuous?: boolean; + /** Live dictation mode (/diktat): auto-restart browser speech recognition on pause, and chunk server recordings every ~8s or ~0.8s of silence. */ + live?: boolean; } +const LIVE_CHUNK_MS = 8000; +const LIVE_SILENCE_MS = 800; +const LIVE_TICK_MS = 200; +const LIVE_SPEECH_LEVEL = 0.02; +const MIN_RESTART_INTERVAL_MS = 300; + /** * Microphone input for one button: start, stop (text arrives), cancel (nothing arrives). How the text * is made depends on the server setting, see lib/voice.ts. mode is null until known, "off" or * "unsupported" means: do not show a microphone. */ -export function useVoiceInput({ onText, onError, continuous = false }: VoiceInputOptions) { +export function useVoiceInput({ onText, onError, continuous = false, live = false }: VoiceInputOptions) { const [mode, setMode] = useState(null); const [state, setState] = useState('idle'); + const [pendingChunks, setPendingChunks] = useState(0); const maxSecondsRef = useRef(600); const recorderRef = useRef(null); const recognitionRef = useRef(null); const timerRef = useRef(null); + const segmentTimerRef = useRef(null); + const restartTimerRef = useRef(null); const cancelledRef = useRef(false); + const stoppedRef = useRef(false); const transcriptRef = useRef(''); + const baseTranscriptRef = useRef(''); + const lastRestartAtRef = useRef(0); + const chunkIndexRef = useRef(0); + const chunkResultsRef = useRef>(new Map()); + const inFlightChunksRef = useRef(0); const mountedRef = useRef(true); const onTextRef = useRef(onText); onTextRef.current = onText; const onErrorRef = useRef(onError); onErrorRef.current = onError; + const clearAllTimers = () => { + if (timerRef.current !== null) window.clearTimeout(timerRef.current); + timerRef.current = null; + if (segmentTimerRef.current !== null) window.clearInterval(segmentTimerRef.current); + segmentTimerRef.current = null; + if (restartTimerRef.current !== null) window.clearTimeout(restartTimerRef.current); + restartTimerRef.current = null; + }; + useEffect(() => { mountedRef.current = true; void loadVoiceConfig().then((cfg) => { @@ -41,7 +67,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu }); return () => { mountedRef.current = false; - if (timerRef.current !== null) window.clearTimeout(timerRef.current); + clearAllTimers(); recorderRef.current?.cancel(); try { recognitionRef.current?.abort?.(); @@ -54,17 +80,65 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu const setStateSafe = (s: VoiceState) => { if (mountedRef.current) setState(s); }; + const setPendingChunksSafe = (n: number) => { + if (mountedRef.current) setPendingChunks(n); + }; const fail = (message: string) => { setStateSafe('idle'); + setPendingChunksSafe(0); onErrorRef.current?.(message); }; - const clearTimer = () => { - if (timerRef.current !== null) window.clearTimeout(timerRef.current); - timerRef.current = null; + + const joinedChunkText = () => + [...chunkResultsRef.current.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([, t]) => t.trim()) + .filter(Boolean) + .join(' '); + + const processSegmentPromise = (idx: number, blobPromise: Promise) => { + inFlightChunksRef.current += 1; + setPendingChunksSafe(inFlightChunksRef.current); + void blobPromise + .then(async (blob) => { + if (cancelledRef.current) return; + if (!blob || blob.size === 0) { + inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1); + setPendingChunksSafe(inFlightChunksRef.current); + if (stoppedRef.current && inFlightChunksRef.current === 0) { + setStateSafe('idle'); + onTextRef.current(joinedChunkText(), true); + } + return; + } + const text = await api.transcribe(blob); + if (cancelledRef.current) return; + chunkResultsRef.current.set(idx, text || ''); + inFlightChunksRef.current = Math.max(0, inFlightChunksRef.current - 1); + setPendingChunksSafe(inFlightChunksRef.current); + const combined = joinedChunkText(); + transcriptRef.current = combined; + if (stoppedRef.current && inFlightChunksRef.current === 0) { + setStateSafe('idle'); + onTextRef.current(combined, true); + } else { + onTextRef.current(combined, false); + } + }) + .catch((err: any) => { + if (cancelledRef.current) return; + cancelledRef.current = true; + inFlightChunksRef.current = 0; + clearAllTimers(); + recorderRef.current?.cancel(); + recorderRef.current = null; + fail(err?.message || 'Spracherkennung fehlgeschlagen'); + }); }; const stop = useCallback(async () => { - clearTimer(); + stoppedRef.current = true; + clearAllTimers(); if (recognitionRef.current) { try { recognitionRef.current.stop(); @@ -74,8 +148,20 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu return; } const rec = recorderRef.current; - if (!rec) return; + if (!rec) { + if (live && state === 'listening') { + setStateSafe('idle'); + onTextRef.current(transcriptRef.current, true); + } + return; + } recorderRef.current = null; + if (live) { + const idx = chunkIndexRef.current++; + setStateSafe('transcribing'); + processSegmentPromise(idx, rec.stop()); + return; + } const blob = await rec.stop(); if (cancelledRef.current) return; if (!blob) { @@ -93,11 +179,15 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu } // fail only touches refs and state setters // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [live, state]); const startServer = async () => { const rec = new VoiceRecorder(); recorderRef.current = rec; + chunkIndexRef.current = 0; + chunkResultsRef.current = new Map(); + inFlightChunksRef.current = 0; + setPendingChunksSafe(0); setStateSafe('listening'); try { await rec.start(); @@ -111,6 +201,104 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu return; } timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000); + if (live) { + let segmentMs = 0; + let silentMs = 0; + let hadSpeech = false; + segmentTimerRef.current = window.setInterval(() => { + if (cancelledRef.current || stoppedRef.current || recorderRef.current !== rec) return; + const lvl = rec.level(); + if (lvl !== null) { + if (lvl >= LIVE_SPEECH_LEVEL) { + hadSpeech = true; + silentMs = 0; + } else if (hadSpeech) { + silentMs += LIVE_TICK_MS; + } + } + segmentMs += LIVE_TICK_MS; + if (segmentMs >= LIVE_CHUNK_MS || (hadSpeech && silentMs >= LIVE_SILENCE_MS)) { + segmentMs = 0; + silentMs = 0; + hadSpeech = false; + const idx = chunkIndexRef.current++; + processSegmentPromise(idx, rec.cutSegment()); + } + }, LIVE_TICK_MS); + } + }; + + const spawnBrowserRecognition = (Recognition: any, isRestart: boolean) => { + const recognition = new Recognition(); + recognition.lang = 'de-DE'; + recognition.interimResults = true; + recognition.continuous = continuous || live; + recognition.onresult = (event: any) => { + let sessionText = ''; + for (let i = 0; i < event.results.length; i++) sessionText += event.results[i][0].transcript; + if (live && baseTranscriptRef.current) { + const base = baseTranscriptRef.current.trimEnd(); + const next = sessionText.trimStart(); + transcriptRef.current = next ? `${base} ${next}` : base; + } else { + transcriptRef.current = sessionText; + } + onTextRef.current(transcriptRef.current, false); + }; + recognition.onerror = (event: any) => { + if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') { + cancelledRef.current = true; + clearAllTimers(); + recognitionRef.current = null; + fail('Mikrofon nicht freigegeben'); + } + }; + recognition.onend = () => { + recognitionRef.current = null; + if (cancelledRef.current) { + clearAllTimers(); + return; + } + if (!live || stoppedRef.current) { + clearAllTimers(); + setStateSafe('idle'); + onTextRef.current(transcriptRef.current, true); + return; + } + baseTranscriptRef.current = transcriptRef.current; + const now = Date.now(); + const elapsed = now - lastRestartAtRef.current; + const doRestart = () => { + if (cancelledRef.current || stoppedRef.current) return; + lastRestartAtRef.current = Date.now(); + spawnBrowserRecognition(Recognition, true); + }; + if (lastRestartAtRef.current === 0 || elapsed >= MIN_RESTART_INTERVAL_MS) { + doRestart(); + } else { + restartTimerRef.current = window.setTimeout(() => { + restartTimerRef.current = null; + doRestart(); + }, MIN_RESTART_INTERVAL_MS - elapsed); + } + }; + recognitionRef.current = recognition; + if (!isRestart) setStateSafe('listening'); + try { + recognition.start(); + if (!isRestart) { + timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000); + } + } catch { + recognitionRef.current = null; + if (!isRestart) { + fail('Spracheingabe konnte nicht gestartet werden'); + } else { + clearAllTimers(); + setStateSafe('idle'); + onTextRef.current(transcriptRef.current, true); + } + } }; const startBrowser = () => { @@ -119,55 +307,31 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu fail('Spracheingabe wird von diesem Browser nicht unterstützt'); return; } - const recognition = new Recognition(); - recognition.lang = 'de-DE'; - recognition.interimResults = true; - recognition.continuous = continuous; - recognition.onresult = (event: any) => { - let combined = ''; - for (let i = 0; i < event.results.length; i++) combined += event.results[i][0].transcript; - transcriptRef.current = combined; - onTextRef.current(combined, false); - }; - recognition.onerror = (event: any) => { - if (event?.error === 'not-allowed' || event?.error === 'service-not-allowed') { - cancelledRef.current = true; - recognitionRef.current = null; - fail('Mikrofon nicht freigegeben'); - } - }; - recognition.onend = () => { - clearTimer(); - recognitionRef.current = null; - if (cancelledRef.current) return; - setStateSafe('idle'); - onTextRef.current(transcriptRef.current, true); - }; - recognitionRef.current = recognition; - setStateSafe('listening'); - try { - recognition.start(); - timerRef.current = window.setTimeout(() => void stop(), maxSecondsRef.current * 1000); - } catch { - recognitionRef.current = null; - fail('Spracheingabe konnte nicht gestartet werden'); - } + baseTranscriptRef.current = ''; + lastRestartAtRef.current = 0; + spawnBrowserRecognition(Recognition, false); }; const start = useCallback(async () => { if (recorderRef.current || recognitionRef.current) return; cancelledRef.current = false; + stoppedRef.current = false; transcriptRef.current = ''; + baseTranscriptRef.current = ''; + lastRestartAtRef.current = 0; const current = mode ?? currentVoiceConfig().mode; if (current === 'server') await startServer(); else if (current === 'browser') startBrowser(); else fail(current === 'off' ? 'Spracheingabe ist ausgeschaltet' : 'Spracheingabe wird von diesem Browser nicht unterstützt'); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [mode, continuous]); + }, [mode, continuous, live]); const cancel = useCallback(() => { cancelledRef.current = true; - clearTimer(); + stoppedRef.current = true; + inFlightChunksRef.current = 0; + setPendingChunksSafe(0); + clearAllTimers(); recorderRef.current?.cancel(); recorderRef.current = null; try { @@ -191,6 +355,7 @@ export function useVoiceInput({ onText, onError, continuous = false }: VoiceInpu mode, available: mode === 'server' || mode === 'browser', state, + pendingChunks, start, stop, cancel, diff --git a/frontend/src/lib/voice.ts b/frontend/src/lib/voice.ts index 9c0bfbe..8e447f4 100644 --- a/frontend/src/lib/voice.ts +++ b/frontend/src/lib/voice.ts @@ -86,19 +86,7 @@ export class VoiceRecorder { this.stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); - const type = recordingType(); - this.recorder = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined); - this.chunks = []; - this.recorder.ondataavailable = (e) => { - if (e.data.size > 0) this.chunks.push(e.data); - }; - this.recorder.onstop = () => { - const blob = this.chunks.length ? new Blob(this.chunks, { type: this.recorder?.mimeType || type || 'audio/webm' }) : null; - this.release(); - this.done?.(blob); - this.done = null; - }; - this.recorder.start(1000); + this.startSegmentRecorder(); try { const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext; this.audioCtx = new Ctx(); @@ -113,6 +101,25 @@ export class VoiceRecorder { } } + private startSegmentRecorder(): void { + if (!this.stream) return; + const type = recordingType(); + const rec = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined); + const chunks: Blob[] = []; + rec.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + rec.onstop = () => { + const blob = chunks.length ? new Blob(chunks, { type: rec.mimeType || type || 'audio/webm' }) : null; + this.release(); + this.done?.(blob); + this.done = null; + }; + this.recorder = rec; + this.chunks = chunks; + rec.start(1000); + } + /** Input level 0..1 (roughly loudness), or null when unknown. */ level(): number | null { if (!this.analyser || !this.samples) return null; @@ -137,6 +144,30 @@ export class VoiceRecorder { }); } + /** + * Stops the current MediaRecorder segment and immediately starts a new one on the same MediaStream. + * Resolves with the finished segment's audio Blob while recording continues seamlessly. + */ + cutSegment(): Promise { + if (!this.stream || !this.recorder || this.recorder.state === 'inactive') { + return Promise.resolve(null); + } + const oldRecorder = this.recorder; + const oldChunks = this.chunks; + + this.startSegmentRecorder(); + + return new Promise((resolve) => { + oldRecorder.onstop = () => { + const blob = oldChunks.length + ? new Blob(oldChunks, { type: oldRecorder.mimeType || 'audio/webm' }) + : null; + resolve(blob); + }; + oldRecorder.stop(); + }); + } + cancel(): void { this.done = null; if (this.recorder && this.recorder.state !== 'inactive') {