feat(voice): all microphone buttons use the server's speech recognition
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s

- New useVoiceInput hook (lib/voice.ts): "server" records with MediaRecorder
  and sends to /voice/transcribe, "browser" keeps the live recognition,
  "off" or missing browser support hides the microphone.
- Quick dump bar, mobile capture sheet and hold-to-record use it; live
  waves from the real input level (VoiceWave) and "Wird erkannt…".
- Recordings stop after STT_MAX_SECONDS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-29 22:23:48 +01:00
1 parent 7801cd13b9
commit 7fa21e485b
9 files changed
+753 -248

No files matched your search

+79 -131
View File
@@ -1,6 +1,8 @@
import React, { useEffect, useRef, useState } from 'react';
import { Mic, Plus } from 'lucide-react';
import { triggerHaptic } from '../../lib/api';
import { useVoiceInput } from '../../lib/useVoiceInput';
import { VoiceWave } from '../../components/VoiceWave';
export interface HoldToRecordFabProps {
onTap: () => void;
@@ -24,14 +26,10 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
const didEnterHoldRef = useRef(false);
const fallbackHandledRef = useRef(false);
const isCancelZoneRef = useRef(false);
const abortedRef = useRef(false);
const isStoppingRef = useRef(false);
const hasEndedRef = useRef(false);
const finalizedRef = useRef(false);
const startYRef = useRef(0);
const transcriptRef = useRef('');
const recognitionRef = useRef<any>(null);
const holdTimerRef = useRef<number | null>(null);
const stopTimerRef = useRef<number | null>(null);
const toastTimerRef = useRef<number | null>(null);
@@ -50,17 +48,66 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
}
};
const showToast = (msg: string) => {
const showToast = (msg: string, sticky = false) => {
if (toastTimerRef.current !== null) {
window.clearTimeout(toastTimerRef.current);
toastTimerRef.current = null;
}
setToastMessage(msg);
if (sticky) return;
toastTimerRef.current = window.setTimeout(() => {
setToastMessage(null);
toastTimerRef.current = null;
}, TOAST_DURATION_MS);
};
const finalizeRecording = async (text: string) => {
if (finalizedRef.current) return;
finalizedRef.current = true;
clearStopTimer();
setLiveTranscript('');
const trimmed = text.trim();
if (!trimmed) {
showToast('Nichts erkannt');
return;
}
try {
await onTranscript(trimmed);
triggerHaptic(25);
showToast('Im Quick Dump gespeichert');
} catch {
showToast('Fehler beim Speichern');
}
};
// Own Whisper server (text after release) or the browser's recognition (text while speaking)
const voice = useVoiceInput({
continuous: true,
onText: (text, final) => {
transcriptRef.current = text;
if (!final) {
setLiveTranscript(text);
return;
}
void finalizeRecording(text);
},
onError: (message) => {
clearStopTimer();
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
if (didEnterHoldRef.current && isPressedRef.current && !fallbackHandledRef.current) {
// Recording could not start: open the normal capture sheet instead
fallbackHandledRef.current = true;
showToast(message);
onTap();
return;
}
finalizedRef.current = true;
showToast(message);
},
});
useEffect(() => {
return () => {
clearHoldTimer();
@@ -68,112 +115,28 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
if (toastTimerRef.current !== null) {
window.clearTimeout(toastTimerRef.current);
}
if (recognitionRef.current) {
try {
recognitionRef.current.abort?.();
} catch {
// ignore cleanup errors
}
}
};
}, []);
const finalizeRecording = async () => {
if (finalizedRef.current) return;
finalizedRef.current = true;
clearStopTimer();
recognitionRef.current = null;
if (abortedRef.current) return;
const text = transcriptRef.current.trim();
setLiveTranscript('');
if (!text) {
showToast('Nichts erkannt');
return;
}
try {
await onTranscript(text);
triggerHaptic(25);
showToast('Im Quick Dump gespeichert');
} catch {
showToast('Fehler beim Speichern');
}
};
useEffect(() => {
if (voice.state === 'transcribing') showToast('Wird erkannt…', true);
}, [voice.state]);
const startVoiceRecording = () => {
const SpeechRecognition =
(window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
if (voice.mode === 'off' || voice.mode === 'unsupported') {
fallbackHandledRef.current = true;
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
if (voice.mode === 'unsupported') showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
onTap();
return;
}
try {
const recognition = new SpeechRecognition();
recognition.lang = 'de-DE';
recognition.interimResults = true;
recognition.continuous = true;
transcriptRef.current = '';
abortedRef.current = false;
isStoppingRef.current = false;
hasEndedRef.current = false;
finalizedRef.current = false;
isCancelZoneRef.current = false;
recognition.onresult = (event: any) => {
let combined = '';
for (let i = 0; i < event.results.length; i++) {
combined += event.results[i][0].transcript;
}
transcriptRef.current = combined;
setLiveTranscript(combined);
};
recognition.onerror = (event: any) => {
const errCode = event?.error;
if (errCode === 'not-allowed' || errCode === 'service-not-allowed') {
abortedRef.current = true;
clearStopTimer();
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
if (!fallbackHandledRef.current) {
fallbackHandledRef.current = true;
showToast('Mikrofon nicht freigegeben');
onTap();
}
return;
}
if (isStoppingRef.current) {
void finalizeRecording();
}
};
recognition.onend = () => {
hasEndedRef.current = true;
if (isStoppingRef.current) {
void finalizeRecording();
}
};
recognitionRef.current = recognition;
setLiveTranscript('');
setIsCancelZone(false);
setIsRecording(true);
triggerHaptic([15, 60, 25]);
recognition.start();
} catch {
fallbackHandledRef.current = true;
setIsRecording(false);
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
onTap();
}
transcriptRef.current = '';
finalizedRef.current = false;
isCancelZoneRef.current = false;
setLiveTranscript('');
setIsCancelZone(false);
setIsRecording(true);
triggerHaptic([15, 60, 25]);
void voice.start();
};
const handlePointerDown = (e: React.PointerEvent<HTMLButtonElement>) => {
@@ -240,34 +203,23 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
setIsCancelZone(false);
if (isCancelZoneRef.current) {
abortedRef.current = true;
finalizedRef.current = true;
clearStopTimer();
setLiveTranscript('');
triggerHaptic(15);
try {
recognitionRef.current?.abort?.();
} catch {
// ignore
}
recognitionRef.current = null;
voice.cancel();
return;
}
isStoppingRef.current = true;
if (hasEndedRef.current || !recognitionRef.current) {
void finalizeRecording();
return;
}
stopTimerRef.current = window.setTimeout(() => {
void finalizeRecording();
}, STOP_TIMEOUT_MS);
try {
recognitionRef.current.stop();
} catch {
void finalizeRecording();
// The browser's recognition sometimes never reports its end: save what was heard after a moment.
// The own server answers when it is done (with its own time limit).
if (voice.mode !== 'server') {
stopTimerRef.current = window.setTimeout(() => {
voice.cancel();
void finalizeRecording(transcriptRef.current);
}, STOP_TIMEOUT_MS);
}
void voice.stop();
};
const handlePointerCancel = (e: React.PointerEvent<HTMLButtonElement>) => {
@@ -283,16 +235,11 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
clearStopTimer();
if (didEnterHoldRef.current && !fallbackHandledRef.current) {
abortedRef.current = true;
finalizedRef.current = true;
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
try {
recognitionRef.current?.abort?.();
} catch {
// ignore
}
recognitionRef.current = null;
voice.cancel();
}
};
@@ -313,7 +260,8 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
<div className="text-xs font-semibold">🗑️ Zum Abbrechen loslassen</div>
) : (
<div className="space-y-1">
<div className="text-xs font-semibold text-main">
<div className="text-xs font-semibold text-main flex items-center justify-center gap-2">
{voice.mode === 'server' && <VoiceWave getLevel={voice.level} bars={7} className="h-4 text-accent" />}
Ich höre zu… (loslassen speichert)
</div>
{displayedTranscript ? (