feat(voice): all microphone buttons use the server's speech recognition
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s
- New useVoiceInput hook (lib/voice.ts): "server" records with MediaRecorder and sends to /voice/transcribe, "browser" keeps the live recognition, "off" or missing browser support hides the microphone. - Quick dump bar, mobile capture sheet and hold-to-record use it; live waves from the real input level (VoiceWave) and "Wird erkannt…". - Recordings stop after STT_MAX_SECONDS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7801cd13b9
commit
7fa21e485b
9 files changed
+753
-248
No files matched your search
@@ -1,6 +1,8 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { Mic, Plus } from 'lucide-react';
|
||||
import { triggerHaptic } from '../../lib/api';
|
||||
import { useVoiceInput } from '../../lib/useVoiceInput';
|
||||
import { VoiceWave } from '../../components/VoiceWave';
|
||||
|
||||
export interface HoldToRecordFabProps {
|
||||
onTap: () => void;
|
||||
@@ -24,14 +26,10 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
const didEnterHoldRef = useRef(false);
|
||||
const fallbackHandledRef = useRef(false);
|
||||
const isCancelZoneRef = useRef(false);
|
||||
const abortedRef = useRef(false);
|
||||
const isStoppingRef = useRef(false);
|
||||
const hasEndedRef = useRef(false);
|
||||
const finalizedRef = useRef(false);
|
||||
const startYRef = useRef(0);
|
||||
const transcriptRef = useRef('');
|
||||
|
||||
const recognitionRef = useRef<any>(null);
|
||||
const holdTimerRef = useRef<number | null>(null);
|
||||
const stopTimerRef = useRef<number | null>(null);
|
||||
const toastTimerRef = useRef<number | null>(null);
|
||||
@@ -50,17 +48,66 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
}
|
||||
};
|
||||
|
||||
const showToast = (msg: string) => {
|
||||
const showToast = (msg: string, sticky = false) => {
|
||||
if (toastTimerRef.current !== null) {
|
||||
window.clearTimeout(toastTimerRef.current);
|
||||
toastTimerRef.current = null;
|
||||
}
|
||||
setToastMessage(msg);
|
||||
if (sticky) return;
|
||||
toastTimerRef.current = window.setTimeout(() => {
|
||||
setToastMessage(null);
|
||||
toastTimerRef.current = null;
|
||||
}, TOAST_DURATION_MS);
|
||||
};
|
||||
|
||||
const finalizeRecording = async (text: string) => {
|
||||
if (finalizedRef.current) return;
|
||||
finalizedRef.current = true;
|
||||
clearStopTimer();
|
||||
setLiveTranscript('');
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
showToast('Nichts erkannt');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await onTranscript(trimmed);
|
||||
triggerHaptic(25);
|
||||
showToast('Im Quick Dump gespeichert');
|
||||
} catch {
|
||||
showToast('Fehler beim Speichern');
|
||||
}
|
||||
};
|
||||
|
||||
// Own Whisper server (text after release) or the browser's recognition (text while speaking)
|
||||
const voice = useVoiceInput({
|
||||
continuous: true,
|
||||
onText: (text, final) => {
|
||||
transcriptRef.current = text;
|
||||
if (!final) {
|
||||
setLiveTranscript(text);
|
||||
return;
|
||||
}
|
||||
void finalizeRecording(text);
|
||||
},
|
||||
onError: (message) => {
|
||||
clearStopTimer();
|
||||
setIsRecording(false);
|
||||
setIsCancelZone(false);
|
||||
setLiveTranscript('');
|
||||
if (didEnterHoldRef.current && isPressedRef.current && !fallbackHandledRef.current) {
|
||||
// Recording could not start: open the normal capture sheet instead
|
||||
fallbackHandledRef.current = true;
|
||||
showToast(message);
|
||||
onTap();
|
||||
return;
|
||||
}
|
||||
finalizedRef.current = true;
|
||||
showToast(message);
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearHoldTimer();
|
||||
@@ -68,112 +115,28 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
if (toastTimerRef.current !== null) {
|
||||
window.clearTimeout(toastTimerRef.current);
|
||||
}
|
||||
if (recognitionRef.current) {
|
||||
try {
|
||||
recognitionRef.current.abort?.();
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const finalizeRecording = async () => {
|
||||
if (finalizedRef.current) return;
|
||||
finalizedRef.current = true;
|
||||
clearStopTimer();
|
||||
recognitionRef.current = null;
|
||||
|
||||
if (abortedRef.current) return;
|
||||
|
||||
const text = transcriptRef.current.trim();
|
||||
setLiveTranscript('');
|
||||
if (!text) {
|
||||
showToast('Nichts erkannt');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await onTranscript(text);
|
||||
triggerHaptic(25);
|
||||
showToast('Im Quick Dump gespeichert');
|
||||
} catch {
|
||||
showToast('Fehler beim Speichern');
|
||||
}
|
||||
};
|
||||
useEffect(() => {
|
||||
if (voice.state === 'transcribing') showToast('Wird erkannt…', true);
|
||||
}, [voice.state]);
|
||||
|
||||
const startVoiceRecording = () => {
|
||||
const SpeechRecognition =
|
||||
(window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
|
||||
|
||||
if (!SpeechRecognition) {
|
||||
if (voice.mode === 'off' || voice.mode === 'unsupported') {
|
||||
fallbackHandledRef.current = true;
|
||||
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
if (voice.mode === 'unsupported') showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
onTap();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const recognition = new SpeechRecognition();
|
||||
recognition.lang = 'de-DE';
|
||||
recognition.interimResults = true;
|
||||
recognition.continuous = true;
|
||||
|
||||
transcriptRef.current = '';
|
||||
abortedRef.current = false;
|
||||
isStoppingRef.current = false;
|
||||
hasEndedRef.current = false;
|
||||
finalizedRef.current = false;
|
||||
isCancelZoneRef.current = false;
|
||||
|
||||
recognition.onresult = (event: any) => {
|
||||
let combined = '';
|
||||
for (let i = 0; i < event.results.length; i++) {
|
||||
combined += event.results[i][0].transcript;
|
||||
}
|
||||
transcriptRef.current = combined;
|
||||
setLiveTranscript(combined);
|
||||
};
|
||||
|
||||
recognition.onerror = (event: any) => {
|
||||
const errCode = event?.error;
|
||||
if (errCode === 'not-allowed' || errCode === 'service-not-allowed') {
|
||||
abortedRef.current = true;
|
||||
clearStopTimer();
|
||||
setIsRecording(false);
|
||||
setIsCancelZone(false);
|
||||
setLiveTranscript('');
|
||||
if (!fallbackHandledRef.current) {
|
||||
fallbackHandledRef.current = true;
|
||||
showToast('Mikrofon nicht freigegeben');
|
||||
onTap();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isStoppingRef.current) {
|
||||
void finalizeRecording();
|
||||
}
|
||||
};
|
||||
|
||||
recognition.onend = () => {
|
||||
hasEndedRef.current = true;
|
||||
if (isStoppingRef.current) {
|
||||
void finalizeRecording();
|
||||
}
|
||||
};
|
||||
|
||||
recognitionRef.current = recognition;
|
||||
setLiveTranscript('');
|
||||
setIsCancelZone(false);
|
||||
setIsRecording(true);
|
||||
triggerHaptic([15, 60, 25]);
|
||||
recognition.start();
|
||||
} catch {
|
||||
fallbackHandledRef.current = true;
|
||||
setIsRecording(false);
|
||||
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
|
||||
onTap();
|
||||
}
|
||||
transcriptRef.current = '';
|
||||
finalizedRef.current = false;
|
||||
isCancelZoneRef.current = false;
|
||||
setLiveTranscript('');
|
||||
setIsCancelZone(false);
|
||||
setIsRecording(true);
|
||||
triggerHaptic([15, 60, 25]);
|
||||
void voice.start();
|
||||
};
|
||||
|
||||
const handlePointerDown = (e: React.PointerEvent<HTMLButtonElement>) => {
|
||||
@@ -240,34 +203,23 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
setIsCancelZone(false);
|
||||
|
||||
if (isCancelZoneRef.current) {
|
||||
abortedRef.current = true;
|
||||
finalizedRef.current = true;
|
||||
clearStopTimer();
|
||||
setLiveTranscript('');
|
||||
triggerHaptic(15);
|
||||
try {
|
||||
recognitionRef.current?.abort?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
recognitionRef.current = null;
|
||||
voice.cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
isStoppingRef.current = true;
|
||||
if (hasEndedRef.current || !recognitionRef.current) {
|
||||
void finalizeRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
stopTimerRef.current = window.setTimeout(() => {
|
||||
void finalizeRecording();
|
||||
}, STOP_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
recognitionRef.current.stop();
|
||||
} catch {
|
||||
void finalizeRecording();
|
||||
// The browser's recognition sometimes never reports its end: save what was heard after a moment.
|
||||
// The own server answers when it is done (with its own time limit).
|
||||
if (voice.mode !== 'server') {
|
||||
stopTimerRef.current = window.setTimeout(() => {
|
||||
voice.cancel();
|
||||
void finalizeRecording(transcriptRef.current);
|
||||
}, STOP_TIMEOUT_MS);
|
||||
}
|
||||
void voice.stop();
|
||||
};
|
||||
|
||||
const handlePointerCancel = (e: React.PointerEvent<HTMLButtonElement>) => {
|
||||
@@ -283,16 +235,11 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
clearStopTimer();
|
||||
|
||||
if (didEnterHoldRef.current && !fallbackHandledRef.current) {
|
||||
abortedRef.current = true;
|
||||
finalizedRef.current = true;
|
||||
setIsRecording(false);
|
||||
setIsCancelZone(false);
|
||||
setLiveTranscript('');
|
||||
try {
|
||||
recognitionRef.current?.abort?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
recognitionRef.current = null;
|
||||
voice.cancel();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -313,7 +260,8 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
|
||||
<div className="text-xs font-semibold">🗑️ Zum Abbrechen loslassen</div>
|
||||
) : (
|
||||
<div className="space-y-1">
|
||||
<div className="text-xs font-semibold text-main">
|
||||
<div className="text-xs font-semibold text-main flex items-center justify-center gap-2">
|
||||
{voice.mode === 'server' && <VoiceWave getLevel={voice.level} bars={7} className="h-4 text-accent" />}
|
||||
Ich höre zu… (loslassen speichert)
|
||||
</div>
|
||||
{displayedTranscript ? (
|
||||
|
||||
Reference in new issue
Block a user