sandrodach/MindSift
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): all microphone buttons use the server's speech recognition
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 1m56s
Details

Browse Source 
- New useVoiceInput hook (lib/voice.ts): "server" records with MediaRecorder
  and sends to /voice/transcribe, "browser" keeps the live recognition,
  "off" or missing browser support hides the microphone.
- Quick dump bar, mobile capture sheet and hold-to-record use it; live
  waves from the real input level (VoiceWave) and "Wird erkannt…".
- Recordings stop after STT_MAX_SECONDS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed 2026-10-29 22:23:48 +01:00
1 parent 7801cd13b9
commit 7fa21e485b
9 files changed
+753 -248
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

frontend/src/features/capture/HoldToRecordFab.tsx
+79 -131
View File
@@ -1,6 +1,8 @@
import React, { useEffect, useRef, useState } from 'react';
import { Mic, Plus } from 'lucide-react';
import { triggerHaptic } from '../../lib/api';
import { useVoiceInput } from '../../lib/useVoiceInput';
import { VoiceWave } from '../../components/VoiceWave';
export interface HoldToRecordFabProps {
onTap: () => void;
@@ -24,14 +26,10 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
const didEnterHoldRef = useRef(false);
const fallbackHandledRef = useRef(false);
const isCancelZoneRef = useRef(false);
const abortedRef = useRef(false);
const isStoppingRef = useRef(false);
const hasEndedRef = useRef(false);
const finalizedRef = useRef(false);
const startYRef = useRef(0);
const transcriptRef = useRef('');
const recognitionRef = useRef<any>(null);
const holdTimerRef = useRef<number | null>(null);
const stopTimerRef = useRef<number | null>(null);
const toastTimerRef = useRef<number | null>(null);
@@ -50,17 +48,66 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
}
};
const showToast = (msg: string) => {
const showToast = (msg: string, sticky = false) => {
if (toastTimerRef.current !== null) {
window.clearTimeout(toastTimerRef.current);
toastTimerRef.current = null;
}
setToastMessage(msg);
if (sticky) return;
toastTimerRef.current = window.setTimeout(() => {
setToastMessage(null);
toastTimerRef.current = null;
}, TOAST_DURATION_MS);
};
const finalizeRecording = async (text: string) => {
if (finalizedRef.current) return;
finalizedRef.current = true;
clearStopTimer();
setLiveTranscript('');
const trimmed = text.trim();
if (!trimmed) {
showToast('Nichts erkannt');
return;
}
try {
await onTranscript(trimmed);
triggerHaptic(25);
showToast('Im Quick Dump gespeichert');
} catch {
showToast('Fehler beim Speichern');
}
};
// Own Whisper server (text after release) or the browser's recognition (text while speaking)
const voice = useVoiceInput({
continuous: true,
onText: (text, final) => {
transcriptRef.current = text;
if (!final) {
setLiveTranscript(text);
return;
}
void finalizeRecording(text);
},
onError: (message) => {
clearStopTimer();
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
if (didEnterHoldRef.current && isPressedRef.current && !fallbackHandledRef.current) {
// Recording could not start: open the normal capture sheet instead
fallbackHandledRef.current = true;
showToast(message);
onTap();
return;
}
finalizedRef.current = true;
showToast(message);
},
});
useEffect(() => {
return () => {
clearHoldTimer();
@@ -68,112 +115,28 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
if (toastTimerRef.current !== null) {
window.clearTimeout(toastTimerRef.current);
}
if (recognitionRef.current) {
try {
recognitionRef.current.abort?.();
} catch {
// ignore cleanup errors
}
}
};
}, []);
const finalizeRecording = async () => {
if (finalizedRef.current) return;
finalizedRef.current = true;
clearStopTimer();
recognitionRef.current = null;
if (abortedRef.current) return;
const text = transcriptRef.current.trim();
setLiveTranscript('');
if (!text) {
showToast('Nichts erkannt');
return;
}
try {
await onTranscript(text);
triggerHaptic(25);
showToast('Im Quick Dump gespeichert');
} catch {
showToast('Fehler beim Speichern');
}
};
useEffect(() => {
if (voice.state === 'transcribing') showToast('Wird erkannt…', true);
}, [voice.state]);
const startVoiceRecording = () => {
const SpeechRecognition =
(window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
if (voice.mode === 'off' || voice.mode === 'unsupported') {
fallbackHandledRef.current = true;
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
if (voice.mode === 'unsupported') showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
onTap();
return;
}
try {
const recognition = new SpeechRecognition();
recognition.lang = 'de-DE';
recognition.interimResults = true;
recognition.continuous = true;
transcriptRef.current = '';
abortedRef.current = false;
isStoppingRef.current = false;
hasEndedRef.current = false;
finalizedRef.current = false;
isCancelZoneRef.current = false;
recognition.onresult = (event: any) => {
let combined = '';
for (let i = 0; i < event.results.length; i++) {
combined += event.results[i][0].transcript;
}
transcriptRef.current = combined;
setLiveTranscript(combined);
};
recognition.onerror = (event: any) => {
const errCode = event?.error;
if (errCode === 'not-allowed' || errCode === 'service-not-allowed') {
abortedRef.current = true;
clearStopTimer();
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
if (!fallbackHandledRef.current) {
fallbackHandledRef.current = true;
showToast('Mikrofon nicht freigegeben');
onTap();
}
return;
}
if (isStoppingRef.current) {
void finalizeRecording();
}
};
recognition.onend = () => {
hasEndedRef.current = true;
if (isStoppingRef.current) {
void finalizeRecording();
}
};
recognitionRef.current = recognition;
setLiveTranscript('');
setIsCancelZone(false);
setIsRecording(true);
triggerHaptic([15, 60, 25]);
recognition.start();
} catch {
fallbackHandledRef.current = true;
setIsRecording(false);
showToast('Spracheingabe wird von diesem Browser nicht unterstützt');
onTap();
}
transcriptRef.current = '';
finalizedRef.current = false;
isCancelZoneRef.current = false;
setLiveTranscript('');
setIsCancelZone(false);
setIsRecording(true);
triggerHaptic([15, 60, 25]);
void voice.start();
};
const handlePointerDown = (e: React.PointerEvent<HTMLButtonElement>) => {
@@ -240,34 +203,23 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
setIsCancelZone(false);
if (isCancelZoneRef.current) {
abortedRef.current = true;
finalizedRef.current = true;
clearStopTimer();
setLiveTranscript('');
triggerHaptic(15);
try {
recognitionRef.current?.abort?.();
} catch {
// ignore
}
recognitionRef.current = null;
voice.cancel();
return;
}
isStoppingRef.current = true;
if (hasEndedRef.current || !recognitionRef.current) {
void finalizeRecording();
return;
}
stopTimerRef.current = window.setTimeout(() => {
void finalizeRecording();
}, STOP_TIMEOUT_MS);
try {
recognitionRef.current.stop();
} catch {
void finalizeRecording();
// The browser's recognition sometimes never reports its end: save what was heard after a moment.
// The own server answers when it is done (with its own time limit).
if (voice.mode !== 'server') {
stopTimerRef.current = window.setTimeout(() => {
voice.cancel();
void finalizeRecording(transcriptRef.current);
}, STOP_TIMEOUT_MS);
}
void voice.stop();
};
const handlePointerCancel = (e: React.PointerEvent<HTMLButtonElement>) => {
@@ -283,16 +235,11 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
clearStopTimer();
if (didEnterHoldRef.current && !fallbackHandledRef.current) {
abortedRef.current = true;
finalizedRef.current = true;
setIsRecording(false);
setIsCancelZone(false);
setLiveTranscript('');
try {
recognitionRef.current?.abort?.();
} catch {
// ignore
}
recognitionRef.current = null;
voice.cancel();
}
};
@@ -313,7 +260,8 @@ export const HoldToRecordFab: React.FC<HoldToRecordFabProps> = ({ onTap, onTrans
<div className="text-xs font-semibold">🗑️ Zum Abbrechen loslassen</div>
) : (
<div className="space-y-1">
<div className="text-xs font-semibold text-main">
<div className="text-xs font-semibold text-main flex items-center justify-center gap-2">
{voice.mode === 'server' && <VoiceWave getLevel={voice.level} bars={7} className="h-4 text-accent" />}
Ich höre zu… (loslassen speichert)
</div>
{displayedTranscript ? (
frontend/src/features/capture/InstantCaptureBar.tsx
+57 -64
View File
@@ -13,9 +13,12 @@ import {
ChevronDown,
AlertTriangle,
Flame,
Loader2,
} from 'lucide-react';
import { PriorityLevel, SharedList, Thought } from '../../lib/types';
import { api, triggerHaptic } from '../../lib/api';
import { useVoiceInput } from '../../lib/useVoiceInput';
import { VoiceWave } from '../../components/VoiceWave';
export type CaptureTarget = 'inbox' | 'task' | 'list_item' | 'document';
@@ -46,17 +49,31 @@ export const InstantCaptureBar: React.FC<InstantCaptureBarProps> = ({
const [target, setTarget] = useState<CaptureTarget>(fixedTarget || 'inbox');
const [targetListId, setTargetListId] = useState<string>('');
const [showOptions, setShowOptions] = useState(defaultShowOptions || hideTargetPicker);
const [isListening, setIsListening] = useState(false);
const [voiceError, setVoiceError] = useState<string | null>(null);
const [isSubmitting, setIsSubmitting] = useState(false);
const [justSavedLabel, setJustSavedLabel] = useState<string | null>(null);
const [isMultiline, setIsMultiline] = useState(false);
const inputRef = useRef<HTMLTextAreaElement>(null);
const containerRef = useRef<HTMLFormElement>(null);
const recognitionRef = useRef<any>(null);
const baseVoiceTextRef = useRef('');
const hasVoiceInputRef = useRef(false);
// Voice input: own Whisper server or the browser's recognition (lib/voice.ts)
const voice = useVoiceInput({
onText: (text, final) => {
const base = baseVoiceTextRef.current;
if (text) setContent(base ? `${base} ${text}` : text);
if (final) inputRef.current?.focus();
},
onError: (message) => {
setVoiceError(message);
window.setTimeout(() => setVoiceError(null), 4000);
},
});
const isListening = voice.state === 'listening';
const isTranscribing = voice.state === 'transcribing';
const adjustHeight = (el?: HTMLTextAreaElement | null) => {
const targetEl = el || inputRef.current;
if (!targetEl) return;
@@ -120,57 +137,13 @@ export const InstantCaptureBar: React.FC<InstantCaptureBarProps> = ({
}, []);
const toggleVoiceDictation = () => {
const SpeechRecognition =
(window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
alert('Spracheingabe wird von diesem Browser nicht direkt unterstützt (bitte Chrome, Edge oder Safari nutzen).');
return;
}
if (isListening && recognitionRef.current) {
recognitionRef.current.stop();
setIsListening(false);
return;
}
try {
if (voice.state === 'idle') {
baseVoiceTextRef.current = content.replace(/\s*\[…\]$/, '').trim();
hasVoiceInputRef.current = true;
const recognition = new SpeechRecognition();
recognition.lang = 'de-DE';
recognition.continuous = false;
recognition.interimResults = true;
recognition.onstart = () => {
setIsListening(true);
triggerHaptic(20);
};
recognition.onresult = (event: any) => {
let transcript = '';
for (let i = 0; i < event.results.length; i++) {
transcript += event.results[i][0].transcript;
}
const base = baseVoiceTextRef.current;
setContent(base ? `${base} ${transcript}` : transcript);
};
recognition.onerror = () => {
setIsListening(false);
};
recognition.onend = () => {
setIsListening(false);
inputRef.current?.focus();
};
recognitionRef.current = recognition;
recognition.start();
} catch {
setIsListening(false);
setVoiceError(null);
triggerHaptic(20);
}
voice.toggle();
};
const executeCapture = async (
@@ -187,6 +160,8 @@ export const InstantCaptureBar: React.FC<InstantCaptureBarProps> = ({
const capturedDueDate = dueDate;
const capturedSource = hasVoiceInputRef.current || isListening ? 'voice' : 'manual';
hasVoiceInputRef.current = false;
// A recognition still running would refill the field after it was saved
if (voice.state !== 'idle') voice.cancel();
// Optimistically clear input immediately so the UI feels instant (<10ms) in all browsers
setContent('');
@@ -285,8 +260,12 @@ export const InstantCaptureBar: React.FC<InstantCaptureBarProps> = ({
}
}}
placeholder={
isListening
voiceError
? voiceError
: isListening
? 'Ich höre zu…'
: isTranscribing
? 'Wird erkannt…'
: hideTargetPicker || fixedTarget === 'inbox'
? 'Quick Dump erfassen… (Enter zum Speichern)'
: target === 'task'
@@ -336,19 +315,33 @@ export const InstantCaptureBar: React.FC<InstantCaptureBarProps> = ({
</button>
)}
{/* Voice Dictation Button */}
<button
type="button"
onClick={toggleVoiceDictation}
title="Spracheingabe"
className={`w-6 h-6 capture-pill rounded-full flex items-center justify-center transition-colors ${
isListening
? 'bg-accent-dim text-accent'
: 'text-muted hover:text-main hover:bg-panel-elevated'
}`}
>
{isListening ? <MicOff className="w-3 h-3" /> : <Mic className="w-3 h-3" />}
</button>
{/* Voice Dictation Button (hidden when voice input is off or not supported) */}
{voice.available && (
<>
{isListening && voice.mode === 'server' && (
<VoiceWave getLevel={voice.level} className="h-3.5 text-accent" />
)}
<button
type="button"
onClick={toggleVoiceDictation}
disabled={isTranscribing}
title={isListening ? 'Aufnahme beenden' : isTranscribing ? 'Wird erkannt…' : 'Spracheingabe'}
className={`w-6 h-6 capture-pill rounded-full flex items-center justify-center transition-colors ${
isListening || isTranscribing
? 'bg-accent-dim text-accent'
: 'text-muted hover:text-main hover:bg-panel-elevated'
}`}
>
{isTranscribing ? (
<Loader2 className="w-3 h-3 animate-spin" />
) : isListening ? (
<MicOff className="w-3 h-3" />
) : (
<Mic className="w-3 h-3" />
)}
</button>
</>
)}
{/* Instant Send Button */}
<button