feat(workspace): voice notes via /audio (M3)
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 2m10s
MindSift CI & Docker Release / Build, Test in Container & Push (push) Successful in 2m10s
- /audio (only in an empty bullet) shows a recording bubble: green round button starts, it turns red with a live wave and timer on its left, a tap saves the recording as an inline audio player. An audio file can be uploaded instead; Esc/✕ restores the empty bullet. - Chrome's WebM recordings get their length measured, so the player shows the total time and seeking works. - With the own Whisper server "Transkribieren" turns the recording into the (searchable) caption. - Loaded files keep their Blob, as the CSP rightly blocks fetch() on blob: URLs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sandrodachandClaude Opus 5.5 committed
1 parent
e8fa14a4cb
commit
3f5e3b2ac1
6 files changed
+378
-9
No files matched your search
@@ -160,6 +160,8 @@ export const HelpView: React.FC<HelpViewProps> = ({ onBack }) => {
|
||||
'callout',
|
||||
'divider',
|
||||
'bild',
|
||||
'audio',
|
||||
'sprachnotiz',
|
||||
'image',
|
||||
'foto',
|
||||
'auto-formatting',
|
||||
@@ -199,6 +201,10 @@ export const HelpView: React.FC<HelpViewProps> = ({ onBack }) => {
|
||||
<span className="text-accent font-bold">🖼️</span>
|
||||
<span>/bild (Bild hochladen, fotografieren, einfügen oder per Link; nur im leeren Punkt)</span>
|
||||
</div>
|
||||
<div className="p-1.5 bg-panel rounded border border-border flex items-center gap-1.5 text-main col-span-2">
|
||||
<span className="text-accent font-bold">🎙️</span>
|
||||
<span>/audio (Sprachnotiz aufnehmen: grüner Knopf startet, roter speichert; oder Audiodatei hochladen)</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="pt-2 border-t border-border">
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { FileAudio, Loader2, Wand2 } from 'lucide-react';
|
||||
import { api } from '../../lib/api';
|
||||
import { mediaBlob, mediaObjectURL, parseMediaContent } from '../../lib/media';
|
||||
import { loadVoiceConfig } from '../../lib/voice';
|
||||
|
||||
interface AudioBlockProps {
|
||||
content: string;
|
||||
canEdit: boolean;
|
||||
onCaptionChange: (caption: string) => void;
|
||||
}
|
||||
|
||||
// Audio block: player plus caption. With the own Whisper server the recording can be turned into text,
|
||||
// which becomes the caption (and so searchable).
|
||||
export const AudioBlock: React.FC<AudioBlockProps> = ({ content, canEdit, onCaptionChange }) => {
|
||||
const ref = parseMediaContent(content);
|
||||
const mediaId = ref?.id;
|
||||
const [src, setSrc] = useState<string | null>(null);
|
||||
const [failed, setFailed] = useState(false);
|
||||
const [caption, setCaption] = useState(ref?.caption || '');
|
||||
const [canTranscribe, setCanTranscribe] = useState(false);
|
||||
const [transcribing, setTranscribing] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
|
||||
// Recordings from Chrome (WebM) carry no length: jump to the end once so the browser measures it,
|
||||
// otherwise the player shows no total time and seeking does not work
|
||||
const fixDuration = () => {
|
||||
const a = audioRef.current;
|
||||
if (!a || Number.isFinite(a.duration)) return;
|
||||
const reset = () => {
|
||||
a.removeEventListener('timeupdate', reset);
|
||||
a.currentTime = 0;
|
||||
};
|
||||
a.addEventListener('timeupdate', reset);
|
||||
a.currentTime = 1e101;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
setCaption(ref?.caption || '');
|
||||
}, [ref?.caption]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mediaId) return;
|
||||
let alive = true;
|
||||
setFailed(false);
|
||||
mediaObjectURL(mediaId)
|
||||
.then((u) => alive && setSrc(u))
|
||||
.catch(() => alive && setFailed(true));
|
||||
return () => {
|
||||
alive = false;
|
||||
};
|
||||
}, [mediaId]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadVoiceConfig().then((cfg) => setCanTranscribe(cfg.mode === 'server'));
|
||||
}, []);
|
||||
|
||||
if (!ref) {
|
||||
return <div className="text-xs text-status-err py-1">Audio-Verweis ist beschädigt: {content}</div>;
|
||||
}
|
||||
|
||||
const commitCaption = (value: string) => {
|
||||
if (value.trim() !== ref.caption) onCaptionChange(value.trim());
|
||||
};
|
||||
|
||||
const transcribe = async () => {
|
||||
if (!src) return;
|
||||
setTranscribing(true);
|
||||
setError(null);
|
||||
try {
|
||||
const blob = await mediaBlob(ref.id);
|
||||
const text = await api.transcribe(blob);
|
||||
if (text) {
|
||||
setCaption(text);
|
||||
commitCaption(text);
|
||||
} else {
|
||||
setError('Nichts erkannt');
|
||||
}
|
||||
} catch (err: any) {
|
||||
setError(err?.message || 'Spracherkennung fehlgeschlagen');
|
||||
} finally {
|
||||
setTranscribing(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="my-1 space-y-1">
|
||||
{failed ? (
|
||||
<div className="flex items-center gap-2 h-12 px-3 rounded-card border border-border bg-panel text-xs text-muted">
|
||||
<FileAudio className="w-4 h-4" /> Aufnahme nicht verfügbar (offline oder gelöscht)
|
||||
</div>
|
||||
) : src ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<audio ref={audioRef} controls preload="metadata" src={src} onLoadedMetadata={fixDuration} className="w-full max-w-md h-11" />
|
||||
{canEdit && canTranscribe && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void transcribe()}
|
||||
disabled={transcribing}
|
||||
title="In Text umwandeln (eigene Spracherkennung)"
|
||||
className="btn-secondary h-11 sm:h-8 px-2.5 text-xs flex items-center gap-1.5 shrink-0 disabled:opacity-40"
|
||||
>
|
||||
{transcribing ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <Wand2 className="w-3.5 h-3.5" />}
|
||||
<span className="hidden sm:inline">{transcribing ? 'Wird erkannt…' : 'Transkribieren'}</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center h-12 px-3 rounded-card border border-border bg-panel w-64">
|
||||
<Loader2 className="w-4 h-4 animate-spin text-muted" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{canEdit ? (
|
||||
<input
|
||||
type="text"
|
||||
value={caption}
|
||||
onChange={(e) => setCaption(e.target.value)}
|
||||
onBlur={() => commitCaption(caption)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
(e.target as HTMLInputElement).blur();
|
||||
}
|
||||
}}
|
||||
placeholder="Beschriftung hinzufügen…"
|
||||
className="w-full bg-transparent border-0 p-0 text-xs text-muted placeholder-faint focus:outline-none focus:ring-0"
|
||||
/>
|
||||
) : (
|
||||
ref.caption && <p className="text-xs text-muted">{ref.caption}</p>
|
||||
)}
|
||||
{error && <p className="text-xs text-status-err">{error}</p>}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,183 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { Loader2, Mic, Square, Upload, X } from 'lucide-react';
|
||||
import { UploadedMedia, uploadMedia } from '../../lib/media';
|
||||
import { VoiceRecorder } from '../../lib/voice';
|
||||
import { VoiceWave } from '../../components/VoiceWave';
|
||||
import { triggerHaptic } from '../../lib/api';
|
||||
|
||||
interface AudioRecorderPickerProps {
|
||||
docId: string;
|
||||
onDone: (media: UploadedMedia) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
const MAX_RECORDING_MS = 30 * 60 * 1000;
|
||||
|
||||
function formatTime(ms: number): string {
|
||||
const s = Math.floor(ms / 1000);
|
||||
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
// Inline bubble an empty bullet turns into after /audio. Green button = start, red button = stop and
|
||||
// save; the live wave runs left of the red button. An existing audio file can be uploaded instead.
|
||||
export const AudioRecorderPicker: React.FC<AudioRecorderPickerProps> = ({ docId, onDone, onCancel }) => {
|
||||
const [state, setState] = useState<'ready' | 'recording' | 'uploading'>('ready');
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const recorderRef = useRef<VoiceRecorder | null>(null);
|
||||
const startedRef = useRef(0);
|
||||
const tickRef = useRef<number | null>(null);
|
||||
const fileRef = useRef<HTMLInputElement>(null);
|
||||
const boxRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const stopTicking = () => {
|
||||
if (tickRef.current !== null) window.clearInterval(tickRef.current);
|
||||
tickRef.current = null;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
boxRef.current?.focus();
|
||||
return () => {
|
||||
stopTicking();
|
||||
recorderRef.current?.cancel();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const upload = async (blob: Blob, name: string) => {
|
||||
setState('uploading');
|
||||
setError(null);
|
||||
try {
|
||||
onDone(await uploadMedia(docId, blob, name));
|
||||
} catch (err: any) {
|
||||
setError(err?.message || 'Hochladen fehlgeschlagen');
|
||||
setState('ready');
|
||||
}
|
||||
};
|
||||
|
||||
const stop = async () => {
|
||||
const rec = recorderRef.current;
|
||||
if (!rec) return;
|
||||
recorderRef.current = null;
|
||||
stopTicking();
|
||||
triggerHaptic(20);
|
||||
const blob = await rec.stop();
|
||||
if (!blob) {
|
||||
setError('Die Aufnahme ist leer');
|
||||
setState('ready');
|
||||
return;
|
||||
}
|
||||
const ext = blob.type.includes('mp4') ? 'm4a' : blob.type.includes('ogg') ? 'ogg' : 'webm';
|
||||
await upload(blob, `sprachnotiz.${ext}`);
|
||||
};
|
||||
|
||||
const start = async () => {
|
||||
setError(null);
|
||||
const rec = new VoiceRecorder();
|
||||
try {
|
||||
await rec.start();
|
||||
} catch {
|
||||
setError('Kein Zugriff auf das Mikrofon (Berechtigung, oder die Seite läuft nicht über HTTPS)');
|
||||
return;
|
||||
}
|
||||
recorderRef.current = rec;
|
||||
startedRef.current = Date.now();
|
||||
setElapsed(0);
|
||||
setState('recording');
|
||||
triggerHaptic([15, 60, 25]);
|
||||
tickRef.current = window.setInterval(() => {
|
||||
const ms = Date.now() - startedRef.current;
|
||||
setElapsed(ms);
|
||||
if (ms >= MAX_RECORDING_MS) void stop();
|
||||
}, 250);
|
||||
};
|
||||
|
||||
const cancel = () => {
|
||||
stopTicking();
|
||||
recorderRef.current?.cancel();
|
||||
recorderRef.current = null;
|
||||
onCancel();
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={boxRef}
|
||||
tabIndex={-1}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault();
|
||||
cancel();
|
||||
}
|
||||
}}
|
||||
className="my-1 inline-flex max-w-full flex-col gap-2 rounded-card border border-border bg-panel-elevated px-3 py-2 focus:outline-none"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
{state === 'recording' && (
|
||||
<>
|
||||
<span className="text-xs font-mono text-main tabular-nums">{formatTime(elapsed)}</span>
|
||||
<VoiceWave getLevel={() => recorderRef.current?.level() ?? null} bars={11} className="h-6 text-status-err" />
|
||||
</>
|
||||
)}
|
||||
{state === 'ready' && <span className="text-xs text-muted">Sprachnotiz aufnehmen</span>}
|
||||
{state === 'uploading' && (
|
||||
<span className="text-xs text-muted flex items-center gap-1.5">
|
||||
<Loader2 className="w-3.5 h-3.5 animate-spin" /> Wird gespeichert…
|
||||
</span>
|
||||
)}
|
||||
|
||||
{state === 'recording' ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void stop()}
|
||||
title="Aufnahme beenden und speichern"
|
||||
aria-label="Aufnahme beenden und speichern"
|
||||
className="w-14 h-14 sm:w-10 sm:h-10 rounded-full bg-status-err text-on-accent flex items-center justify-center shadow-panel shrink-0"
|
||||
>
|
||||
<Square className="w-5 h-5 sm:w-4 sm:h-4 fill-current" />
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void start()}
|
||||
disabled={state === 'uploading'}
|
||||
title="Aufnahme starten"
|
||||
aria-label="Aufnahme starten"
|
||||
className="w-14 h-14 sm:w-10 sm:h-10 rounded-full bg-status-ok text-on-accent flex items-center justify-center shadow-panel shrink-0 disabled:opacity-40"
|
||||
>
|
||||
<Mic className="w-6 h-6 sm:w-5 sm:h-5" />
|
||||
</button>
|
||||
)}
|
||||
|
||||
{state === 'ready' && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => fileRef.current?.click()}
|
||||
title="Audiodatei hochladen"
|
||||
className="btn-secondary h-11 sm:h-8 px-3 text-xs flex items-center gap-1.5 shrink-0"
|
||||
>
|
||||
<Upload className="w-3.5 h-3.5" /> <span className="hidden sm:inline">Datei</span>
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancel}
|
||||
disabled={state === 'uploading'}
|
||||
title="Abbrechen (Esc)"
|
||||
className="w-11 h-11 sm:w-7 sm:h-7 flex items-center justify-center rounded-btn text-muted hover:text-main hover:bg-panel shrink-0 disabled:opacity-40"
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
{error && <p className="text-xs text-status-err">{error}</p>}
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept="audio/*"
|
||||
hidden
|
||||
onChange={(e) => {
|
||||
const f = e.target.files?.[0];
|
||||
if (f) void upload(f, f.name || 'audio');
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -21,12 +21,15 @@ import {
|
||||
Lock,
|
||||
AlertTriangle,
|
||||
Image as ImageIcon,
|
||||
Mic,
|
||||
} from 'lucide-react';
|
||||
import { DocumentBlock, BlockType } from '../../lib/types';
|
||||
import { LinkModal } from './LinkModal';
|
||||
import { CodeBlockEditor, resolveLanguageId, isUnambiguousLanguageShortcut } from './CodeBlockEditor';
|
||||
import { ImagePicker } from './ImagePicker';
|
||||
import { ImageBlock } from './ImageBlock';
|
||||
import { AudioRecorderPicker } from './AudioRecorderPicker';
|
||||
import { AudioBlock } from './AudioBlock';
|
||||
import { mediaContent, parseMediaContent } from '../../lib/media';
|
||||
import { renderInlineMarkdown } from '../../lib/markdown';
|
||||
import { api } from '../../lib/api';
|
||||
@@ -51,6 +54,7 @@ const SLASH_COMMANDS: SlashCommand[] = [
|
||||
{ type: 'callout', label: '/callout', desc: 'Hinweisbox', icon: <MessageSquare className="w-3.5 h-3.5" /> },
|
||||
{ type: 'divider', label: '/divider', desc: 'Trennlinie', icon: <Minus className="w-3.5 h-3.5" /> },
|
||||
{ type: 'image', label: '/bild', desc: 'Bild hochladen, fotografieren oder verlinken (image)', icon: <ImageIcon className="w-3.5 h-3.5" />, media: true },
|
||||
{ type: 'audio', label: '/audio', desc: 'Sprachnotiz aufnehmen oder Audiodatei hochladen', icon: <Mic className="w-3.5 h-3.5" />, media: true },
|
||||
];
|
||||
|
||||
export interface OutlinerBlockProps {
|
||||
@@ -142,7 +146,7 @@ export const OutlinerBlock: React.FC<OutlinerBlockProps> = ({
|
||||
const canFocus = canEdit || (!!lockedBy && !!canAppendBelow);
|
||||
const [slashSearch, setSlashSearch] = useState<string | null>(null);
|
||||
// Inline picker an empty bullet turns into after /bild (local only, nothing synced until done)
|
||||
const [mediaPicker, setMediaPicker] = useState<'image' | null>(null);
|
||||
const [mediaPicker, setMediaPicker] = useState<'image' | 'audio' | null>(null);
|
||||
const [slashIndex, setSlashIndex] = useState(0);
|
||||
const [showLinkModal, setShowLinkModal] = useState<boolean>(false);
|
||||
const [showTodoMenu, setShowTodoMenu] = useState<boolean>(false);
|
||||
@@ -361,7 +365,7 @@ export const OutlinerBlock: React.FC<OutlinerBlockProps> = ({
|
||||
setLocalContent('');
|
||||
localContentRef.current = '';
|
||||
flushDebounce('');
|
||||
setMediaPicker(cmd.type as 'image');
|
||||
setMediaPicker(cmd.type as 'image' | 'audio');
|
||||
return;
|
||||
}
|
||||
const langOverride =
|
||||
@@ -850,6 +854,34 @@ export const OutlinerBlock: React.FC<OutlinerBlockProps> = ({
|
||||
requestAnimationFrame(() => textareaRef.current?.focus());
|
||||
}}
|
||||
/>
|
||||
) : mediaPicker === 'audio' ? (
|
||||
<AudioRecorderPicker
|
||||
docId={docId}
|
||||
onDone={(media) => {
|
||||
setMediaPicker(null);
|
||||
const content = mediaContent('audio', media.id);
|
||||
setLocalContent(content);
|
||||
localContentRef.current = content;
|
||||
flushDebounce(content, { type: 'audio' });
|
||||
}}
|
||||
onCancel={() => {
|
||||
setMediaPicker(null);
|
||||
requestAnimationFrame(() => textareaRef.current?.focus());
|
||||
}}
|
||||
/>
|
||||
) : block.type === 'audio' ? (
|
||||
<AudioBlock
|
||||
content={localContent}
|
||||
canEdit={canEdit}
|
||||
onCaptionChange={(caption) => {
|
||||
const ref = parseMediaContent(localContent);
|
||||
if (!ref) return;
|
||||
const content = mediaContent('audio', ref.id, caption);
|
||||
setLocalContent(content);
|
||||
localContentRef.current = content;
|
||||
flushDebounce(content);
|
||||
}}
|
||||
/>
|
||||
) : block.type === 'image' ? (
|
||||
<ImageBlock
|
||||
content={localContent}
|
||||
|
||||
@@ -23,11 +23,11 @@ export function mediaContent(kind: 'image' | 'audio', id: string, caption = ''):
|
||||
return `${kind === 'image' ? '!' : ''}[${safe}](media:${id})`;
|
||||
}
|
||||
|
||||
const objectURLs = new Map<string, Promise<string>>();
|
||||
// Loaded files, once per page. The Blob is kept as well: the CSP does not allow fetch() on blob: URLs.
|
||||
const loaded = new Map<string, Promise<{ blob: Blob; url: string }>>();
|
||||
|
||||
/** blob: URL of a file; loaded once per page. */
|
||||
export function mediaObjectURL(id: string): Promise<string> {
|
||||
let p = objectURLs.get(id);
|
||||
function loadMedia(id: string): Promise<{ blob: Blob; url: string }> {
|
||||
let p = loaded.get(id);
|
||||
if (!p) {
|
||||
p = (async () => {
|
||||
const token = getToken();
|
||||
@@ -35,14 +35,25 @@ export function mediaObjectURL(id: string): Promise<string> {
|
||||
headers: token ? { Authorization: `Bearer ${token}` } : {},
|
||||
});
|
||||
if (!res.ok) throw new Error(res.status === 404 ? 'Datei nicht gefunden' : `HTTP ${res.status}`);
|
||||
return URL.createObjectURL(await res.blob());
|
||||
const blob = await res.blob();
|
||||
return { blob, url: URL.createObjectURL(blob) };
|
||||
})();
|
||||
p.catch(() => objectURLs.delete(id)); // try again next time (e.g. was offline)
|
||||
objectURLs.set(id, p);
|
||||
p.catch(() => loaded.delete(id)); // try again next time (e.g. was offline)
|
||||
loaded.set(id, p);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/** blob: URL of a file for <img>/<audio>. */
|
||||
export function mediaObjectURL(id: string): Promise<string> {
|
||||
return loadMedia(id).then((m) => m.url);
|
||||
}
|
||||
|
||||
/** The file itself, e.g. to send a recording to the speech recognition. */
|
||||
export function mediaBlob(id: string): Promise<Blob> {
|
||||
return loadMedia(id).then((m) => m.blob);
|
||||
}
|
||||
|
||||
export interface UploadedMedia {
|
||||
id: string;
|
||||
kind: 'image' | 'audio';
|
||||
|
||||
Reference in new issue
Block a user