From 3f5e3b2ac1b7e17b26d360d8944f273e7783d4c8 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 29 Oct 2026 23:23:54 +0100 Subject: [PATCH] feat(workspace): voice notes via /audio (M3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /audio (only in an empty bullet) shows a recording bubble: green round button starts, it turns red with a live wave and timer on its left, a tap saves the recording as an inline audio player. An audio file can be uploaded instead; Esc/✕ restores the empty bullet. - Chrome's WebM recordings get their length measured, so the player shows the total time and seeking works. - With the own Whisper server "Transkribieren" turns the recording into the (searchable) caption. - Loaded files keep their Blob, as the CSP rightly blocks fetch() on blob: URLs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + frontend/src/features/help/HelpView.tsx | 6 + .../src/features/workspace/AudioBlock.tsx | 136 +++++++++++++ .../workspace/AudioRecorderPicker.tsx | 183 ++++++++++++++++++ .../src/features/workspace/OutlinerBlock.tsx | 36 +++- frontend/src/lib/media.ts | 25 ++- 6 files changed, 378 insertions(+), 9 deletions(-) create mode 100644 frontend/src/features/workspace/AudioBlock.tsx create mode 100644 frontend/src/features/workspace/AudioRecorderPicker.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2015d..92a4f0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Das Format folgt [Keep a Changelog](https://keepachangelog.com/de/1.1.0/), die V - Eigene Spracherkennung (optional): Mit `STT_URL` schickt MindSift Aufnahmen an einen selbst betriebenen Whisper-Server (OpenAI-kompatibel, z. B. speaches) statt an Google oder Microsoft. Einstellung nur per Umgebungsvariablen (`STT_*`) inklusive Grenzwerten; `docker-compose.yml` enthält den Dienst `whisper` als auskommentierten Block zum Einschalten, das Modell lädt MindSift beim ersten Start selbst. Administration → Server & Isolation zeigt den aktiven Modus und bietet eine Testaufnahme. In der Demo bleibt es bei der Browser-Erkennung. - API: `POST /voice/transcribe`, `GET /admin/voice`; `/auth/status` meldet `voice.mode`. - Bilder in Notizen: `/bild` in einem leeren Punkt öffnet direkt im Punkt eine Auswahl – Datei, Kamera (Handy), Hineinziehen, Einfügen mit Strg+V oder Bild-Link (der Server übernimmt eine Kopie). Fotos werden vor dem Hochladen auf höchstens 2048 px verkleinert und als WebP gespeichert, Standortdaten entfallen dabei. Klick vergrößert, darunter eine optionale Beschriftung. Esc oder ✕ macht den Punkt wieder zum leeren Punkt. +- Sprachnotizen in Notizen: `/audio` in einem leeren Punkt zeigt eine Aufnahme-Blase – grüner Knopf startet, er wird rot und links laufen Live-Wellen mit der Zeit, ein Tipp speichert die Aufnahme als Audio-Player im Punkt. Alternativ eine Audiodatei hochladen. Mit eigener Spracherkennung macht „Transkribieren“ aus der Aufnahme die (durchsuchbare) Beschriftung. - Medienablage für Notizen (Grundlage für Bilder und Audio): Hochladen, Kopieren externer Bilder, Abruf nur mit Leserecht; Grenzen per `MEDIA_*`; nicht mehr verwendete Dateien werden automatisch entfernt. API: `POST /documents/{id}/media`, `POST /documents/{id}/media/fetch`, `GET /media/{id}`. - Code-Blöcke in Notizen: Sprachauswahl (rund 50 Sprachen, bei Bedarf nachgeladen), Farben beim Tippen im Stil von VS Code Dark+/Light+ für alle Themes, „Kopieren“, Tab/Shift+Tab rückt ein und aus. `/code yaml` oder ```` ```yaml ```` legt die Sprache direkt fest. diff --git a/frontend/src/features/help/HelpView.tsx b/frontend/src/features/help/HelpView.tsx index 2902c0a..b85dbc0 100644 --- a/frontend/src/features/help/HelpView.tsx +++ b/frontend/src/features/help/HelpView.tsx @@ -160,6 +160,8 @@ export const HelpView: React.FC = ({ onBack }) => { 'callout', 'divider', 'bild', + 'audio', + 'sprachnotiz', 'image', 'foto', 'auto-formatting', @@ -199,6 +201,10 @@ export const HelpView: React.FC = ({ onBack }) => { 🖼️ /bild (Bild hochladen, fotografieren, einfügen oder per Link; nur im leeren Punkt) +
+ 🎙️ + /audio (Sprachnotiz aufnehmen: grüner Knopf startet, roter speichert; oder Audiodatei hochladen) +
diff --git a/frontend/src/features/workspace/AudioBlock.tsx b/frontend/src/features/workspace/AudioBlock.tsx new file mode 100644 index 0000000..9a07df2 --- /dev/null +++ b/frontend/src/features/workspace/AudioBlock.tsx @@ -0,0 +1,136 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { FileAudio, Loader2, Wand2 } from 'lucide-react'; +import { api } from '../../lib/api'; +import { mediaBlob, mediaObjectURL, parseMediaContent } from '../../lib/media'; +import { loadVoiceConfig } from '../../lib/voice'; + +interface AudioBlockProps { + content: string; + canEdit: boolean; + onCaptionChange: (caption: string) => void; +} + +// Audio block: player plus caption. With the own Whisper server the recording can be turned into text, +// which becomes the caption (and so searchable). +export const AudioBlock: React.FC = ({ content, canEdit, onCaptionChange }) => { + const ref = parseMediaContent(content); + const mediaId = ref?.id; + const [src, setSrc] = useState(null); + const [failed, setFailed] = useState(false); + const [caption, setCaption] = useState(ref?.caption || ''); + const [canTranscribe, setCanTranscribe] = useState(false); + const [transcribing, setTranscribing] = useState(false); + const [error, setError] = useState(null); + const audioRef = useRef(null); + + // Recordings from Chrome (WebM) carry no length: jump to the end once so the browser measures it, + // otherwise the player shows no total time and seeking does not work + const fixDuration = () => { + const a = audioRef.current; + if (!a || Number.isFinite(a.duration)) return; + const reset = () => { + a.removeEventListener('timeupdate', reset); + a.currentTime = 0; + }; + a.addEventListener('timeupdate', reset); + a.currentTime = 1e101; + }; + + useEffect(() => { + setCaption(ref?.caption || ''); + }, [ref?.caption]); + + useEffect(() => { + if (!mediaId) return; + let alive = true; + setFailed(false); + mediaObjectURL(mediaId) + .then((u) => alive && setSrc(u)) + .catch(() => alive && setFailed(true)); + return () => { + alive = false; + }; + }, [mediaId]); + + useEffect(() => { + void loadVoiceConfig().then((cfg) => setCanTranscribe(cfg.mode === 'server')); + }, []); + + if (!ref) { + return
Audio-Verweis ist beschädigt: {content}
; + } + + const commitCaption = (value: string) => { + if (value.trim() !== ref.caption) onCaptionChange(value.trim()); + }; + + const transcribe = async () => { + if (!src) return; + setTranscribing(true); + setError(null); + try { + const blob = await mediaBlob(ref.id); + const text = await api.transcribe(blob); + if (text) { + setCaption(text); + commitCaption(text); + } else { + setError('Nichts erkannt'); + } + } catch (err: any) { + setError(err?.message || 'Spracherkennung fehlgeschlagen'); + } finally { + setTranscribing(false); + } + }; + + return ( +
+ {failed ? ( +
+ Aufnahme nicht verfügbar (offline oder gelöscht) +
+ ) : src ? ( +
+
+ ) : ( +
+ +
+ )} + + {canEdit ? ( + setCaption(e.target.value)} + onBlur={() => commitCaption(caption)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + (e.target as HTMLInputElement).blur(); + } + }} + placeholder="Beschriftung hinzufügen…" + className="w-full bg-transparent border-0 p-0 text-xs text-muted placeholder-faint focus:outline-none focus:ring-0" + /> + ) : ( + ref.caption &&

{ref.caption}

+ )} + {error &&

{error}

} +
+ ); +}; diff --git a/frontend/src/features/workspace/AudioRecorderPicker.tsx b/frontend/src/features/workspace/AudioRecorderPicker.tsx new file mode 100644 index 0000000..ebe4b08 --- /dev/null +++ b/frontend/src/features/workspace/AudioRecorderPicker.tsx @@ -0,0 +1,183 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { Loader2, Mic, Square, Upload, X } from 'lucide-react'; +import { UploadedMedia, uploadMedia } from '../../lib/media'; +import { VoiceRecorder } from '../../lib/voice'; +import { VoiceWave } from '../../components/VoiceWave'; +import { triggerHaptic } from '../../lib/api'; + +interface AudioRecorderPickerProps { + docId: string; + onDone: (media: UploadedMedia) => void; + onCancel: () => void; +} + +const MAX_RECORDING_MS = 30 * 60 * 1000; + +function formatTime(ms: number): string { + const s = Math.floor(ms / 1000); + return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; +} + +// Inline bubble an empty bullet turns into after /audio. Green button = start, red button = stop and +// save; the live wave runs left of the red button. An existing audio file can be uploaded instead. +export const AudioRecorderPicker: React.FC = ({ docId, onDone, onCancel }) => { + const [state, setState] = useState<'ready' | 'recording' | 'uploading'>('ready'); + const [elapsed, setElapsed] = useState(0); + const [error, setError] = useState(null); + const recorderRef = useRef(null); + const startedRef = useRef(0); + const tickRef = useRef(null); + const fileRef = useRef(null); + const boxRef = useRef(null); + + const stopTicking = () => { + if (tickRef.current !== null) window.clearInterval(tickRef.current); + tickRef.current = null; + }; + + useEffect(() => { + boxRef.current?.focus(); + return () => { + stopTicking(); + recorderRef.current?.cancel(); + }; + }, []); + + const upload = async (blob: Blob, name: string) => { + setState('uploading'); + setError(null); + try { + onDone(await uploadMedia(docId, blob, name)); + } catch (err: any) { + setError(err?.message || 'Hochladen fehlgeschlagen'); + setState('ready'); + } + }; + + const stop = async () => { + const rec = recorderRef.current; + if (!rec) return; + recorderRef.current = null; + stopTicking(); + triggerHaptic(20); + const blob = await rec.stop(); + if (!blob) { + setError('Die Aufnahme ist leer'); + setState('ready'); + return; + } + const ext = blob.type.includes('mp4') ? 'm4a' : blob.type.includes('ogg') ? 'ogg' : 'webm'; + await upload(blob, `sprachnotiz.${ext}`); + }; + + const start = async () => { + setError(null); + const rec = new VoiceRecorder(); + try { + await rec.start(); + } catch { + setError('Kein Zugriff auf das Mikrofon (Berechtigung, oder die Seite läuft nicht über HTTPS)'); + return; + } + recorderRef.current = rec; + startedRef.current = Date.now(); + setElapsed(0); + setState('recording'); + triggerHaptic([15, 60, 25]); + tickRef.current = window.setInterval(() => { + const ms = Date.now() - startedRef.current; + setElapsed(ms); + if (ms >= MAX_RECORDING_MS) void stop(); + }, 250); + }; + + const cancel = () => { + stopTicking(); + recorderRef.current?.cancel(); + recorderRef.current = null; + onCancel(); + }; + + return ( +
{ + if (e.key === 'Escape') { + e.preventDefault(); + cancel(); + } + }} + className="my-1 inline-flex max-w-full flex-col gap-2 rounded-card border border-border bg-panel-elevated px-3 py-2 focus:outline-none" + > +
+ {state === 'recording' && ( + <> + {formatTime(elapsed)} + recorderRef.current?.level() ?? null} bars={11} className="h-6 text-status-err" /> + + )} + {state === 'ready' && Sprachnotiz aufnehmen} + {state === 'uploading' && ( + + Wird gespeichert… + + )} + + {state === 'recording' ? ( + + ) : ( + + )} + + {state === 'ready' && ( + + )} + +
+ {error &&

{error}

} + { + const f = e.target.files?.[0]; + if (f) void upload(f, f.name || 'audio'); + }} + /> +
+ ); +}; diff --git a/frontend/src/features/workspace/OutlinerBlock.tsx b/frontend/src/features/workspace/OutlinerBlock.tsx index 6279722..f556ecd 100644 --- a/frontend/src/features/workspace/OutlinerBlock.tsx +++ b/frontend/src/features/workspace/OutlinerBlock.tsx @@ -21,12 +21,15 @@ import { Lock, AlertTriangle, Image as ImageIcon, + Mic, } from 'lucide-react'; import { DocumentBlock, BlockType } from '../../lib/types'; import { LinkModal } from './LinkModal'; import { CodeBlockEditor, resolveLanguageId, isUnambiguousLanguageShortcut } from './CodeBlockEditor'; import { ImagePicker } from './ImagePicker'; import { ImageBlock } from './ImageBlock'; +import { AudioRecorderPicker } from './AudioRecorderPicker'; +import { AudioBlock } from './AudioBlock'; import { mediaContent, parseMediaContent } from '../../lib/media'; import { renderInlineMarkdown } from '../../lib/markdown'; import { api } from '../../lib/api'; @@ -51,6 +54,7 @@ const SLASH_COMMANDS: SlashCommand[] = [ { type: 'callout', label: '/callout', desc: 'Hinweisbox', icon: }, { type: 'divider', label: '/divider', desc: 'Trennlinie', icon: }, { type: 'image', label: '/bild', desc: 'Bild hochladen, fotografieren oder verlinken (image)', icon: , media: true }, + { type: 'audio', label: '/audio', desc: 'Sprachnotiz aufnehmen oder Audiodatei hochladen', icon: , media: true }, ]; export interface OutlinerBlockProps { @@ -142,7 +146,7 @@ export const OutlinerBlock: React.FC = ({ const canFocus = canEdit || (!!lockedBy && !!canAppendBelow); const [slashSearch, setSlashSearch] = useState(null); // Inline picker an empty bullet turns into after /bild (local only, nothing synced until done) - const [mediaPicker, setMediaPicker] = useState<'image' | null>(null); + const [mediaPicker, setMediaPicker] = useState<'image' | 'audio' | null>(null); const [slashIndex, setSlashIndex] = useState(0); const [showLinkModal, setShowLinkModal] = useState(false); const [showTodoMenu, setShowTodoMenu] = useState(false); @@ -361,7 +365,7 @@ export const OutlinerBlock: React.FC = ({ setLocalContent(''); localContentRef.current = ''; flushDebounce(''); - setMediaPicker(cmd.type as 'image'); + setMediaPicker(cmd.type as 'image' | 'audio'); return; } const langOverride = @@ -850,6 +854,34 @@ export const OutlinerBlock: React.FC = ({ requestAnimationFrame(() => textareaRef.current?.focus()); }} /> + ) : mediaPicker === 'audio' ? ( + { + setMediaPicker(null); + const content = mediaContent('audio', media.id); + setLocalContent(content); + localContentRef.current = content; + flushDebounce(content, { type: 'audio' }); + }} + onCancel={() => { + setMediaPicker(null); + requestAnimationFrame(() => textareaRef.current?.focus()); + }} + /> + ) : block.type === 'audio' ? ( + { + const ref = parseMediaContent(localContent); + if (!ref) return; + const content = mediaContent('audio', ref.id, caption); + setLocalContent(content); + localContentRef.current = content; + flushDebounce(content); + }} + /> ) : block.type === 'image' ? ( >(); +// Loaded files, once per page. The Blob is kept as well: the CSP does not allow fetch() on blob: URLs. +const loaded = new Map>(); -/** blob: URL of a file; loaded once per page. */ -export function mediaObjectURL(id: string): Promise { - let p = objectURLs.get(id); +function loadMedia(id: string): Promise<{ blob: Blob; url: string }> { + let p = loaded.get(id); if (!p) { p = (async () => { const token = getToken(); @@ -35,14 +35,25 @@ export function mediaObjectURL(id: string): Promise { headers: token ? { Authorization: `Bearer ${token}` } : {}, }); if (!res.ok) throw new Error(res.status === 404 ? 'Datei nicht gefunden' : `HTTP ${res.status}`); - return URL.createObjectURL(await res.blob()); + const blob = await res.blob(); + return { blob, url: URL.createObjectURL(blob) }; })(); - p.catch(() => objectURLs.delete(id)); // try again next time (e.g. was offline) - objectURLs.set(id, p); + p.catch(() => loaded.delete(id)); // try again next time (e.g. was offline) + loaded.set(id, p); } return p; } +/** blob: URL of a file for /