MindSift/frontend/src/lib/voice.ts

// Voice input for all places with a microphone button. The server decides how it works (STT_* in the
// environment, reported by /auth/status):
//   server   record with MediaRecorder, the own Whisper server turns it into text after stopping
//   browser  the browser's speech recognition (Chrome, Edge, Safari; text appears while speaking)
//   off      no voice input
import { api } from './api';
import type { VoiceMode } from './types';

export type EffectiveVoiceMode = VoiceMode | 'unsupported';

export interface VoiceConfig {
  mode: EffectiveVoiceMode;
  maxSeconds: number;
}

let configPromise: Promise<VoiceConfig> | null = null;
let loadedConfig: VoiceConfig | null = null;

export function browserRecognition(): any {
  if (typeof window === 'undefined') return null;
  return (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition || null;
}

export function canRecord(): boolean {
  return typeof window !== 'undefined' && typeof (window as any).MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia;
}

/** What this browser can actually do with the server's setting. */
export function effectiveMode(mode: VoiceMode | undefined): EffectiveVoiceMode {
  if (mode === 'off') return 'off';
  if (mode === 'server') return canRecord() ? 'server' : 'unsupported';
  return browserRecognition() ? 'browser' : 'unsupported';
}

/** Loaded once per page; the setting only changes with a server restart. */
export function loadVoiceConfig(): Promise<VoiceConfig> {
  if (!configPromise) {
    configPromise = api
      .getAuthStatus()
      .then((st) => {
        loadedConfig = { mode: effectiveMode(st.voice?.mode), maxSeconds: st.voice?.maxSeconds || 600 };
        return loadedConfig;
      })
      .catch(() => {
        configPromise = null; // offline at start: ask again next time
        return { mode: effectiveMode('browser'), maxSeconds: 600 };
      });
  }
  return configPromise;
}

/** The setting if already loaded; until then the browser's recognition (a button must react at once). */
export function currentVoiceConfig(): VoiceConfig {
  return loadedConfig ?? { mode: effectiveMode('browser'), maxSeconds: 600 };
}

/** For tests. */
export function resetVoiceConfig(): void {
  configPromise = null;
  loadedConfig = null;
}

// Containers in order of preference; Safari only records MP4/AAC
const RECORDING_TYPES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus'];

function recordingType(): string | undefined {
  const MR = (window as any).MediaRecorder;
  if (!MR?.isTypeSupported) return undefined;
  return RECORDING_TYPES.find((t) => MR.isTypeSupported(t));
}

/**
 * One recording from the microphone, with a live input level for wave animations. stop() resolves
 * with the audio; cancel() throws it away.
 */
export class VoiceRecorder {
  private stream: MediaStream | null = null;
  private recorder: MediaRecorder | null = null;
  private chunks: Blob[] = [];
  private audioCtx: AudioContext | null = null;
  private analyser: AnalyserNode | null = null;
  private samples: Uint8Array | null = null;
  private done: ((blob: Blob | null) => void) | null = null;

  async start(): Promise<void> {
    this.stream = await navigator.mediaDevices.getUserMedia({
      audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
    });
    this.startSegmentRecorder();
    try {
      const Ctx = (window as any).AudioContext || (window as any).webkitAudioContext;
      this.audioCtx = new Ctx();
      const source = this.audioCtx!.createMediaStreamSource(this.stream);
      this.analyser = this.audioCtx!.createAnalyser();
      this.analyser.fftSize = 512;
      source.connect(this.analyser);
      this.samples = new Uint8Array(this.analyser.fftSize);
      void this.audioCtx!.resume?.();
    } catch {
      this.analyser = null; // no level, the recording still works
    }
  }

  private startSegmentRecorder(): void {
    if (!this.stream) return;
    const type = recordingType();
    const rec = new MediaRecorder(this.stream, type ? { mimeType: type } : undefined);
    const chunks: Blob[] = [];
    rec.ondataavailable = (e) => {
      if (e.data.size > 0) chunks.push(e.data);
    };
    rec.onstop = () => {
      const blob = chunks.length ? new Blob(chunks, { type: rec.mimeType || type || 'audio/webm' }) : null;
      this.release();
      this.done?.(blob);
      this.done = null;
    };
    this.recorder = rec;
    this.chunks = chunks;
    rec.start(1000);
  }

  /** Input level 0..1 (roughly loudness), or null when unknown. */
  level(): number | null {
    if (!this.analyser || !this.samples) return null;
    this.analyser.getByteTimeDomainData(this.samples as Uint8Array<ArrayBuffer>);
    let sum = 0;
    for (const v of this.samples) {
      const x = (v - 128) / 128;
      sum += x * x;
    }
    return Math.min(1, Math.sqrt(sum / this.samples.length) * 4);
  }

  stop(): Promise<Blob | null> {
    return new Promise((resolve) => {
      if (!this.recorder || this.recorder.state === 'inactive') {
        this.release();
        resolve(null);
        return;
      }
      this.done = resolve;
      this.recorder.stop();
    });
  }

  /**
   * Stops the current MediaRecorder segment and immediately starts a new one on the same MediaStream.
   * Resolves with the finished segment's audio Blob while recording continues seamlessly.
   */
  cutSegment(): Promise<Blob | null> {
    if (!this.stream || !this.recorder || this.recorder.state === 'inactive') {
      return Promise.resolve(null);
    }
    const oldRecorder = this.recorder;
    const oldChunks = this.chunks;

    this.startSegmentRecorder();

    return new Promise((resolve) => {
      oldRecorder.onstop = () => {
        const blob = oldChunks.length
          ? new Blob(oldChunks, { type: oldRecorder.mimeType || 'audio/webm' })
          : null;
        resolve(blob);
      };
      oldRecorder.stop();
    });
  }

  cancel(): void {
    this.done = null;
    if (this.recorder && this.recorder.state !== 'inactive') {
      this.recorder.onstop = () => this.release();
      this.recorder.stop();
    } else {
      this.release();
    }
  }

  private release(): void {
    this.stream?.getTracks().forEach((t) => t.stop());
    this.stream = null;
    void this.audioCtx?.close?.().catch(() => {});
    this.audioCtx = null;
    this.analyser = null;
  }
}