import { spawn, execFile } from 'node:child_process'; import { promisify } from 'node:util'; import fs from 'node:fs'; import path from 'node:path'; import readline from 'node:readline'; import * as diar from './diar.js'; import * as llm from './llm.js'; const pexec = promisify(execFile); const WHISPER = '/opt/whisper.cpp/build/bin/whisper-cli'; // medium since 2026-07-08: user reported small mis-hears words on the USB-headset // call audio (cleaner than the old BT-mSBC, so the extra accuracy now pays off). // ~2x slower than small + more heat on long recordings (thermal throttle >85 C) — // acceptable trade for accuracy. Override with WHISPER_MODEL to switch back. const MODEL = process.env.WHISPER_MODEL || '/opt/whisper.cpp/models/ggml-medium.bin'; // NPU turbo transcriber: large-v3-turbo ENCODER on the RK3588 NPU (rknn fp16), // decoder on CPU. Benchmarked ~2.4x faster than whisper.cpp medium on the R1 // (RTF 0.64 vs 1.51) and the encoder runs off-CPU -> far less A76 heat. The // Python CLI emits {text, words:[{word,t_start_ms,t_end_ms}]}. Set TRANSCRIBER=cpu // to fall back to whisper.cpp medium (the block below). const TRANSCRIBER = process.env.TRANSCRIBER || 'npu'; const NPU_PY = '/opt/aime-venv/bin/python'; const NPU_CLI = '/opt/callrec/inference/transcribe_cli.py'; // SHORT clips (<= SHORT_SEC): a warm CPU daemon + resident rkllm instead of // the NPU path, whose per-call model load (~28-40s) + rkllm swap (~40s) dwarf // the actual work on a tiny clip. See the daemon client below. const TD_DAEMON = '/opt/callrec/inference/transcribe_daemon_cpu.py'; const SHORT_SEC = +(process.env.SHORT_TRANSCRIBE_SEC || 30); const REC_DIR = '/opt/callrec/recordings'; fs.mkdirSync(REC_DIR, { recursive: true }); // PipeWire/Pulse runs in the callrec user session; the systemd unit already // sets XDG_RUNTIME_DIR + DBUS_SESSION_BUS_ADDRESS, so just inherit the env. function penv() { return process.env; } // --- Thermal safety for whisper on the fanless board ----------------------- // A 79-min recording transcribed with ggml-medium pinned to all 4 A76 cores at // max clock drove the big cores to CRITICAL temp -> the kernel did a HARDWARE // PROTECTION shutdown (auto-reboot), losing the in-flight recording. Two guards: // 1. cap the A76 max clock while transcribing (biggest heat lever), and // 2. a hard backstop that SIGSTOPs whisper if the big cores still cross 90 C // and SIGCONTs it once they fall back under 78 C. const A76_MAX = 2256000; // restore-to clock (from scaling_available_frequencies) const A76_CAP = 1608000; // 1.6 GHz cap during whisper const A76_CAP_NPU = 1800000; // relaxed 1.8 GHz for the NPU decoder (encoder is off-CPU; // measured only ~2% slower at 1.6 but ~20 C cooler than full 2.25) const TEMP_ZONES = ['/sys/class/thermal/thermal_zone1/temp', // bigcore0 '/sys/class/thermal/thermal_zone2/temp']; // bigcore1 const TEMP_PAUSE = 90000; // >90 C -> pause whisper const TEMP_RESUME = 78000; // <78 C -> resume function readMaxBigTemp() { let m = 0; for (const z of TEMP_ZONES) { try { const v = parseInt(fs.readFileSync(z, 'utf8'), 10); if (v > m) m = v; } catch {} } return m; } // callrec has a NOPASSWD sudo rule for exactly /opt/callrec/aime-cpucap.sh. function setA76Max(freq) { return new Promise(res => execFile('sudo', ['-n', '/opt/callrec/aime-cpucap.sh', String(freq)], () => res())); } // Spawn whisper on the A76 cores with the SIGSTOP/SIGCONT temp backstop. The A76 // CLOCK CAP is NOT managed here — the caller (transcribeWav) holds it across the // WHOLE heavy pipeline (whisper + diarization), because diarization also pins the // A76 cores and crashed the board when it ran at full clock after whisper's cap // was released. Rejects on non-zero exit. async function runWhisperGuarded(args) { let guard = null, paused = false; try { await new Promise((resolve, reject) => { const p = spawn('taskset', ['-c', '4-7', WHISPER, ...args], { stdio: ['ignore', 'ignore', 'pipe'] }); let err = ''; p.stderr.on('data', d => { err += d; if (err.length > (1 << 20)) err = err.slice(-(1 << 19)); }); guard = setInterval(() => { const t = readMaxBigTemp(); if (!paused && t >= TEMP_PAUSE) { try { process.kill(p.pid, 'SIGSTOP'); paused = true; console.warn(`whisper PAUSED (A76 ${t / 1000} C)`); } catch {} } else if (paused && t > 0 && t <= TEMP_RESUME) { try { process.kill(p.pid, 'SIGCONT'); paused = false; console.warn(`whisper resumed (A76 ${t / 1000} C)`); } catch {} } }, 3000); p.on('error', reject); p.on('close', code => code === 0 ? resolve() : reject(new Error('whisper exit ' + code + ': ' + err.slice(-400)))); }); } finally { if (guard) clearInterval(guard); } } // Run the NPU turbo transcriber (Python) under the same A76 temp backstop as // whisper. taskset execs python -> p.pid IS python, so SIGSTOP/SIGCONT reach it. // Returns the parsed CLI JSON. The A76 CLOCK CAP is held by the caller. async function runNpuTranscribe(wavPath) { let guard = null, paused = false, out = '', err = ''; try { await new Promise((resolve, reject) => { const p = spawn('taskset', ['-c', '4-7', NPU_PY, NPU_CLI, wavPath], { stdio: ['ignore', 'pipe', 'pipe'], cwd: '/opt/callrec' }); p.stdout.on('data', d => { out += d; }); p.stderr.on('data', d => { err += d; if (err.length > (1 << 20)) err = err.slice(-(1 << 19)); }); guard = setInterval(() => { const t = readMaxBigTemp(); if (!paused && t >= TEMP_PAUSE) { try { process.kill(p.pid, 'SIGSTOP'); paused = true; console.warn(`transcribe PAUSED (A76 ${t / 1000} C)`); } catch {} } else if (paused && t > 0 && t <= TEMP_RESUME) { try { process.kill(p.pid, 'SIGCONT'); paused = false; console.warn(`transcribe resumed (A76 ${t / 1000} C)`); } catch {} } }, 3000); p.on('error', reject); p.on('close', code => code === 0 ? resolve() : reject(new Error('npu-transcribe exit ' + code + ': ' + err.slice(-400)))); }); } finally { if (guard) clearInterval(guard); } const line = out.trim(); if (!line) throw new Error('npu-transcribe: leere Ausgabe. stderr: ' + err.slice(-400)); return JSON.parse(line); } // --- Warm CPU transcriber daemon (SHORT clips) ----------------------------- // large-v3-turbo CT2 on the CPU, loaded ONCE and kept warm. CPU-only, so it // coexists with a resident rkllm_server on the NPU (no memory-wall conflict): // a short clip needs neither a per-call model load nor an rkllm swap. Lazy- // spawned on the first short clip, then persistent. base64 line protocol. let tdProc = null, tdReady = null, tdRl = null, tdPending = null, tdChain = Promise.resolve(); function startTd() { if (tdProc) return tdReady; tdProc = spawn('taskset', ['-c', '4-7', NPU_PY, TD_DAEMON], { stdio: ['pipe', 'pipe', 'pipe'], cwd: '/opt/callrec', env: { ...process.env, WHISPER_BATCH: process.env.WHISPER_BATCH || '1' } }); tdProc.stderr.on('data', () => {}); // onnxruntime GPU-probe warnings, ignore tdReady = new Promise((res, rej) => { const to = setTimeout(() => rej(new Error('transcribe-daemon READY timeout')), 120000); tdRl = readline.createInterface({ input: tdProc.stdout }); tdRl.on('line', line => { if (line.startsWith('READY')) { clearTimeout(to); res(); return; } if (!tdPending) return; const p = tdPending; tdPending = null; if (line.startsWith('OK ')) { try { p.resolve(JSON.parse(Buffer.from(line.slice(3), 'base64').toString('utf8'))); } catch (e) { p.reject(e); } } else p.reject(new Error(Buffer.from(line.slice(4), 'base64').toString('utf8') || 'daemon error')); }); }); const cleanup = () => { tdProc = null; tdReady = null; tdRl = null; if (tdPending) { tdPending.reject(new Error('transcribe-daemon beendet')); tdPending = null; } }; tdProc.on('exit', cleanup); tdProc.on('error', cleanup); return tdReady; } // One request at a time (single warm process), chained like spk.embed. function runDaemonTranscribe(wavPath) { tdChain = tdChain.then(async () => { await startTd(); return await new Promise((res, rej) => { tdPending = { resolve: res, reject: rej }; tdProc.stdin.write(wavPath + '\n'); }); }); return tdChain; } // Wav duration from file size (16 kHz mono s16 = 32000 B/s, 44 B header). function wavDurationSec(wavPath) { try { return Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch { return 0; } } let current = null; // { proc, wavPath, startTs, source, title, markers } export function status() { return current ? { recording: true, startedAt: current.startTs, source: current.source, markers: current.markers } : { recording: false }; } export function isRecording() { return !!current; } // --- live speaker markers --------------------------------------------------- // While recording, the user types the name of whoever is talking. We stamp the // offset from recording start; after diarization the name is attached to the // acoustic cluster that was speaking around that offset (see diar.resolveMarkers). // This is the only reliable way to name 20+ participants of a conference call. export function addMarker(name) { if (!current) throw new Error('not recording'); const n = String(name || '').trim(); if (!n) throw new Error('Name fehlt'); const m = { t_ms: Date.now() - current.startTs, name: n.slice(0, 128) }; current.markers.push(m); return { ...m, index: current.markers.length - 1, markers: current.markers }; } export function listMarkers() { return current ? current.markers : []; } export function removeMarker(index) { if (!current) throw new Error('not recording'); const i = +index; if (i >= 0 && i < current.markers.length) current.markers.splice(i, 1); return current.markers; } // List real capture devices (one entry per recording device). A "source" in // PulseAudio/PipeWire includes monitor sources (the loopback of every OUTPUT) // and both A2DP/HFP variants of a BT card — confusing for the user. We only // want actual microphones/inputs, labelled by their friendly device name. export async function listSources() { try { const { stdout } = await pexec('pactl', ['list', 'sources'], { env: penv() }); const out = []; const seen = new Set(); for (const block of stdout.split(/\nSource #/).map(b => b.trim()).filter(Boolean)) { const name = (block.match(/^\s*Name:\s*(.+)$/m) || [])[1]; if (!name) continue; // Skip monitors (loopback of an output) — not a recording device. // Every block has a "Monitor of Sink:" line; a real input reads "n/a". if (/\.monitor$/i.test(name) || /Monitor of Sink:\s*(?!n\/a)\S/i.test(block)) continue; const isBluetooth = /bluez/i.test(name); // For BT the physical device is one card; both bluez_input.* names map to // the same headset, so collapse by the MAC to a single entry. const key = isBluetooth ? (name.match(/([0-9A-F]{2}[:_]){5}[0-9A-F]{2}/i) || [name])[0] : name; if (seen.has(key)) continue; seen.add(key); const label = (block.match(/^\s*Description:\s*(.+)$/m) || [])[1] || name; out.push({ name, label, isBluetooth }); } return out; } catch (e) { return []; } } // Find the bluez card name (e.g. bluez_card.98_34_8C_FD_24_17). async function bluezCard() { const { stdout } = await pexec('pactl', ['list', 'cards', 'short'], { env: penv() }); return stdout.split('\n').map(l => l.split('\t')[1]).find(n => /bluez/i.test(n || '')); } // List available profiles of the bluez card (for diagnostics / picking HFP). async function cardProfiles(card) { const { stdout } = await pexec('pactl', ['list', 'cards'], { env: penv() }); const block = stdout.split('Card #').find(b => b.includes(card)); if (!block) return []; const m = block.match(/Profiles:[\s\S]*?(?=\n\tActive Profile:|\n\tPorts:|$)/); return m ? [...m[0].matchAll(/^\t\t([\w-]+):/gm)].map(x => x[1]) : []; } // Switch the connected BT headset to an HFP profile (bidirectional, mic-capable). // A2DP has no microphone, so recording a call REQUIRES HFP (headset-head-unit). // codec: 'msbc' (16 kHz wideband), 'cvsd' (8 kHz narrowband), or 'auto' (mSBC first). // On this card: headset-head-unit = mSBC, headset-head-unit-cvsd = CVSD. export async function enableHeadsetMic(codec = 'auto') { try { const card = await bluezCard(); if (!card) return { ok: false, msg: 'no bluez card (headset connected?)' }; const avail = await cardProfiles(card); const lists = { // mSBC = the plain "headset-head-unit" here (some stacks name it -msbc). msbc: ['headset-head-unit-msbc', 'headset-head-unit', 'handsfree-head-unit-msbc', 'handsfree-head-unit'], cvsd: ['headset-head-unit-cvsd', 'handsfree-head-unit-cvsd'], auto: ['headset-head-unit-msbc', 'headset-head-unit', 'handsfree-head-unit-msbc', 'handsfree-head-unit', 'headset-head-unit-cvsd', 'handsfree-head-unit-cvsd'], }; const prefer = lists[codec] || lists.auto; const pick = prefer.find(p => avail.includes(p)); if (!pick) return { ok: false, msg: `codec ${codec} nicht verfügbar (${avail.join(', ')})`, available: avail }; await pexec('pactl', ['set-card-profile', card, pick], { env: penv() }); return { ok: true, card, profile: pick, codec, available: avail }; } catch (e) { return { ok: false, msg: String(e && e.message || e) }; } } export async function startRecording({ source, title, codec, numSpeakers } = {}) { if (current) throw new Error('already recording'); // Bluetooth headset: force HFP so the microphone actually delivers audio // (A2DP is playback-only). Do this right before capture; the profile holds // while the source is open. codec = 'msbc' | 'cvsd' | 'auto' (A/B testing). let hfp = null; if (source && /bluez/i.test(source)) { hfp = await enableHeadsetMic(codec || 'auto'); if (!hfp.ok) throw new Error('HFP-Profil konnte nicht aktiviert werden: ' + hfp.msg); // the source object is recreated under the new profile; wait, then re-resolve // the actual bluez input source name (it may differ from the A2DP-era name). await new Promise(r => setTimeout(r, 900)); const srcs = await listSources(); const bz = srcs.find(s => s.isBluetooth && !/monitor/i.test(s.name)); if (bz) source = bz.name; } const ts = Date.now(); const wavPath = path.join('/tmp', `rec_${ts}.wav`); // 16 kHz mono s16 = whisper's native format. NOTE: parecord takes the output // filename as a POSITIONAL arg (there is no --file= option). const args = ['--file-format=wav', '--rate=16000', '--channels=1', '--format=s16le']; if (source) args.push(`--device=${source}`); args.push(wavPath); const proc = spawn('parecord', args, { env: penv() }); let err = ''; proc.stderr.on('data', d => { err += d; }); proc.on('error', e => { err += String(e); }); // Label the recording with the actual codec used so A/B tests are obvious // in the list (mSBC 16 kHz vs CVSD 8 kHz). const codecTag = hfp && hfp.profile ? (/cvsd/i.test(hfp.profile) ? ' [CVSD 8k]' : ' [mSBC 16k]') : ''; current = { proc, wavPath, startTs: ts, source: source || 'default', numSpeakers, markers: [], title: (title || `Aufnahme ${new Date(ts).toLocaleString('de-DE')}`) + codecTag, err: () => err }; return { startedAt: ts }; } // Stop capture and return the raw wav path only (no whisper/mp3/diarization). // Used for voiceprint enrollment. export async function stopRaw() { if (!current) throw new Error('not recording'); const c = current; current = null; c.proc.kill('SIGINT'); await new Promise(resolve => { if (c.proc.exitCode !== null || c.proc.signalCode !== null) return resolve(); let done = false; const fin = () => { if (!done) { done = true; resolve(); } }; c.proc.on('close', fin); setTimeout(fin, 4000); }); if (!fs.existsSync(c.wavPath) || fs.statSync(c.wavPath).size < 1000) throw new Error('Aufnahme leer/fehlgeschlagen. parecord: ' + c.err()); return c.wavPath; } // Stop capture and PERSIST the wav + mp3 to REC_DIR immediately — BEFORE any // transcription. This is the crash-safety fix: whisper on a long clip can crash // the board (thermal), so the audio must already be on rootfs (not /tmp) and the // DB row created before whisper runs. Call transcribeWav() afterwards (async). export async function stopAndSave() { if (!current) throw new Error('not recording'); const c = current; current = null; c.proc.kill('SIGINT'); await new Promise(resolve => { if (c.proc.exitCode !== null || c.proc.signalCode !== null) return resolve(); let done = false; const fin = () => { if (!done) { done = true; resolve(); } }; c.proc.on('close', fin); setTimeout(fin, 4000); }); const dur = Date.now() - c.startTs; if (!fs.existsSync(c.wavPath) || fs.statSync(c.wavPath).size < 1000) throw new Error('Aufnahme leer/fehlgeschlagen. parecord: ' + c.err()); const base = `rec_${c.startTs}`; const mp3Path = path.join(REC_DIR, `${base}.mp3`); // Gentle cleanup for playback: cut low rumble + normalize loudness. Applied to // the MP3 only; whisper still transcribes the untouched WAV. // lame's psymodel asserts (`calc_energy: el >= 0`) on pure digital silence — // real since the gated mic path: nobody spoke = all-zero samples. Fall back // to a filterless encode instead of failing the whole stop. try { await pexec('ffmpeg', ['-y', '-i', c.wavPath, '-af', 'highpass=f=80,loudnorm=I=-16:TP=-1.5:LRA=11', '-codec:a', 'libmp3lame', '-qscale:a', '3', mp3Path]); } catch (e) { console.warn('mp3 encode with filters failed (silence?), retrying plain:', String(e.message).slice(0, 200)); await pexec('ffmpeg', ['-y', '-i', c.wavPath, '-codec:a', 'libmp3lame', '-qscale:a', '3', mp3Path]); } // Move the wav into REC_DIR (rootfs — survives a reboot); it is NOT left in // /tmp, so a crash during transcription can no longer lose it. const wavFile = `${base}.wav`; try { fs.renameSync(c.wavPath, path.join(REC_DIR, wavFile)); } catch { fs.copyFileSync(c.wavPath, path.join(REC_DIR, wavFile)); try { fs.unlinkSync(c.wavPath); } catch {} } return { mp3_file: `${base}.mp3`, wav_file: wavFile, duration_ms: dur, language: 'de', title: c.title, device: c.source, numSpeakers: c.numSpeakers, markers: c.markers }; } // Transcribe a saved wav (in REC_DIR) with whisper (thermal-guarded) + diarize. // Returns { transcript, words }. Idempotent — safe to re-run for recovery of a // recording whose transcription was interrupted (crash) or failed. export async function transcribeWav(wavFile, numSpeakers, opts = {}) { const wavPath = path.isAbsolute(wavFile) ? wavFile : path.join(REC_DIR, wavFile); if (!fs.existsSync(wavPath)) throw new Error('wav-Datei fehlt: ' + wavFile); const outBase = `/tmp/${path.basename(wavPath, '.wav')}`; const words = []; let transcript = ''; // Diarization (sherpa-onnx over the whole recording) pins all 4 A76 cores and // crashed the board at full clock on a 46-min call, so it ALWAYS runs under the // A76 clock cap. Transcription is capped ONLY on the whisper.cpp (medium) path // (CPU-hot); the NPU-turbo path runs the encoder on the NPU (CPU only decodes, // ~59-69 C) so it needs NO cap. rkllm_server (~3.1 GB) is freed first on the NPU // path (turbo rknn + CT2 decoder OOM-killed next to it); it respawns for analysis. const wavSec = wavDurationSec(wavPath); try { if (TRANSCRIBER === 'npu' && wavSec > 0 && wavSec <= SHORT_SEC) { // SHORT clip: warm CPU daemon + keep rkllm RESIDENT (no NPU swap), at // full clock (a few seconds of compute won't overheat the fanless board). // Cuts a 10s clip from ~120s (per-call model load + rkllm swap) to ~30s. const r = await runDaemonTranscribe(wavPath); for (const w of r.words || []) { const t = (w.word || '').trim(); if (!t) continue; words.push({ word: t, t_start_ms: w.t_start_ms, t_end_ms: w.t_end_ms }); transcript += (transcript ? ' ' : '') + t; } console.log(`cpu-daemon-transcribe: ${words.length} words, RTF ${r.rtf} (${r.transcribe_s}s / ${r.duration}s audio, warm, rkllm kept resident)`); } else if (TRANSCRIBER === 'npu') { await llm.stopServer(); await setA76Max(A76_CAP_NPU); // large-v3-turbo, encoder on NPU. CLI already emits word-level timestamps. const r = await runNpuTranscribe(wavPath); for (const w of r.words || []) { const t = (w.word || '').trim(); if (!t) continue; words.push({ word: t, t_start_ms: w.t_start_ms, t_end_ms: w.t_end_ms }); transcript += (transcript ? ' ' : '') + t; } console.log(`npu-transcribe: ${words.length} words, RTF ${r.rtf} (load ${r.load_s}s + transcribe ${r.transcribe_s}s / ${r.duration}s audio)`); } else { await setA76Max(A76_CAP); // Fallback: whisper.cpp medium. Greedy (-bs 1 -bo 1) ~= beam-5 on German, // ~30% faster + less heat. whisper also has a SIGSTOP temp guard. await runWhisperGuarded(['-m', MODEL, '-f', wavPath, '-l', 'de', '-t', '4', '-bs', '1', '-bo', '1', '-ml', '1', '-sow', '-oj', '-of', outBase]); const json = JSON.parse(fs.readFileSync(`${outBase}.json`, 'utf8')); for (const seg of json.transcription || []) { const t = (seg.text || '').trim(); if (!t) continue; words.push({ word: t, t_start_ms: seg.offsets.from, t_end_ms: seg.offsets.to }); transcript += (transcript ? ' ' : '') + t; } } // Acoustic diarization under the A76 clock cap (both paths). Non-fatal. await setA76Max(A76_CAP); try { const segs = await diar.diarize(wavPath, numSpeakers, { markers: opts.markers || [] }); diar.assignSpeakers(words, segs); } catch (e) { console.error('diarize failed:', e.message); } try { fs.unlinkSync(`${outBase}.json`); } catch {} } finally { await setA76Max(A76_MAX); } return { transcript, words }; }