Sascha/Aime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(diar): sherpa abschnittsweise laufen lassen (OOM bei langen Aufnahmen)

Browse Source 
Der sherpa-Diarisierungslauf brauchte RAM quadratisch zur Gesprächslänge
(gemessen 349 MB bei 10 min, 1389 MB bei 20 min, 2643 MB bei 30 min). Die
54-min-Aufnahme #99 riss 5,4 GB, wurde OOM-getötet und nahm wegen
OOMPolicy=stop die ganze App mit — in der GUI "Transkription fehlgeschlagen"
und "failed to fetch".

Die v2-Pipeline verwirft sherpas Labels und nutzt nur die Turn-Grenzen, also
laeuft der Pass jetzt in 600-s-Abschnitten (je +-5 s Kontext) und die Grenzen
werden mit Zeit-Offset zusammengesetzt. Zusätzlich OOMPolicy=continue, damit
ein OOM-getöteter Helfer die App nicht mehr stoppt.

Verifiziert an #99: sherpa-Spitze 363 MB, 726 Turns, 8 Sprecher, Analyse
vollstaendig (done/done).
This commit is contained in:
Sascha
2026-08-07 13:50:01 +02:00
parent 0aa6a37325
commit c5daee25af
4 changed files with 161 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/diar.js
+58 -10
View File
@@ -36,11 +36,11 @@ const MIN_TOTAL_SEC = +(process.env.DIAR_MIN_TOTAL_SEC || 20);
// 40 min). The old fixed 10-min timeout killed every recording over ~11 min and
// surfaced as an opaque "Command failed" (only sherpa's argv echo in stderr).
// Scale the timeout with the audio length: ~2.5x its duration, floor 20 min.
function diarTimeoutMs(wavPath) {
let sec = 0;
try { sec = Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch {} // 16kHz mono s16
return Math.max(20 * 60 * 1000, Math.ceil(sec * 2.5) * 1000);
function wavSec(wavPath) {
try { return Math.max(0, (fs.statSync(wavPath).size - 44) / 32000); } catch { return 0; } // 16kHz mono s16
}
function diarTimeoutMs(wavPath) { return timeoutForSec(wavSec(wavPath)); }
function timeoutForSec(sec) { return Math.max(20 * 60 * 1000, Math.ceil(sec * 2.5) * 1000); }
const DIR = '/opt/sherpa/sherpa-onnx-v1.13.3-linux-aarch64-shared-cpu';
const BIN = `${DIR}/bin/sherpa-onnx-offline-speaker-diarization`;
@@ -98,12 +98,26 @@ const COS_FINAL_MAX = +(process.env.DIAR_COS_FINAL_MAX || 0.35);
// want fine turn boundaries (and min-duration-off=0 so adjacent turns of the
// same label are NOT glued together).
const FINE_THRESHOLD = process.env.DIAR_FINE_THRESHOLD || '0.2';
// sherpa keeps the whole recording plus its clustering state in RAM, and the
// peak grows QUADRATICALLY with the audio length — measured on #99 at 1.6 GHz:
// 10 min → 349 MB (130 turns, 130 s wall)
// 20 min → 1389 MB (242 turns, 300 s wall) = 4x RAM for 2x audio
// 30 min → 2643 MB (361 turns, 459 s wall)
// so the 54-min #99 hit 5.4 GB, the kernel OOM-killer took sherpa out and
// systemd's default OOMPolicy=stop tore the whole app down with it (GUI:
// "Transkription fehlgeschlagen" + "failed to fetch").
// v2 throws sherpa's LABELS away and keeps only the pyannote turn BOUNDARIES, so
// the pass can run on slices whose boundaries are stitched back with an offset —
// the footprint then stays at one slice's level no matter how long the call is,
// and because the cost is quadratic it is also FASTER (6x 130 s beats one
// 54-min run). Cost: a turn crossing a seam is cut in two; the re-clustering
// gives both halves the same speaker anyway. DIAR_CHUNK_SEC=0 disables chunking.
const CHUNK_SEC = process.env.DIAR_CHUNK_SEC != null ? +process.env.DIAR_CHUNK_SEC : 600;
const CHUNK_CTX_SEC = 5;
// Raw sherpa turns. mode 'fine' = boundaries only (v2), else the legacy call
// with its own clustering (threshold or pinned count).
async function sherpaTurns(wavPath, numSpeakers, fine) {
const clusterArg = fine ? `--clustering.cluster-threshold=${FINE_THRESHOLD}`
: (numSpeakers > 0 ? `--clustering.num-clusters=${numSpeakers}` : `--clustering.cluster-threshold=${THRESHOLD}`);
// One sherpa call over a whole wav file. `fine` = boundaries only (v2), else the
// legacy call with its own clustering (threshold or pinned count).
async function sherpaRun(wavPath, clusterArg, fine, timeoutMs) {
const args = [
'taskset', '-c', '4-7', BIN,
`--segmentation.pyannote-model=${SEG}`,
@@ -114,7 +128,7 @@ async function sherpaTurns(wavPath, numSpeakers, fine) {
...(fine ? ['--min-duration-on=0.2', '--min-duration-off=0.0'] : []),
wavPath,
];
const { stdout } = await pexec(args[0], args.slice(1), { env: env(), maxBuffer: 1 << 22, timeout: diarTimeoutMs(wavPath) });
const { stdout } = await pexec(args[0], args.slice(1), { env: env(), maxBuffer: 1 << 22, timeout: timeoutMs });
const segs = [];
for (const line of stdout.split('\n')) {
const m = line.match(/([\d.]+)\s*--\s*([\d.]+)\s*(speaker_\d+)/);
@@ -124,6 +138,40 @@ async function sherpaTurns(wavPath, numSpeakers, fine) {
return segs;
}
// Raw sherpa turns, chunked on long recordings (see CHUNK_SEC).
async function sherpaTurns(wavPath, numSpeakers, fine) {
const clusterArg = fine ? `--clustering.cluster-threshold=${FINE_THRESHOLD}`
: (numSpeakers > 0 ? `--clustering.num-clusters=${numSpeakers}` : `--clustering.cluster-threshold=${THRESHOLD}`);
const dur = wavSec(wavPath);
// The legacy pass needs ONE global clustering, so it can never be chunked.
if (!fine || !CHUNK_SEC || dur <= CHUNK_SEC * 1.25) {
return sherpaRun(wavPath, clusterArg, fine, timeoutForSec(dur));
}
const wav = loadWav(wavPath);
const tmp = `/tmp/diarchunk_${process.pid}.wav`;
const out = [];
try {
for (let t0 = 0; t0 < dur; t0 += CHUNK_SEC) {
const t1 = Math.min(dur, t0 + CHUNK_SEC);
// Context on both sides: pyannote decides on a sliding 10 s window, so a
// hard cut would blur the turns right at the seam.
const a = Math.max(0, t0 - CHUNK_CTX_SEC), b = Math.min(dur, t1 + CHUNK_CTX_SEC);
writeSlice(wav, a, b, tmp);
const segs = await sherpaRun(tmp, clusterArg, fine, timeoutForSec(b - a));
let kept = 0;
for (const s of segs) {
// Keep only what belongs to THIS chunk's own range (the context is
// already covered by its neighbour) and shift into absolute time.
const start = Math.max(t0, s.start + a), end = Math.min(t1, s.end + a);
if (end - start >= 0.1) { out.push({ start, end, speaker: s.speaker }); kept++; }
}
console.log(`diar: sherpa-Abschnitt ${Math.round(t0)}-${Math.round(t1)}s → ${kept} Turns`);
}
} finally { try { fs.unlinkSync(tmp); } catch {} }
out.sort((x, y) => x.start - y.start);
return out;
}
// Returns [{ start, end, speaker }] (seconds, speaker = "speaker_00"…).
// numSpeakers > 0 fixes the count when the user knows it, otherwise the count
// is found automatically (people can join/leave a conference call).