Sascha/Aime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Diarization v2: Fenster-Clustering mit gelernter Schwelle + Live-Namensmarker

Browse Source 
#74 (22 Teilnehmer) ergab automatisch 44 Sprecher, dieselbe Stimme mehrfach.
Ursache gemessen: ein Zentroid pro sherpa-Cluster aus max. 3 Schnitten, fester
Kosinus auf rohen Embeddings (Kreuz-Turn-Kosinus ~0,4 = keine Trennschaerfe).

v2: sherpa liefert nur Turn-Grenzen, Identitaet aus 3-s-Fenster-Embeddings,
Mittelwert-Zentrierung, Average-Linkage-AHC, Weak-Fold, Schwelle pro Aufnahme
per 2-Gauss-EM gelernt. Validiert: #74 44->20, #70 ->4, #68 ->2.

Live-Namensmarker: Namen waehrend der Aufnahme tippen; Zeitstempel benennt
spaeter den Cluster und dient als must-/cannot-link-Zwangsbedingung.
This commit is contained in:
Claude
2026-07-27 17:00:45 +02:00
parent 432c4b14dd
commit 48fa6b4f9e
17 changed files with 1357 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/server.js
+115 -7
View File
@@ -56,7 +56,7 @@ async function identifySpeakers(recData) {
if (!speakers.length || !recData.wav_file) return {};
const wavPath = path.join(REC_DIR2, recData.wav_file);
if (!fs.existsSync(wavPath)) return {};
const vps = await db.listVoiceprints();
const vps = await db.listVoiceprints(spk.modelId());
if (!vps.length) return {};
const best = {}; let cur = null;
for (const w of words) {
@@ -80,6 +80,73 @@ async function identifySpeakers(recData) {
return map;
}
// Fold a speaker's longest clean run into the named voiceprint centroid, so the
// next recording can auto-identify them. Shared by the manual rename popup and
// the live-marker resolution. Returns true if a sample was learned.
async function enrollSpeakerVoice(recData, speaker, name) {
if (!name || !recData.wav_file) return false;
const seg = longestRun(recData.words, speaker);
if (!seg || (seg.end - seg.start) < 1500) return false;
const wavPath = path.join(REC_DIR2, recData.wav_file);
if (!fs.existsSync(wavPath)) return false;
const cut = `/tmp/enr_${recData.id}_${speaker}.wav`;
try {
await pexec('ffmpeg', ['-y', '-i', wavPath, '-ss', String(seg.start / 1000),
'-to', String(Math.min(seg.end, seg.start + 20000) / 1000), '-ar', '16000', '-ac', '1', cut]);
const emb = await spk.embed(cut);
await db.upsertVoiceprint(name, emb, spk.modelId());
return true;
} catch (e) { console.error('enroll failed:', e.message); return false; }
finally { try { fs.unlinkSync(cut); } catch {} }
}
// Apply the names the user typed DURING the recording to the diarized clusters.
// Runs after every (re-)diarization, before the LLM analysis, so the summary
// already talks about real people. Two effects:
// 1. each resolved cluster gets its typed name (authoritative, manual-level),
// 2. if ONE name confidently lands on SEVERAL clusters, those clusters are the
// same person split by the diarizer → they are merged (user-supervised fix
// for the residual over-segmentation).
async function applyMarkers(id) {
const rec = await db.getRecording(id);
if (!rec || !(rec.markers || []).length || !(rec.words || []).some(w => w.speaker)) return null;
const resolved = diar.resolveMarkers(rec.words, rec.markers);
for (const m of resolved) await db.setMarkerSpeaker(m.id, m.speaker, m.conf).catch(() => {});
// name -> clusters (only confident hits drive the merge)
const byName = {};
for (const m of resolved) {
if (!m.speaker || m.conf === 'low' || m.conf === null) continue;
(byName[m.name] = byName[m.name] || new Set()).add(m.speaker);
}
const talk = {};
for (const w of rec.words) if (w.speaker) talk[w.speaker] = (talk[w.speaker] || 0) + (w.t_end_ms - w.t_start_ms);
const fold = {}; // extra cluster -> primary cluster
const nameOf = {}; // cluster -> typed name
for (const [name, set] of Object.entries(byName)) {
const cls = [...set].sort((a, b) => (talk[b] || 0) - (talk[a] || 0));
const primary = cls[0];
nameOf[primary] = name;
for (const c of cls.slice(1)) fold[c] = primary;
}
let merged = 0;
if (Object.keys(fold).length) {
for (const w of rec.words) if (fold[w.speaker]) w.speaker = fold[w.speaker];
await db.replaceWords(id, rec.words);
merged = Object.keys(fold).length;
console.log(`markers: merged ${merged} cluster(s) into marker-named speakers on #${id}`);
}
for (const [sp, name] of Object.entries(nameOf)) await db.setSpeakerName(id, sp, name).catch(() => {});
// Learn the voices so the NEXT recording identifies them automatically.
const rec2 = await db.getRecording(id);
let learned = 0;
for (const [sp, name] of Object.entries(nameOf)) if (await enrollSpeakerVoice(rec2, sp, name)) learned++;
const named = Object.keys(nameOf).length;
console.log(`markers on #${id}: ${resolved.length} marker(s) → ${named} named speaker(s), ${merged} merged, ${learned} voice(s) learned`);
return { resolved, named, merged, learned };
}
// Run LLM analysis for a recording and persist title/summary/todos/participants/
// clean dialogue. Background (NPU ~30-90s) so record/stop stays fast.
// A76 clock cap for the whole run: rkllm burns ~1.7 CPU cores alongside the NPU
@@ -189,15 +256,25 @@ function withNpuLock(label, fn) {
// --- Recording ---
app.post('/api/record/start', wrap(async (req, res) => res.json(await rec.startRecording(req.body || {}))));
// Live speaker markers: type the name of whoever is talking WHILE recording.
app.post('/api/record/marker', wrap(async (req, res) => res.json(rec.addMarker((req.body && req.body.name) || ''))));
app.get('/api/record/markers', (req, res) => res.json(rec.listMarkers()));
app.delete('/api/record/marker/:index', wrap(async (req, res) => res.json({ markers: rec.removeMarker(req.params.index) })));
// Transcribe (thermal-guarded whisper + diarize) then analyze, in the background.
// The wav + DB row already exist, so a crash here loses nothing — the recording
// stays with status 'transcribing' and can be recovered via /retranscribe.
async function transcribeAndAnalyze(id, wavFile, numSpeakers) {
await db.markAnalysisStart(id).catch(() => {});
try {
const { transcript, words } = await rec.transcribeWav(wavFile, numSpeakers);
// Live markers constrain the diarization itself (same name = one speaker,
// different names = never merged), so they are loaded before transcribing.
const markers = await db.listMarkers(id).catch(() => []);
const { transcript, words } = await rec.transcribeWav(wavFile, numSpeakers, { markers });
await db.setTranscript(id, transcript, words);
console.log(`transcription done for #${id}: ${words.length} segments`);
// Names typed live → cluster names (+ user-supervised cluster merge) BEFORE
// the LLM sees the dialogue.
await applyMarkers(id).catch(e => console.error('applyMarkers:', e.message));
} catch (e) {
console.error(`transcription failed for #${id}:`, e.message);
await db.setStatus(id, 'error').catch(() => {});
@@ -214,8 +291,10 @@ app.post('/api/record/stop', wrap(async (req, res) => {
// analysis_status === 'pending', so the transcript + analysis fill in live.
const r = await rec.stopAndSave();
const id = await db.addRecording({ ...r, transcript: '', status: 'transcribing' });
await db.addMarkers(id, r.markers).catch(e => console.error('addMarkers:', e.message));
res.json({ id, title: r.title, mp3_file: r.mp3_file, duration_ms: r.duration_ms,
transcript: '', status: 'transcribing', analysis_status: 'pending' });
transcript: '', status: 'transcribing', analysis_status: 'pending',
markers: (r.markers || []).length });
withNpuLock(`transcribe #${id}`, () => transcribeAndAnalyze(id, r.wav_file, r.numSpeakers));
}));
@@ -230,7 +309,9 @@ app.delete('/api/recordings/:id', wrap(async (req, res) => { await db.deleteReco
// --- Voiceprints / enrollment ---
app.get('/api/voiceprints', wrap(async (req, res) => {
const vps = await db.listVoiceprints();
res.json(vps.map(v => ({ id: v.id, name: v.name, dim: v.dim, created_at: v.created_at })));
const cur = spk.modelId();
res.json(vps.map(v => ({ id: v.id, name: v.name, dim: v.dim, model: v.model,
stale: v.model !== cur, created_at: v.created_at })));
}));
app.delete('/api/voiceprints/:id', wrap(async (req, res) => { await db.deleteVoiceprint(+req.params.id); res.json({ ok: true }); }));
app.post('/api/enroll/start', wrap(async (req, res) => res.json(await rec.startRecording({ source: req.body && req.body.source }))));
@@ -240,7 +321,7 @@ app.post('/api/enroll/stop', wrap(async (req, res) => {
const wav = await rec.stopRaw();
try {
const emb = await spk.embed(wav);
const id = await db.addVoiceprint(name, emb);
const id = await db.addVoiceprint(name, emb, spk.modelId());
res.json({ id, name, dim: emb.length });
} finally { try { fs.unlinkSync(wav); } catch {} }
}));
@@ -283,9 +364,13 @@ async function rediarize(id, numSpeakers) {
// is the other entry point and must do it too, or a long re-diar overheats).
await pexec('sudo', ['-n', '/opt/callrec/aime-cpucap.sh', '1608000']).catch(() => {});
try {
const segs = await diar.diarize(wav, numSpeakers);
const segs = await diar.diarize(wav, numSpeakers, { markers: r.markers || [] });
diar.assignSpeakers(words, segs);
await db.replaceWords(id, words);
// Re-diarization RENUMBERS the clusters, so every previously stored name
// ("speaker_02" = Fabio) would now point at a different voice. Drop them; the
// live markers and the voiceprint identification re-fill them right after.
await db.pool.query('UPDATE recordings SET speaker_names=NULL WHERE id=?', [id]).catch(() => {});
console.log(`re-diarized #${id} (numSpeakers=${numSpeakers || 'auto'}) → ${new Set(segs.map(s => s.speaker)).size} speakers`);
} finally {
await pexec('sudo', ['-n', '/opt/callrec/aime-cpucap.sh', '2256000']).catch(() => {});
@@ -307,10 +392,33 @@ app.post('/api/recordings/:id/reanalyze', wrap(async (req, res) => {
withNpuLock(`reanalyze #${id}`, async () => {
try { await rediarize(id, numSpeakers); }
catch (e) { console.error(`re-diarize failed for #${id}:`, e.message); }
await applyMarkers(id).catch(e => console.error('applyMarkers:', e.message));
await analyzeRecording(id);
});
}));
// --- markers of a stored recording -----------------------------------------
app.get('/api/recordings/:id/markers', wrap(async (req, res) => res.json(await db.listMarkers(+req.params.id))));
// Add a marker after the fact (e.g. while listening back) — same resolution path.
app.post('/api/recordings/:id/markers', wrap(async (req, res) => {
const name = String((req.body && req.body.name) || '').trim();
const tMs = +(req.body && req.body.t_ms);
if (!name || !Number.isFinite(tMs)) return res.status(400).json({ error: 'name/t_ms fehlt' });
await db.addMarker(+req.params.id, tMs, name);
res.json({ ok: true, markers: await db.listMarkers(+req.params.id) });
}));
app.delete('/api/recordings/:id/markers/:mid', wrap(async (req, res) => {
await db.deleteMarker(+req.params.mid);
res.json({ ok: true, markers: await db.listMarkers(+req.params.id) });
}));
// Re-resolve the markers against the current diarization (no LLM, no sherpa).
app.post('/api/recordings/:id/apply-markers', wrap(async (req, res) => {
const out = await applyMarkers(+req.params.id);
const r = await db.getRecording(+req.params.id);
res.json({ ok: true, ...(out || { named: 0, merged: 0, learned: 0 }),
speaker_names: (r && r.speaker_names) || {}, markers: (r && r.markers) || [] });
}));
// LLM-only re-analysis: re-run title/summary/todos on the EXISTING words +
// speaker labels, WITHOUT re-diarizing (fast — skips the ~40min sherpa pass when
// the speakers are already correct and only the LLM/prompts changed).
@@ -378,7 +486,7 @@ app.post('/api/recordings/:id/speaker', wrap(async (req, res) => {
await pexec('ffmpeg', ['-y', '-i', wavPath, '-ss', String(seg.start / 1000),
'-to', String(seg.end / 1000), '-ar', '16000', '-ac', '1', cut]);
const emb = await spk.embed(cut);
await db.upsertVoiceprint(name, emb);
await db.upsertVoiceprint(name, emb, spk.modelId());
enrolled = true;
} catch (e) { console.error('enroll-from-recording failed:', e.message); learnMsg = 'Lernen fehlgeschlagen'; }
finally { try { fs.unlinkSync(cut); } catch {} }