Diarization v2: Fenster-Clustering mit gelernter Schwelle + Live-Namensmarker
#74 (22 Teilnehmer) ergab automatisch 44 Sprecher, dieselbe Stimme mehrfach. Ursache gemessen: ein Zentroid pro sherpa-Cluster aus max. 3 Schnitten, fester Kosinus auf rohen Embeddings (Kreuz-Turn-Kosinus ~0,4 = keine Trennschaerfe). v2: sherpa liefert nur Turn-Grenzen, Identitaet aus 3-s-Fenster-Embeddings, Mittelwert-Zentrierung, Average-Linkage-AHC, Weak-Fold, Schwelle pro Aufnahme per 2-Gauss-EM gelernt. Validiert: #74 44->20, #70 ->4, #68 ->2. Live-Namensmarker: Namen waehrend der Aufnahme tippen; Zeitstempel benennt spaeter den Cluster und dient als must-/cannot-link-Zwangsbedingung.
This commit is contained in:
@@ -27,6 +27,11 @@ export async function ensureSchema() {
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)`);
|
||||
// n = number of samples averaged into this voiceprint (running centroid).
|
||||
await pool.query(`ALTER TABLE voiceprints ADD COLUMN IF NOT EXISTS n INT NOT NULL DEFAULT 1`);
|
||||
// Embedding model the vector came from: prints of different models are NOT
|
||||
// comparable (cosine across models is noise), so identification only ever
|
||||
// uses the rows of the model currently loaded. NULL = pre-2026-07-27 TitaNet.
|
||||
await pool.query(`ALTER TABLE voiceprints ADD COLUMN IF NOT EXISTS model VARCHAR(64) NULL`);
|
||||
await pool.query(`UPDATE voiceprints SET model='nemo_titanet_small' WHERE model IS NULL`);
|
||||
// Domain glossary: known proper nouns / terms the ASR mis-hears (e.g. "Aime"
|
||||
// heard as "Amy"). Fed into the LLM correction step so soundalikes get mapped
|
||||
// to the correct spelling. `soundalikes` = optional comma-list of mis-hearings.
|
||||
@@ -36,6 +41,41 @@ export async function ensureSchema() {
|
||||
soundalikes TEXT NULL,
|
||||
enabled TINYINT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)`);
|
||||
// Live speaker markers: names typed DURING the recording ("who is talking
|
||||
// right now"). t_ms = offset from recording start. After diarization each
|
||||
// marker is resolved to the acoustic cluster that was speaking around t_ms
|
||||
// (`speaker`), which names that cluster — ground truth instead of guessing.
|
||||
await pool.query(`CREATE TABLE IF NOT EXISTS markers (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
recording_id INT NOT NULL,
|
||||
t_ms INT NOT NULL,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
speaker VARCHAR(32) NULL,
|
||||
conf VARCHAR(8) NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX rec_idx (recording_id))`);
|
||||
}
|
||||
|
||||
// --- live speaker markers ---------------------------------------------------
|
||||
export async function addMarkers(recId, markers) {
|
||||
const list = (markers || []).filter(m => m && m.name);
|
||||
if (!list.length) return;
|
||||
await pool.query('INSERT INTO markers (recording_id, t_ms, name) VALUES ?',
|
||||
[list.map(m => [recId, Math.max(0, Math.round(m.t_ms || 0)), String(m.name).slice(0, 128)])]);
|
||||
}
|
||||
export async function listMarkers(recId) {
|
||||
const [rows] = await pool.query(
|
||||
'SELECT id, t_ms, name, speaker, conf FROM markers WHERE recording_id=? ORDER BY t_ms', [recId]);
|
||||
return rows;
|
||||
}
|
||||
export async function setMarkerSpeaker(id, speaker, conf) {
|
||||
await pool.execute('UPDATE markers SET speaker=?, conf=? WHERE id=?', [speaker || null, conf || null, id]);
|
||||
}
|
||||
export async function deleteMarker(id) { await pool.execute('DELETE FROM markers WHERE id=?', [id]); }
|
||||
export async function addMarker(recId, tMs, name) {
|
||||
const [res] = await pool.execute('INSERT INTO markers (recording_id, t_ms, name) VALUES (?,?,?)',
|
||||
[recId, Math.max(0, Math.round(tMs || 0)), String(name).slice(0, 128)]);
|
||||
return res.insertId;
|
||||
}
|
||||
|
||||
export async function listGlossary() {
|
||||
@@ -58,9 +98,10 @@ export async function updateGlossary(id, term, soundalikes) {
|
||||
return rows[0] ? { ...rows[0], enabled: !!rows[0].enabled } : null;
|
||||
}
|
||||
|
||||
export async function addVoiceprint(name, vec) {
|
||||
export async function addVoiceprint(name, vec, model) {
|
||||
const [res] = await pool.execute(
|
||||
'INSERT INTO voiceprints (name, dim, vec) VALUES (?,?,?)', [name, vec.length, JSON.stringify(vec)]);
|
||||
'INSERT INTO voiceprints (name, dim, vec, model) VALUES (?,?,?,?)',
|
||||
[name, vec.length, JSON.stringify(vec), model || null]);
|
||||
return res.insertId;
|
||||
}
|
||||
|
||||
@@ -68,11 +109,13 @@ export async function addVoiceprint(name, vec) {
|
||||
// centroid (mean of all samples). This is how the app "learns who is who": every
|
||||
// time a speaker is named in a recording, their voice is folded into the centroid,
|
||||
// so future recordings match them more reliably. New name -> fresh row.
|
||||
export async function upsertVoiceprint(name, vec) {
|
||||
const [rows] = await pool.query('SELECT id, vec, n FROM voiceprints WHERE name=? ORDER BY id LIMIT 1', [name]);
|
||||
export async function upsertVoiceprint(name, vec, model) {
|
||||
const [rows] = await pool.query(
|
||||
'SELECT id, vec, n FROM voiceprints WHERE name=? AND (model=? OR (model IS NULL AND ? IS NULL)) ORDER BY id LIMIT 1',
|
||||
[name, model || null, model || null]);
|
||||
if (!rows.length) {
|
||||
const [res] = await pool.execute('INSERT INTO voiceprints (name, dim, vec, n) VALUES (?,?,?,1)',
|
||||
[name, vec.length, JSON.stringify(vec)]);
|
||||
const [res] = await pool.execute('INSERT INTO voiceprints (name, dim, vec, n, model) VALUES (?,?,?,1,?)',
|
||||
[name, vec.length, JSON.stringify(vec), model || null]);
|
||||
return res.insertId;
|
||||
}
|
||||
const old = rows[0];
|
||||
@@ -83,8 +126,12 @@ export async function upsertVoiceprint(name, vec) {
|
||||
[JSON.stringify(mean), on + 1, mean.length, old.id]);
|
||||
return old.id;
|
||||
}
|
||||
export async function listVoiceprints() {
|
||||
const [rows] = await pool.query('SELECT id,name,dim,vec,created_at FROM voiceprints ORDER BY name');
|
||||
// model given → only the prints of that embedding model (what identification
|
||||
// must use); omitted → all rows (GUI listing).
|
||||
export async function listVoiceprints(model) {
|
||||
const [rows] = model
|
||||
? await pool.query('SELECT id,name,dim,vec,model,created_at FROM voiceprints WHERE model=? ORDER BY name', [model])
|
||||
: await pool.query('SELECT id,name,dim,vec,model,created_at FROM voiceprints ORDER BY name');
|
||||
return rows.map(r => ({ ...r, vec: JSON.parse(r.vec) }));
|
||||
}
|
||||
export async function deleteVoiceprint(id) { await pool.query('DELETE FROM voiceprints WHERE id=?', [id]); }
|
||||
@@ -185,9 +232,11 @@ export async function getRecording(id) {
|
||||
'SELECT word,t_start_ms,t_end_ms,idx,speaker FROM words WHERE recording_id=? ORDER BY idx', [id]);
|
||||
let speaker_names = {};
|
||||
try { speaker_names = JSON.parse(rec.speaker_names || '{}') || {}; } catch {}
|
||||
return { ...parseTodos(rec), speaker_names, words };
|
||||
const markers = await listMarkers(id).catch(() => []);
|
||||
return { ...parseTodos(rec), speaker_names, words, markers };
|
||||
}
|
||||
|
||||
export async function deleteRecording(id) {
|
||||
await pool.query('DELETE FROM markers WHERE recording_id=?', [id]).catch(() => {});
|
||||
await pool.query('DELETE FROM recordings WHERE id=?', [id]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user