Diarisierung: Marker-Zwangsbedingungen absichern, Voiceprints als cannot-link; LLM: Sektions-Pfad ab 9k Zeichen
#79 verlor einen von vier Sprechern und lieferte eine 1082-Zeichen-Zusammenfassung. Diarisierung: ein zweiter, falsch verankerter Marker verschmolz Saschas Cluster in Merles (Kosinus -0,727). must-link prueft jetzt die akustische Aehnlichkeit, Anker werden nach Trefferqualitaet gewichtet, und enrolled Stimmen wirken als cannot-link -- damit ist der Box-Besitzer immer ein eigener Sprecher. LLM: der Whole-Pass komprimierte das erste Thema weg. Sektions-Pfad jetzt ab 9000 Zeichen, kleinere Chunks, woertlicher Kontext aus dem Vorgaenger-Chunk und ein gezielter Pruef-Aufruf gegen ueberdehnte Einzelaussagen. #79: 4 Sprecher, Zusammenfassung ~5300 Zeichen, 7 Themenbloecke. Regression: #70 -> 4, #68 -> 2, #74 -> 20 Sprecher unveraendert.
This commit is contained in:
+96
-14
@@ -132,7 +132,7 @@ export async function diarize(wavPath, numSpeakers, opts = {}) {
|
||||
if (MODE === 'v2') {
|
||||
const turns = await sherpaTurns(wavPath, n, true);
|
||||
if (turns.length) {
|
||||
try { return await reclusterTurns(wavPath, turns, n, opts.markers || []); }
|
||||
try { return await reclusterTurns(wavPath, turns, n, opts.markers || [], opts.voiceprints || []); }
|
||||
catch (e) { console.log('diar: v2 re-clustering failed, falling back to legacy:', e.message); }
|
||||
}
|
||||
}
|
||||
@@ -297,9 +297,9 @@ function ahcAverage(X, threshold, kStop = 0, initGroups = null, names = null) {
|
||||
}
|
||||
|
||||
// v2 entry: embed the windows, then cluster them.
|
||||
async function reclusterTurns(wavPath, turns, numSpeakers = 0, markers = []) {
|
||||
async function reclusterTurns(wavPath, turns, numSpeakers = 0, markers = [], voiceprints = []) {
|
||||
const ok = await embedWindows(wavPath, turns);
|
||||
return clusterWindows(turns, ok, numSpeakers, markers);
|
||||
return clusterWindows(turns, ok, numSpeakers, markers, voiceprints);
|
||||
}
|
||||
|
||||
// Map each live marker to the turn the named person was speaking in: the turn
|
||||
@@ -309,6 +309,11 @@ async function reclusterTurns(wavPath, turns, numSpeakers = 0, markers = []) {
|
||||
const ANCHOR_BACK_SEC = +(process.env.MARK_PRE_MS || 25000) / 1000;
|
||||
const ANCHOR_FWD_SEC = +(process.env.MARK_POST_MS || 8000) / 1000;
|
||||
const ANCHOR_MIN_SEC = 1.0; // too short to identify a voice from
|
||||
// Each anchor carries how it was found: `d` = 0 when the marker fell INSIDE the
|
||||
// turn (certain), else the seconds of typing lag (the further back, the more
|
||||
// likely the name belongs to somebody else who spoke in between). The vote
|
||||
// weight below is derived from it — a covering anchor must outweigh a 20-second
|
||||
// guess, otherwise one mistyped marker decides a whole cluster (#79).
|
||||
function markerAnchors(turns, markers) {
|
||||
const byName = new Map();
|
||||
const usable = turns.map((t, i) => ({ i, ...t })).filter(t => t.end - t.start >= ANCHOR_MIN_SEC);
|
||||
@@ -325,11 +330,35 @@ function markerAnchors(turns, markers) {
|
||||
if (d < bd) { bd = d; pick = c.i; }
|
||||
}
|
||||
if (pick < 0) continue;
|
||||
const key = String(mk.name).trim().toLowerCase();
|
||||
(byName.get(key) || byName.set(key, []).get(key)).push(pick);
|
||||
const key = normName(mk.name);
|
||||
(byName.get(key) || byName.set(key, []).get(key)).push({ ti: pick, d: bd });
|
||||
}
|
||||
return byName;
|
||||
}
|
||||
// Constraint key for a person. First name only, lowercased: the same human is
|
||||
// enrolled/typed as both "Sascha" and "Sascha Wolf" (two voiceprint rows exist),
|
||||
// and treating those as two people would forbid merging their clusters.
|
||||
const normName = (s) => String(s || '').trim().toLowerCase().split(/\s+/)[0];
|
||||
// Anchor vote weight: 1.0 for a marker inside the turn, decaying to 0.2 at the
|
||||
// far end of the look-back window.
|
||||
const anchorWeight = (d) => d <= 0 ? 1 : Math.max(0.2, 1 - 0.8 * (d / ANCHOR_BACK_SEC));
|
||||
// A name typed twice may legitimately point at two stage-A clusters of the SAME
|
||||
// voice (that is the must-link the markers exist for) — but only if the two
|
||||
// clusters actually sound alike. Below this fraction of the per-recording merge
|
||||
// threshold the second anchor is treated as mistyped/mis-anchored instead of
|
||||
// gluing two different people together (#79: "Merle" typed once while Sascha
|
||||
// was talking merged Sascha into Merle → 3 speakers instead of 4).
|
||||
const MARK_LINK_REL = +(process.env.DIAR_MARK_LINK_REL || 0.6);
|
||||
// Enrolled voiceprints as a SECOND cannot-link source. Voice identification used
|
||||
// to run only AFTER the clustering (identifySpeakers), so it could name what came
|
||||
// out but never prevent two known people from being merged into one cluster —
|
||||
// and the box owner, who is enrolled and speaks in every recording, silently
|
||||
// disappeared into somebody else's cluster (#79: Sascha inside "Merle"). Matching
|
||||
// happens in RAW embedding space (voiceprints are stored un-centred), so it needs
|
||||
// a strict score AND a margin over the runner-up before it is allowed to
|
||||
// constrain anything.
|
||||
const VP_MIN = +(process.env.DIAR_VP_MIN || 0.55);
|
||||
const VP_MARGIN = +(process.env.DIAR_VP_MARGIN || 0.05);
|
||||
|
||||
// Slice + embed every window of every turn (the only part that touches audio).
|
||||
export async function embedWindows(wavPath, turns) {
|
||||
@@ -368,7 +397,7 @@ export async function embedWindows(wavPath, turns) {
|
||||
// The heart of v2: window embeddings → clusters → per-turn speaker labels.
|
||||
// Split from the audio/embedding part so the clustering can be replayed offline
|
||||
// on cached embeddings (lab/lab-eval.mjs) — the tuning ran against THIS code.
|
||||
export function clusterWindows(turns, ok, numSpeakers = 0, markers = []) {
|
||||
export function clusterWindows(turns, ok, numSpeakers = 0, markers = [], voiceprints = []) {
|
||||
// Channel compensation: L2 → subtract the recording mean → L2 again.
|
||||
let X = ok.map(w => l2(w.vec));
|
||||
const mu = meanVec(X);
|
||||
@@ -432,34 +461,87 @@ export function clusterWindows(turns, ok, numSpeakers = 0, markers = []) {
|
||||
let tags = null;
|
||||
if (anchors.size) {
|
||||
const { lab } = labelTurns(groups);
|
||||
const nameOfGroup = new Map(); // groupIdx -> name (first/most-voted wins)
|
||||
const nameOfGroup = new Map(); // groupIdx -> name (strongest vote wins)
|
||||
const votes = new Map();
|
||||
for (const [name, tis] of anchors) for (const ti of tis) {
|
||||
for (const [name, hits] of anchors) for (const { ti, d } of hits) {
|
||||
const g = lab[ti];
|
||||
if (g < 0) continue;
|
||||
const key = g + '\0' + name;
|
||||
votes.set(key, (votes.get(key) || 0) + 1);
|
||||
votes.set(key, (votes.get(key) || 0) + anchorWeight(d));
|
||||
}
|
||||
const bestPerGroup = new Map();
|
||||
for (const [key, v] of votes) {
|
||||
const [g, name] = key.split('\0');
|
||||
const cur = bestPerGroup.get(g);
|
||||
if (!cur || v > cur.v) bestPerGroup.set(g, { name, v });
|
||||
else cur.rival = true;
|
||||
}
|
||||
for (const [g, { name }] of bestPerGroup) nameOfGroup.set(+g, name);
|
||||
// must-link: all groups tagged with the same name become ONE group
|
||||
for (const [g, { name, rival }] of bestPerGroup) {
|
||||
nameOfGroup.set(+g, name);
|
||||
// Two different names anchored in ONE cluster means stage A already glued
|
||||
// two people together — the cannot-link can no longer separate them, so at
|
||||
// least say so instead of silently keeping the louder vote.
|
||||
if (rival) console.log(`diar v2: WARNUNG Cluster ${g} trägt Marker mehrerer Namen → "${name}" gewinnt (Sprecher evtl. verschmolzen)`);
|
||||
}
|
||||
// must-link: groups carrying the SAME name become one — but only as far as
|
||||
// they sound alike (see MARK_LINK_REL). The strongest-vote group is the seed;
|
||||
// an acoustically distant namesake stays separate and UNTAGGED, so stage B
|
||||
// may still merge it on its own evidence.
|
||||
const linkMin = cosFinal * MARK_LINK_REL;
|
||||
const byName = new Map();
|
||||
for (const [g, name] of nameOfGroup) (byName.get(name) || byName.set(name, []).get(name)).push(g);
|
||||
const used = new Set();
|
||||
const merged = [], mtags = [];
|
||||
let anchored = 0, rejected = 0;
|
||||
for (const [name, gs] of byName) {
|
||||
merged.push(gs.flatMap(g => groups[g]));
|
||||
const ranked = [...gs].sort((a, b) => (bestPerGroup.get(String(b))?.v || 0) - (bestPerGroup.get(String(a))?.v || 0));
|
||||
const seed = ranked[0];
|
||||
const seedCent = centroidOf(groups[seed]);
|
||||
const take = [seed];
|
||||
for (const g of ranked.slice(1)) {
|
||||
const c = dot(seedCent, centroidOf(groups[g]));
|
||||
if (c >= linkMin) { take.push(g); continue; }
|
||||
rejected++;
|
||||
console.log(`diar v2: Marker "${name}" auf Cluster ${g} verworfen (cos ${c.toFixed(3)} < ${linkMin.toFixed(3)}) — vermutlich falsch getippter Zeitpunkt`);
|
||||
}
|
||||
merged.push(take.flatMap(g => groups[g]));
|
||||
mtags.push(name);
|
||||
gs.forEach(g => used.add(g));
|
||||
take.forEach(g => used.add(g));
|
||||
anchored += take.length;
|
||||
}
|
||||
groups.forEach((g, i) => { if (!used.has(i)) { merged.push(g); mtags.push(null); } });
|
||||
groups = merged; tags = mtags;
|
||||
console.log(`diar v2: ${anchors.size} Marker-Name(n) als Zwangsbedingung (${[...byName.values()].reduce((a, g) => a + g.length, 0)} Cluster verankert)`);
|
||||
console.log(`diar v2: ${anchors.size} Marker-Name(n) als Zwangsbedingung (${anchored} Cluster verankert, ${rejected} verworfen)`);
|
||||
}
|
||||
|
||||
// Known voices → cannot-link for every cluster the markers did NOT already
|
||||
// claim. Typed markers stay authoritative: a name that is already a marker tag
|
||||
// is skipped here, and one enrolled person can claim at most one cluster (best
|
||||
// score wins), so a shaky second match cannot split a speaker in two.
|
||||
if ((voiceprints || []).length) {
|
||||
if (!tags) tags = groups.map(() => null);
|
||||
const rawCent = g => l2(meanVec(g.map(i => ok[i].vec)));
|
||||
const taken = new Set(tags.filter(Boolean));
|
||||
const cand = [];
|
||||
groups.forEach((g, gi) => {
|
||||
if (tags[gi]) return;
|
||||
const c = rawCent(g);
|
||||
let best = null, s1 = -Infinity, s2 = -Infinity;
|
||||
for (const vp of voiceprints) {
|
||||
const s = spk.cosine(c, vp.vec);
|
||||
if (s > s1) { s2 = s1; s1 = s; best = normName(vp.name); }
|
||||
else if (s > s2) s2 = s;
|
||||
}
|
||||
if (best && s1 >= VP_MIN && (s1 - s2) >= VP_MARGIN && !taken.has(best)) cand.push({ gi, name: best, s: s1 });
|
||||
});
|
||||
cand.sort((a, b) => b.s - a.s);
|
||||
let vpTags = 0;
|
||||
for (const c of cand) {
|
||||
if (taken.has(c.name) || tags[c.gi]) continue;
|
||||
tags[c.gi] = c.name; taken.add(c.name); vpTags++;
|
||||
console.log(`diar v2: Cluster ${c.gi} = bekannte Stimme "${c.name}" (cos ${c.s.toFixed(3)}) → Trennungs-Zwangsbedingung`);
|
||||
}
|
||||
if (!vpTags) console.log(`diar v2: keine Stimme sicher genug erkannt (${voiceprints.length} Abdrücke, Schwelle ${VP_MIN})`);
|
||||
}
|
||||
|
||||
// Stage B: merge the remaining strong clusters up to the final threshold — or
|
||||
|
||||
Reference in New Issue
Block a user