diff --git a/app/lab/ab-llm.mjs b/app/lab/ab-llm.mjs new file mode 100644 index 0000000..96407b2 --- /dev/null +++ b/app/lab/ab-llm.mjs @@ -0,0 +1,160 @@ +// A/B-Messstrecke fuer den LLM-Wechsel (Qwen3-4B vs. Qwen3.5-4B). +// +// Misst NUR das Modell: laeuft ueber /api/recordings/:id/reanalyze-llm, also +// ohne Neu-Diarisierung -- gleiches Transkript, gleiche Sprecher, gleicher +// Kontextvermerk. Verglichen werden Titel, Zusammenfassung, Todos und die +// Analysedauer. +// +// WICHTIG: reanalyze-llm ueberschreibt die gespeicherte Analyse. Deshalb IMMER +// zuerst "snapshot" laufen lassen; "restore" schreibt den Originalstand zurueck. +// +// node lab/ab-llm.mjs snapshot 101 114 133 +// node lab/ab-llm.mjs run qwen3 101 114 133 # aktuelles Modell +// ... Modell umstellen, callrec neu starten ... +// node lab/ab-llm.mjs run qwen35 101 114 133 +// node lab/ab-llm.mjs report qwen3 qwen35 +// node lab/ab-llm.mjs restore + +import fs from 'node:fs'; +import path from 'node:path'; +import { pool } from '../db.js'; + +const OUT = '/opt/callrec/lab/ab-llm'; +const API = 'http://127.0.0.1'; +const POLL_MS = 5000; +const TIMEOUT_MS = 45 * 60 * 1000; // eine Analyse darf lange dauern (Whole-Pass) + +fs.mkdirSync(OUT, { recursive: true }); + +const sleep = ms => new Promise(r => setTimeout(r, ms)); +const jread = f => JSON.parse(fs.readFileSync(f, 'utf8')); +const jwrite = (f, o) => fs.writeFileSync(f, JSON.stringify(o, null, 1)); + +async function row(id) { + const [rows] = await pool.query( + `SELECT id, title, title_locked, summary, todos, participants, transcript_clean, + analysis_status, analysis_ms, char_length(transcript) AS tr_chars + FROM recordings WHERE id=?`, [id]); + return rows[0] || null; +} + +async function activeModel() { + const r = await fetch(`${API}/api/llm/status`).then(r => r.json()).catch(() => ({})); + return r.model || '(unbekannt)'; +} + +async function snapshot(ids) { + for (const id of ids) { + const r = await row(id); + if (!r) { console.log(`#${id}: nicht gefunden`); continue; } + jwrite(path.join(OUT, `orig-${id}.json`), r); + console.log(`#${id}: gesichert (${r.tr_chars} Zeichen Transkript, analysis_ms=${r.analysis_ms})`); + } +} + +async function restore() { + const files = fs.readdirSync(OUT).filter(f => /^orig-\d+\.json$/.test(f)); + if (!files.length) return console.log('Nichts zu restaurieren.'); + for (const f of files) { + const o = jread(path.join(OUT, f)); + await pool.execute( + `UPDATE recordings SET title=?, summary=?, todos=?, participants=?, + transcript_clean=?, analysis_status=?, analysis_ms=? WHERE id=?`, + [o.title, o.summary, o.todos, o.participants, o.transcript_clean, + o.analysis_status, o.analysis_ms, o.id]); + console.log(`#${o.id}: Originalanalyse zurueckgeschrieben`); + } +} + +async function run(label, ids) { + const model = await activeModel(); + console.log(`== Lauf "${label}" mit Modell ${model}`); + const results = []; + for (const id of ids) { + const before = await row(id); + if (!before) { console.log(`#${id}: nicht gefunden`); continue; } + if (!fs.existsSync(path.join(OUT, `orig-${id}.json`))) + throw new Error(`#${id}: kein Snapshot -- erst "snapshot" laufen lassen!`); + + process.stdout.write(`#${id} (${before.tr_chars} Zeichen): starte ... `); + const t0 = Date.now(); + const res = await fetch(`${API}/api/recordings/${id}/reanalyze-llm`, { method: 'POST' }); + if (!res.ok) { console.log(`FEHLER HTTP ${res.status}`); continue; } + + let r, waited = 0; + do { + await sleep(POLL_MS); waited = Date.now() - t0; + r = await row(id); + process.stdout.write('.'); + } while (r && r.analysis_status === 'pending' && waited < TIMEOUT_MS); + + const wall = Date.now() - t0; + console.log(` ${r.analysis_status} nach ${(wall / 1000).toFixed(0)} s (analysis_ms=${r.analysis_ms})`); + results.push({ + id, model, label, + status: r.analysis_status, + wall_ms: wall, + analysis_ms: r.analysis_ms, + tr_chars: r.tr_chars, + title: r.title, + summary: r.summary, + summary_chars: (r.summary || '').length, + todos: r.todos, + todo_count: (() => { try { return JSON.parse(r.todos || '[]').length; } catch { return null; } })(), + participants: r.participants, + }); + jwrite(path.join(OUT, `run-${label}.json`), results); + } + console.log(`\nGeschrieben: ${OUT}/run-${label}.json`); +} + +function report(a, b) { + const A = jread(path.join(OUT, `run-${a}.json`)); + const B = jread(path.join(OUT, `run-${b}.json`)); + const byId = x => Object.fromEntries(x.map(r => [r.id, r])); + const ma = byId(A), mb = byId(B); + const ids = [...new Set([...A.map(r => r.id), ...B.map(r => r.id)])].sort((x, y) => x - y); + + console.log(`\nModell A (${a}): ${A[0]?.model}`); + console.log(`Modell B (${b}): ${B[0]?.model}\n`); + console.log('ID Zeichen | Dauer A | Dauer B | Summary A | Summary B | Todos A/B | Status'); + console.log('-'.repeat(92)); + for (const id of ids) { + const x = ma[id] || {}, y = mb[id] || {}; + const s = v => v == null ? ' -' : (v / 1000).toFixed(0).padStart(5) + ' s'; + console.log( + String(id).padStart(4), String(x.tr_chars ?? y.tr_chars ?? '').padStart(8), + '|', s(x.analysis_ms), '|', s(y.analysis_ms), + '|', String(x.summary_chars ?? '-').padStart(9), '|', String(y.summary_chars ?? '-').padStart(9), + '|', `${x.todo_count ?? '-'}/${y.todo_count ?? '-'}`.padStart(8), + '|', `${x.status ?? '-'}/${y.status ?? '-'}`); + } + + // Volltext zum Danebenlegen -- die inhaltliche Bewertung macht der Mensch. + const md = [`# A/B: ${a} vs. ${b}`, '', + `- **A (${a})**: \`${A[0]?.model}\``, `- **B (${b})**: \`${B[0]?.model}\``, '']; + for (const id of ids) { + const x = ma[id] || {}, y = mb[id] || {}; + md.push(`## Aufnahme #${id} (${x.tr_chars ?? y.tr_chars} Zeichen Transkript)`, '', + `### A — ${a} (${x.analysis_ms ? (x.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '', + `**Titel:** ${x.title ?? '-'}`, '', (x.summary || '_keine_'), '', + `**Todos:** ${x.todos ?? '-'}`, '', + `### B — ${b} (${y.analysis_ms ? (y.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '', + `**Titel:** ${y.title ?? '-'}`, '', (y.summary || '_keine_'), '', + `**Todos:** ${y.todos ?? '-'}`, ''); + } + const f = path.join(OUT, `vergleich-${a}-vs-${b}.md`); + fs.writeFileSync(f, md.join('\n')); + console.log(`\nVolltext-Vergleich: ${f}`); +} + +const [cmd, ...rest] = process.argv.slice(2); +try { + if (cmd === 'snapshot') await snapshot(rest.map(Number)); + else if (cmd === 'run') await run(rest[0], rest.slice(1).map(Number)); + else if (cmd === 'restore') await restore(); + else if (cmd === 'report') report(rest[0], rest[1]); + else console.log('Befehle: snapshot | run