A/B-Aufbau fuer Qwen3.5-4B: Messstrecke und Konvertierungs-Kit
lab/ab-llm.mjs vergleicht zwei LLMs ueber reanalyze-llm (ohne Neu-Diarisierung), mit snapshot/restore der Originalanalysen und einem Report aus Tabelle plus Volltext-Vergleich. tools/rkllm-qwen35/ konvertiert Qwen3.5-4B auf dem PC mit rkllm-toolkit 1.3.0 (muss zur Board-Runtime passen), Parameter identisch zum produktiven Modell: w8a8_g128, 3 NPU-Kerne, max_context 16384. Doku: Changelog 2026-09-29 (2) mit Recherchestand, Sicherung und GOTCHAs, plus neuer Abschnitt "Modellwechsel und A/B-Tests" in Kapitel 05 samt Mermaid-Ablauf. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,160 @@
|
||||
// A/B-Messstrecke fuer den LLM-Wechsel (Qwen3-4B vs. Qwen3.5-4B).
|
||||
//
|
||||
// Misst NUR das Modell: laeuft ueber /api/recordings/:id/reanalyze-llm, also
|
||||
// ohne Neu-Diarisierung -- gleiches Transkript, gleiche Sprecher, gleicher
|
||||
// Kontextvermerk. Verglichen werden Titel, Zusammenfassung, Todos und die
|
||||
// Analysedauer.
|
||||
//
|
||||
// WICHTIG: reanalyze-llm ueberschreibt die gespeicherte Analyse. Deshalb IMMER
|
||||
// zuerst "snapshot" laufen lassen; "restore" schreibt den Originalstand zurueck.
|
||||
//
|
||||
// node lab/ab-llm.mjs snapshot 101 114 133
|
||||
// node lab/ab-llm.mjs run qwen3 101 114 133 # aktuelles Modell
|
||||
// ... Modell umstellen, callrec neu starten ...
|
||||
// node lab/ab-llm.mjs run qwen35 101 114 133
|
||||
// node lab/ab-llm.mjs report qwen3 qwen35
|
||||
// node lab/ab-llm.mjs restore
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { pool } from '../db.js';
|
||||
|
||||
const OUT = '/opt/callrec/lab/ab-llm';
|
||||
const API = 'http://127.0.0.1';
|
||||
const POLL_MS = 5000;
|
||||
const TIMEOUT_MS = 45 * 60 * 1000; // eine Analyse darf lange dauern (Whole-Pass)
|
||||
|
||||
fs.mkdirSync(OUT, { recursive: true });
|
||||
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
const jread = f => JSON.parse(fs.readFileSync(f, 'utf8'));
|
||||
const jwrite = (f, o) => fs.writeFileSync(f, JSON.stringify(o, null, 1));
|
||||
|
||||
async function row(id) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, title, title_locked, summary, todos, participants, transcript_clean,
|
||||
analysis_status, analysis_ms, char_length(transcript) AS tr_chars
|
||||
FROM recordings WHERE id=?`, [id]);
|
||||
return rows[0] || null;
|
||||
}
|
||||
|
||||
async function activeModel() {
|
||||
const r = await fetch(`${API}/api/llm/status`).then(r => r.json()).catch(() => ({}));
|
||||
return r.model || '(unbekannt)';
|
||||
}
|
||||
|
||||
async function snapshot(ids) {
|
||||
for (const id of ids) {
|
||||
const r = await row(id);
|
||||
if (!r) { console.log(`#${id}: nicht gefunden`); continue; }
|
||||
jwrite(path.join(OUT, `orig-${id}.json`), r);
|
||||
console.log(`#${id}: gesichert (${r.tr_chars} Zeichen Transkript, analysis_ms=${r.analysis_ms})`);
|
||||
}
|
||||
}
|
||||
|
||||
async function restore() {
|
||||
const files = fs.readdirSync(OUT).filter(f => /^orig-\d+\.json$/.test(f));
|
||||
if (!files.length) return console.log('Nichts zu restaurieren.');
|
||||
for (const f of files) {
|
||||
const o = jread(path.join(OUT, f));
|
||||
await pool.execute(
|
||||
`UPDATE recordings SET title=?, summary=?, todos=?, participants=?,
|
||||
transcript_clean=?, analysis_status=?, analysis_ms=? WHERE id=?`,
|
||||
[o.title, o.summary, o.todos, o.participants, o.transcript_clean,
|
||||
o.analysis_status, o.analysis_ms, o.id]);
|
||||
console.log(`#${o.id}: Originalanalyse zurueckgeschrieben`);
|
||||
}
|
||||
}
|
||||
|
||||
async function run(label, ids) {
|
||||
const model = await activeModel();
|
||||
console.log(`== Lauf "${label}" mit Modell ${model}`);
|
||||
const results = [];
|
||||
for (const id of ids) {
|
||||
const before = await row(id);
|
||||
if (!before) { console.log(`#${id}: nicht gefunden`); continue; }
|
||||
if (!fs.existsSync(path.join(OUT, `orig-${id}.json`)))
|
||||
throw new Error(`#${id}: kein Snapshot -- erst "snapshot" laufen lassen!`);
|
||||
|
||||
process.stdout.write(`#${id} (${before.tr_chars} Zeichen): starte ... `);
|
||||
const t0 = Date.now();
|
||||
const res = await fetch(`${API}/api/recordings/${id}/reanalyze-llm`, { method: 'POST' });
|
||||
if (!res.ok) { console.log(`FEHLER HTTP ${res.status}`); continue; }
|
||||
|
||||
let r, waited = 0;
|
||||
do {
|
||||
await sleep(POLL_MS); waited = Date.now() - t0;
|
||||
r = await row(id);
|
||||
process.stdout.write('.');
|
||||
} while (r && r.analysis_status === 'pending' && waited < TIMEOUT_MS);
|
||||
|
||||
const wall = Date.now() - t0;
|
||||
console.log(` ${r.analysis_status} nach ${(wall / 1000).toFixed(0)} s (analysis_ms=${r.analysis_ms})`);
|
||||
results.push({
|
||||
id, model, label,
|
||||
status: r.analysis_status,
|
||||
wall_ms: wall,
|
||||
analysis_ms: r.analysis_ms,
|
||||
tr_chars: r.tr_chars,
|
||||
title: r.title,
|
||||
summary: r.summary,
|
||||
summary_chars: (r.summary || '').length,
|
||||
todos: r.todos,
|
||||
todo_count: (() => { try { return JSON.parse(r.todos || '[]').length; } catch { return null; } })(),
|
||||
participants: r.participants,
|
||||
});
|
||||
jwrite(path.join(OUT, `run-${label}.json`), results);
|
||||
}
|
||||
console.log(`\nGeschrieben: ${OUT}/run-${label}.json`);
|
||||
}
|
||||
|
||||
function report(a, b) {
|
||||
const A = jread(path.join(OUT, `run-${a}.json`));
|
||||
const B = jread(path.join(OUT, `run-${b}.json`));
|
||||
const byId = x => Object.fromEntries(x.map(r => [r.id, r]));
|
||||
const ma = byId(A), mb = byId(B);
|
||||
const ids = [...new Set([...A.map(r => r.id), ...B.map(r => r.id)])].sort((x, y) => x - y);
|
||||
|
||||
console.log(`\nModell A (${a}): ${A[0]?.model}`);
|
||||
console.log(`Modell B (${b}): ${B[0]?.model}\n`);
|
||||
console.log('ID Zeichen | Dauer A | Dauer B | Summary A | Summary B | Todos A/B | Status');
|
||||
console.log('-'.repeat(92));
|
||||
for (const id of ids) {
|
||||
const x = ma[id] || {}, y = mb[id] || {};
|
||||
const s = v => v == null ? ' -' : (v / 1000).toFixed(0).padStart(5) + ' s';
|
||||
console.log(
|
||||
String(id).padStart(4), String(x.tr_chars ?? y.tr_chars ?? '').padStart(8),
|
||||
'|', s(x.analysis_ms), '|', s(y.analysis_ms),
|
||||
'|', String(x.summary_chars ?? '-').padStart(9), '|', String(y.summary_chars ?? '-').padStart(9),
|
||||
'|', `${x.todo_count ?? '-'}/${y.todo_count ?? '-'}`.padStart(8),
|
||||
'|', `${x.status ?? '-'}/${y.status ?? '-'}`);
|
||||
}
|
||||
|
||||
// Volltext zum Danebenlegen -- die inhaltliche Bewertung macht der Mensch.
|
||||
const md = [`# A/B: ${a} vs. ${b}`, '',
|
||||
`- **A (${a})**: \`${A[0]?.model}\``, `- **B (${b})**: \`${B[0]?.model}\``, ''];
|
||||
for (const id of ids) {
|
||||
const x = ma[id] || {}, y = mb[id] || {};
|
||||
md.push(`## Aufnahme #${id} (${x.tr_chars ?? y.tr_chars} Zeichen Transkript)`, '',
|
||||
`### A — ${a} (${x.analysis_ms ? (x.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '',
|
||||
`**Titel:** ${x.title ?? '-'}`, '', (x.summary || '_keine_'), '',
|
||||
`**Todos:** ${x.todos ?? '-'}`, '',
|
||||
`### B — ${b} (${y.analysis_ms ? (y.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '',
|
||||
`**Titel:** ${y.title ?? '-'}`, '', (y.summary || '_keine_'), '',
|
||||
`**Todos:** ${y.todos ?? '-'}`, '');
|
||||
}
|
||||
const f = path.join(OUT, `vergleich-${a}-vs-${b}.md`);
|
||||
fs.writeFileSync(f, md.join('\n'));
|
||||
console.log(`\nVolltext-Vergleich: ${f}`);
|
||||
}
|
||||
|
||||
const [cmd, ...rest] = process.argv.slice(2);
|
||||
try {
|
||||
if (cmd === 'snapshot') await snapshot(rest.map(Number));
|
||||
else if (cmd === 'run') await run(rest[0], rest.slice(1).map(Number));
|
||||
else if (cmd === 'restore') await restore();
|
||||
else if (cmd === 'report') report(rest[0], rest[1]);
|
||||
else console.log('Befehle: snapshot <ids...> | run <label> <ids...> | report <a> <b> | restore');
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
Reference in New Issue
Block a user