Aime/app/lab/ab-llm.mjs

// A/B-Messstrecke fuer den LLM-Wechsel (Qwen3-4B vs. Qwen3.5-4B).
//
// Misst NUR das Modell: laeuft ueber /api/recordings/:id/reanalyze-llm, also
// ohne Neu-Diarisierung -- gleiches Transkript, gleiche Sprecher, gleicher
// Kontextvermerk. Verglichen werden Titel, Zusammenfassung, Todos und die
// Analysedauer.
//
// WICHTIG: reanalyze-llm ueberschreibt die gespeicherte Analyse. Deshalb IMMER
// zuerst "snapshot" laufen lassen; "restore" schreibt den Originalstand zurueck.
//
//   node lab/ab-llm.mjs snapshot 101 114 133
//   node lab/ab-llm.mjs run qwen3 101 114 133      # aktuelles Modell
//   ... Modell umstellen, callrec neu starten ...
//   node lab/ab-llm.mjs run qwen35 101 114 133
//   node lab/ab-llm.mjs report qwen3 qwen35
//   node lab/ab-llm.mjs restore

import fs from 'node:fs';
import path from 'node:path';
import { pool } from '../db.js';

const OUT = '/opt/callrec/lab/ab-llm';
const API = 'http://127.0.0.1';
const POLL_MS = 5000;
const TIMEOUT_MS = 45 * 60 * 1000;   // eine Analyse darf lange dauern (Whole-Pass)

fs.mkdirSync(OUT, { recursive: true });

const sleep = ms => new Promise(r => setTimeout(r, ms));
const jread = f => JSON.parse(fs.readFileSync(f, 'utf8'));
const jwrite = (f, o) => fs.writeFileSync(f, JSON.stringify(o, null, 1));

async function row(id) {
  const [rows] = await pool.query(
    `SELECT id, title, title_locked, summary, todos, participants, transcript_clean,
            analysis_status, analysis_ms, char_length(transcript) AS tr_chars
       FROM recordings WHERE id=?`, [id]);
  return rows[0] || null;
}

async function activeModel() {
  const r = await fetch(`${API}/api/llm/status`).then(r => r.json()).catch(() => ({}));
  return r.model || '(unbekannt)';
}

async function snapshot(ids) {
  for (const id of ids) {
    const r = await row(id);
    if (!r) { console.log(`#${id}: nicht gefunden`); continue; }
    jwrite(path.join(OUT, `orig-${id}.json`), r);
    console.log(`#${id}: gesichert (${r.tr_chars} Zeichen Transkript, analysis_ms=${r.analysis_ms})`);
  }
}

async function restore() {
  const files = fs.readdirSync(OUT).filter(f => /^orig-\d+\.json$/.test(f));
  if (!files.length) return console.log('Nichts zu restaurieren.');
  for (const f of files) {
    const o = jread(path.join(OUT, f));
    await pool.execute(
      `UPDATE recordings SET title=?, summary=?, todos=?, participants=?,
              transcript_clean=?, analysis_status=?, analysis_ms=? WHERE id=?`,
      [o.title, o.summary, o.todos, o.participants, o.transcript_clean,
       o.analysis_status, o.analysis_ms, o.id]);
    console.log(`#${o.id}: Originalanalyse zurueckgeschrieben`);
  }
}

async function run(label, ids) {
  const model = await activeModel();
  console.log(`== Lauf "${label}" mit Modell ${model}`);
  const results = [];
  for (const id of ids) {
    const before = await row(id);
    if (!before) { console.log(`#${id}: nicht gefunden`); continue; }
    if (!fs.existsSync(path.join(OUT, `orig-${id}.json`)))
      throw new Error(`#${id}: kein Snapshot -- erst "snapshot" laufen lassen!`);

    process.stdout.write(`#${id} (${before.tr_chars} Zeichen): starte ... `);
    const t0 = Date.now();
    const res = await fetch(`${API}/api/recordings/${id}/reanalyze-llm`, { method: 'POST' });
    if (!res.ok) { console.log(`FEHLER HTTP ${res.status}`); continue; }

    let r, waited = 0;
    do {
      await sleep(POLL_MS); waited = Date.now() - t0;
      r = await row(id);
      process.stdout.write('.');
    } while (r && r.analysis_status === 'pending' && waited < TIMEOUT_MS);

    const wall = Date.now() - t0;
    console.log(` ${r.analysis_status} nach ${(wall / 1000).toFixed(0)} s (analysis_ms=${r.analysis_ms})`);
    results.push({
      id, model, label,
      status: r.analysis_status,
      wall_ms: wall,
      analysis_ms: r.analysis_ms,
      tr_chars: r.tr_chars,
      title: r.title,
      summary: r.summary,
      summary_chars: (r.summary || '').length,
      todos: r.todos,
      todo_count: (() => { try { return JSON.parse(r.todos || '[]').length; } catch { return null; } })(),
      participants: r.participants,
    });
    jwrite(path.join(OUT, `run-${label}.json`), results);
  }
  console.log(`\nGeschrieben: ${OUT}/run-${label}.json`);
}

function report(a, b) {
  const A = jread(path.join(OUT, `run-${a}.json`));
  const B = jread(path.join(OUT, `run-${b}.json`));
  const byId = x => Object.fromEntries(x.map(r => [r.id, r]));
  const ma = byId(A), mb = byId(B);
  const ids = [...new Set([...A.map(r => r.id), ...B.map(r => r.id)])].sort((x, y) => x - y);

  console.log(`\nModell A (${a}): ${A[0]?.model}`);
  console.log(`Modell B (${b}): ${B[0]?.model}\n`);
  console.log('ID    Zeichen |   Dauer A |   Dauer B |  Summary A |  Summary B | Todos A/B | Status');
  console.log('-'.repeat(92));
  for (const id of ids) {
    const x = ma[id] || {}, y = mb[id] || {};
    const s = v => v == null ? '     -' : (v / 1000).toFixed(0).padStart(5) + ' s';
    console.log(
      String(id).padStart(4), String(x.tr_chars ?? y.tr_chars ?? '').padStart(8),
      '|', s(x.analysis_ms), '|', s(y.analysis_ms),
      '|', String(x.summary_chars ?? '-').padStart(9), '|', String(y.summary_chars ?? '-').padStart(9),
      '|', `${x.todo_count ?? '-'}/${y.todo_count ?? '-'}`.padStart(8),
      '|', `${x.status ?? '-'}/${y.status ?? '-'}`);
  }

  // Volltext zum Danebenlegen -- die inhaltliche Bewertung macht der Mensch.
  const md = [`# A/B: ${a} vs. ${b}`, '',
    `- **A (${a})**: \`${A[0]?.model}\``, `- **B (${b})**: \`${B[0]?.model}\``, ''];
  for (const id of ids) {
    const x = ma[id] || {}, y = mb[id] || {};
    md.push(`## Aufnahme #${id} (${x.tr_chars ?? y.tr_chars} Zeichen Transkript)`, '',
      `### A — ${a} (${x.analysis_ms ? (x.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '',
      `**Titel:** ${x.title ?? '-'}`, '', (x.summary || '_keine_'), '',
      `**Todos:** ${x.todos ?? '-'}`, '',
      `### B — ${b} (${y.analysis_ms ? (y.analysis_ms / 1000).toFixed(0) + ' s' : 'n/a'})`, '',
      `**Titel:** ${y.title ?? '-'}`, '', (y.summary || '_keine_'), '',
      `**Todos:** ${y.todos ?? '-'}`, '');
  }
  const f = path.join(OUT, `vergleich-${a}-vs-${b}.md`);
  fs.writeFileSync(f, md.join('\n'));
  console.log(`\nVolltext-Vergleich: ${f}`);
}

const [cmd, ...rest] = process.argv.slice(2);
try {
  if (cmd === 'snapshot') await snapshot(rest.map(Number));
  else if (cmd === 'run') await run(rest[0], rest.slice(1).map(Number));
  else if (cmd === 'restore') await restore();
  else if (cmd === 'report') report(rest[0], rest[1]);
  else console.log('Befehle: snapshot <ids...> | run <label> <ids...> | report <a> <b> | restore');
} finally {
  await pool.end();
}