Aime/app/search.js

// Content search across all meetings: "which project did X present last week?"
//
// Retrieval is deliberately split from the answer. The passage hits come from
// SQL and are on screen in milliseconds; the written answer is a separate,
// slow (~5 tok/s on this NPU) LLM call the GUI loads afterwards. Asking the LLM
// first would mean staring at a spinner for a minute before seeing anything.
//
// Ranking is hybrid:
//   full-text  — MariaDB FULLTEXT over the passage text. Nails proper nouns
//                ("Nordlicht", "Jeckel") that an embedding blurs away.
//   semantic   — cosine over text embeddings, so a question worded differently
//                than the meeting still hits. Optional: without the model
//                (see emb.js) `vec` stays NULL and search runs full-text only.
//   boosts     — speaker match, exact phrase, recency.
// Neither alone is enough: full-text misses paraphrases, embeddings miss names.
import { pool } from './db.js';
import * as emb from './emb.js';
import * as llm from './llm.js';

// A passage is the retrieval unit. Too short and a hit has no context to show
// or to answer from; too long and the timestamp stops meaning anything and the
// embedding averages several topics into mush. ~2-4 sentences of one speaker.
const PASSAGE_CHARS = +(process.env.AIME_PASSAGE_CHARS || 420);
const PASSAGE_MAX_CHARS = +(process.env.AIME_PASSAGE_MAX_CHARS || 700);
// A speaker change always ends a passage, but a long monologue is still split.
const TOPK = +(process.env.AIME_SEARCH_TOPK || 20);
const ANSWER_PASSAGES = +(process.env.AIME_ANSWER_PASSAGES || 8);

// German stop words. Natural-language FULLTEXT would rank on "welches"/"hat"
// and drown the two words that carry the question. MariaDB's built-in stopword
// list is English only, so the filtering has to happen here.
const STOP = new Set(`aber alle allem allen aller alles als also am an ander andere anderem anderen anderer anderes
auch auf aus bei beim bin bis bist da damit dann der den des dem die das dass daß derselbe derselben denn dazu dein
deine dem den der des dessen deshalb dies diese diesem diesen dieser dieses doch dort du durch ein eine einem einen
einer eines einig einige einigem einigen einiger einiges einmal er ihn ihm es etwas euer eure für gegen gewesen hab
habe haben hat hatte hatten hier hin hinter ich mich mir ihr ihre ihrem ihren ihrer ihres euch im in indem ins ist
jede jedem jeden jeder jedes jene jenem jenen jener jenes jetzt kann kannst können könnt machen man manche manchem
manchen mancher manches mein meine mit muss musst müssen nach nicht nichts noch nun nur ob oder ohne sehr sein seine
seinem seinen seiner seines selbst sich sie sind so solche solchem solchen solcher solches soll sollte sondern sonst
über um und uns unse unsem unsen unser unses unter viel vom von vor während war waren warst weg weil weiter
wenn werde werden wieder will wir wird wirst wollen wollte würde würden zu zum zur zwar zwischen mal schon ja nein
okay halt eben eigentlich quasi irgendwie
was wer wen wem wessen wann wo wohin woher wie warum wieso weshalb welche welchem welchen welcher welches
wurde wurden worden gibt gab geht ging steht stand macht gemacht gesagt sagt sagte bitte danke`.split(/\s+/));

// Query words worth searching for.
//
// Capitalisation is NOT a licence to keep a stop word: German capitalises every
// noun, and a question starts with a capitalised "Welches"/"Was"/"Wer". Keeping
// those was what made "Welches Projekt hat X letzte Woche vorgestellt" search
// for "Welches" and "Woche" and rank the wrong passages to the top.
function contentTerms(q) {
  const raw = (q || '').split(/[^\p{L}\p{N}äöüßÄÖÜ-]+/u).filter(Boolean);
  const out = [];
  for (const w of raw) {
    if (w.length < 3) continue;                 // below innodb_ft_min_token_size
    if (STOP.has(w.toLowerCase())) continue;
    out.push(w);
  }
  return [...new Set(out)];
}

// --- relative dates ----------------------------------------------------------
// "letzte Woche" is answered deterministically, not by the LLM: a wrong week is
// worse than no filter at all, and date arithmetic is exactly what a 4B model
// gets subtly wrong. Weeks are Mon-Sun (ISO), matching how people say it here.
function startOfDay(d) { const x = new Date(d); x.setHours(0, 0, 0, 0); return x; }
function mondayOf(d) {
  const x = startOfDay(d);
  x.setDate(x.getDate() - ((x.getDay() + 6) % 7));   // getDay(): 0=Sun
  return x;
}
const MONTHS = ['januar', 'februar', 'märz', 'april', 'mai', 'juni', 'juli', 'august', 'september', 'oktober', 'november', 'dezember'];

// Returns {from, to, label, match} — `match` is the text that expressed the date
// so search() can strip it before extracting search terms. Leaving "letzte
// Woche" in the terms makes every passage containing "Woche" a candidate.
export function parseDateRange(q, now = new Date()) {
  const s = (q || '').toLowerCase();
  const day = 86400000;
  const rules = [
    [/\bvorgestern\b/, () => [new Date(+startOfDay(now) - 2 * day), new Date(+startOfDay(now) - day), 'vorgestern']],
    [/\bgestern\b/, () => [new Date(+startOfDay(now) - day), startOfDay(now), 'gestern']],
    [/\bheute\b/, () => [startOfDay(now), new Date(+startOfDay(now) + day), 'heute']],
    [/\b(diese|dieser)\s+woche\b/, () => [mondayOf(now), new Date(+mondayOf(now) + 7 * day), 'diese Woche']],
    [/\b(letzte|letzter|letzten|vergangene|vergangener|vergangenen|vorige|voriger|vorigen)\s+woche\b/, () => {
      const from = new Date(+mondayOf(now) - 7 * day);
      return [from, new Date(+from + 7 * day), 'letzte Woche'];
    }],
    [/\b(diesen|diesem)\s+monat\b/, () =>
      [new Date(now.getFullYear(), now.getMonth(), 1), new Date(now.getFullYear(), now.getMonth() + 1, 1), 'diesen Monat']],
    [/\b(letzten|letztem|vergangenen|vorigen)\s+monat\b/, () =>
      [new Date(now.getFullYear(), now.getMonth() - 1, 1), new Date(now.getFullYear(), now.getMonth(), 1), 'letzten Monat']],
    [new RegExp(`\\b(im\\s+)?(${MONTHS.join('|')})\\b`), (m) => {
      const mi = MONTHS.indexOf(m[2]);
      // A month still ahead of us this year must mean last year's.
      const year = mi > now.getMonth() ? now.getFullYear() - 1 : now.getFullYear();
      return [new Date(year, mi, 1), new Date(year, mi + 1, 1), MONTHS[mi][0].toUpperCase() + MONTHS[mi].slice(1)];
    }],
  ];
  for (const [re, fn] of rules) {
    const m = re.exec(s);
    if (!m) continue;
    const [from, to, label] = fn(m);
    return { from, to, label, match: m[0] };
  }
  return null;
}

// --- indexing ----------------------------------------------------------------
// Passages are cut from the `words` rows, not from transcript_clean: only the
// words carry per-word timestamps, and the timestamp is the whole point of a
// hit. A passage never spans two speakers.
export function buildPassages(words, speakerNames = {}) {
  const out = [];
  let cur = null;
  const flush = () => {
    if (cur && cur.text.trim().length >= 25) {   // drop "Ja." / "Mhm." fragments
      cur.text = cur.text.trim();
      out.push(cur);
    }
    cur = null;
  };
  for (const w of words) {
    const sp = w.speaker || null;
    if (cur && cur.speaker !== sp) flush();
    if (!cur) cur = { speaker: sp, speaker_name: speakerNames[sp] || null, t_start_ms: w.t_start_ms, t_end_ms: w.t_end_ms, text: '' };
    cur.text += (cur.text ? ' ' : '') + w.word;
    cur.t_end_ms = w.t_end_ms;
    // Prefer to break on a sentence end once the target length is reached;
    // force a break at the hard cap so one monologue is not a single passage.
    const endsSentence = /[.!?…]["»)]?$/.test(w.word);
    if ((cur.text.length >= PASSAGE_CHARS && endsSentence) || cur.text.length >= PASSAGE_MAX_CHARS) flush();
  }
  flush();
  return out.map((p, i) => ({ ...p, idx: i }));
}

// Rebuild one recording's passages. Called after transcription/diarization and
// after a speaker rename (the speaker name is denormalised into the row so a
// search can filter by person without joining back).
export async function indexRecording(recId) {
  const [recRows] = await pool.query('SELECT speaker_names, title, summary FROM recordings WHERE id=?', [recId]);
  if (!recRows.length) return 0;
  let names = {};
  try { names = JSON.parse(recRows[0].speaker_names || '{}') || {}; } catch {}
  await indexDoc(recId, recRows[0]);
  const [words] = await pool.query(
    'SELECT word,t_start_ms,t_end_ms,speaker FROM words WHERE recording_id=? ORDER BY idx', [recId]);
  const passages = buildPassages(words, names);
  await pool.query('DELETE FROM passages WHERE recording_id=?', [recId]);
  if (!passages.length) return 0;
  const vecs = await emb.embedMany(passages.map(p => p.text)).catch(() => null);
  const rows = passages.map((p, i) => [
    recId, p.idx, p.speaker, p.speaker_name, p.t_start_ms, p.t_end_ms, p.text,
    vecs && vecs[i] ? JSON.stringify(vecs[i]) : null]);
  await pool.query(
    'INSERT INTO passages (recording_id, idx, speaker, speaker_name, t_start_ms, t_end_ms, text, vec) VALUES ?', [rows]);
  console.log(`search: #${recId} indexiert — ${passages.length} Passagen${vecs ? ' (mit Embeddings)' : ' (nur Volltext)'}`);
  return passages.length;
}

// Document-level embedding over title + summary.
//
// Measured on this corpus: passage-level cosine over raw ASR text separates
// badly — for "wie werden unsere Daten gesichert" the whole candidate field sat
// between 0.809 and 0.856, and an off-topic passage outranked the two passages
// that literally describe the backup job. The transcript is conversational and
// the ASR breaks compounds ("Backup -Job"), which is exactly what an embedding
// handles worst. Title and summary are clean LLM prose about the MEETING'S
// topic, so a second similarity on that level says "this recording is about
// your question" even when no single passage embeds well. Cheap: one vector per
// recording, 22 of them.
async function indexDoc(recId, row) {
  const doc = [row.title || '', row.summary || ''].filter(Boolean).join('. ').slice(0, 4000);
  if (!doc.trim()) return;
  const v = await emb.embedMany([doc]).catch(() => null);
  await pool.execute('UPDATE recordings SET doc_vec=? WHERE id=?',
    [v && v[0] ? JSON.stringify(v[0]) : null, recId]);
}

// Refresh only the document vectors (seconds, vs ~9 min for a full passage
// re-embed). Used after installing the model or changing titles/summaries.
export async function reindexDocs() {
  const [rows] = await pool.query(
    'SELECT id, title, summary FROM recordings WHERE deleted_at IS NULL');
  let n = 0;
  for (const r of rows) {
    try { await indexDoc(r.id, r); n++; } catch (e) { console.error(`search: doc #${r.id} — ${e.message}`); }
  }
  console.log(`search: ${n} Dokument-Vektoren aktualisiert`);
  return n;
}

// Index everything that has words but no passages (first run after the update),
// or everything when force=true. Sequential on purpose — this competes with
// transcription for the same cores.
export async function reindexAll({ force = false, onProgress = null } = {}) {
  const [rows] = await pool.query(
    force
      ? 'SELECT id FROM recordings WHERE deleted_at IS NULL ORDER BY id'
      : `SELECT r.id FROM recordings r
         WHERE r.deleted_at IS NULL
           AND EXISTS (SELECT 1 FROM words w WHERE w.recording_id=r.id)
           AND NOT EXISTS (SELECT 1 FROM passages p WHERE p.recording_id=r.id)
         ORDER BY r.id`);
  let done = 0;
  for (const r of rows) {
    try { await indexRecording(r.id); } catch (e) { console.error(`search: #${r.id} fehlgeschlagen — ${e.message}`); }
    if (onProgress) onProgress(++done, rows.length);
  }
  return { indexed: done, total: rows.length };
}

// --- retrieval ---------------------------------------------------------------
const norm = s => (s || '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ' ').trim();

function cosine(a, b) {
  let s = 0, na = 0, nb = 0;
  for (let i = 0; i < a.length; i++) { s += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
  return s / (Math.sqrt(na) * Math.sqrt(nb) + 1e-9);
}

export async function search(q, { limit = TOPK, recordingId = null } = {}) {
  const range = parseDateRange(q);
  // The date phrase is a FILTER, not a search term — strip it before tokenising.
  // Case-insensitively but WITHOUT lowercasing the rest: the capitalisation of
  // the remaining words is what identifies names for the speaker boost below.
  const terms = contentTerms(
    range ? q.replace(new RegExp(range.match.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i'), ' ') : q);
  const where = ['r.deleted_at IS NULL'];
  const args = [];
  if (recordingId) { where.push('p.recording_id=?'); args.push(recordingId); }
  if (range) { where.push('r.created_at >= ? AND r.created_at < ?'); args.push(range.from, range.to); }

  // "Was lief letzte Woche?" — a date with nothing else to match on. Returning
  // "no hits" would be misleading (there ARE meetings), so show that week's
  // recordings from their opening passages instead of an empty page.
  if (!terms.length && range) {
    const [rows] = await pool.query(
      `SELECT p.id, p.recording_id, p.speaker, p.speaker_name, p.t_start_ms, p.t_end_ms, p.text,
              r.title, r.created_at, r.duration_ms
       FROM passages p JOIN recordings r ON r.id=p.recording_id
       WHERE ${where.join(' AND ')} AND p.idx < 3
       ORDER BY r.created_at DESC, p.idx ASC LIMIT ?`, [...args, limit]);
    return {
      hits: rows.map(r => ({ ...r, score: 0, ft: 0, vec: 0 })),
      terms, range: { label: range.label, from: range.from, to: range.to },
      semantic: false, total: rows.length, dateOnly: true,
    };
  }

  // Full-text candidates. BOOLEAN MODE with OR'd terms: a passage matching only
  // "Jeckel" must still be a candidate, the ranking sorts it out afterwards.
  let ftRows = [];
  if (terms.length) {
    const clean = terms.map(t => t.replace(/[+\-><()~*"@]/g, ''));
    const boolExpr = clean.map(t => `${t}*`).join(' ');       // candidate selection (prefix, OR)
    // Scored in NATURAL LANGUAGE mode, which is IDF-weighted; BOOLEAN MODE only
    // counts matches, so a passage repeating a common word like "Daten" beat the
    // passage that actually contained the rare, meaning-carrying term.
    const nlExpr = clean.join(' ');
    const [rows] = await pool.query(
      `SELECT p.id, p.recording_id, p.speaker, p.speaker_name, p.t_start_ms, p.t_end_ms, p.text, p.vec,
              r.title, r.created_at, r.duration_ms, r.doc_vec,
              MATCH(p.text) AGAINST(?) AS ft
       FROM passages p JOIN recordings r ON r.id=p.recording_id
       WHERE ${where.join(' AND ')} AND MATCH(p.text) AGAINST(? IN BOOLEAN MODE)
       ORDER BY ft DESC LIMIT 200`,
      [nlExpr, ...args, boolExpr]);
    ftRows = rows;
  }

  // Semantic candidates. Only worth loading when the model is there AND some
  // passage actually carries a vector.
  let vecRows = [];
  let qvec = null;
  if (emb.available()) {
    qvec = await emb.embedQuery(q).catch(() => null);
    if (qvec) {
      const [rows] = await pool.query(
        `SELECT p.id, p.recording_id, p.speaker, p.speaker_name, p.t_start_ms, p.t_end_ms, p.text, p.vec,
                r.title, r.created_at, r.duration_ms, r.doc_vec
         FROM passages p JOIN recordings r ON r.id=p.recording_id
         WHERE ${where.join(' AND ')} AND p.vec IS NOT NULL`, args);
      vecRows = rows;
    }
  }

  // Merge both candidate sets, then score every candidate on both axes.
  const byId = new Map();
  for (const r of [...ftRows, ...vecRows]) if (!byId.has(r.id)) byId.set(r.id, r);
  const ftMax = Math.max(1e-9, ...ftRows.map(r => r.ft || 0));
  const ftById = new Map(ftRows.map(r => [r.id, (r.ft || 0) / ftMax]));

  const qn = norm(q);
  const nameTerms = terms.filter(t => t[0] === t[0].toUpperCase());
  const now = Date.now();

  // Raw e5 cosines sit in a narrow band (measured here: 0.82-0.86 across ALL
  // passages, relevant or not) because the model puts everything in the same
  // cone. Feeding that raw into the score makes the semantic axis worth ~0.01
  // while full-text is worth up to 0.55 — the ranking then ignores semantics
  // entirely except when full-text finds nothing. So spread the band: rank by
  // how far above the field a passage is, not by its absolute cosine.
  const cand = [...byId.values()];
  const cosById = new Map();
  if (qvec) {
    for (const r of cand) {
      if (!r.vec) continue;
      try { cosById.set(r.id, cosine(qvec, JSON.parse(r.vec))); } catch {}
    }
  }
  const cosVals = [...cosById.values()];
  const cosMean = cosVals.length ? cosVals.reduce((a, b) => a + b, 0) / cosVals.length : 0;
  const cosSd = cosVals.length
    ? Math.sqrt(cosVals.reduce((a, b) => a + (b - cosMean) ** 2, 0) / cosVals.length) : 0;
  const cosMin = cosVals.length ? Math.min(...cosVals) : 0;
  const cosMax = cosVals.length ? Math.max(...cosVals) : 0;
  // Min-max, not standard deviations: with a 0.047-wide band an sd-based squash
  // saturates at 1.0 for dozens of passages and throws away the ordering
  // exactly where it matters — at the top.
  const vecNorm = id => {
    if (!cosById.has(id) || cosMax - cosMin < 1e-9) return 0;
    return (cosById.get(id) - cosMin) / (cosMax - cosMin);
  };

  // Document level: how close is the RECORDING (title + summary) to the
  // question. One value per recording, normalised over the recordings actually
  // in play.
  const docCos = new Map();
  if (qvec) {
    for (const r of cand) {
      if (docCos.has(r.recording_id) || !r.doc_vec) continue;
      try { docCos.set(r.recording_id, cosine(qvec, JSON.parse(r.doc_vec))); } catch {}
    }
  }
  const dVals = [...docCos.values()];
  const dMin = dVals.length ? Math.min(...dVals) : 0;
  const dMax = dVals.length ? Math.max(...dVals) : 0;
  const docNorm = recId => {
    if (!docCos.has(recId) || dMax - dMin < 1e-9) return 0;
    return (docCos.get(recId) - dMin) / (dMax - dMin);
  };

  // Relevance floor for semantic-only candidates, still in sd terms (a scale
  // that means "stands out from the field" rather than "is the best of a bad
  // field", which is what min-max would say).
  const semFloor = cosMean + cosSd;

  const hits = [];
  for (const r of cand) {
    const ft = ftById.get(r.id) || 0;
    const vec = vecNorm(r.id);
    const doc = docNorm(r.recording_id);
    if (!ft && (cosById.get(r.id) || 0) < semFloor) continue;
    // Speaker boost: the person asked about is usually the one TALKING, and
    // their own words never contain their name — without this the passage that
    // actually answers the question ranks below every passage mentioning them.
    const spk = nameTerms.some(t => norm(r.speaker_name).includes(norm(t))) ? 0.25 : 0;
    const phrase = qn.length > 8 && norm(r.text).includes(qn) ? 0.35 : 0;
    // Mild recency tilt — "last week" questions are the common case. Half a
    // point of decay per year; never enough to beat a clearly better match.
    const ageDays = (now - new Date(r.created_at).getTime()) / 86400000;
    const recency = 0.1 / (1 + ageDays / 180);
    // Passage semantics get less weight than the document level because the
    // measurement above showed them to be the noisier of the two signals.
    const score = 0.45 * ft + 0.25 * vec + 0.30 * doc + spk + phrase + recency;
    hits.push({
      id: r.id, recording_id: r.recording_id, title: r.title, created_at: r.created_at,
      duration_ms: r.duration_ms, speaker: r.speaker, speaker_name: r.speaker_name,
      t_start_ms: r.t_start_ms, t_end_ms: r.t_end_ms, text: r.text,
      score, ft, vec: +vec.toFixed(4), doc: +doc.toFixed(4),
      cos: +(cosById.get(r.id) || 0).toFixed(4),
    });
  }
  hits.sort((a, b) => b.score - a.score);
  return {
    hits: hits.slice(0, limit),
    terms,
    range: range ? { label: range.label, from: range.from, to: range.to } : null,
    semantic: !!qvec,
    total: hits.length,
  };
}

// --- answer ------------------------------------------------------------------
const ts = ms => `${String(Math.floor(ms / 60000)).padStart(2, '0')}:${String(Math.floor(ms / 1000) % 60).padStart(2, '0')}`;
const dateDe = d => new Date(d).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit', year: 'numeric' });

// Writes the prose answer over the top hits. Every passage is numbered and the
// model is told to cite those numbers, so each claim can be traced back to a
// recording and a minute — an uncited sentence is the tell for a hallucination.
export async function answer(q, hits) {
  const use = hits.slice(0, ANSWER_PASSAGES);
  if (!use.length) return { answer: 'Dazu finde ich in den Aufnahmen nichts.', sources: [] };
  const ctx = use.map((h, i) =>
    `[${i + 1}] Meeting "${h.title || 'Aufnahme #' + h.recording_id}" (#${h.recording_id}) vom ${dateDe(h.created_at)}, `
    + `${h.speaker_name || h.speaker || 'unbekannt'} bei ${ts(h.t_start_ms)}:\n${h.text}`).join('\n\n');
  const prompt = `Du beantwortest eine Frage AUSSCHLIESSLICH aus den folgenden Ausschnitten aus Meeting-Mitschriften.

${ctx}

Frage: ${q}

Regeln:
- Antworte kurz und konkret auf Deutsch, in höchstens 4 Sätzen.
- Belege JEDE Aussage mit der Nummer des Ausschnitts, z.B. [2].
- Steht die Antwort nicht in den Ausschnitten, schreibe genau: "Dazu finde ich in den Aufnahmen nichts."
- Rate nicht und ergänze nichts aus eigenem Wissen.

Antwort:`;
  const text = (await llm.askRaw(prompt) || '').trim();
  // Map the [n] the model used back to real recordings + timestamps.
  const cited = new Set((text.match(/\[(\d+)\]/g) || []).map(m => +m.slice(1, -1)));
  const sources = use
    .map((h, i) => ({ n: i + 1, ...h }))
    .filter(h => cited.size === 0 || cited.has(h.n));
  return { answer: text, sources };
}