Scanner: Stellenanzeigen aus Jobbörsen (DACH, max. 21 Tage)

Neuer Menüpunkt Scanner mit Dashboard und Suchagenten, Zeitplan und API-Schlüssel unter Einstellungen -> Scanner. - Quellen: Agentur für Arbeit (API), StepStone, karriere.at, freelancermap (HTML) sowie Adzuna und Jooble (kostenlose Schlüssel) - Quote = gefundene Schlagwörter, Titel zählt doppelt; keine KI - Abgleich mit icon_companies, bekannte Firmen oben mit Abzeichen - Zeitplan Vorgabe stündlich Mo-Fr 06-20 Uhr, einstellbar - Sperre gegen parallele Läufe wie im perinco-Scanner Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 10:49:44 +02:00
parent 6fe5b18c11
commit 1b82f580db
23 changed files with 2894 additions and 2 deletions
@@ -0,0 +1,347 @@
+import express from 'express';
+import { pool } from '../db.js';
+import { requireAdmin, requireAuth } from '../middleware/auth.js';
+import { logHistory } from '../history.js';
+import { SOURCES, SOURCE_IDS, sourceReady } from '../scanner/sources.js';
+import { parseList, runScan, scanStatus } from '../scanner/pipeline.js';
+import {
+  getLastScanAt,
+  loadScannerSecrets,
+  loadScannerSettings,
+  maskedScannerSecrets,
+  saveScannerSettings,
+} from '../scanner/settings.js';
+
+/*
+ * Scanner: Stellenanzeigen aus Jobboersen (siehe docs/12-scanner.md).
+ *
+ * Suchagenten und Treffer pflegt jeder Benutzer; Zeitplan und API-Schluessel
+ * nur Administratoren (Einstellungen -> Scanner).
+ */
+
+const router = express.Router();
+router.use(requireAuth);
+
+const STATUSES = ['neu', 'gesehen', 'in_bearbeitung', 'verworfen'];
+
+const cleanList = (value) =>
+  [...new Set((Array.isArray(value) ? value : []).map((item) => String(item).trim()).filter(Boolean))].slice(0, 50);
+
+const agentOut = (row) => ({
+  id: row.id,
+  name: row.name,
+  keywords: parseList(row.keywords),
+  sources: parseList(row.sources),
+  excludes: parseList(row.excludes),
+  is_active: Boolean(row.is_active),
+  matches: Number(row.matches ?? 0),
+  created_at: row.created_at,
+  updated_at: row.updated_at,
+});
+
+/* ------------------------------------------------------------------------
+ * Uebersicht: Kennzahlen, Quellen, Zustand des Zeitplans
+ * ---------------------------------------------------------------------- */
+router.get('/overview', async (req, res, next) => {
+  try {
+    const settings = await loadScannerSettings();
+    const secrets = await loadScannerSecrets();
+    const [[agents]] = await pool.query(
+      'SELECT COUNT(*) AS active FROM icon_scanner_agents WHERE is_active = 1'
+    );
+    const [[jobs]] = await pool.query('SELECT COUNT(*) AS total FROM icon_scanner_jobs');
+    const [[matches]] = await pool.query(
+      `SELECT COUNT(*) AS total,
+              SUM(m.status = 'neu') AS fresh,
+              SUM(j.company_id IS NOT NULL) AS known
+         FROM icon_scanner_matches m JOIN icon_scanner_jobs j ON j.id = m.job_id`
+    );
+    const [[lastRun]] = await pool.query(
+      'SELECT * FROM icon_scanner_runs WHERE finished_at IS NOT NULL ORDER BY id DESC LIMIT 1'
+    );
+    const perSource = lastRun?.per_source ? JSON.parse(lastRun.per_source) : {};
+
+    res.json({
+      stats: {
+        agents: Number(agents.active),
+        jobs: Number(jobs.total),
+        matches: Number(matches.total ?? 0),
+        fresh: Number(matches.fresh ?? 0),
+        known: Number(matches.known ?? 0),
+      },
+      sources: SOURCE_IDS.map((id) => ({
+        id,
+        label: SOURCES[id].label,
+        countries: SOURCES[id].countries,
+        ready: sourceReady(id, secrets),
+        needsKey: SOURCES[id].needsKey,
+        last: perSource[id] ?? null,
+      })),
+      schedule: settings,
+      lastScanAt: (await getLastScanAt())?.toISOString() ?? null,
+      lastRun: lastRun
+        ? {
+            id: lastRun.id,
+            trigger: lastRun.trigger_kind,
+            started_at: lastRun.started_at,
+            finished_at: lastRun.finished_at,
+            jobs_found: lastRun.jobs_found,
+            matches_new: lastRun.matches_new,
+          }
+        : null,
+      status: scanStatus(),
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+/* ------------------------------------------------------------------------
+ * Suchagenten
+ * ---------------------------------------------------------------------- */
+router.get('/agents', async (req, res, next) => {
+  try {
+    const [rows] = await pool.query(
+      `SELECT a.*, (SELECT COUNT(*) FROM icon_scanner_matches m WHERE m.agent_id = a.id) AS matches
+         FROM icon_scanner_agents a ORDER BY a.name`
+    );
+    res.json({ agents: rows.map(agentOut) });
+  } catch (err) {
+    next(err);
+  }
+});
+
+function readAgent(body) {
+  const name = String(body?.name ?? '').trim().slice(0, 190);
+  const keywords = cleanList(body?.keywords);
+  const sources = cleanList(body?.sources).filter((id) => SOURCE_IDS.includes(id));
+  const excludes = cleanList(body?.excludes);
+  if (!name) return { error: 'Bitte einen Namen angeben.' };
+  if (keywords.length === 0) return { error: 'Mindestens ein Schlagwort angeben.' };
+  if (sources.length === 0) return { error: 'Mindestens eine Börse auswählen.' };
+  return { name, keywords, sources, excludes, is_active: body?.is_active !== false };
+}
+
+router.post('/agents', async (req, res, next) => {
+  const agent = readAgent(req.body);
+  if (agent.error) return res.status(400).json({ error: agent.error });
+  try {
+    const [result] = await pool.query(
+      `INSERT INTO icon_scanner_agents (name, keywords, sources, excludes, is_active, created_by)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [
+        agent.name,
+        JSON.stringify(agent.keywords),
+        JSON.stringify(agent.sources),
+        JSON.stringify(agent.excludes),
+        agent.is_active ? 1 : 0,
+        req.user.id,
+      ]
+    );
+    await logHistory({
+      entityType: 'system',
+      entityLabel: 'Scanner',
+      action: 'created',
+      text: `Suchagent „${agent.name}" angelegt`,
+      userId: req.user.id,
+    });
+    const [[row]] = await pool.query('SELECT * FROM icon_scanner_agents WHERE id = ?', [result.insertId]);
+    res.status(201).json({ agent: agentOut(row) });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.put('/agents/:id', async (req, res, next) => {
+  const agent = readAgent(req.body);
+  if (agent.error) return res.status(400).json({ error: agent.error });
+  try {
+    const [result] = await pool.query(
+      `UPDATE icon_scanner_agents
+          SET name = ?, keywords = ?, sources = ?, excludes = ?, is_active = ?
+        WHERE id = ?`,
+      [
+        agent.name,
+        JSON.stringify(agent.keywords),
+        JSON.stringify(agent.sources),
+        JSON.stringify(agent.excludes),
+        agent.is_active ? 1 : 0,
+        Number(req.params.id),
+      ]
+    );
+    if (result.affectedRows === 0) return res.status(404).json({ error: 'Suchagent nicht gefunden.' });
+    const [[row]] = await pool.query('SELECT * FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
+    res.json({ agent: agentOut(row) });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.delete('/agents/:id', async (req, res, next) => {
+  try {
+    const [[row]] = await pool.query('SELECT name FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
+    if (!row) return res.status(404).json({ error: 'Suchagent nicht gefunden.' });
+    // Treffer gehen per ON DELETE CASCADE mit.
+    await pool.query('DELETE FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
+    await logHistory({
+      entityType: 'system',
+      entityLabel: 'Scanner',
+      action: 'deleted',
+      text: `Suchagent „${row.name}" gelöscht`,
+      userId: req.user.id,
+    });
+    res.json({ ok: true });
+  } catch (err) {
+    next(err);
+  }
+});
+
+/* ------------------------------------------------------------------------
+ * Treffer
+ * ---------------------------------------------------------------------- */
+router.get('/matches', async (req, res, next) => {
+  const where = [];
+  const params = [];
+
+  const hours = Number(req.query.hours);
+  if (Number.isFinite(hours) && hours > 0) {
+    where.push('m.created_at >= NOW() - INTERVAL ? HOUR');
+    params.push(Math.min(hours, 24 * 90));
+  }
+  if (req.query.agentId) {
+    where.push('m.agent_id = ?');
+    params.push(Number(req.query.agentId));
+  }
+  if (req.query.source && SOURCE_IDS.includes(String(req.query.source))) {
+    where.push('j.source = ?');
+    params.push(String(req.query.source));
+  }
+  if (req.query.known === '1') where.push('j.company_id IS NOT NULL');
+  if (req.query.status && STATUSES.includes(String(req.query.status))) {
+    where.push('m.status = ?');
+    params.push(String(req.query.status));
+  } else if (req.query.status !== 'alle') {
+    // Verworfenes nur auf ausdruecklichen Wunsch.
+    where.push("m.status <> 'verworfen'");
+  }
+  const minScore = Number(req.query.minScore);
+  if (Number.isFinite(minScore) && minScore > 0) {
+    where.push('m.score >= ?');
+    params.push(minScore);
+  }
+  const q = String(req.query.q ?? '').trim();
+  if (q) {
+    where.push('(j.title LIKE ? OR j.company LIKE ? OR j.location LIKE ?)');
+    params.push(`%${q}%`, `%${q}%`, `%${q}%`);
+  }
+
+  const order =
+    req.query.sort === 'score'
+      ? 'm.score DESC, m.created_at DESC'
+      : req.query.sort === 'published'
+        ? 'j.published_at IS NULL, j.published_at DESC'
+        : // bekannte Firmen zuerst, dann neueste
+          'known DESC, m.created_at DESC';
+
+  const limit = Math.min(Math.max(Number(req.query.limit) || 300, 1), 1000);
+  const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';
+
+  try {
+    const [rows] = await pool.query(
+      `SELECT m.id, m.score, m.hits, m.status, m.created_at,
+              a.id AS agent_id, a.name AS agent_name,
+              j.id AS job_id, j.source, j.title, j.company, j.location, j.country,
+              j.employment_type, j.url, j.description, j.published_at,
+              j.company_id, c.name AS company_name, c.is_customer,
+              (j.company_id IS NOT NULL) AS known
+         FROM icon_scanner_matches m
+         JOIN icon_scanner_agents a ON a.id = m.agent_id
+         JOIN icon_scanner_jobs j ON j.id = m.job_id
+         LEFT JOIN icon_companies c ON c.id = j.company_id
+         ${whereSql}
+        ORDER BY ${order}
+        LIMIT ${limit}`,
+      params
+    );
+    const [[count]] = await pool.query(
+      `SELECT COUNT(*) AS total
+         FROM icon_scanner_matches m JOIN icon_scanner_jobs j ON j.id = m.job_id
+         ${whereSql}`,
+      params
+    );
+    res.json({
+      total: Number(count.total),
+      matches: rows.map((row) => ({
+        ...row,
+        hits: parseList(row.hits),
+        known: Boolean(row.known),
+        is_customer: row.is_customer == null ? null : Boolean(row.is_customer),
+      })),
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.patch('/matches/:id', async (req, res, next) => {
+  const status = String(req.body?.status ?? '');
+  if (!STATUSES.includes(status)) return res.status(400).json({ error: 'Unbekannter Status.' });
+  try {
+    const [result] = await pool.query('UPDATE icon_scanner_matches SET status = ? WHERE id = ?', [
+      status,
+      Number(req.params.id),
+    ]);
+    if (result.affectedRows === 0) return res.status(404).json({ error: 'Treffer nicht gefunden.' });
+    res.json({ ok: true });
+  } catch (err) {
+    next(err);
+  }
+});
+
+/* ------------------------------------------------------------------------
+ * Jetzt scannen - laeuft im Hintergrund, die Oberflaeche fragt /overview ab.
+ * Ein Lauf dauert je nach Zahl der Schlagwoerter Minuten; darauf zu warten
+ * hiesse, gegen das Zeitlimit des Proxys zu laufen.
+ * ---------------------------------------------------------------------- */
+router.post('/scan', (req, res) => {
+  if (scanStatus().running) return res.status(409).json({ error: 'Es läuft bereits ein Suchlauf.' });
+  runScan('manual').catch((err) => console.error('[Scanner] Manueller Lauf:', err));
+  res.status(202).json({ started: true });
+});
+
+/* ------------------------------------------------------------------------
+ * Einstellungen (nur Administratoren)
+ * ---------------------------------------------------------------------- */
+router.get('/settings', requireAdmin, async (req, res, next) => {
+  try {
+    res.json({ settings: await loadScannerSettings(), secrets: await maskedScannerSecrets() });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.patch('/settings', requireAdmin, async (req, res, next) => {
+  try {
+    const settings = await saveScannerSettings(req.body ?? {});
+    const touchedKeys = ['adzunaId', 'adzunaKey', 'joobleKey'].filter((key) => req.body?.[key] !== undefined);
+    await logHistory({
+      entityType: 'system',
+      entityLabel: 'Scanner',
+      action: 'updated',
+      // Nie die Schluessel selbst - nur, dass sie geaendert wurden.
+      text: [
+        `Zeitplan: ${settings.active ? 'an' : 'aus'}, alle ${settings.interval} min, ${settings.from}–${settings.to} Uhr`,
+        touchedKeys.length ? 'API-Schlüssel geändert' : null,
+      ]
+        .filter(Boolean)
+        .join(', '),
+      userId: req.user.id,
+    });
+    res.json({ settings, secrets: await maskedScannerSecrets() });
+  } catch (err) {
+    if (err?.status === 400) return res.status(400).json({ error: err.message });
+    next(err);
+  }
+});
+
+export default router;