Scanner: Stellenanzeigen aus Jobbörsen (DACH, max. 21 Tage)
Neuer Menüpunkt Scanner mit Dashboard und Suchagenten, Zeitplan und API-Schlüssel unter Einstellungen -> Scanner. - Quellen: Agentur für Arbeit (API), StepStone, karriere.at, freelancermap (HTML) sowie Adzuna und Jooble (kostenlose Schlüssel) - Quote = gefundene Schlagwörter, Titel zählt doppelt; keine KI - Abgleich mit icon_companies, bekannte Firmen oben mit Abzeichen - Zeitplan Vorgabe stündlich Mo-Fr 06-20 Uhr, einstellbar - Sperre gegen parallele Läufe wie im perinco-Scanner Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
schakal00783
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6fe5b18c11
commit
1b82f580db
@@ -11,6 +11,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"bcryptjs": "^2.4.3",
|
||||
"cheerio": "^1.2.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
|
||||
@@ -27,7 +27,8 @@ router.get('/', async (req, res, next) => {
|
||||
(SELECT COUNT(*) FROM icon_tenders) AS tenders,
|
||||
(SELECT COUNT(*) FROM icon_projects WHERE status = 'Aktiv') AS projects_active,
|
||||
(SELECT COUNT(*) FROM icon_leads WHERE archived = 0) AS leads_open,
|
||||
(SELECT COUNT(*) FROM icon_leads WHERE archived = 1) AS leads_archived`
|
||||
(SELECT COUNT(*) FROM icon_leads WHERE archived = 1) AS leads_archived,
|
||||
(SELECT COUNT(*) FROM icon_scanner_matches WHERE status = 'neu') AS scanner_new`
|
||||
);
|
||||
|
||||
const [byStatus] = await pool.query(
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
import express from 'express';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAdmin, requireAuth } from '../middleware/auth.js';
|
||||
import { logHistory } from '../history.js';
|
||||
import { SOURCES, SOURCE_IDS, sourceReady } from '../scanner/sources.js';
|
||||
import { parseList, runScan, scanStatus } from '../scanner/pipeline.js';
|
||||
import {
|
||||
getLastScanAt,
|
||||
loadScannerSecrets,
|
||||
loadScannerSettings,
|
||||
maskedScannerSecrets,
|
||||
saveScannerSettings,
|
||||
} from '../scanner/settings.js';
|
||||
|
||||
/*
|
||||
* Scanner: Stellenanzeigen aus Jobboersen (siehe docs/12-scanner.md).
|
||||
*
|
||||
* Suchagenten und Treffer pflegt jeder Benutzer; Zeitplan und API-Schluessel
|
||||
* nur Administratoren (Einstellungen -> Scanner).
|
||||
*/
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const STATUSES = ['neu', 'gesehen', 'in_bearbeitung', 'verworfen'];
|
||||
|
||||
const cleanList = (value) =>
|
||||
[...new Set((Array.isArray(value) ? value : []).map((item) => String(item).trim()).filter(Boolean))].slice(0, 50);
|
||||
|
||||
const agentOut = (row) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
keywords: parseList(row.keywords),
|
||||
sources: parseList(row.sources),
|
||||
excludes: parseList(row.excludes),
|
||||
is_active: Boolean(row.is_active),
|
||||
matches: Number(row.matches ?? 0),
|
||||
created_at: row.created_at,
|
||||
updated_at: row.updated_at,
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Uebersicht: Kennzahlen, Quellen, Zustand des Zeitplans
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.get('/overview', async (req, res, next) => {
|
||||
try {
|
||||
const settings = await loadScannerSettings();
|
||||
const secrets = await loadScannerSecrets();
|
||||
const [[agents]] = await pool.query(
|
||||
'SELECT COUNT(*) AS active FROM icon_scanner_agents WHERE is_active = 1'
|
||||
);
|
||||
const [[jobs]] = await pool.query('SELECT COUNT(*) AS total FROM icon_scanner_jobs');
|
||||
const [[matches]] = await pool.query(
|
||||
`SELECT COUNT(*) AS total,
|
||||
SUM(m.status = 'neu') AS fresh,
|
||||
SUM(j.company_id IS NOT NULL) AS known
|
||||
FROM icon_scanner_matches m JOIN icon_scanner_jobs j ON j.id = m.job_id`
|
||||
);
|
||||
const [[lastRun]] = await pool.query(
|
||||
'SELECT * FROM icon_scanner_runs WHERE finished_at IS NOT NULL ORDER BY id DESC LIMIT 1'
|
||||
);
|
||||
const perSource = lastRun?.per_source ? JSON.parse(lastRun.per_source) : {};
|
||||
|
||||
res.json({
|
||||
stats: {
|
||||
agents: Number(agents.active),
|
||||
jobs: Number(jobs.total),
|
||||
matches: Number(matches.total ?? 0),
|
||||
fresh: Number(matches.fresh ?? 0),
|
||||
known: Number(matches.known ?? 0),
|
||||
},
|
||||
sources: SOURCE_IDS.map((id) => ({
|
||||
id,
|
||||
label: SOURCES[id].label,
|
||||
countries: SOURCES[id].countries,
|
||||
ready: sourceReady(id, secrets),
|
||||
needsKey: SOURCES[id].needsKey,
|
||||
last: perSource[id] ?? null,
|
||||
})),
|
||||
schedule: settings,
|
||||
lastScanAt: (await getLastScanAt())?.toISOString() ?? null,
|
||||
lastRun: lastRun
|
||||
? {
|
||||
id: lastRun.id,
|
||||
trigger: lastRun.trigger_kind,
|
||||
started_at: lastRun.started_at,
|
||||
finished_at: lastRun.finished_at,
|
||||
jobs_found: lastRun.jobs_found,
|
||||
matches_new: lastRun.matches_new,
|
||||
}
|
||||
: null,
|
||||
status: scanStatus(),
|
||||
});
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Suchagenten
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.get('/agents', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT a.*, (SELECT COUNT(*) FROM icon_scanner_matches m WHERE m.agent_id = a.id) AS matches
|
||||
FROM icon_scanner_agents a ORDER BY a.name`
|
||||
);
|
||||
res.json({ agents: rows.map(agentOut) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
function readAgent(body) {
|
||||
const name = String(body?.name ?? '').trim().slice(0, 190);
|
||||
const keywords = cleanList(body?.keywords);
|
||||
const sources = cleanList(body?.sources).filter((id) => SOURCE_IDS.includes(id));
|
||||
const excludes = cleanList(body?.excludes);
|
||||
if (!name) return { error: 'Bitte einen Namen angeben.' };
|
||||
if (keywords.length === 0) return { error: 'Mindestens ein Schlagwort angeben.' };
|
||||
if (sources.length === 0) return { error: 'Mindestens eine Börse auswählen.' };
|
||||
return { name, keywords, sources, excludes, is_active: body?.is_active !== false };
|
||||
}
|
||||
|
||||
router.post('/agents', async (req, res, next) => {
|
||||
const agent = readAgent(req.body);
|
||||
if (agent.error) return res.status(400).json({ error: agent.error });
|
||||
try {
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO icon_scanner_agents (name, keywords, sources, excludes, is_active, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
agent.name,
|
||||
JSON.stringify(agent.keywords),
|
||||
JSON.stringify(agent.sources),
|
||||
JSON.stringify(agent.excludes),
|
||||
agent.is_active ? 1 : 0,
|
||||
req.user.id,
|
||||
]
|
||||
);
|
||||
await logHistory({
|
||||
entityType: 'system',
|
||||
entityLabel: 'Scanner',
|
||||
action: 'created',
|
||||
text: `Suchagent „${agent.name}" angelegt`,
|
||||
userId: req.user.id,
|
||||
});
|
||||
const [[row]] = await pool.query('SELECT * FROM icon_scanner_agents WHERE id = ?', [result.insertId]);
|
||||
res.status(201).json({ agent: agentOut(row) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/agents/:id', async (req, res, next) => {
|
||||
const agent = readAgent(req.body);
|
||||
if (agent.error) return res.status(400).json({ error: agent.error });
|
||||
try {
|
||||
const [result] = await pool.query(
|
||||
`UPDATE icon_scanner_agents
|
||||
SET name = ?, keywords = ?, sources = ?, excludes = ?, is_active = ?
|
||||
WHERE id = ?`,
|
||||
[
|
||||
agent.name,
|
||||
JSON.stringify(agent.keywords),
|
||||
JSON.stringify(agent.sources),
|
||||
JSON.stringify(agent.excludes),
|
||||
agent.is_active ? 1 : 0,
|
||||
Number(req.params.id),
|
||||
]
|
||||
);
|
||||
if (result.affectedRows === 0) return res.status(404).json({ error: 'Suchagent nicht gefunden.' });
|
||||
const [[row]] = await pool.query('SELECT * FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
|
||||
res.json({ agent: agentOut(row) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/agents/:id', async (req, res, next) => {
|
||||
try {
|
||||
const [[row]] = await pool.query('SELECT name FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
|
||||
if (!row) return res.status(404).json({ error: 'Suchagent nicht gefunden.' });
|
||||
// Treffer gehen per ON DELETE CASCADE mit.
|
||||
await pool.query('DELETE FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
|
||||
await logHistory({
|
||||
entityType: 'system',
|
||||
entityLabel: 'Scanner',
|
||||
action: 'deleted',
|
||||
text: `Suchagent „${row.name}" gelöscht`,
|
||||
userId: req.user.id,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Treffer
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.get('/matches', async (req, res, next) => {
|
||||
const where = [];
|
||||
const params = [];
|
||||
|
||||
const hours = Number(req.query.hours);
|
||||
if (Number.isFinite(hours) && hours > 0) {
|
||||
where.push('m.created_at >= NOW() - INTERVAL ? HOUR');
|
||||
params.push(Math.min(hours, 24 * 90));
|
||||
}
|
||||
if (req.query.agentId) {
|
||||
where.push('m.agent_id = ?');
|
||||
params.push(Number(req.query.agentId));
|
||||
}
|
||||
if (req.query.source && SOURCE_IDS.includes(String(req.query.source))) {
|
||||
where.push('j.source = ?');
|
||||
params.push(String(req.query.source));
|
||||
}
|
||||
if (req.query.known === '1') where.push('j.company_id IS NOT NULL');
|
||||
if (req.query.status && STATUSES.includes(String(req.query.status))) {
|
||||
where.push('m.status = ?');
|
||||
params.push(String(req.query.status));
|
||||
} else if (req.query.status !== 'alle') {
|
||||
// Verworfenes nur auf ausdruecklichen Wunsch.
|
||||
where.push("m.status <> 'verworfen'");
|
||||
}
|
||||
const minScore = Number(req.query.minScore);
|
||||
if (Number.isFinite(minScore) && minScore > 0) {
|
||||
where.push('m.score >= ?');
|
||||
params.push(minScore);
|
||||
}
|
||||
const q = String(req.query.q ?? '').trim();
|
||||
if (q) {
|
||||
where.push('(j.title LIKE ? OR j.company LIKE ? OR j.location LIKE ?)');
|
||||
params.push(`%${q}%`, `%${q}%`, `%${q}%`);
|
||||
}
|
||||
|
||||
const order =
|
||||
req.query.sort === 'score'
|
||||
? 'm.score DESC, m.created_at DESC'
|
||||
: req.query.sort === 'published'
|
||||
? 'j.published_at IS NULL, j.published_at DESC'
|
||||
: // bekannte Firmen zuerst, dann neueste
|
||||
'known DESC, m.created_at DESC';
|
||||
|
||||
const limit = Math.min(Math.max(Number(req.query.limit) || 300, 1), 1000);
|
||||
const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT m.id, m.score, m.hits, m.status, m.created_at,
|
||||
a.id AS agent_id, a.name AS agent_name,
|
||||
j.id AS job_id, j.source, j.title, j.company, j.location, j.country,
|
||||
j.employment_type, j.url, j.description, j.published_at,
|
||||
j.company_id, c.name AS company_name, c.is_customer,
|
||||
(j.company_id IS NOT NULL) AS known
|
||||
FROM icon_scanner_matches m
|
||||
JOIN icon_scanner_agents a ON a.id = m.agent_id
|
||||
JOIN icon_scanner_jobs j ON j.id = m.job_id
|
||||
LEFT JOIN icon_companies c ON c.id = j.company_id
|
||||
${whereSql}
|
||||
ORDER BY ${order}
|
||||
LIMIT ${limit}`,
|
||||
params
|
||||
);
|
||||
const [[count]] = await pool.query(
|
||||
`SELECT COUNT(*) AS total
|
||||
FROM icon_scanner_matches m JOIN icon_scanner_jobs j ON j.id = m.job_id
|
||||
${whereSql}`,
|
||||
params
|
||||
);
|
||||
res.json({
|
||||
total: Number(count.total),
|
||||
matches: rows.map((row) => ({
|
||||
...row,
|
||||
hits: parseList(row.hits),
|
||||
known: Boolean(row.known),
|
||||
is_customer: row.is_customer == null ? null : Boolean(row.is_customer),
|
||||
})),
|
||||
});
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
router.patch('/matches/:id', async (req, res, next) => {
|
||||
const status = String(req.body?.status ?? '');
|
||||
if (!STATUSES.includes(status)) return res.status(400).json({ error: 'Unbekannter Status.' });
|
||||
try {
|
||||
const [result] = await pool.query('UPDATE icon_scanner_matches SET status = ? WHERE id = ?', [
|
||||
status,
|
||||
Number(req.params.id),
|
||||
]);
|
||||
if (result.affectedRows === 0) return res.status(404).json({ error: 'Treffer nicht gefunden.' });
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Jetzt scannen - laeuft im Hintergrund, die Oberflaeche fragt /overview ab.
|
||||
* Ein Lauf dauert je nach Zahl der Schlagwoerter Minuten; darauf zu warten
|
||||
* hiesse, gegen das Zeitlimit des Proxys zu laufen.
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.post('/scan', (req, res) => {
|
||||
if (scanStatus().running) return res.status(409).json({ error: 'Es läuft bereits ein Suchlauf.' });
|
||||
runScan('manual').catch((err) => console.error('[Scanner] Manueller Lauf:', err));
|
||||
res.status(202).json({ started: true });
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Einstellungen (nur Administratoren)
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.get('/settings', requireAdmin, async (req, res, next) => {
|
||||
try {
|
||||
res.json({ settings: await loadScannerSettings(), secrets: await maskedScannerSecrets() });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
router.patch('/settings', requireAdmin, async (req, res, next) => {
|
||||
try {
|
||||
const settings = await saveScannerSettings(req.body ?? {});
|
||||
const touchedKeys = ['adzunaId', 'adzunaKey', 'joobleKey'].filter((key) => req.body?.[key] !== undefined);
|
||||
await logHistory({
|
||||
entityType: 'system',
|
||||
entityLabel: 'Scanner',
|
||||
action: 'updated',
|
||||
// Nie die Schluessel selbst - nur, dass sie geaendert wurden.
|
||||
text: [
|
||||
`Zeitplan: ${settings.active ? 'an' : 'aus'}, alle ${settings.interval} min, ${settings.from}–${settings.to} Uhr`,
|
||||
touchedKeys.length ? 'API-Schlüssel geändert' : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(', '),
|
||||
userId: req.user.id,
|
||||
});
|
||||
res.json({ settings, secrets: await maskedScannerSecrets() });
|
||||
} catch (err) {
|
||||
if (err?.status === 400) return res.status(400).json({ error: err.message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Bewertung einer Anzeige gegen einen Suchagenten - ohne KI.
|
||||
*
|
||||
* Wunsch vom 29.09.2026: kein Abgleich mit Lebenslaeufen, sondern "je mehr
|
||||
* Schlagwoerter gefunden werden, desto besser der Treffer". Gezaehlt wird in
|
||||
* Titel und Text; ein Schlagwort im Titel zaehlt doppelt, weil der Titel die
|
||||
* Rolle benennt und der Text oft nur Umfeld beschreibt.
|
||||
*
|
||||
* score = Summe(Gewicht je gefundenem Schlagwort) / (2 * Anzahl Schlagwoerter)
|
||||
* Gewicht 2 = im Titel, 1 = nur im Text
|
||||
*/
|
||||
|
||||
const normalize = (text) => String(text ?? '').normalize('NFC').toLowerCase();
|
||||
|
||||
const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/**
|
||||
* Kurze Begriffe ("IT", "SAP", "HR") nur als ganzes Wort - sonst steckt "it" in
|
||||
* jedem "Mitarbeiter". Laengere als Teilwort, damit "Projektleiter" auch in
|
||||
* "IT-Projektleiterin" und "Projektleiterstelle" trifft.
|
||||
*/
|
||||
export function containsTerm(text, term) {
|
||||
const needle = normalize(term).trim();
|
||||
if (!needle) return false;
|
||||
const haystack = normalize(text);
|
||||
if (needle.length <= 3) {
|
||||
return new RegExp(`(?<![\\p{L}\\p{N}])${escape(needle)}(?![\\p{L}\\p{N}])`, 'u').test(haystack);
|
||||
}
|
||||
return haystack.includes(needle);
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {{ score: number, hits: string[] } | null} null = ausgeschlossen
|
||||
* oder kein einziges Schlagwort gefunden
|
||||
*/
|
||||
export function scoreJob(agent, job) {
|
||||
const title = job.title ?? '';
|
||||
const all = `${title}\n${job.company ?? ''}\n${job.description ?? ''}`;
|
||||
|
||||
if ((agent.excludes ?? []).some((term) => containsTerm(all, term))) return null;
|
||||
|
||||
const keywords = agent.keywords ?? [];
|
||||
if (keywords.length === 0) return null;
|
||||
|
||||
let points = 0;
|
||||
const hits = [];
|
||||
for (const keyword of keywords) {
|
||||
if (containsTerm(title, keyword)) {
|
||||
points += 2;
|
||||
hits.push(keyword);
|
||||
} else if (containsTerm(all, keyword)) {
|
||||
points += 1;
|
||||
hits.push(keyword);
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null;
|
||||
|
||||
return { score: Math.round((100 * points) / (2 * keywords.length)), hits };
|
||||
}
|
||||
|
||||
/*
|
||||
* Firmennamen der Boersen und des CRM auf einen Nenner bringen:
|
||||
* "Müller Holding GmbH & Co. KG" -> "müller holding".
|
||||
*/
|
||||
const LEGAL_FORMS =
|
||||
/\b(gmbh|mbh|ag|se|kg|kgaa|ohg|gbr|ug|e\.?\s?v|e\.?\s?k|co|inc|ltd|llc|plc|sa|sarl|bv|nv|haftungsbeschränkt|gesellschaft mit beschränkter haftung|aktiengesellschaft)\b\.?/g;
|
||||
|
||||
export function normalizeCompany(name) {
|
||||
return normalize(name)
|
||||
.replace(/\(.*?\)/g, ' ')
|
||||
.replace(/&/g, ' ')
|
||||
.replace(LEGAL_FORMS, ' ')
|
||||
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sucht das Unternehmen im CRM. Gleicher Name gewinnt; sonst darf der Name der
|
||||
* Anzeige mit dem CRM-Namen beginnen ("Siemens" -> "Siemens Mobility"), aber
|
||||
* nur ab fuenf Zeichen, sonst wird aus "ABC" jeder, der mit "ABC" anfaengt.
|
||||
*
|
||||
* @param {Array<{id:number, key:string}>} companies vorberechnet, laengste zuerst
|
||||
*/
|
||||
export function findCompany(companies, name) {
|
||||
const key = normalizeCompany(name);
|
||||
if (!key) return null;
|
||||
const exact = companies.find((company) => company.key === key);
|
||||
if (exact) return exact.id;
|
||||
const prefix = companies.find((company) => company.key.length >= 5 && key.startsWith(`${company.key} `));
|
||||
return prefix ? prefix.id : null;
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import { pool } from '../db.js';
|
||||
import { SOURCES, sourceReady } from './sources.js';
|
||||
import { findCompany, normalizeCompany, scoreJob } from './matching.js';
|
||||
import { loadScannerSecrets, loadScannerSettings, setLastScanAt } from './settings.js';
|
||||
|
||||
/*
|
||||
* Ein Suchlauf.
|
||||
*
|
||||
* 1. aktive Agenten laden
|
||||
* 2. je Boerse die Schlagwoerter aller Agenten zusammenfuehren - dasselbe
|
||||
* Wort wird nur einmal abgefragt, egal wie viele Agenten es tragen
|
||||
* 3. Anzeigen abrufen, alles ueber der Altersgrenze verwerfen
|
||||
* 4. Anzeigen speichern, Firma gegen das CRM abgleichen
|
||||
* 5. jede Anzeige gegen jeden Agenten bewerten, der diese Boerse durchsucht
|
||||
*
|
||||
* Zwei Aufrufer (Zeitplan und Knopf "Jetzt scannen") teilen sich die Sperre
|
||||
* unten. Lehre aus dem perinco-Scanner (26.08.2026): ohne sie stapeln sich die
|
||||
* Laeufe. Das Freigeben steht zwingend im finally, sonst bliebe die Sperre
|
||||
* nach einer Ausnahme fuer immer stehen.
|
||||
*/
|
||||
|
||||
let running = false;
|
||||
let runningSince = null;
|
||||
// Notventil: haengt ein Lauf, gibt der naechste Aufruf die Sperre frei.
|
||||
const MAX_RUNTIME_MS = 30 * 60 * 1000;
|
||||
// Pause zwischen zwei Abrufen derselben Boerse - freundlich bleiben.
|
||||
const REQUEST_GAP_MS = 1500;
|
||||
// Anzeigen, die so lange nicht mehr gesehen wurden, fliegen samt Treffern raus.
|
||||
const KEEP_DAYS = 60;
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export function scanStatus() {
|
||||
return { running, since: runningSince ? new Date(runningSince).toISOString() : null };
|
||||
}
|
||||
|
||||
export async function runScan(trigger = 'manual') {
|
||||
if (running && Date.now() - runningSince < MAX_RUNTIME_MS) {
|
||||
return { skipped: true, reason: 'Es läuft bereits ein Suchlauf.' };
|
||||
}
|
||||
running = true;
|
||||
runningSince = Date.now();
|
||||
try {
|
||||
return await execute(trigger);
|
||||
} finally {
|
||||
running = false;
|
||||
runningSince = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function execute(trigger) {
|
||||
const started = new Date();
|
||||
// Beginn des Laufs, nicht sein Ende: das Intervall ist ein Takt.
|
||||
await setLastScanAt(started);
|
||||
|
||||
const [runResult] = await pool.query('INSERT INTO icon_scanner_runs (trigger_kind) VALUES (?)', [trigger]);
|
||||
const runId = runResult.insertId;
|
||||
|
||||
const settings = await loadScannerSettings();
|
||||
const secrets = await loadScannerSecrets();
|
||||
const agents = await loadActiveAgents();
|
||||
|
||||
// Boerse -> Schlagwoerter (ohne Doppelte, Gross/klein egal)
|
||||
const plan = new Map();
|
||||
for (const agent of agents) {
|
||||
for (const source of agent.sources) {
|
||||
if (!SOURCES[source] || !sourceReady(source, secrets)) continue;
|
||||
const words = plan.get(source) ?? new Map();
|
||||
for (const keyword of agent.keywords) words.set(keyword.toLowerCase(), keyword);
|
||||
plan.set(source, words);
|
||||
}
|
||||
}
|
||||
|
||||
const perSource = {};
|
||||
const jobs = new Map();
|
||||
const oldest = Date.now() - settings.maxAge * 24 * 3_600_000;
|
||||
|
||||
for (const [source, words] of plan) {
|
||||
const stats = { ok: true, count: 0, errors: [] };
|
||||
let first = true;
|
||||
for (const keyword of words.values()) {
|
||||
if (!first) await sleep(REQUEST_GAP_MS);
|
||||
first = false;
|
||||
try {
|
||||
const found = await SOURCES[source].fetch(keyword, { maxAge: settings.maxAge, secrets });
|
||||
for (const job of found) {
|
||||
// Ohne Datum bleibt die Anzeige drin: die Boersen sortieren bzw.
|
||||
// filtern selbst nach Alter, das fehlende Feld ist ein Parserluecke.
|
||||
if (job.publishedAt && job.publishedAt.getTime() < oldest) continue;
|
||||
jobs.set(`${job.source}:${job.externalId}`, job);
|
||||
stats.count += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
stats.ok = false;
|
||||
stats.errors.push(`${keyword}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
// Teilweise Fehler sind kein Totalausfall - nur, wenn gar nichts kam.
|
||||
if (stats.count > 0) stats.ok = true;
|
||||
stats.errors = stats.errors.slice(0, 5);
|
||||
perSource[source] = stats;
|
||||
}
|
||||
|
||||
const companies = await loadCompanyKeys();
|
||||
let matchesNew = 0;
|
||||
|
||||
for (const job of jobs.values()) {
|
||||
const jobId = await saveJob(job, findCompany(companies, job.company));
|
||||
for (const agent of agents) {
|
||||
if (!agent.sources.includes(job.source)) continue;
|
||||
const result = scoreJob(agent, job);
|
||||
if (!result) continue;
|
||||
const [insert] = await pool.query(
|
||||
`INSERT IGNORE INTO icon_scanner_matches (agent_id, job_id, score, hits) VALUES (?, ?, ?, ?)`,
|
||||
[agent.id, jobId, result.score, JSON.stringify(result.hits)]
|
||||
);
|
||||
matchesNew += insert.affectedRows;
|
||||
}
|
||||
}
|
||||
|
||||
// Seit dem letzten Lauf angelegte Unternehmen auch bei aelteren Anzeigen erkennen.
|
||||
await relinkCompanies();
|
||||
await cleanup();
|
||||
|
||||
await pool.query(
|
||||
`UPDATE icon_scanner_runs
|
||||
SET finished_at = CURRENT_TIMESTAMP, jobs_found = ?, matches_new = ?, per_source = ?
|
||||
WHERE id = ?`,
|
||||
[jobs.size, matchesNew, JSON.stringify(perSource), runId]
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[Scanner] Lauf ${runId} (${trigger}): ${jobs.size} Anzeigen, ${matchesNew} neue Treffer, ` +
|
||||
Object.entries(perSource)
|
||||
.map(([source, s]) => `${source}=${s.ok ? s.count : 'Fehler'}`)
|
||||
.join(' ')
|
||||
);
|
||||
|
||||
return { skipped: false, runId, jobs: jobs.size, matchesNew, perSource };
|
||||
}
|
||||
|
||||
export function parseList(value) {
|
||||
try {
|
||||
const list = JSON.parse(value ?? '[]');
|
||||
return Array.isArray(list) ? list.map((item) => String(item).trim()).filter(Boolean) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function loadActiveAgents() {
|
||||
const [rows] = await pool.query('SELECT * FROM icon_scanner_agents WHERE is_active = 1');
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
keywords: parseList(row.keywords),
|
||||
sources: parseList(row.sources),
|
||||
excludes: parseList(row.excludes),
|
||||
}));
|
||||
}
|
||||
|
||||
async function loadCompanyKeys() {
|
||||
const [rows] = await pool.query('SELECT id, name FROM icon_companies');
|
||||
return rows
|
||||
.map((row) => ({ id: row.id, key: normalizeCompany(row.name) }))
|
||||
.filter((row) => row.key)
|
||||
.sort((a, b) => b.key.length - a.key.length);
|
||||
}
|
||||
|
||||
/** Speichert oder aktualisiert eine Anzeige und liefert ihre Kennung. */
|
||||
async function saveJob(job, companyId) {
|
||||
const cut = (value, max) => (value == null ? null : String(value).slice(0, max));
|
||||
// LAST_INSERT_ID(id) im Update-Zweig: so kommt auch bei einer vorhandenen
|
||||
// Anzeige deren Kennung als insertId zurueck - ohne zweite Abfrage.
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO icon_scanner_jobs
|
||||
(source, external_id, title, company, company_id, location, country,
|
||||
employment_type, url, description, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
title = VALUES(title),
|
||||
company = VALUES(company),
|
||||
company_id = VALUES(company_id),
|
||||
location = VALUES(location),
|
||||
employment_type = COALESCE(VALUES(employment_type), employment_type),
|
||||
description = COALESCE(VALUES(description), description),
|
||||
published_at = COALESCE(published_at, VALUES(published_at)),
|
||||
last_seen_at = CURRENT_TIMESTAMP`,
|
||||
[
|
||||
job.source,
|
||||
cut(job.externalId, 190),
|
||||
cut(job.title, 500),
|
||||
cut(job.company, 300),
|
||||
companyId,
|
||||
cut(job.location, 300),
|
||||
job.country,
|
||||
cut(job.employmentType, 60),
|
||||
cut(job.url, 1000),
|
||||
cut(job.description, 60_000),
|
||||
job.publishedAt,
|
||||
]
|
||||
);
|
||||
return result.insertId;
|
||||
}
|
||||
|
||||
/*
|
||||
* Alte Anzeigen raeumen. Was jemand "in Bearbeitung" hat, bleibt stehen.
|
||||
*/
|
||||
async function cleanup() {
|
||||
await pool.query(
|
||||
`DELETE j FROM icon_scanner_jobs j
|
||||
WHERE j.last_seen_at < NOW() - INTERVAL ? DAY
|
||||
AND NOT EXISTS (SELECT 1 FROM icon_scanner_matches m
|
||||
WHERE m.job_id = j.id AND m.status = 'in_bearbeitung')`,
|
||||
[KEEP_DAYS]
|
||||
);
|
||||
await pool.query('DELETE FROM icon_scanner_runs WHERE started_at < NOW() - INTERVAL 30 DAY');
|
||||
}
|
||||
|
||||
/**
|
||||
* Nach dem Anlegen oder Umbenennen von Unternehmen: vorhandene Anzeigen neu
|
||||
* zuordnen. Billig - reiner Namensvergleich.
|
||||
*/
|
||||
export async function relinkCompanies() {
|
||||
const companies = await loadCompanyKeys();
|
||||
const [jobs] = await pool.query('SELECT id, company, company_id FROM icon_scanner_jobs WHERE company IS NOT NULL');
|
||||
let changed = 0;
|
||||
for (const job of jobs) {
|
||||
const id = findCompany(companies, job.company);
|
||||
if (id === job.company_id) continue;
|
||||
await pool.query('UPDATE icon_scanner_jobs SET company_id = ? WHERE id = ?', [id, job.id]);
|
||||
changed += 1;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { runScan } from './pipeline.js';
|
||||
import { getLastScanAt, loadScannerSettings } from './settings.js';
|
||||
|
||||
/*
|
||||
* Zeitplan des Scanners - einstellbar unter Einstellungen -> Scanner.
|
||||
*
|
||||
* Ein Takt pro Minute prueft: Scanner an? Wochentag erlaubt? Uhrzeit im
|
||||
* Fenster? Intervall seit dem letzten Lauf um? Der letzte Lauf steht in der
|
||||
* Datenbank (scanner_last_scan_at) und wird auch vom Knopf "Jetzt scannen"
|
||||
* geschrieben - so zieht der Zeitplan nicht direkt hinter einem manuellen Lauf
|
||||
* noch einmal los, und ein Neustart des Dienstes verschiebt nichts.
|
||||
*
|
||||
* Gerechnet wird in deutscher Zeit, unabhaengig von der Zeitzone des Servers.
|
||||
*/
|
||||
|
||||
const TICK_MS = 60_000;
|
||||
// Ein Takt kommt nie auf die Sekunde - ohne Toleranz wuerde aus 60 Minuten 61.
|
||||
const SLACK_MS = 45_000;
|
||||
const WEEKDAYS = { Mon: 1, Tue: 2, Wed: 3, Thu: 4, Fri: 5, Sat: 6, Sun: 7 };
|
||||
|
||||
let timer = null;
|
||||
|
||||
export function berlinNow(date = new Date()) {
|
||||
const parts = Object.fromEntries(
|
||||
new Intl.DateTimeFormat('en-GB', {
|
||||
timeZone: 'Europe/Berlin',
|
||||
weekday: 'short',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hourCycle: 'h23',
|
||||
})
|
||||
.formatToParts(date)
|
||||
.map((part) => [part.type, part.value])
|
||||
);
|
||||
return { weekday: WEEKDAYS[parts.weekday], time: `${parts.hour}:${parts.minute}` };
|
||||
}
|
||||
|
||||
/** Liegt der Zeitpunkt in einem erlaubten Fenster? (Ende einschliesslich) */
|
||||
export function inWindow(settings, date = new Date()) {
|
||||
const { weekday, time } = berlinNow(date);
|
||||
return settings.days.includes(weekday) && time >= settings.from && time <= settings.to;
|
||||
}
|
||||
|
||||
async function tick() {
|
||||
try {
|
||||
const settings = await loadScannerSettings();
|
||||
if (!settings.active || !inWindow(settings)) return;
|
||||
|
||||
const last = await getLastScanAt();
|
||||
if (last && Date.now() - last.getTime() < settings.interval * 60_000 - SLACK_MS) return;
|
||||
|
||||
await runScan('auto');
|
||||
} catch (err) {
|
||||
console.error('[Scanner] Zeitplan:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export function startScheduler() {
|
||||
if (timer) return;
|
||||
if (process.env.SCANNER_DISABLED === 'true') {
|
||||
console.log('[Scanner] Zeitplan abgeschaltet (SCANNER_DISABLED).');
|
||||
return;
|
||||
}
|
||||
timer = setInterval(() => void tick(), TICK_MS);
|
||||
console.log('[Scanner] Zeitplan aktiv.');
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { getSetting, setSetting } from '../db.js';
|
||||
|
||||
/*
|
||||
* Einstellungen des Scanners (Einstellungen -> Scanner).
|
||||
*
|
||||
* Alles liegt in icon_settings unter dem Praefix `scanner_`. Die Vorgaben
|
||||
* entsprechen dem Wunsch vom 29.09.2026: stuendlich, Mo-Fr, 06-20 Uhr,
|
||||
* Anzeigen hoechstens 21 Tage alt.
|
||||
*/
|
||||
|
||||
export const DEFAULTS = {
|
||||
active: true,
|
||||
// ISO-Wochentage, 1 = Montag ... 7 = Sonntag
|
||||
days: [1, 2, 3, 4, 5],
|
||||
from: '06:00',
|
||||
to: '20:00',
|
||||
interval: 60,
|
||||
maxAge: 21,
|
||||
};
|
||||
|
||||
/** Zugangsdaten der kostenlosen APIs - gehen nie im Klartext zum Browser. */
|
||||
const SECRET_KEYS = {
|
||||
adzunaId: 'scanner_adzuna_id',
|
||||
adzunaKey: 'scanner_adzuna_key',
|
||||
joobleKey: 'scanner_jooble_key',
|
||||
};
|
||||
|
||||
const TIME = /^([01]\d|2[0-3]):[0-5]\d$/;
|
||||
|
||||
export async function loadScannerSettings() {
|
||||
const [active, days, from, to, interval, maxAge] = await Promise.all([
|
||||
getSetting('scanner_active'),
|
||||
getSetting('scanner_days'),
|
||||
getSetting('scanner_from'),
|
||||
getSetting('scanner_to'),
|
||||
getSetting('scanner_interval'),
|
||||
getSetting('scanner_max_age'),
|
||||
]);
|
||||
|
||||
const parsedDays = (days ?? '')
|
||||
.split(',')
|
||||
.map(Number)
|
||||
.filter((d) => d >= 1 && d <= 7);
|
||||
|
||||
return {
|
||||
active: active === null ? DEFAULTS.active : active === 'true',
|
||||
days: days === null ? DEFAULTS.days : parsedDays,
|
||||
from: TIME.test(from ?? '') ? from : DEFAULTS.from,
|
||||
to: TIME.test(to ?? '') ? to : DEFAULTS.to,
|
||||
interval: clampInt(interval, 15, 24 * 60, DEFAULTS.interval),
|
||||
maxAge: clampInt(maxAge, 1, 90, DEFAULTS.maxAge),
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveScannerSettings(input) {
|
||||
const current = await loadScannerSettings();
|
||||
const next = { ...current };
|
||||
|
||||
if (typeof input.active === 'boolean') next.active = input.active;
|
||||
if (Array.isArray(input.days)) {
|
||||
next.days = [...new Set(input.days.map(Number).filter((d) => d >= 1 && d <= 7))].sort();
|
||||
}
|
||||
if (input.from !== undefined) {
|
||||
if (!TIME.test(String(input.from))) throw badRequest('Startzeit bitte als HH:MM.');
|
||||
next.from = String(input.from);
|
||||
}
|
||||
if (input.to !== undefined) {
|
||||
if (!TIME.test(String(input.to))) throw badRequest('Endzeit bitte als HH:MM.');
|
||||
next.to = String(input.to);
|
||||
}
|
||||
if (next.from >= next.to) throw badRequest('Die Startzeit muss vor der Endzeit liegen.');
|
||||
if (input.interval !== undefined) next.interval = clampInt(input.interval, 15, 24 * 60, current.interval);
|
||||
if (input.maxAge !== undefined) next.maxAge = clampInt(input.maxAge, 1, 90, current.maxAge);
|
||||
|
||||
await Promise.all([
|
||||
setSetting('scanner_active', String(next.active)),
|
||||
setSetting('scanner_days', next.days.join(',')),
|
||||
setSetting('scanner_from', next.from),
|
||||
setSetting('scanner_to', next.to),
|
||||
setSetting('scanner_interval', String(next.interval)),
|
||||
setSetting('scanner_max_age', String(next.maxAge)),
|
||||
]);
|
||||
|
||||
// Schluessel: undefined = unveraendert, '' = entfernen
|
||||
for (const [field, key] of Object.entries(SECRET_KEYS)) {
|
||||
if (input[field] === undefined) continue;
|
||||
await setSetting(key, String(input[field] ?? '').trim());
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function loadScannerSecrets() {
|
||||
const entries = await Promise.all(
|
||||
Object.entries(SECRET_KEYS).map(async ([field, key]) => [field, (await getSetting(key)) || ''])
|
||||
);
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
|
||||
/** Fuer die Oberflaeche: nur, ob etwas hinterlegt ist, plus verkuerzte Form. */
|
||||
export async function maskedScannerSecrets() {
|
||||
const secrets = await loadScannerSecrets();
|
||||
return Object.fromEntries(
|
||||
Object.entries(secrets).map(([field, value]) => [
|
||||
field,
|
||||
value ? `${value.slice(0, 3)}…${value.slice(-3)}` : null,
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
export async function getLastScanAt() {
|
||||
const value = await getSetting('scanner_last_scan_at');
|
||||
const date = value ? new Date(value) : null;
|
||||
return date && !Number.isNaN(date.getTime()) ? date : null;
|
||||
}
|
||||
|
||||
export async function setLastScanAt(date) {
|
||||
await setSetting('scanner_last_scan_at', date.toISOString());
|
||||
}
|
||||
|
||||
function clampInt(value, min, max, fallback) {
|
||||
const n = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(n)) return fallback;
|
||||
return Math.min(max, Math.max(min, n));
|
||||
}
|
||||
|
||||
function badRequest(message) {
|
||||
const err = new Error(message);
|
||||
err.status = 400;
|
||||
return err;
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
/*
|
||||
* Die Jobbörsen des Scanners.
|
||||
*
|
||||
* Jede Quelle liefert fuer ein Schlagwort eine Liste einheitlicher Anzeigen:
|
||||
*
|
||||
* { source, externalId, title, company, location, country,
|
||||
* employmentType, url, description, publishedAt: Date|null }
|
||||
*
|
||||
* Bewusst nur kostenlose Wege (Entscheidung 29.09.2026): LinkedIn und Indeed
|
||||
* sind ohne kostenpflichtigen Dienst nicht sauber abrufbar, Monster.de antwortet
|
||||
* mit CAPTCHA. Kein Chromium - die Seiten hier liefern ihre Liste als HTML bzw.
|
||||
* JSON aus, ein einfacher Abruf reicht und belastet den Server kaum.
|
||||
*
|
||||
* Eine Quelle, die scheitert, wirft. Die Pipeline faengt das je Quelle ab, der
|
||||
* Lauf geht mit den anderen weiter.
|
||||
*/
|
||||
|
||||
const UA =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36';
|
||||
const TIMEOUT_MS = 20_000;
|
||||
|
||||
export const SOURCES = {
|
||||
ba: { label: 'Agentur für Arbeit', countries: 'DE', needsKey: null, fetch: fetchBa },
|
||||
stepstone: { label: 'StepStone', countries: 'DE', needsKey: null, fetch: fetchStepstone },
|
||||
karriere: { label: 'karriere.at', countries: 'AT', needsKey: null, fetch: fetchKarriere },
|
||||
adzuna: { label: 'Adzuna', countries: 'DE · AT · CH', needsKey: 'adzuna', fetch: fetchAdzuna },
|
||||
jooble: { label: 'Jooble', countries: 'DE · AT · CH', needsKey: 'jooble', fetch: fetchJooble },
|
||||
freelancermap: { label: 'freelancermap', countries: 'DE', needsKey: null, fetch: fetchFreelancermap },
|
||||
};
|
||||
|
||||
export const SOURCE_IDS = Object.keys(SOURCES);
|
||||
|
||||
/** Kann die Quelle mit den hinterlegten Schluesseln ueberhaupt laufen? */
|
||||
export function sourceReady(id, secrets) {
|
||||
const needs = SOURCES[id]?.needsKey;
|
||||
if (!needs) return true;
|
||||
if (needs === 'adzuna') return Boolean(secrets.adzunaId && secrets.adzunaKey);
|
||||
if (needs === 'jooble') return Boolean(secrets.joobleKey);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------ */
|
||||
|
||||
async function request(url, { json = false, headers = {}, method = 'GET', body } = {}) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
body,
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Accept-Language': 'de-DE,de;q=0.9',
|
||||
Accept: json ? 'application/json' : 'text/html,application/xhtml+xml',
|
||||
...headers,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
return json ? await response.json() : await response.text();
|
||||
} catch (err) {
|
||||
if (err?.name === 'AbortError') throw new Error('Zeitüberschreitung');
|
||||
throw err;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
const clean = (value) =>
|
||||
String(value ?? '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
const absolute = (href, base) => {
|
||||
if (!href) return null;
|
||||
try {
|
||||
return new URL(href, base).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const toDate = (value) => {
|
||||
if (!value) return null;
|
||||
const date = new Date(value);
|
||||
return Number.isNaN(date.getTime()) ? null : date;
|
||||
};
|
||||
|
||||
/**
|
||||
* "vor 5 Tagen", "vor 3 Stunden", "Gestern", "Heute", "vor 2 Wochen",
|
||||
* "28.09.2026" - die Listen zeigen das Alter meist nur so an.
|
||||
*/
|
||||
export function parseRelativeDate(text, now = new Date()) {
|
||||
const value = clean(text).toLowerCase();
|
||||
if (!value) return null;
|
||||
|
||||
const dotted = value.match(/(\d{1,2})\.(\d{1,2})\.(\d{2,4})/);
|
||||
if (dotted) {
|
||||
const year = dotted[3].length === 2 ? 2000 + Number(dotted[3]) : Number(dotted[3]);
|
||||
return new Date(year, Number(dotted[2]) - 1, Number(dotted[1]), 12);
|
||||
}
|
||||
|
||||
const shift = (ms) => new Date(now.getTime() - ms);
|
||||
const HOUR = 3_600_000;
|
||||
const DAY = 24 * HOUR;
|
||||
|
||||
if (/heute|gerade|neu|minute/.test(value)) return now;
|
||||
if (/gestern/.test(value)) return shift(DAY);
|
||||
const number = Number(value.match(/(\d+)/)?.[1] ?? 1);
|
||||
if (/stunde/.test(value)) return shift(number * HOUR);
|
||||
if (/tag/.test(value)) return shift(number * DAY);
|
||||
if (/woche/.test(value)) return shift(number * 7 * DAY);
|
||||
if (/monat/.test(value)) return shift(number * 30 * DAY);
|
||||
return null;
|
||||
}
|
||||
|
||||
const slug = (keyword) =>
|
||||
clean(keyword)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9äöüß]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Bundesagentur fuer Arbeit - offizielle Jobsuche-API (JSON, ohne Anmeldung;
|
||||
* der Schluessel "jobboerse-jobsuche" ist der oeffentliche der Jobsuche-App).
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchBa(keyword, { maxAge }) {
|
||||
const url =
|
||||
'https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v6/jobs?' +
|
||||
new URLSearchParams({ was: keyword, veroeffentlichtseit: String(maxAge), size: '100', page: '1' });
|
||||
const data = await request(url, { json: true, headers: { 'X-API-Key': 'jobboerse-jobsuche' } });
|
||||
const list = data?.ergebnisliste ?? data?.stellenangebote ?? [];
|
||||
|
||||
return list
|
||||
.filter((job) => !/AUSBILDUNG|PRAKTIKUM/i.test(job.stellenangebotsart ?? job.angebotsart ?? ''))
|
||||
.map((job) => {
|
||||
const refnr = job.refnr ?? job.referenznummer ?? job.hashId;
|
||||
const place = job.arbeitsort ?? job.arbeitsorte?.[0] ?? {};
|
||||
const land = clean(place.land);
|
||||
return {
|
||||
source: 'ba',
|
||||
externalId: String(refnr ?? job.stellenangebotsTitel ?? job.titel),
|
||||
title: clean(job.stellenangebotsTitel ?? job.titel ?? job.beruf),
|
||||
company: clean(job.arbeitgeber ?? job.arbeitgeberName) || null,
|
||||
location: clean([place.plz, place.ort].filter(Boolean).join(' ')) || clean(place.region) || null,
|
||||
// Die BA fuehrt auch Stellen ausserhalb DACH - die fallen unten raus.
|
||||
country: !land || /deutschland/i.test(land) ? 'DE' : /österreich/i.test(land) ? 'AT' : /schweiz/i.test(land) ? 'CH' : null,
|
||||
employmentType: /SELBST/i.test(job.stellenangebotsart ?? '') ? 'Freiberuflich' : 'Anstellung',
|
||||
url: job.externeUrl || `https://www.arbeitsagentur.de/jobsuche/jobdetail/${encodeURIComponent(refnr)}`,
|
||||
description: clean(job.beruf) || null,
|
||||
publishedAt: toDate(job.aktuelleVeroeffentlichungsdatum ?? job.veroeffentlichungsdatum ?? job.modifikationsTimestamp),
|
||||
};
|
||||
})
|
||||
.filter((job) => job.title && job.externalId && job.country);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* StepStone.de - Trefferliste als HTML, jede Karte traegt data-at-Attribute.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchStepstone(keyword) {
|
||||
const base = 'https://www.stepstone.de';
|
||||
const html = await request(`${base}/jobs/${encodeURIComponent(slug(keyword))}?action=sort_publish`);
|
||||
const $ = cheerio.load(html);
|
||||
const jobs = [];
|
||||
|
||||
$('[data-at="job-item-title"]').each((_, element) => {
|
||||
const card = $(element).closest('article').length ? $(element).closest('article') : $(element).parent().parent();
|
||||
const link =
|
||||
$(element).attr('href') ||
|
||||
$(element).closest('a').attr('href') ||
|
||||
card.find('a[href*="stellenangebote--"]').first().attr('href');
|
||||
const url = absolute(link, base);
|
||||
if (!url) return;
|
||||
|
||||
const id = url.match(/--(\d+)(?:-inline)?\.html/)?.[1] ?? url.split('?')[0];
|
||||
jobs.push({
|
||||
source: 'stepstone',
|
||||
externalId: id,
|
||||
title: clean($(element).text()),
|
||||
company: clean(card.find('[data-at="job-item-company-name"]').first().text()) || null,
|
||||
location: clean(card.find('[data-at="job-item-location"]').first().text()) || null,
|
||||
country: 'DE',
|
||||
employmentType: clean(card.find('[data-at="job-item-work-type"], [data-at="job-item-contract-type"]').first().text()) || null,
|
||||
url: url.split('?')[0],
|
||||
description: clean(card.find('[data-at="jobcard-content"], [data-at="job-item-middle"]').first().text()) || null,
|
||||
publishedAt: parseRelativeDate(card.find('time').first().text()) ?? toDate(card.find('time').first().attr('datetime')),
|
||||
});
|
||||
});
|
||||
|
||||
return jobs.filter((job) => job.title);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* karriere.at - groesste Jobboerse in Oesterreich, Liste als HTML.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchKarriere(keyword) {
|
||||
const base = 'https://www.karriere.at';
|
||||
const html = await request(`${base}/jobs/${encodeURIComponent(slug(keyword))}`);
|
||||
const $ = cheerio.load(html);
|
||||
const jobs = [];
|
||||
|
||||
$('.m-jobsListItem').each((_, element) => {
|
||||
const card = $(element);
|
||||
const link = card.find('.m-jobsListItem__titleLink').first();
|
||||
const url = absolute(link.attr('href'), base);
|
||||
if (!url) return;
|
||||
|
||||
jobs.push({
|
||||
source: 'karriere',
|
||||
externalId: url.match(/\/jobs\/(\d+)/)?.[1] ?? url.split('?')[0],
|
||||
title: clean(link.text()),
|
||||
company: clean(card.find('.m-jobsListItem__company').first().text()) || null,
|
||||
location: clean(card.find('.m-jobsListItem__locations').first().text()) || null,
|
||||
country: 'AT',
|
||||
employmentType: clean(card.find('.m-jobsListItem__pill').first().text()) || null,
|
||||
url: url.split('?')[0],
|
||||
description: clean(card.find('.m-jobsListItem__snippet').first().text()) || null,
|
||||
publishedAt: parseRelativeDate(card.find('.m-jobsListItem__date').first().text()),
|
||||
});
|
||||
});
|
||||
|
||||
return jobs.filter((job) => job.title);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Adzuna - kostenloser API-Schluessel (developer.adzuna.com), sammelt Anzeigen
|
||||
* vieler Boersen fuer DE, AT und CH. Die einzige Quelle fuer die Schweiz.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchAdzuna(keyword, { maxAge, secrets }) {
|
||||
const jobs = [];
|
||||
for (const country of ['de', 'at', 'ch']) {
|
||||
const url =
|
||||
`https://api.adzuna.com/v1/api/jobs/${country}/search/1?` +
|
||||
new URLSearchParams({
|
||||
app_id: secrets.adzunaId,
|
||||
app_key: secrets.adzunaKey,
|
||||
what_phrase: keyword,
|
||||
max_days_old: String(maxAge),
|
||||
results_per_page: '50',
|
||||
sort_by: 'date',
|
||||
'content-type': 'application/json',
|
||||
});
|
||||
const data = await request(url, { json: true });
|
||||
for (const job of data?.results ?? []) {
|
||||
jobs.push({
|
||||
source: 'adzuna',
|
||||
externalId: String(job.id),
|
||||
title: clean(job.title).replace(/<\/?strong>/g, ''),
|
||||
company: clean(job.company?.display_name) || null,
|
||||
location: clean(job.location?.display_name) || null,
|
||||
country: country.toUpperCase(),
|
||||
employmentType:
|
||||
job.contract_type === 'permanent'
|
||||
? 'Festanstellung'
|
||||
: job.contract_type === 'contract'
|
||||
? 'Befristet / Projekt'
|
||||
: null,
|
||||
url: job.redirect_url,
|
||||
description: clean(job.description).replace(/<\/?strong>/g, '') || null,
|
||||
publishedAt: toDate(job.created),
|
||||
});
|
||||
}
|
||||
}
|
||||
return jobs.filter((job) => job.title && job.url);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Jooble - kostenloser API-Schluessel (jooble.org/api/about), Sammler fuer DACH.
|
||||
* Je Land ein eigener Host.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchJooble(keyword, { maxAge, secrets }) {
|
||||
const since = new Date(Date.now() - maxAge * 24 * 3_600_000).toISOString().slice(0, 10);
|
||||
const jobs = [];
|
||||
for (const country of ['de', 'at', 'ch']) {
|
||||
const data = await request(`https://${country}.jooble.org/api/${encodeURIComponent(secrets.joobleKey)}`, {
|
||||
json: true,
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ keywords: keyword, datecreatedfrom: since, page: '1' }),
|
||||
});
|
||||
for (const job of data?.jobs ?? []) {
|
||||
jobs.push({
|
||||
source: 'jooble',
|
||||
externalId: String(job.id ?? job.link),
|
||||
title: clean(job.title).replace(/<[^>]+>/g, ''),
|
||||
company: clean(job.company) || null,
|
||||
location: clean(job.location) || null,
|
||||
country: country.toUpperCase(),
|
||||
employmentType: clean(job.type) || null,
|
||||
url: job.link,
|
||||
description: clean(job.snippet).replace(/<[^>]+>/g, '').replace(/ /g, ' ') || null,
|
||||
publishedAt: toDate(job.updated),
|
||||
});
|
||||
}
|
||||
}
|
||||
return jobs.filter((job) => job.title && job.url);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* freelancermap - Projekte fuer Freiberufler und Interim, neueste zuerst.
|
||||
* Selektoren wie im Scanner des perinco CRM.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchFreelancermap(keyword) {
|
||||
const base = 'https://www.freelancermap.de';
|
||||
const html = await request(`${base}/projekte?q=${encodeURIComponent(keyword)}&sort=created_desc`);
|
||||
const $ = cheerio.load(html);
|
||||
const jobs = [];
|
||||
|
||||
$('.project-card').each((_, element) => {
|
||||
const card = $(element);
|
||||
const link = card.find('a[data-testid="title"], a[data-id="project-card-title"]').first();
|
||||
const url = absolute(link.attr('href'), base);
|
||||
if (!url) return;
|
||||
|
||||
const slugPart = url.match(/\/projekt\/([^/?#]+)/)?.[1];
|
||||
const meta = ['type', 'duration', 'beginningText', 'remoteInPercent']
|
||||
.map((id) => clean(card.find(`[data-testid="${id}"]`).first().text()))
|
||||
.filter(Boolean)
|
||||
.join(' | ');
|
||||
|
||||
jobs.push({
|
||||
source: 'freelancermap',
|
||||
externalId: slugPart?.match(/-(\d+)$/)?.[1] ?? slugPart ?? url,
|
||||
title: clean(link.text()),
|
||||
company: clean(card.find('.project-info div').first().text()) || null,
|
||||
location: clean(card.find('[data-testid="city"]').first().text()) || null,
|
||||
country: 'DE',
|
||||
employmentType: 'Freiberuflich',
|
||||
url: url.split('?')[0],
|
||||
description: meta || null,
|
||||
publishedAt:
|
||||
toDate(card.find('time').first().attr('datetime')) ??
|
||||
parseRelativeDate(card.find('[data-testid="created"], time').first().text()),
|
||||
});
|
||||
});
|
||||
|
||||
return jobs.filter((job) => job.title);
|
||||
}
|
||||
@@ -419,3 +419,83 @@ CREATE TABLE IF NOT EXISTS icon_settings (
|
||||
value MEDIUMTEXT NULL,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Scanner: Stellenanzeigen aus Jobbörsen (DACH, höchstens 21 Tage alt)
|
||||
--
|
||||
-- Ein Suchagent ist eine Liste von Schlagwörtern plus der Börsen, in denen
|
||||
-- gesucht wird, plus Ausschlussbegriffen. Jede Anzeige liegt genau einmal in
|
||||
-- icon_scanner_jobs (eindeutig je Börse und deren Kennung); welcher Agent sie
|
||||
-- mit welcher Güte gefunden hat, steht in icon_scanner_matches.
|
||||
--
|
||||
-- Bewertet wird nicht per KI, sondern per Zählung: je mehr Schlagwörter eines
|
||||
-- Agenten in Titel und Text vorkommen, desto höher der Treffer.
|
||||
-- ---------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS icon_scanner_agents (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(190) NOT NULL,
|
||||
-- JSON-Arrays: ["SAP FI", "Interim CFO"], ["ba", "stepstone"], ["Praktikum"]
|
||||
keywords TEXT NOT NULL,
|
||||
sources TEXT NOT NULL,
|
||||
excludes TEXT NULL,
|
||||
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
||||
created_by INT UNSIGNED NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
CONSTRAINT fk_icon_scanner_agents_creator FOREIGN KEY (created_by)
|
||||
REFERENCES icon_users (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS icon_scanner_jobs (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
source VARCHAR(30) NOT NULL,
|
||||
external_id VARCHAR(190) NOT NULL,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
company VARCHAR(300) NULL,
|
||||
-- Treffer im CRM: Unternehmen mit gleichem (normalisiertem) Namen.
|
||||
company_id INT UNSIGNED NULL,
|
||||
location VARCHAR(300) NULL,
|
||||
country CHAR(2) NULL,
|
||||
employment_type VARCHAR(60) NULL,
|
||||
url VARCHAR(1000) NOT NULL,
|
||||
description MEDIUMTEXT NULL,
|
||||
published_at DATETIME NULL,
|
||||
first_seen_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uq_icon_scanner_jobs (source, external_id),
|
||||
KEY idx_icon_scanner_jobs_published (published_at),
|
||||
KEY idx_icon_scanner_jobs_company (company_id),
|
||||
CONSTRAINT fk_icon_scanner_jobs_company FOREIGN KEY (company_id)
|
||||
REFERENCES icon_companies (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS icon_scanner_matches (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
agent_id INT UNSIGNED NOT NULL,
|
||||
job_id INT UNSIGNED NOT NULL,
|
||||
score TINYINT UNSIGNED NOT NULL DEFAULT 0,
|
||||
-- JSON-Array der gefundenen Schlagwörter
|
||||
hits TEXT NULL,
|
||||
status ENUM('neu','gesehen','in_bearbeitung','verworfen') NOT NULL DEFAULT 'neu',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uq_icon_scanner_matches (agent_id, job_id),
|
||||
KEY idx_icon_scanner_matches_created (created_at),
|
||||
CONSTRAINT fk_icon_scanner_matches_agent FOREIGN KEY (agent_id)
|
||||
REFERENCES icon_scanner_agents (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_icon_scanner_matches_job FOREIGN KEY (job_id)
|
||||
REFERENCES icon_scanner_jobs (id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Protokoll der Läufe: wann, wodurch ausgelöst, was je Börse herauskam.
|
||||
CREATE TABLE IF NOT EXISTS icon_scanner_runs (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
trigger_kind ENUM('auto','manual') NOT NULL,
|
||||
started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
finished_at TIMESTAMP NULL,
|
||||
jobs_found INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
matches_new INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
-- JSON: { ba: { ok: true, count: 12 }, stepstone: { ok: false, error: "..." } }
|
||||
per_source TEXT NULL,
|
||||
KEY idx_icon_scanner_runs_started (started_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
@@ -21,6 +21,8 @@ import searchRoutes from './routes/search.js';
|
||||
import networkRoutes from './routes/network.js';
|
||||
import connectionRoutes from './routes/connections.js';
|
||||
import dashboardRoutes from './routes/dashboard.js';
|
||||
import scannerRoutes from './routes/scanner.js';
|
||||
import { startScheduler } from './scanner/scheduler.js';
|
||||
|
||||
const app = express();
|
||||
const PORT = Number(process.env.PORT || 5007);
|
||||
@@ -71,6 +73,8 @@ app.use('/api/search', searchRoutes);
|
||||
app.use('/api/network', networkRoutes);
|
||||
app.use('/api/connections', connectionRoutes);
|
||||
app.use('/api/dashboard', dashboardRoutes);
|
||||
// Stellenanzeigen aus Jobbörsen - Suchagenten, Treffer, Zeitplan.
|
||||
app.use('/api/scanner', scannerRoutes);
|
||||
|
||||
app.use('/api', (req, res) => res.status(404).json({ error: 'Endpunkt nicht gefunden.' }));
|
||||
|
||||
@@ -109,6 +113,7 @@ try {
|
||||
app.listen(PORT, HOST, () => {
|
||||
console.log(`[ICON CRM] Backend läuft auf http://${HOST}:${PORT}`);
|
||||
});
|
||||
startScheduler();
|
||||
} catch (err) {
|
||||
console.error('[ICON CRM] Start fehlgeschlagen:', err);
|
||||
process.exit(1);
|
||||
|
||||
Reference in New Issue
Block a user