Scanner: ohne Altersgrenze suchen, Filter Anzeigenalter, Alarm-Schlagwörter
- Suche ohne 21-Tage-Grenze (BA ohne veroeffentlichtseit, Adzuna ohne max_days_old); Einstellung Höchstalter entfällt - Dashboard: Filter Anzeigenalter (bis/älter als 21 Tage), Sortierung älteste Anzeige, Filter nur Alarm, Zeitraum heißt jetzt Gefunden - Suchagent: Schlagwörter mit Alarm; Treffer mit alarm = 1 melden sich über den AlarmWatcher (App-weit, jede Minute) mit Ton und Pop-up, Quittieren gilt für alle Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
schakal00783
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
b34a1968df
commit
1ad774fdd2
@@ -185,6 +185,19 @@ async function migrateColumns() {
|
||||
'email',
|
||||
'ALTER TABLE icon_suspects ADD COLUMN email VARCHAR(190) NULL AFTER company',
|
||||
],
|
||||
// Scanner: Schlagwoerter mit Alarm (Ton + Pop-up beim Fund), 29.09.2026.
|
||||
[
|
||||
'icon_scanner_agents',
|
||||
'alarm_keywords',
|
||||
'ALTER TABLE icon_scanner_agents ADD COLUMN alarm_keywords TEXT NULL AFTER excludes',
|
||||
],
|
||||
[
|
||||
'icon_scanner_matches',
|
||||
'alarm',
|
||||
`ALTER TABLE icon_scanner_matches
|
||||
ADD COLUMN alarm TINYINT(1) NOT NULL DEFAULT 0 AFTER status,
|
||||
ADD COLUMN alarm_ack TINYINT(1) NOT NULL DEFAULT 0 AFTER alarm`,
|
||||
],
|
||||
[
|
||||
'icon_cv_imports',
|
||||
'photo',
|
||||
|
||||
@@ -24,6 +24,8 @@ const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const STATUSES = ['neu', 'gesehen', 'in_bearbeitung', 'verworfen'];
|
||||
/** Grenze des Filters "Anzeigenalter" im Dashboard. */
|
||||
const AGE_DAYS = 21;
|
||||
|
||||
const cleanList = (value) =>
|
||||
[...new Set((Array.isArray(value) ? value : []).map(stripQuotes).filter(Boolean))].slice(0, 50);
|
||||
@@ -35,6 +37,7 @@ const agentOut = (row) => ({
|
||||
// Nur Quellen, die es noch gibt (Jooble ist am 29.09.2026 entfallen).
|
||||
sources: parseList(row.sources).filter((id) => SOURCE_IDS.includes(id)),
|
||||
excludes: parseList(row.excludes),
|
||||
alarm_keywords: parseList(row.alarm_keywords),
|
||||
is_active: Boolean(row.is_active),
|
||||
matches: Number(row.matches ?? 0),
|
||||
created_at: row.created_at,
|
||||
@@ -118,27 +121,33 @@ function readAgent(body) {
|
||||
const keywords = cleanList(body?.keywords);
|
||||
const sources = cleanList(body?.sources).filter((id) => SOURCE_IDS.includes(id));
|
||||
const excludes = cleanList(body?.excludes);
|
||||
const alarmKeywords = cleanList(body?.alarm_keywords);
|
||||
if (!name) return { error: 'Bitte einen Namen angeben.' };
|
||||
if (keywords.length === 0) return { error: 'Mindestens ein Schlagwort angeben.' };
|
||||
if (keywords.length === 0 && alarmKeywords.length === 0) {
|
||||
return { error: 'Mindestens ein Schlagwort oder Alarm-Schlagwort angeben.' };
|
||||
}
|
||||
if (sources.length === 0) return { error: 'Mindestens eine Börse auswählen.' };
|
||||
return { name, keywords, sources, excludes, is_active: body?.is_active !== false };
|
||||
return { name, keywords, alarmKeywords, sources, excludes, is_active: body?.is_active !== false };
|
||||
}
|
||||
|
||||
/** Spaltenwerte in der Reihenfolge name, keywords, alarm_keywords, sources, excludes, is_active. */
|
||||
const agentValues = (agent) => [
|
||||
agent.name,
|
||||
JSON.stringify(agent.keywords),
|
||||
JSON.stringify(agent.alarmKeywords),
|
||||
JSON.stringify(agent.sources),
|
||||
JSON.stringify(agent.excludes),
|
||||
agent.is_active ? 1 : 0,
|
||||
];
|
||||
|
||||
router.post('/agents', async (req, res, next) => {
|
||||
const agent = readAgent(req.body);
|
||||
if (agent.error) return res.status(400).json({ error: agent.error });
|
||||
try {
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO icon_scanner_agents (name, keywords, sources, excludes, is_active, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
agent.name,
|
||||
JSON.stringify(agent.keywords),
|
||||
JSON.stringify(agent.sources),
|
||||
JSON.stringify(agent.excludes),
|
||||
agent.is_active ? 1 : 0,
|
||||
req.user.id,
|
||||
]
|
||||
`INSERT INTO icon_scanner_agents (name, keywords, alarm_keywords, sources, excludes, is_active, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[...agentValues(agent), req.user.id]
|
||||
);
|
||||
await logHistory({
|
||||
entityType: 'system',
|
||||
@@ -160,16 +169,9 @@ router.put('/agents/:id', async (req, res, next) => {
|
||||
try {
|
||||
const [result] = await pool.query(
|
||||
`UPDATE icon_scanner_agents
|
||||
SET name = ?, keywords = ?, sources = ?, excludes = ?, is_active = ?
|
||||
SET name = ?, keywords = ?, alarm_keywords = ?, sources = ?, excludes = ?, is_active = ?
|
||||
WHERE id = ?`,
|
||||
[
|
||||
agent.name,
|
||||
JSON.stringify(agent.keywords),
|
||||
JSON.stringify(agent.sources),
|
||||
JSON.stringify(agent.excludes),
|
||||
agent.is_active ? 1 : 0,
|
||||
Number(req.params.id),
|
||||
]
|
||||
[...agentValues(agent), Number(req.params.id)]
|
||||
);
|
||||
if (result.affectedRows === 0) return res.status(404).json({ error: 'Suchagent nicht gefunden.' });
|
||||
const [[row]] = await pool.query('SELECT * FROM icon_scanner_agents WHERE id = ?', [Number(req.params.id)]);
|
||||
@@ -236,21 +238,37 @@ router.get('/matches', async (req, res, next) => {
|
||||
where.push('(j.title LIKE ? OR j.company LIKE ? OR j.location LIKE ?)');
|
||||
params.push(`%${q}%`, `%${q}%`, `%${q}%`);
|
||||
}
|
||||
/*
|
||||
* Alter der Anzeige (nicht des Fundes). Gesucht wird seit 29.09.2026 ohne
|
||||
* Altersgrenze; die 21 Tage sind nur noch Filter. Anzeigen ohne Datum
|
||||
* (manche Boersen nennen keins) fallen bei beiden Filtern heraus.
|
||||
*/
|
||||
if (req.query.age === 'recent' || req.query.age === 'old') {
|
||||
where.push(
|
||||
req.query.age === 'recent'
|
||||
? 'j.published_at >= NOW() - INTERVAL ? DAY'
|
||||
: 'j.published_at < NOW() - INTERVAL ? DAY'
|
||||
);
|
||||
params.push(AGE_DAYS);
|
||||
}
|
||||
if (req.query.alarm === '1') where.push('m.alarm = 1');
|
||||
|
||||
const order =
|
||||
req.query.sort === 'score'
|
||||
? 'm.score DESC, m.created_at DESC'
|
||||
: req.query.sort === 'published'
|
||||
? 'j.published_at IS NULL, j.published_at DESC'
|
||||
: // bekannte Firmen zuerst, dann neueste
|
||||
'known DESC, m.created_at DESC';
|
||||
: req.query.sort === 'oldest'
|
||||
? 'j.published_at IS NULL, j.published_at ASC'
|
||||
: // bekannte Firmen zuerst, dann neueste
|
||||
'known DESC, m.created_at DESC';
|
||||
|
||||
const limit = Math.min(Math.max(Number(req.query.limit) || 300, 1), 1000);
|
||||
const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT m.id, m.score, m.hits, m.status, m.created_at,
|
||||
`SELECT m.id, m.score, m.hits, m.status, m.alarm, m.created_at,
|
||||
a.id AS agent_id, a.name AS agent_name,
|
||||
j.id AS job_id, j.source, j.title, j.company, j.location, j.country,
|
||||
j.employment_type, j.url, j.description, j.published_at,
|
||||
@@ -276,6 +294,7 @@ router.get('/matches', async (req, res, next) => {
|
||||
matches: rows.map((row) => ({
|
||||
...row,
|
||||
hits: parseList(row.hits),
|
||||
alarm: Boolean(row.alarm),
|
||||
known: Boolean(row.known),
|
||||
is_customer: row.is_customer == null ? null : Boolean(row.is_customer),
|
||||
})),
|
||||
@@ -300,6 +319,43 @@ router.patch('/matches/:id', async (req, res, next) => {
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Alarme - Treffer mit Alarm-Schlagwort, die noch niemand quittiert hat.
|
||||
*
|
||||
* Der AlarmWatcher der Oberflaeche fragt jede Minute, egal welche Ansicht
|
||||
* offen ist, spielt einen Ton und zeigt ein Pop-up. Quittiert wird fuer alle:
|
||||
* wer das Pop-up schliesst, nimmt es auch den Kollegen ab - sonst klingelte
|
||||
* derselbe Fund bei jedem einzeln.
|
||||
* ---------------------------------------------------------------------- */
|
||||
router.get('/alarms', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT m.id, m.score, m.hits, m.created_at, a.name AS agent_name,
|
||||
j.source, j.title, j.company, j.location, j.url, j.published_at, j.company_id
|
||||
FROM icon_scanner_matches m
|
||||
JOIN icon_scanner_agents a ON a.id = m.agent_id
|
||||
JOIN icon_scanner_jobs j ON j.id = m.job_id
|
||||
WHERE m.alarm = 1 AND m.alarm_ack = 0 AND m.status <> 'verworfen'
|
||||
ORDER BY m.id DESC
|
||||
LIMIT 50`
|
||||
);
|
||||
res.json({ alarms: rows.map((row) => ({ ...row, hits: parseList(row.hits) })) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/alarms/ack', async (req, res, next) => {
|
||||
const ids = (Array.isArray(req.body?.ids) ? req.body.ids : []).map(Number).filter(Number.isInteger).slice(0, 200);
|
||||
if (ids.length === 0) return res.status(400).json({ error: 'Keine Treffer übergeben.' });
|
||||
try {
|
||||
await pool.query('UPDATE icon_scanner_matches SET alarm_ack = 1 WHERE id IN (?)', [ids]);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------------
|
||||
* Jetzt scannen - laeuft im Hintergrund, die Oberflaeche fragt /overview ab.
|
||||
* Ein Lauf dauert je nach Zahl der Schlagwoerter Minuten; darauf zu warten
|
||||
|
||||
@@ -41,8 +41,23 @@ export function containsTerm(text, term) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {{ score: number, hits: string[] } | null} null = ausgeschlossen
|
||||
* oder kein einziges Schlagwort gefunden
|
||||
* Alle Suchbegriffe eines Agenten: Schlagwoerter plus Alarm-Schlagwoerter,
|
||||
* ohne Doppelte (Gross/klein egal).
|
||||
*/
|
||||
export function agentTerms(agent) {
|
||||
const seen = new Set();
|
||||
return [...(agent.keywords ?? []), ...(agent.alarmKeywords ?? [])].filter((term) => {
|
||||
const key = term.toLowerCase();
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {{ score: number, hits: string[], alarm: boolean } | null}
|
||||
* null = ausgeschlossen oder kein einziges Schlagwort gefunden.
|
||||
* `alarm` = mindestens ein Alarm-Schlagwort steht in der Anzeige.
|
||||
*/
|
||||
export function scoreJob(agent, job) {
|
||||
const title = job.title ?? '';
|
||||
@@ -50,7 +65,7 @@ export function scoreJob(agent, job) {
|
||||
|
||||
if ((agent.excludes ?? []).some((term) => containsTerm(all, term))) return null;
|
||||
|
||||
const keywords = agent.keywords ?? [];
|
||||
const keywords = agentTerms(agent);
|
||||
if (keywords.length === 0) return null;
|
||||
|
||||
let points = 0;
|
||||
@@ -66,7 +81,12 @@ export function scoreJob(agent, job) {
|
||||
}
|
||||
if (hits.length === 0) return null;
|
||||
|
||||
return { score: Math.round((100 * points) / (2 * keywords.length)), hits };
|
||||
const alarmSet = new Set((agent.alarmKeywords ?? []).map((term) => term.toLowerCase()));
|
||||
return {
|
||||
score: Math.round((100 * points) / (2 * keywords.length)),
|
||||
hits,
|
||||
alarm: hits.some((hit) => alarmSet.has(hit.toLowerCase())),
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { pool } from '../db.js';
|
||||
import { SOURCES, sourceReady } from './sources.js';
|
||||
import { findCompany, normalizeCompany, scoreJob, stripQuotes } from './matching.js';
|
||||
import { loadScannerSecrets, loadScannerSettings, setLastScanAt } from './settings.js';
|
||||
import { agentTerms, findCompany, normalizeCompany, scoreJob, stripQuotes } from './matching.js';
|
||||
import { loadScannerSecrets, setLastScanAt } from './settings.js';
|
||||
|
||||
/*
|
||||
* Ein Suchlauf.
|
||||
@@ -9,7 +9,7 @@ import { loadScannerSecrets, loadScannerSettings, setLastScanAt } from './settin
|
||||
* 1. aktive Agenten laden
|
||||
* 2. je Boerse die Schlagwoerter aller Agenten zusammenfuehren - dasselbe
|
||||
* Wort wird nur einmal abgefragt, egal wie viele Agenten es tragen
|
||||
* 3. Anzeigen abrufen, alles ueber der Altersgrenze verwerfen
|
||||
* 3. Anzeigen abrufen - jeden Alters (das Dashboard filtert nach Alter)
|
||||
* 4. Anzeigen speichern, Firma gegen das CRM abgleichen
|
||||
* 5. jede Anzeige gegen jeden Agenten bewerten, der diese Boerse durchsucht
|
||||
*
|
||||
@@ -56,24 +56,22 @@ async function execute(trigger) {
|
||||
const [runResult] = await pool.query('INSERT INTO icon_scanner_runs (trigger_kind) VALUES (?)', [trigger]);
|
||||
const runId = runResult.insertId;
|
||||
|
||||
const settings = await loadScannerSettings();
|
||||
const secrets = await loadScannerSecrets();
|
||||
const agents = await loadActiveAgents();
|
||||
|
||||
// Boerse -> Schlagwoerter (ohne Doppelte, Gross/klein egal)
|
||||
// Boerse -> Schlagwoerter inkl. Alarm-Schlagwoerter (ohne Doppelte)
|
||||
const plan = new Map();
|
||||
for (const agent of agents) {
|
||||
for (const source of agent.sources) {
|
||||
if (!SOURCES[source] || !sourceReady(source, secrets)) continue;
|
||||
const words = plan.get(source) ?? new Map();
|
||||
for (const keyword of agent.keywords) words.set(keyword.toLowerCase(), keyword);
|
||||
for (const keyword of agentTerms(agent)) words.set(keyword.toLowerCase(), keyword);
|
||||
plan.set(source, words);
|
||||
}
|
||||
}
|
||||
|
||||
const perSource = {};
|
||||
const jobs = new Map();
|
||||
const oldest = Date.now() - settings.maxAge * 24 * 3_600_000;
|
||||
|
||||
for (const [source, words] of plan) {
|
||||
const stats = { ok: true, count: 0, errors: [] };
|
||||
@@ -82,11 +80,10 @@ async function execute(trigger) {
|
||||
if (!first) await sleep(REQUEST_GAP_MS);
|
||||
first = false;
|
||||
try {
|
||||
const found = await SOURCES[source].fetch(keyword, { maxAge: settings.maxAge, secrets });
|
||||
// Seit 29.09.2026 ohne Altersgrenze: auch aeltere Anzeigen kommen
|
||||
// herein, gefiltert wird im Dashboard ("Anzeigenalter").
|
||||
const found = await SOURCES[source].fetch(keyword, { secrets });
|
||||
for (const job of found) {
|
||||
// Ohne Datum bleibt die Anzeige drin: die Boersen sortieren bzw.
|
||||
// filtern selbst nach Alter, das fehlende Feld ist ein Parserluecke.
|
||||
if (job.publishedAt && job.publishedAt.getTime() < oldest) continue;
|
||||
jobs.set(`${job.source}:${job.externalId}`, job);
|
||||
stats.count += 1;
|
||||
}
|
||||
@@ -111,8 +108,8 @@ async function execute(trigger) {
|
||||
const result = scoreJob(agent, job);
|
||||
if (!result) continue;
|
||||
const [insert] = await pool.query(
|
||||
`INSERT IGNORE INTO icon_scanner_matches (agent_id, job_id, score, hits) VALUES (?, ?, ?, ?)`,
|
||||
[agent.id, jobId, result.score, JSON.stringify(result.hits)]
|
||||
`INSERT IGNORE INTO icon_scanner_matches (agent_id, job_id, score, hits, alarm) VALUES (?, ?, ?, ?, ?)`,
|
||||
[agent.id, jobId, result.score, JSON.stringify(result.hits), result.alarm ? 1 : 0]
|
||||
);
|
||||
matchesNew += insert.affectedRows;
|
||||
}
|
||||
@@ -153,6 +150,7 @@ async function loadActiveAgents() {
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
keywords: parseList(row.keywords),
|
||||
alarmKeywords: parseList(row.alarm_keywords),
|
||||
sources: parseList(row.sources),
|
||||
excludes: parseList(row.excludes),
|
||||
}));
|
||||
|
||||
@@ -5,7 +5,7 @@ import { getSetting, setSetting } from '../db.js';
|
||||
*
|
||||
* Alles liegt in icon_settings unter dem Praefix `scanner_`. Die Vorgaben
|
||||
* entsprechen dem Wunsch vom 29.09.2026: stuendlich, Mo-Fr, 06-20 Uhr,
|
||||
* Anzeigen hoechstens 21 Tage alt.
|
||||
* ohne Altersgrenze (seit 29.09.2026 - das Alter filtert das Dashboard).
|
||||
*/
|
||||
|
||||
export const DEFAULTS = {
|
||||
@@ -15,7 +15,6 @@ export const DEFAULTS = {
|
||||
from: '06:00',
|
||||
to: '20:00',
|
||||
interval: 60,
|
||||
maxAge: 21,
|
||||
};
|
||||
|
||||
/** Zugangsdaten der kostenlosen APIs - gehen nie im Klartext zum Browser. */
|
||||
@@ -27,13 +26,12 @@ const SECRET_KEYS = {
|
||||
const TIME = /^([01]\d|2[0-3]):[0-5]\d$/;
|
||||
|
||||
export async function loadScannerSettings() {
|
||||
const [active, days, from, to, interval, maxAge] = await Promise.all([
|
||||
const [active, days, from, to, interval] = await Promise.all([
|
||||
getSetting('scanner_active'),
|
||||
getSetting('scanner_days'),
|
||||
getSetting('scanner_from'),
|
||||
getSetting('scanner_to'),
|
||||
getSetting('scanner_interval'),
|
||||
getSetting('scanner_max_age'),
|
||||
]);
|
||||
|
||||
const parsedDays = (days ?? '')
|
||||
@@ -47,7 +45,6 @@ export async function loadScannerSettings() {
|
||||
from: TIME.test(from ?? '') ? from : DEFAULTS.from,
|
||||
to: TIME.test(to ?? '') ? to : DEFAULTS.to,
|
||||
interval: clampInt(interval, 15, 24 * 60, DEFAULTS.interval),
|
||||
maxAge: clampInt(maxAge, 1, 90, DEFAULTS.maxAge),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -69,7 +66,6 @@ export async function saveScannerSettings(input) {
|
||||
}
|
||||
if (next.from >= next.to) throw badRequest('Die Startzeit muss vor der Endzeit liegen.');
|
||||
if (input.interval !== undefined) next.interval = clampInt(input.interval, 15, 24 * 60, current.interval);
|
||||
if (input.maxAge !== undefined) next.maxAge = clampInt(input.maxAge, 1, 90, current.maxAge);
|
||||
|
||||
await Promise.all([
|
||||
setSetting('scanner_active', String(next.active)),
|
||||
@@ -77,7 +73,6 @@ export async function saveScannerSettings(input) {
|
||||
setSetting('scanner_from', next.from),
|
||||
setSetting('scanner_to', next.to),
|
||||
setSetting('scanner_interval', String(next.interval)),
|
||||
setSetting('scanner_max_age', String(next.maxAge)),
|
||||
]);
|
||||
|
||||
// Schluessel: undefined = unveraendert, '' = entfernen
|
||||
|
||||
@@ -129,10 +129,10 @@ const slug = (keyword) =>
|
||||
* Bundesagentur fuer Arbeit - offizielle Jobsuche-API (JSON, ohne Anmeldung;
|
||||
* der Schluessel "jobboerse-jobsuche" ist der oeffentliche der Jobsuche-App).
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchBa(keyword, { maxAge }) {
|
||||
async function fetchBa(keyword) {
|
||||
const url =
|
||||
'https://rest.arbeitsagentur.de/jobboerse/jobsuche-service/pc/v6/jobs?' +
|
||||
new URLSearchParams({ was: keyword, veroeffentlichtseit: String(maxAge), size: '100', page: '1' });
|
||||
new URLSearchParams({ was: keyword, size: '100', page: '1' });
|
||||
const data = await request(url, { json: true, headers: { 'X-API-Key': 'jobboerse-jobsuche' } });
|
||||
const list = data?.ergebnisliste ?? data?.stellenangebote ?? [];
|
||||
|
||||
@@ -264,7 +264,7 @@ async function fetchKarriere(keyword) {
|
||||
* Adzuna - kostenloser API-Schluessel (developer.adzuna.com), sammelt Anzeigen
|
||||
* vieler Boersen fuer DE, AT und CH. Die einzige Quelle fuer die Schweiz.
|
||||
* ---------------------------------------------------------------------- */
|
||||
async function fetchAdzuna(keyword, { maxAge, secrets }) {
|
||||
async function fetchAdzuna(keyword, { secrets }) {
|
||||
const jobs = [];
|
||||
for (const country of ['de', 'at', 'ch']) {
|
||||
const url =
|
||||
@@ -273,7 +273,6 @@ async function fetchAdzuna(keyword, { maxAge, secrets }) {
|
||||
app_id: secrets.adzunaId,
|
||||
app_key: secrets.adzunaKey,
|
||||
what_phrase: keyword,
|
||||
max_days_old: String(maxAge),
|
||||
results_per_page: '50',
|
||||
sort_by: 'date',
|
||||
'content-type': 'application/json',
|
||||
|
||||
+543
-537
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user