feat: Projekt Scanner integrieren, Whisper in die Einstellungen
Der eigenstaendige Projekt Scanner zieht komplett ins CRM ein und laeuft parallel zur bestehenden App weiter - jede geteilte Ressource ist verdoppelt statt geteilt, damit die alte Installation unberuehrt bleibt: - eigene Datenbank perinco_scanner, legt sich beim Start selbst an - eigener SCANNER_ENCRYPTION_KEY, eigenes Upload-Verzeichnis - eigener Scheduler mit Zeitversatz, damit nicht beide Instanzen gleichzeitig mit denselben Portal-Cookies scrapen (Rate-Limits, CAPTCHA, doppelter RAM) - Sperre gegen ueberlappende Laeufe: Scheduler und "Jetzt scannen" teilen sie, ein zweiter Aufruf bekommt 409 statt doppelter OpenAI-Kosten Backend unter /api/scanner hinter der CRM-Anmeldung; die HTTP-Basic-Auth des Originals entfaellt. CV-PDFs kommen als Stream mit Token statt aus einem statischen Verzeichnis. Bestandsdaten holt "npm run scanner:import" aus der alten Datenbank, ohne sie zu veraendern. Oberflaeche in Tailwind und CI-Farben neu gebaut: Menuepunkt Scanner mit aufklappbaren Unterpunkten, Einstellungen als eigener Reiter. Whisper hat keinen eigenen Menuepunkt mehr, sondern ist ein Reiter in den Einstellungen. Damit diese Geraeteeinstellung jeder Rolle offensteht, sind die Einstellungen jetzt fuer alle sichtbar - welche Reiter erscheinen, entscheidet die Rolle. Ausserdem: kein festes Admin-Passwort mehr im Quelltext, playwright exakt auf 1.60.0 gepinnt (nutzt den vorhandenen Chromium-Build des Servers), technische Dokumentation mit Funktionszeichnungen unter docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sascha Wolf
@@ -0,0 +1,71 @@
|
||||
# perinco CRM Backend
|
||||
# Kopie als .env anlegen und Werte setzen. .env niemals committen.
|
||||
|
||||
PORT=5006
|
||||
# Nur lokal binden - von aussen erreichbar ist die App ausschliesslich
|
||||
# ueber den Apache-Reverse-Proxy.
|
||||
HOST=127.0.0.1
|
||||
|
||||
DB_HOST=localhost
|
||||
DB_PORT=3306
|
||||
DB_USER=root
|
||||
DB_PASSWORD=changeme
|
||||
DB_NAME=perinco_CRM
|
||||
|
||||
# Signiert die JWT-Sessions. Muss ein langer Zufallswert sein.
|
||||
JWT_SECRET=changeme-langer-zufallswert
|
||||
JWT_EXPIRES_IN=12h
|
||||
|
||||
# Wird beim ersten Start angelegt, falls noch kein Benutzer existiert.
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_NAME=Administrator
|
||||
ADMIN_EMAIL=admin@perinco.de
|
||||
ADMIN_PASSWORD=changeme-starkes-passwort
|
||||
|
||||
# Das Diktat braucht hier keine Konfiguration: Whisper laeuft im Browser des
|
||||
# Benutzers, eingestellt wird es im Reiter "Whisper" der Anwendung.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Projekt Scanner (portiert, laeuft parallel zur alten App)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Diese Instanz ist vollstaendig eigenstaendig: eigene Datenbank, eigener
|
||||
# Schluessel, eigenes Upload-Verzeichnis, eigener Scheduler. Sie fasst die Daten
|
||||
# des alten Scanners nicht an.
|
||||
|
||||
# Eigene Datenbank - wird beim Start automatisch angelegt, falls sie fehlt.
|
||||
# NICHT auf project_scanner zeigen lassen, das ist die Datenbank der alten App.
|
||||
SCANNER_DB_NAME=perinco_scanner
|
||||
# SCANNER_DB_HOST=localhost
|
||||
# SCANNER_DB_PORT=3306
|
||||
# SCANNER_DB_USER=root
|
||||
# SCANNER_DB_PASSWORD=
|
||||
|
||||
# Eigener Schluessel fuer die Portal-Passwoerter, unabhaengig von der alten App.
|
||||
# Langer Zufallswert, z. B.: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
||||
SCANNER_ENCRYPTION_KEY=
|
||||
|
||||
# Eigenes Verzeichnis fuer die CV-PDFs. Leer = <server>/uploads.
|
||||
SCANNER_UPLOAD_DIR=
|
||||
|
||||
# Hintergrund-Scraper (Playwright + OpenAI). Standard ist AN; nur "false"
|
||||
# schaltet ihn ab. Achtung: solange die alte App laeuft, bewerten beide Instanzen
|
||||
# dieselben Projekte unabhaengig voneinander - die OpenAI-Kosten fallen doppelt an.
|
||||
SCANNER_SCHEDULER=true
|
||||
|
||||
# Minuten Versatz fuer den ersten Lauf nach einem Neustart. Verhindert, dass
|
||||
# beide Scanner gleichzeitig mit denselben Portal-Sessions losziehen (CAPTCHA).
|
||||
# 0 = sofort loslegen. Wird ignoriert, wenn der Wert >= Suchintervall ist.
|
||||
SCANNER_SCHEDULER_OFFSET=30
|
||||
|
||||
# Fallback fuer den OpenAI-Key; bevorzugt wird der Wert aus den Einstellungen.
|
||||
# OPENAI_API_KEY=
|
||||
|
||||
# --- Einmalige Uebernahme aus der alten Scanner-Datenbank ---
|
||||
# npm run scanner:import [-- --with-matches] [-- --force] [-- --dry-run]
|
||||
LEGACY_DB_NAME=project_scanner
|
||||
# LEGACY_DB_HOST / _PORT / _USER / _PASSWORD (leer = CRM-Werte von oben)
|
||||
# Verzeichnis der CV-PDFs der alten App - von dort werden die Dateien kopiert.
|
||||
# LEGACY_UPLOAD_DIR=/opt/project-scanner/backend/uploads
|
||||
# ENCRYPTION_KEY der alten App - nur damit koennen die Portal-Passwoerter
|
||||
# uebernommen (und hier neu verschluesselt) werden.
|
||||
# LEGACY_ENCRYPTION_KEY=
|
||||
Generated
+1796
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"name": "perinco-crm-backend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "REST-Backend fuer perinco CRM (MariaDB perinco_CRM) inkl. portiertem Projekt Scanner",
|
||||
"comments": {
|
||||
"playwright": "Exakt 1.60.0 - dieselbe Version wie die laufende Scanner-App. Deren Chromium-Build (chromium-1223) liegt auf dem Server schon im Cache, ein Caret wuerde eine neuere Version und damit einen zweiten Browser-Download ziehen."
|
||||
},
|
||||
"main": "src/server.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/server.js",
|
||||
"dev": "node --watch src/server.js",
|
||||
"migrate": "node src/migrate.js",
|
||||
"scanner:import": "node src/scanner/importLegacy.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"bcryptjs": "^2.4.3",
|
||||
"cors": "^2.8.5",
|
||||
"crypto-js": "^4.2.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"multer": "^2.0.2",
|
||||
"mysql2": "^3.11.5",
|
||||
"node-cron": "^3.0.3",
|
||||
"nodemailer": "^6.9.13",
|
||||
"openai": "^4.47.1",
|
||||
"pdf-parse": "^1.1.1",
|
||||
"playwright": "1.60.0",
|
||||
"sequelize": "^6.37.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import mysql from 'mysql2/promise';
|
||||
import bcrypt from 'bcryptjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export const pool = mysql.createPool({
|
||||
host: process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.DB_PORT || 3306),
|
||||
user: process.env.DB_USER || 'root',
|
||||
password: process.env.DB_PASSWORD || '',
|
||||
database: process.env.DB_NAME || 'perinco_CRM',
|
||||
waitForConnections: true,
|
||||
// Bewusst klein gehalten: der Server hostet weitere Dienste auf derselben
|
||||
// MariaDB-Instanz.
|
||||
connectionLimit: 8,
|
||||
queueLimit: 0,
|
||||
charset: 'utf8mb4',
|
||||
dateStrings: true,
|
||||
});
|
||||
|
||||
/** Fuehrt schema.sql aus und legt bei leerer Benutzertabelle den Admin an. */
|
||||
export async function initDatabase() {
|
||||
const sqlPath = path.join(__dirname, 'schema.sql');
|
||||
const sql = await fs.readFile(sqlPath, 'utf8');
|
||||
|
||||
// schema.sql enthaelt mehrere Statements; die splitten wir an Semikolons
|
||||
// am Zeilenende, damit multipleStatements nicht aktiviert werden muss.
|
||||
// Kommentarzeilen fliegen vorher raus, sonst besteht ein Statement nach dem
|
||||
// Split nur aus Kommentar und wuerde einen Syntaxfehler ausloesen.
|
||||
const statements = sql
|
||||
.split('\n')
|
||||
.filter((line) => !line.trim().startsWith('--'))
|
||||
.join('\n')
|
||||
.split(/;\s*$/m)
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
const conn = await pool.getConnection();
|
||||
try {
|
||||
for (const statement of statements) {
|
||||
await conn.query(statement);
|
||||
}
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
|
||||
await ensureAdminUser();
|
||||
}
|
||||
|
||||
async function ensureAdminUser() {
|
||||
const [rows] = await pool.query('SELECT COUNT(*) AS count FROM crm_users');
|
||||
if (rows[0].count > 0) return;
|
||||
|
||||
const username = process.env.ADMIN_USERNAME || 'admin';
|
||||
const name = process.env.ADMIN_NAME || 'Administrator';
|
||||
const email = process.env.ADMIN_EMAIL || null;
|
||||
|
||||
// Kein festes Ersatzpasswort im Quelltext: ohne ADMIN_PASSWORD wird ein
|
||||
// Zufallswert erzeugt und einmalig ins Log geschrieben. Ein bekannter Default
|
||||
// waere sonst auf jeder Installation derselbe.
|
||||
const generated = !process.env.ADMIN_PASSWORD;
|
||||
const password = process.env.ADMIN_PASSWORD || randomBytes(12).toString('base64url');
|
||||
const hash = await bcrypt.hash(password, 12);
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO crm_users (username, email, name, role, password_hash, is_active)
|
||||
VALUES (?, ?, ?, 'admin', ?, 1)`,
|
||||
[username, email, name, hash]
|
||||
);
|
||||
|
||||
console.log(`[perinco CRM] Admin-Benutzer "${username}" wurde initial angelegt.`);
|
||||
if (generated) {
|
||||
console.log(
|
||||
`[perinco CRM] ADMIN_PASSWORD war nicht gesetzt - erzeugtes Passwort: ${password}\n` +
|
||||
' Jetzt notieren und in der Anwendung unter "Mein Konto" aendern.'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { pool } from '../db.js';
|
||||
|
||||
function secret() {
|
||||
const value = process.env.JWT_SECRET;
|
||||
if (!value || value.length < 16) {
|
||||
throw new Error('JWT_SECRET fehlt oder ist zu kurz (min. 16 Zeichen).');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function signToken(user) {
|
||||
return jwt.sign({ sub: user.id, role: user.role }, secret(), {
|
||||
expiresIn: process.env.JWT_EXPIRES_IN || '12h',
|
||||
});
|
||||
}
|
||||
|
||||
/** Prueft das Bearer-Token und laedt den aktuellen Benutzer aus der DB. */
|
||||
export async function requireAuth(req, res, next) {
|
||||
const header = req.headers.authorization || '';
|
||||
const token = header.startsWith('Bearer ') ? header.slice(7) : null;
|
||||
if (!token) return res.status(401).json({ error: 'Nicht angemeldet.' });
|
||||
|
||||
let payload;
|
||||
try {
|
||||
payload = jwt.verify(token, secret());
|
||||
} catch {
|
||||
return res.status(401).json({ error: 'Sitzung abgelaufen. Bitte neu anmelden.' });
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, username, email, name, role, color, avatar, is_active, created_at, updated_at
|
||||
FROM crm_users WHERE id = ? LIMIT 1`,
|
||||
[payload.sub]
|
||||
);
|
||||
const user = rows[0];
|
||||
if (!user) return res.status(401).json({ error: 'Benutzer existiert nicht mehr.' });
|
||||
if (!user.is_active) return res.status(403).json({ error: 'Benutzerkonto ist deaktiviert.' });
|
||||
|
||||
req.user = { ...user, is_active: Boolean(user.is_active) };
|
||||
next();
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Nur Rolle admin - schuetzt Einstellungen und Nutzerverwaltung. */
|
||||
export function requireAdmin(req, res, next) {
|
||||
if (req.user?.role !== 'admin') {
|
||||
return res.status(403).json({ error: 'Diese Aktion ist Administratoren vorbehalten.' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import express from 'express';
|
||||
import bcrypt from 'bcryptjs';
|
||||
import { pool } from '../db.js';
|
||||
import { signToken, requireAuth } from '../middleware/auth.js';
|
||||
import { normalizeAvatar, normalizeColor } from '../userFields.js';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
function publicUser(row) {
|
||||
return {
|
||||
id: row.id,
|
||||
username: row.username,
|
||||
email: row.email,
|
||||
name: row.name,
|
||||
role: row.role,
|
||||
color: row.color ?? null,
|
||||
avatar: row.avatar ?? null,
|
||||
is_active: Boolean(row.is_active),
|
||||
created_at: row.created_at,
|
||||
updated_at: row.updated_at,
|
||||
};
|
||||
}
|
||||
|
||||
// POST /api/auth/login { identifier, password }
|
||||
router.post('/login', async (req, res, next) => {
|
||||
const identifier = String(req.body?.identifier || '').trim();
|
||||
const password = String(req.body?.password || '');
|
||||
|
||||
if (!identifier || !password) {
|
||||
return res.status(400).json({ error: 'Benutzername und Passwort sind erforderlich.' });
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT * FROM crm_users WHERE username = ? OR email = ? LIMIT 1`,
|
||||
[identifier, identifier.toLowerCase()]
|
||||
);
|
||||
const user = rows[0];
|
||||
|
||||
// Bewusst identische Meldung fuer "kein Benutzer" und "falsches Passwort",
|
||||
// damit ueber die API keine Benutzernamen aufgezaehlt werden koennen.
|
||||
const invalid = () => res.status(401).json({ error: 'Benutzername oder Passwort ist falsch.' });
|
||||
if (!user) return invalid();
|
||||
|
||||
const ok = await bcrypt.compare(password, user.password_hash);
|
||||
if (!ok) return invalid();
|
||||
if (!user.is_active) {
|
||||
return res.status(403).json({ error: 'Dieses Benutzerkonto ist deaktiviert.' });
|
||||
}
|
||||
|
||||
res.json({ token: signToken(user), user: publicUser(user) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/auth/me
|
||||
router.get('/me', requireAuth, (req, res) => {
|
||||
res.json({ user: publicUser(req.user) });
|
||||
});
|
||||
|
||||
// PATCH /api/auth/me { color?, avatar? }
|
||||
// Jeder darf seine eigene Kalenderfarbe und sein Foto setzen - dafuer braucht
|
||||
// es keine Adminrechte. null bzw. "" setzt das Feld zurueck (Farbe: automatisch
|
||||
// aus der Palette, Foto: Initiale).
|
||||
router.patch('/me', requireAuth, async (req, res, next) => {
|
||||
const fields = [];
|
||||
const values = [];
|
||||
const changed = {};
|
||||
|
||||
for (const [field, normalize] of [
|
||||
['color', normalizeColor],
|
||||
['avatar', normalizeAvatar],
|
||||
]) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const result = normalize(req.body[field]);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(result.value);
|
||||
changed[field] = result.value;
|
||||
}
|
||||
|
||||
if (fields.length === 0) {
|
||||
return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
}
|
||||
|
||||
try {
|
||||
await pool.query(`UPDATE crm_users SET ${fields.join(', ')} WHERE id = ?`, [
|
||||
...values,
|
||||
req.user.id,
|
||||
]);
|
||||
res.json({ user: publicUser({ ...req.user, ...changed }) });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/auth/change-password { current_password, new_password }
|
||||
router.post('/change-password', requireAuth, async (req, res, next) => {
|
||||
const current = String(req.body?.current_password || '');
|
||||
const next_ = String(req.body?.new_password || '');
|
||||
|
||||
if (!current || !next_) {
|
||||
return res.status(400).json({ error: 'Aktuelles und neues Passwort sind erforderlich.' });
|
||||
}
|
||||
if (next_.length < 8) {
|
||||
return res.status(400).json({ error: 'Das neue Passwort muss mindestens 8 Zeichen haben.' });
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query('SELECT password_hash FROM crm_users WHERE id = ? LIMIT 1', [
|
||||
req.user.id,
|
||||
]);
|
||||
if (!rows[0]) return res.status(404).json({ error: 'Benutzer nicht gefunden.' });
|
||||
|
||||
const ok = await bcrypt.compare(current, rows[0].password_hash);
|
||||
if (!ok) return res.status(400).json({ error: 'Das aktuelle Passwort ist falsch.' });
|
||||
|
||||
const hash = await bcrypt.hash(next_, 12);
|
||||
await pool.query('UPDATE crm_users SET password_hash = ? WHERE id = ?', [hash, req.user.id]);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
export { publicUser };
|
||||
@@ -0,0 +1,154 @@
|
||||
import express from 'express';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAuth } from '../middleware/auth.js';
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const TYPES = new Set([
|
||||
'Kollege',
|
||||
'Vorgesetzter',
|
||||
'Kunde',
|
||||
'Partner',
|
||||
'Empfehlung',
|
||||
'Netzwerk',
|
||||
'Sonstige',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Verbindungen sind ungerichtet gedacht. Damit "A-B" und "B-A" nicht doppelt
|
||||
* in der DB landen, wird das Paar immer aufsteigend sortiert gespeichert - der
|
||||
* Unique-Key auf (from_lead_id, to_lead_id) greift dann zuverlaessig.
|
||||
*/
|
||||
function orderPair(a, b) {
|
||||
return a <= b ? [a, b] : [b, a];
|
||||
}
|
||||
|
||||
async function leadsExist(ids) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id FROM crm_leads WHERE id IN (${ids.map(() => '?').join(', ')})`,
|
||||
ids
|
||||
);
|
||||
return rows.length === ids.length;
|
||||
}
|
||||
|
||||
// GET /api/connections
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, from_lead_id, to_lead_id, type, label, strength, created_by, created_at
|
||||
FROM crm_lead_connections ORDER BY id ASC`
|
||||
);
|
||||
res.json(rows);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/connections
|
||||
router.post('/', async (req, res, next) => {
|
||||
const fromId = Number(req.body?.from_lead_id);
|
||||
const toId = Number(req.body?.to_lead_id);
|
||||
|
||||
if (!Number.isInteger(fromId) || !Number.isInteger(toId)) {
|
||||
return res.status(400).json({ error: 'Beide Kontakte muessen angegeben werden.' });
|
||||
}
|
||||
if (fromId === toId) {
|
||||
return res.status(400).json({ error: 'Ein Kontakt kann nicht mit sich selbst verbunden werden.' });
|
||||
}
|
||||
|
||||
const type = TYPES.has(req.body?.type) ? req.body.type : 'Netzwerk';
|
||||
const label = String(req.body?.label ?? '').trim() || null;
|
||||
const strengthRaw = Number(req.body?.strength ?? 2);
|
||||
const strength = [1, 2, 3].includes(strengthRaw) ? strengthRaw : 2;
|
||||
const [a, b] = orderPair(fromId, toId);
|
||||
|
||||
try {
|
||||
if (!(await leadsExist([a, b]))) {
|
||||
return res.status(400).json({ error: 'Mindestens einer der Kontakte existiert nicht.' });
|
||||
}
|
||||
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO crm_lead_connections (from_lead_id, to_lead_id, type, label, strength, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
[a, b, type, label, strength, req.user.id]
|
||||
);
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, from_lead_id, to_lead_id, type, label, strength, created_by, created_at
|
||||
FROM crm_lead_connections WHERE id = ?`,
|
||||
[result.insertId]
|
||||
);
|
||||
res.status(201).json(rows[0]);
|
||||
} catch (err) {
|
||||
if (err?.code === 'ER_DUP_ENTRY') {
|
||||
return res.status(409).json({ error: 'Diese Verbindung existiert bereits.' });
|
||||
}
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// PATCH /api/connections/:id
|
||||
router.patch('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Verbindungs-ID.' });
|
||||
|
||||
const fields = [];
|
||||
const values = [];
|
||||
|
||||
if (req.body?.type !== undefined) {
|
||||
if (!TYPES.has(req.body.type)) return res.status(400).json({ error: 'Ungueltiger Verbindungstyp.' });
|
||||
fields.push('type = ?');
|
||||
values.push(req.body.type);
|
||||
}
|
||||
if (req.body?.label !== undefined) {
|
||||
fields.push('label = ?');
|
||||
values.push(String(req.body.label).trim() || null);
|
||||
}
|
||||
if (req.body?.strength !== undefined) {
|
||||
const strength = Number(req.body.strength);
|
||||
if (![1, 2, 3].includes(strength)) {
|
||||
return res.status(400).json({ error: 'Verbindungsstaerke muss 1, 2 oder 3 sein.' });
|
||||
}
|
||||
fields.push('strength = ?');
|
||||
values.push(strength);
|
||||
}
|
||||
|
||||
if (fields.length === 0) return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
|
||||
try {
|
||||
values.push(id);
|
||||
const [result] = await pool.query(
|
||||
`UPDATE crm_lead_connections SET ${fields.join(', ')} WHERE id = ?`,
|
||||
values
|
||||
);
|
||||
if (result.affectedRows === 0) {
|
||||
return res.status(404).json({ error: 'Verbindung nicht gefunden.' });
|
||||
}
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, from_lead_id, to_lead_id, type, label, strength, created_by, created_at
|
||||
FROM crm_lead_connections WHERE id = ?`,
|
||||
[id]
|
||||
);
|
||||
res.json(rows[0]);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/connections/:id
|
||||
router.delete('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Verbindungs-ID.' });
|
||||
|
||||
try {
|
||||
const [result] = await pool.query('DELETE FROM crm_lead_connections WHERE id = ?', [id]);
|
||||
if (result.affectedRows === 0) {
|
||||
return res.status(404).json({ error: 'Verbindung nicht gefunden.' });
|
||||
}
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,287 @@
|
||||
import express from 'express';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAuth } from '../middleware/auth.js';
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
// Akzeptiert "YYYY-MM-DD HH:MM[:SS]" und die Variante mit T, wie sie ein
|
||||
// <input type="datetime-local"> liefert.
|
||||
const DATETIME_RE = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?$/;
|
||||
|
||||
/** Normalisiert auf das MariaDB-DATETIME-Format oder gibt null zurueck. */
|
||||
function normalizeDateTime(value) {
|
||||
const match = DATETIME_RE.exec(String(value ?? '').trim());
|
||||
if (!match) return null;
|
||||
|
||||
const [, year, month, day, hour, minute, second = '00'] = match;
|
||||
const probe = new Date(
|
||||
Number(year),
|
||||
Number(month) - 1,
|
||||
Number(day),
|
||||
Number(hour),
|
||||
Number(minute),
|
||||
Number(second)
|
||||
);
|
||||
// Faengt Unsinn wie den 31. Februar ab - Date rollt sonst stillschweigend um.
|
||||
if (
|
||||
probe.getFullYear() !== Number(year) ||
|
||||
probe.getMonth() !== Number(month) - 1 ||
|
||||
probe.getDate() !== Number(day) ||
|
||||
Number(hour) > 23 ||
|
||||
Number(minute) > 59 ||
|
||||
Number(second) > 59
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return `${year}-${month}-${day} ${hour}:${minute}:${second}`;
|
||||
}
|
||||
|
||||
function emptyToNull(value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
const text = String(value).trim();
|
||||
return text === '' ? null : text;
|
||||
}
|
||||
|
||||
/** Teilnehmer-IDs aus dem Request pruefen und deduplizieren. */
|
||||
async function normalizeParticipants(raw) {
|
||||
if (raw === undefined) return undefined;
|
||||
if (!Array.isArray(raw)) return { error: 'Teilnehmer muessen als Liste uebergeben werden.' };
|
||||
|
||||
const ids = [...new Set(raw.map((value) => Number(value)))];
|
||||
if (ids.some((id) => !Number.isInteger(id) || id <= 0)) {
|
||||
return { error: 'Ungueltige Teilnehmer-ID.' };
|
||||
}
|
||||
if (ids.length === 0) return { ids: [] };
|
||||
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id FROM crm_users WHERE id IN (${ids.map(() => '?').join(', ')})`,
|
||||
ids
|
||||
);
|
||||
if (rows.length !== ids.length) return { error: 'Mindestens ein Teilnehmer existiert nicht.' };
|
||||
|
||||
return { ids };
|
||||
}
|
||||
|
||||
const SELECT_EVENTS = `
|
||||
SELECT e.*,
|
||||
creator.name AS created_by_name,
|
||||
creator.color AS created_by_color,
|
||||
creator.avatar AS created_by_avatar
|
||||
FROM crm_events e
|
||||
LEFT JOIN crm_users creator ON creator.id = e.created_by
|
||||
`;
|
||||
|
||||
/** Laedt die Teilnehmer der uebergebenen Termine als Map eventId -> Liste. */
|
||||
async function loadParticipants(eventIds) {
|
||||
if (eventIds.length === 0) return new Map();
|
||||
|
||||
const [rows] = await pool.query(
|
||||
`SELECT p.event_id, u.id, u.username, u.name, u.color, u.avatar
|
||||
FROM crm_event_participants p
|
||||
JOIN crm_users u ON u.id = p.user_id
|
||||
WHERE p.event_id IN (${eventIds.map(() => '?').join(', ')})
|
||||
ORDER BY u.name ASC`,
|
||||
eventIds
|
||||
);
|
||||
|
||||
const map = new Map();
|
||||
for (const row of rows) {
|
||||
const list = map.get(row.event_id) ?? [];
|
||||
list.push({
|
||||
id: row.id,
|
||||
username: row.username,
|
||||
name: row.name,
|
||||
color: row.color,
|
||||
avatar: row.avatar,
|
||||
});
|
||||
map.set(row.event_id, list);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function mapEvent(row, participants) {
|
||||
return { ...row, all_day: Boolean(row.all_day), participants: participants ?? [] };
|
||||
}
|
||||
|
||||
async function fetchEvent(id) {
|
||||
const [rows] = await pool.query(`${SELECT_EVENTS} WHERE e.id = ? LIMIT 1`, [id]);
|
||||
if (!rows[0]) return null;
|
||||
const participants = await loadParticipants([id]);
|
||||
return mapEvent(rows[0], participants.get(id));
|
||||
}
|
||||
|
||||
/** Ersetzt die Teilnehmerliste eines Termins komplett. */
|
||||
async function replaceParticipants(conn, eventId, ids) {
|
||||
await conn.query('DELETE FROM crm_event_participants WHERE event_id = ?', [eventId]);
|
||||
if (ids.length === 0) return;
|
||||
await conn.query(
|
||||
`INSERT INTO crm_event_participants (event_id, user_id)
|
||||
VALUES ${ids.map(() => '(?, ?)').join(', ')}`,
|
||||
ids.flatMap((userId) => [eventId, userId])
|
||||
);
|
||||
}
|
||||
|
||||
// GET /api/events - der Kalender ist geteilt, jeder sieht alle Termine
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(`${SELECT_EVENTS} ORDER BY e.starts_at ASC, e.id ASC`);
|
||||
const participants = await loadParticipants(rows.map((row) => row.id));
|
||||
res.json(rows.map((row) => mapEvent(row, participants.get(row.id))));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/events
|
||||
router.post('/', async (req, res, next) => {
|
||||
const title = emptyToNull(req.body?.title);
|
||||
if (!title) return res.status(400).json({ error: 'Ein Titel ist erforderlich.' });
|
||||
|
||||
const startsAt = normalizeDateTime(req.body?.starts_at);
|
||||
const endsAt = normalizeDateTime(req.body?.ends_at);
|
||||
if (!startsAt || !endsAt) {
|
||||
return res.status(400).json({ error: 'Start und Ende muessen gueltige Zeitpunkte sein.' });
|
||||
}
|
||||
if (endsAt < startsAt) {
|
||||
return res.status(400).json({ error: 'Das Ende darf nicht vor dem Start liegen.' });
|
||||
}
|
||||
|
||||
const participants = await normalizeParticipants(req.body?.participant_ids ?? []);
|
||||
if (participants.error) return res.status(400).json({ error: participants.error });
|
||||
|
||||
const conn = await pool.getConnection();
|
||||
try {
|
||||
await conn.beginTransaction();
|
||||
const [result] = await conn.query(
|
||||
`INSERT INTO crm_events (title, description, location, starts_at, ends_at, all_day, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
title,
|
||||
emptyToNull(req.body?.description),
|
||||
emptyToNull(req.body?.location),
|
||||
startsAt,
|
||||
endsAt,
|
||||
req.body?.all_day ? 1 : 0,
|
||||
req.user.id,
|
||||
]
|
||||
);
|
||||
await replaceParticipants(conn, result.insertId, participants.ids);
|
||||
await conn.commit();
|
||||
res.status(201).json(await fetchEvent(result.insertId));
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {});
|
||||
next(err);
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
});
|
||||
|
||||
/** Bearbeiten und Loeschen darf der Ersteller - und jeder Administrator. */
|
||||
function mayModify(user, event) {
|
||||
return user.role === 'admin' || event.created_by === user.id;
|
||||
}
|
||||
|
||||
// PATCH /api/events/:id
|
||||
router.patch('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Termin-ID.' });
|
||||
|
||||
const conn = await pool.getConnection();
|
||||
try {
|
||||
const [existingRows] = await conn.query('SELECT * FROM crm_events WHERE id = ? LIMIT 1', [id]);
|
||||
const existing = existingRows[0];
|
||||
if (!existing) return res.status(404).json({ error: 'Termin nicht gefunden.' });
|
||||
if (!mayModify(req.user, existing)) {
|
||||
return res
|
||||
.status(403)
|
||||
.json({ error: 'Diesen Termin darf nur der Ersteller oder ein Administrator aendern.' });
|
||||
}
|
||||
|
||||
const fields = [];
|
||||
const values = [];
|
||||
|
||||
if (req.body?.title !== undefined) {
|
||||
const title = emptyToNull(req.body.title);
|
||||
if (!title) return res.status(400).json({ error: 'Der Titel darf nicht leer sein.' });
|
||||
fields.push('title = ?');
|
||||
values.push(title);
|
||||
}
|
||||
for (const field of ['description', 'location']) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(emptyToNull(req.body[field]));
|
||||
}
|
||||
|
||||
let startsAt = existing.starts_at;
|
||||
if (req.body?.starts_at !== undefined) {
|
||||
startsAt = normalizeDateTime(req.body.starts_at);
|
||||
if (!startsAt) return res.status(400).json({ error: 'Ungueltiger Startzeitpunkt.' });
|
||||
fields.push('starts_at = ?');
|
||||
values.push(startsAt);
|
||||
}
|
||||
|
||||
let endsAt = existing.ends_at;
|
||||
if (req.body?.ends_at !== undefined) {
|
||||
endsAt = normalizeDateTime(req.body.ends_at);
|
||||
if (!endsAt) return res.status(400).json({ error: 'Ungueltiger Endzeitpunkt.' });
|
||||
fields.push('ends_at = ?');
|
||||
values.push(endsAt);
|
||||
}
|
||||
|
||||
if (String(endsAt) < String(startsAt)) {
|
||||
return res.status(400).json({ error: 'Das Ende darf nicht vor dem Start liegen.' });
|
||||
}
|
||||
|
||||
if (req.body?.all_day !== undefined) {
|
||||
fields.push('all_day = ?');
|
||||
values.push(req.body.all_day ? 1 : 0);
|
||||
}
|
||||
|
||||
const participants = await normalizeParticipants(req.body?.participant_ids);
|
||||
if (participants?.error) return res.status(400).json({ error: participants.error });
|
||||
|
||||
if (fields.length === 0 && !participants) {
|
||||
return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
}
|
||||
|
||||
await conn.beginTransaction();
|
||||
if (fields.length > 0) {
|
||||
await conn.query(`UPDATE crm_events SET ${fields.join(', ')} WHERE id = ?`, [...values, id]);
|
||||
}
|
||||
if (participants) await replaceParticipants(conn, id, participants.ids);
|
||||
await conn.commit();
|
||||
|
||||
res.json(await fetchEvent(id));
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {});
|
||||
next(err);
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/events/:id - Teilnehmer entfernt die DB per ON DELETE CASCADE
|
||||
router.delete('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Termin-ID.' });
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query('SELECT created_by FROM crm_events WHERE id = ? LIMIT 1', [id]);
|
||||
const existing = rows[0];
|
||||
if (!existing) return res.status(404).json({ error: 'Termin nicht gefunden.' });
|
||||
if (!mayModify(req.user, existing)) {
|
||||
return res
|
||||
.status(403)
|
||||
.json({ error: 'Diesen Termin darf nur der Ersteller oder ein Administrator loeschen.' });
|
||||
}
|
||||
|
||||
await pool.query('DELETE FROM crm_events WHERE id = ?', [id]);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,205 @@
|
||||
import express from 'express';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAuth } from '../middleware/auth.js';
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const STATUSES = new Set([
|
||||
'Neu',
|
||||
'Kontaktiert',
|
||||
'Qualifiziert',
|
||||
'Angebot',
|
||||
'Gewonnen',
|
||||
'Verloren',
|
||||
]);
|
||||
|
||||
// Felder, die per API gesetzt werden duerfen. owner_id/created_by werden
|
||||
// separat behandelt, alles andere ist nicht beschreibbar.
|
||||
const TEXT_FIELDS = [
|
||||
'salutation',
|
||||
'first_name',
|
||||
'last_name',
|
||||
'company',
|
||||
'position',
|
||||
'email',
|
||||
'phone',
|
||||
'mobile',
|
||||
'linkedin',
|
||||
'website',
|
||||
'street',
|
||||
'zip',
|
||||
'city',
|
||||
'country',
|
||||
'source',
|
||||
'notes',
|
||||
'avatar',
|
||||
];
|
||||
|
||||
function emptyToNull(value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
const text = String(value).trim();
|
||||
return text === '' ? null : text;
|
||||
}
|
||||
|
||||
function parseTags(raw) {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return Array.isArray(parsed) ? parsed.filter((t) => typeof t === 'string') : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function serializeTags(value) {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const tags = [...new Set(value.map((t) => String(t).trim()).filter(Boolean))];
|
||||
return JSON.stringify(tags);
|
||||
}
|
||||
|
||||
function mapLead(row) {
|
||||
return { ...row, tags: parseTags(row.tags), owner_id: row.owner_id, created_by: row.created_by };
|
||||
}
|
||||
|
||||
const SELECT_LEADS = `
|
||||
SELECT l.*,
|
||||
creator.name AS created_by_name,
|
||||
owner.name AS owner_name
|
||||
FROM crm_leads l
|
||||
LEFT JOIN crm_users creator ON creator.id = l.created_by
|
||||
LEFT JOIN crm_users owner ON owner.id = l.owner_id
|
||||
`;
|
||||
|
||||
async function fetchLead(id) {
|
||||
const [rows] = await pool.query(`${SELECT_LEADS} WHERE l.id = ? LIMIT 1`, [id]);
|
||||
return rows[0] ? mapLead(rows[0]) : null;
|
||||
}
|
||||
|
||||
/** Prueft, ob eine uebergebene Benutzer-ID existiert. */
|
||||
async function userExists(id) {
|
||||
if (id === null) return true;
|
||||
const [rows] = await pool.query('SELECT id FROM crm_users WHERE id = ? LIMIT 1', [id]);
|
||||
return Boolean(rows[0]);
|
||||
}
|
||||
|
||||
// GET /api/leads
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(`${SELECT_LEADS} ORDER BY l.updated_at DESC, l.id DESC`);
|
||||
res.json(rows.map(mapLead));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/leads
|
||||
router.post('/', async (req, res, next) => {
|
||||
const firstName = emptyToNull(req.body?.first_name);
|
||||
const lastName = emptyToNull(req.body?.last_name);
|
||||
if (!firstName || !lastName) {
|
||||
return res.status(400).json({ error: 'Vorname und Nachname sind erforderlich.' });
|
||||
}
|
||||
|
||||
const status = STATUSES.has(req.body?.status) ? req.body.status : 'Neu';
|
||||
const ownerId =
|
||||
req.body?.owner_id === undefined || req.body.owner_id === null || req.body.owner_id === ''
|
||||
? req.user.id
|
||||
: Number(req.body.owner_id);
|
||||
|
||||
if (Number.isNaN(ownerId) || !(await userExists(ownerId))) {
|
||||
return res.status(400).json({ error: 'Zustaendiger Benutzer existiert nicht.' });
|
||||
}
|
||||
|
||||
const columns = [...TEXT_FIELDS, 'status', 'tags', 'owner_id', 'created_by'];
|
||||
const values = [
|
||||
...TEXT_FIELDS.map((field) =>
|
||||
field === 'first_name' ? firstName : field === 'last_name' ? lastName : emptyToNull(req.body?.[field])
|
||||
),
|
||||
status,
|
||||
serializeTags(req.body?.tags) ?? '[]',
|
||||
ownerId,
|
||||
req.user.id,
|
||||
];
|
||||
|
||||
try {
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO crm_leads (${columns.join(', ')})
|
||||
VALUES (${columns.map(() => '?').join(', ')})`,
|
||||
values
|
||||
);
|
||||
res.status(201).json(await fetchLead(result.insertId));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// PATCH /api/leads/:id
|
||||
router.patch('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Lead-ID.' });
|
||||
|
||||
const fields = [];
|
||||
const values = [];
|
||||
|
||||
for (const field of TEXT_FIELDS) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const value = emptyToNull(req.body[field]);
|
||||
if ((field === 'first_name' || field === 'last_name') && !value) {
|
||||
return res.status(400).json({ error: 'Vorname und Nachname duerfen nicht leer sein.' });
|
||||
}
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(value);
|
||||
}
|
||||
|
||||
if (req.body?.status !== undefined) {
|
||||
if (!STATUSES.has(req.body.status)) return res.status(400).json({ error: 'Ungueltiger Status.' });
|
||||
fields.push('status = ?');
|
||||
values.push(req.body.status);
|
||||
}
|
||||
|
||||
if (req.body?.tags !== undefined) {
|
||||
fields.push('tags = ?');
|
||||
values.push(serializeTags(req.body.tags) ?? '[]');
|
||||
}
|
||||
|
||||
if (req.body?.owner_id !== undefined) {
|
||||
const ownerId =
|
||||
req.body.owner_id === null || req.body.owner_id === '' ? null : Number(req.body.owner_id);
|
||||
if (ownerId !== null && (Number.isNaN(ownerId) || !(await userExists(ownerId)))) {
|
||||
return res.status(400).json({ error: 'Zustaendiger Benutzer existiert nicht.' });
|
||||
}
|
||||
fields.push('owner_id = ?');
|
||||
values.push(ownerId);
|
||||
}
|
||||
|
||||
if (fields.length === 0) return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
|
||||
try {
|
||||
values.push(id);
|
||||
const [result] = await pool.query(
|
||||
`UPDATE crm_leads SET ${fields.join(', ')} WHERE id = ?`,
|
||||
values
|
||||
);
|
||||
if (result.affectedRows === 0) return res.status(404).json({ error: 'Lead nicht gefunden.' });
|
||||
res.json(await fetchLead(id));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/leads/:id - Verbindungen entfernt die DB per ON DELETE CASCADE
|
||||
router.delete('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Lead-ID.' });
|
||||
|
||||
try {
|
||||
const [result] = await pool.query('DELETE FROM crm_leads WHERE id = ?', [id]);
|
||||
if (result.affectedRows === 0) return res.status(404).json({ error: 'Lead nicht gefunden.' });
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,427 @@
|
||||
import express from 'express';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAuth } from '../middleware/auth.js';
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const STAGES = new Set([
|
||||
'Neu',
|
||||
'Qualifizierung',
|
||||
'Angebot',
|
||||
'Verhandlung',
|
||||
'Gewonnen',
|
||||
'Verloren',
|
||||
]);
|
||||
|
||||
const TEXT_FIELDS = ['title', 'company', 'description', 'last_interaction_note', 'next_step', 'notes'];
|
||||
const DATE_FIELDS = ['start_date', 'end_date', 'last_interaction_at'];
|
||||
const NUMBER_FIELDS = ['daily_rate', 'duration_days'];
|
||||
|
||||
function emptyToNull(value) {
|
||||
if (value === null || value === undefined) return null;
|
||||
const text = String(value).trim();
|
||||
return text === '' ? null : text;
|
||||
}
|
||||
|
||||
/** Akzeptiert `YYYY-MM-DD` (auch mit angehaengter Uhrzeit) oder null. */
|
||||
function normalizeDate(value) {
|
||||
if (value === null || value === undefined || String(value).trim() === '') return { value: null };
|
||||
const match = /^(\d{4})-(\d{2})-(\d{2})/.exec(String(value).trim());
|
||||
if (!match) return { error: 'Ungueltiges Datum (erwartet JJJJ-MM-TT).' };
|
||||
|
||||
const [, year, month, day] = match;
|
||||
const probe = new Date(Number(year), Number(month) - 1, Number(day));
|
||||
if (
|
||||
probe.getFullYear() !== Number(year) ||
|
||||
probe.getMonth() !== Number(month) - 1 ||
|
||||
probe.getDate() !== Number(day)
|
||||
) {
|
||||
return { error: 'Dieses Datum gibt es nicht.' };
|
||||
}
|
||||
return { value: `${year}-${month}-${day}` };
|
||||
}
|
||||
|
||||
function normalizeNumber(value, label, max) {
|
||||
if (value === null || value === undefined || String(value).trim() === '') return { value: null };
|
||||
const number = Number(String(value).replace(',', '.'));
|
||||
if (!Number.isFinite(number) || number < 0) return { error: `${label} muss eine Zahl ab 0 sein.` };
|
||||
if (number > max) return { error: `${label} ist unrealistisch hoch.` };
|
||||
return { value: number };
|
||||
}
|
||||
|
||||
function parseTags(raw) {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return Array.isArray(parsed) ? parsed.filter((tag) => typeof tag === 'string') : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function serializeTags(value) {
|
||||
if (!Array.isArray(value)) return null;
|
||||
return JSON.stringify([...new Set(value.map((tag) => String(tag).trim()).filter(Boolean))]);
|
||||
}
|
||||
|
||||
const SELECT_OPPORTUNITIES = `
|
||||
SELECT o.*,
|
||||
creator.name AS created_by_name,
|
||||
owner.name AS owner_name,
|
||||
owner.color AS owner_color,
|
||||
owner.avatar AS owner_avatar
|
||||
FROM crm_opportunities o
|
||||
LEFT JOIN crm_users creator ON creator.id = o.created_by
|
||||
LEFT JOIN crm_users owner ON owner.id = o.owner_id
|
||||
`;
|
||||
|
||||
/**
|
||||
* DECIMAL kommt aus mysql2 als String zurueck - fuer das Frontend werden daraus
|
||||
* Zahlen, damit dort nicht ueberall geparst werden muss.
|
||||
*/
|
||||
function mapOpportunity(row, contacts) {
|
||||
return {
|
||||
...row,
|
||||
daily_rate: row.daily_rate === null ? null : Number(row.daily_rate),
|
||||
duration_days: row.duration_days === null ? null : Number(row.duration_days),
|
||||
probability: Number(row.probability),
|
||||
tags: parseTags(row.tags),
|
||||
contacts: contacts ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Ansprechpartner der uebergebenen Opportunities als Map id -> Liste. */
|
||||
async function loadContacts(opportunityIds) {
|
||||
if (opportunityIds.length === 0) return new Map();
|
||||
|
||||
const [rows] = await pool.query(
|
||||
`SELECT c.opportunity_id, c.role, c.lead_id,
|
||||
l.first_name, l.last_name, l.company, l.position, l.email, l.phone, l.avatar
|
||||
FROM crm_opportunity_contacts c
|
||||
JOIN crm_leads l ON l.id = c.lead_id
|
||||
WHERE c.opportunity_id IN (${opportunityIds.map(() => '?').join(', ')})
|
||||
ORDER BY l.last_name ASC, l.first_name ASC`,
|
||||
opportunityIds
|
||||
);
|
||||
|
||||
const map = new Map();
|
||||
for (const row of rows) {
|
||||
const list = map.get(row.opportunity_id) ?? [];
|
||||
list.push({
|
||||
lead_id: row.lead_id,
|
||||
role: row.role,
|
||||
first_name: row.first_name,
|
||||
last_name: row.last_name,
|
||||
company: row.company,
|
||||
position: row.position,
|
||||
email: row.email,
|
||||
phone: row.phone,
|
||||
avatar: row.avatar,
|
||||
});
|
||||
map.set(row.opportunity_id, list);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function fetchOpportunity(id) {
|
||||
const [rows] = await pool.query(`${SELECT_OPPORTUNITIES} WHERE o.id = ? LIMIT 1`, [id]);
|
||||
if (!rows[0]) return null;
|
||||
const contacts = await loadContacts([id]);
|
||||
return mapOpportunity(rows[0], contacts.get(id));
|
||||
}
|
||||
|
||||
async function userExists(id) {
|
||||
if (id === null) return true;
|
||||
const [rows] = await pool.query('SELECT id FROM crm_users WHERE id = ? LIMIT 1', [id]);
|
||||
return Boolean(rows[0]);
|
||||
}
|
||||
|
||||
/** Prueft die Ansprechpartner-Liste `[{ lead_id, role }]`. */
|
||||
async function normalizeContacts(raw) {
|
||||
if (raw === undefined) return undefined;
|
||||
if (!Array.isArray(raw)) return { error: 'Ansprechpartner muessen als Liste uebergeben werden.' };
|
||||
|
||||
const seen = new Map();
|
||||
for (const entry of raw) {
|
||||
const leadId = Number(entry?.lead_id ?? entry);
|
||||
if (!Number.isInteger(leadId) || leadId <= 0) {
|
||||
return { error: 'Ungueltige Kontakt-ID bei den Ansprechpartnern.' };
|
||||
}
|
||||
const role = emptyToNull(entry?.role);
|
||||
if (role !== null && role.length > 120) {
|
||||
return { error: 'Die Rolle eines Ansprechpartners ist zu lang (max. 120 Zeichen).' };
|
||||
}
|
||||
seen.set(leadId, role);
|
||||
}
|
||||
|
||||
const ids = [...seen.keys()];
|
||||
if (ids.length === 0) return { contacts: [] };
|
||||
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id FROM crm_leads WHERE id IN (${ids.map(() => '?').join(', ')})`,
|
||||
ids
|
||||
);
|
||||
if (rows.length !== ids.length) return { error: 'Mindestens ein Ansprechpartner existiert nicht.' };
|
||||
|
||||
return { contacts: ids.map((leadId) => ({ lead_id: leadId, role: seen.get(leadId) })) };
|
||||
}
|
||||
|
||||
async function replaceContacts(conn, opportunityId, contacts) {
|
||||
await conn.query('DELETE FROM crm_opportunity_contacts WHERE opportunity_id = ?', [opportunityId]);
|
||||
if (contacts.length === 0) return;
|
||||
await conn.query(
|
||||
`INSERT INTO crm_opportunity_contacts (opportunity_id, lead_id, role)
|
||||
VALUES ${contacts.map(() => '(?, ?, ?)').join(', ')}`,
|
||||
contacts.flatMap((contact) => [opportunityId, contact.lead_id, contact.role])
|
||||
);
|
||||
}
|
||||
|
||||
/** Bearbeiten und Loeschen darf der Ersteller, der Zustaendige und jeder Admin. */
|
||||
function mayModify(user, opportunity) {
|
||||
return (
|
||||
user.role === 'admin' ||
|
||||
opportunity.created_by === user.id ||
|
||||
opportunity.owner_id === user.id
|
||||
);
|
||||
}
|
||||
|
||||
// GET /api/opportunities
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`${SELECT_OPPORTUNITIES} ORDER BY o.updated_at DESC, o.id DESC`
|
||||
);
|
||||
const contacts = await loadContacts(rows.map((row) => row.id));
|
||||
res.json(rows.map((row) => mapOpportunity(row, contacts.get(row.id))));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/opportunities
|
||||
router.post('/', async (req, res, next) => {
|
||||
const title = emptyToNull(req.body?.title);
|
||||
if (!title) return res.status(400).json({ error: 'Ein Titel ist erforderlich.' });
|
||||
|
||||
// Fehlt die Phase, gilt "Neu" - eine falsch geschriebene Phase wird aber
|
||||
// abgewiesen statt still ueberschrieben, sonst glaubt der Client, er haette
|
||||
// etwas anderes gesetzt.
|
||||
const stageGiven = req.body?.stage !== undefined && req.body.stage !== null && req.body.stage !== '';
|
||||
if (stageGiven && !STAGES.has(req.body.stage)) {
|
||||
return res.status(400).json({ error: 'Ungueltige Phase.' });
|
||||
}
|
||||
const stage = stageGiven ? req.body.stage : 'Neu';
|
||||
|
||||
const probability = normalizeNumber(req.body?.probability ?? 10, 'Die Wahrscheinlichkeit', 100);
|
||||
if (probability.error) return res.status(400).json({ error: probability.error });
|
||||
|
||||
const numbers = {};
|
||||
for (const [field, label, max] of [
|
||||
['daily_rate', 'Der Tagessatz', 1_000_000],
|
||||
['duration_days', 'Die Laufzeit', 100_000],
|
||||
]) {
|
||||
const result = normalizeNumber(req.body?.[field], label, max);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
numbers[field] = result.value;
|
||||
}
|
||||
|
||||
const dates = {};
|
||||
for (const field of DATE_FIELDS) {
|
||||
const result = normalizeDate(req.body?.[field]);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
dates[field] = result.value;
|
||||
}
|
||||
if (dates.start_date && dates.end_date && dates.end_date < dates.start_date) {
|
||||
return res.status(400).json({ error: 'Das Projektende darf nicht vor dem Start liegen.' });
|
||||
}
|
||||
|
||||
const ownerId =
|
||||
req.body?.owner_id === undefined || req.body.owner_id === null || req.body.owner_id === ''
|
||||
? req.user.id
|
||||
: Number(req.body.owner_id);
|
||||
if (Number.isNaN(ownerId) || !(await userExists(ownerId))) {
|
||||
return res.status(400).json({ error: 'Zustaendiger Benutzer existiert nicht.' });
|
||||
}
|
||||
|
||||
const contacts = await normalizeContacts(req.body?.contacts ?? []);
|
||||
if (contacts.error) return res.status(400).json({ error: contacts.error });
|
||||
|
||||
const conn = await pool.getConnection();
|
||||
try {
|
||||
await conn.beginTransaction();
|
||||
const [result] = await conn.query(
|
||||
`INSERT INTO crm_opportunities
|
||||
(title, company, description, stage, probability, daily_rate, duration_days,
|
||||
start_date, end_date, last_interaction_at, last_interaction_note, next_step,
|
||||
tags, notes, owner_id, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
title,
|
||||
emptyToNull(req.body?.company),
|
||||
emptyToNull(req.body?.description),
|
||||
stage,
|
||||
Math.round(probability.value ?? 10),
|
||||
numbers.daily_rate,
|
||||
numbers.duration_days,
|
||||
dates.start_date,
|
||||
dates.end_date,
|
||||
dates.last_interaction_at,
|
||||
emptyToNull(req.body?.last_interaction_note),
|
||||
emptyToNull(req.body?.next_step),
|
||||
serializeTags(req.body?.tags) ?? '[]',
|
||||
emptyToNull(req.body?.notes),
|
||||
ownerId,
|
||||
req.user.id,
|
||||
]
|
||||
);
|
||||
await replaceContacts(conn, result.insertId, contacts.contacts);
|
||||
await conn.commit();
|
||||
res.status(201).json(await fetchOpportunity(result.insertId));
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {});
|
||||
next(err);
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
});
|
||||
|
||||
// PATCH /api/opportunities/:id
|
||||
router.patch('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Opportunity-ID.' });
|
||||
|
||||
const conn = await pool.getConnection();
|
||||
try {
|
||||
const [existingRows] = await conn.query(
|
||||
'SELECT * FROM crm_opportunities WHERE id = ? LIMIT 1',
|
||||
[id]
|
||||
);
|
||||
const existing = existingRows[0];
|
||||
if (!existing) return res.status(404).json({ error: 'Opportunity nicht gefunden.' });
|
||||
if (!mayModify(req.user, existing)) {
|
||||
return res.status(403).json({
|
||||
error: 'Diese Opportunity darf nur der Ersteller, der Zustaendige oder ein Administrator aendern.',
|
||||
});
|
||||
}
|
||||
|
||||
const fields = [];
|
||||
const values = [];
|
||||
|
||||
for (const field of TEXT_FIELDS) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const value = emptyToNull(req.body[field]);
|
||||
if (field === 'title' && !value) {
|
||||
return res.status(400).json({ error: 'Der Titel darf nicht leer sein.' });
|
||||
}
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(value);
|
||||
}
|
||||
|
||||
if (req.body?.stage !== undefined) {
|
||||
if (!STAGES.has(req.body.stage)) return res.status(400).json({ error: 'Ungueltige Phase.' });
|
||||
fields.push('stage = ?');
|
||||
values.push(req.body.stage);
|
||||
}
|
||||
|
||||
if (req.body?.probability !== undefined) {
|
||||
const result = normalizeNumber(req.body.probability, 'Die Wahrscheinlichkeit', 100);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
fields.push('probability = ?');
|
||||
values.push(Math.round(result.value ?? 0));
|
||||
}
|
||||
|
||||
for (const [field, label, max] of [
|
||||
['daily_rate', 'Der Tagessatz', 1_000_000],
|
||||
['duration_days', 'Die Laufzeit', 100_000],
|
||||
]) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const result = normalizeNumber(req.body[field], label, max);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(result.value);
|
||||
}
|
||||
|
||||
const nextDates = {
|
||||
start_date: existing.start_date,
|
||||
end_date: existing.end_date,
|
||||
};
|
||||
for (const field of DATE_FIELDS) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const result = normalizeDate(req.body[field]);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(result.value);
|
||||
if (field in nextDates) nextDates[field] = result.value;
|
||||
}
|
||||
if (nextDates.start_date && nextDates.end_date && nextDates.end_date < nextDates.start_date) {
|
||||
return res.status(400).json({ error: 'Das Projektende darf nicht vor dem Start liegen.' });
|
||||
}
|
||||
|
||||
if (req.body?.tags !== undefined) {
|
||||
fields.push('tags = ?');
|
||||
values.push(serializeTags(req.body.tags) ?? '[]');
|
||||
}
|
||||
|
||||
if (req.body?.owner_id !== undefined) {
|
||||
const ownerId =
|
||||
req.body.owner_id === null || req.body.owner_id === '' ? null : Number(req.body.owner_id);
|
||||
if (ownerId !== null && (Number.isNaN(ownerId) || !(await userExists(ownerId)))) {
|
||||
return res.status(400).json({ error: 'Zustaendiger Benutzer existiert nicht.' });
|
||||
}
|
||||
fields.push('owner_id = ?');
|
||||
values.push(ownerId);
|
||||
}
|
||||
|
||||
const contacts = await normalizeContacts(req.body?.contacts);
|
||||
if (contacts?.error) return res.status(400).json({ error: contacts.error });
|
||||
|
||||
if (fields.length === 0 && !contacts) {
|
||||
return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
}
|
||||
|
||||
await conn.beginTransaction();
|
||||
if (fields.length > 0) {
|
||||
await conn.query(`UPDATE crm_opportunities SET ${fields.join(', ')} WHERE id = ?`, [
|
||||
...values,
|
||||
id,
|
||||
]);
|
||||
}
|
||||
if (contacts) await replaceContacts(conn, id, contacts.contacts);
|
||||
await conn.commit();
|
||||
|
||||
res.json(await fetchOpportunity(id));
|
||||
} catch (err) {
|
||||
await conn.rollback().catch(() => {});
|
||||
next(err);
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/opportunities/:id - Ansprechpartner raeumt die DB per CASCADE
|
||||
router.delete('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Opportunity-ID.' });
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
'SELECT created_by, owner_id FROM crm_opportunities WHERE id = ? LIMIT 1',
|
||||
[id]
|
||||
);
|
||||
const existing = rows[0];
|
||||
if (!existing) return res.status(404).json({ error: 'Opportunity nicht gefunden.' });
|
||||
if (!mayModify(req.user, existing)) {
|
||||
return res.status(403).json({
|
||||
error: 'Diese Opportunity darf nur der Ersteller, der Zustaendige oder ein Administrator loeschen.',
|
||||
});
|
||||
}
|
||||
|
||||
await pool.query('DELETE FROM crm_opportunities WHERE id = ?', [id]);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,552 @@
|
||||
import express from 'express';
|
||||
import multer from 'multer';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Op } from 'sequelize';
|
||||
import { Agent, Portal, Project, Match, Setting } from '../scanner/models/index.js';
|
||||
import { parsePdfCv } from '../scanner/services/pdfParser.js';
|
||||
import { encrypt } from '../scanner/utils/crypto.js';
|
||||
import { ensureUploadDir, resolveCvPath } from '../scanner/uploadDir.js';
|
||||
import { isPipelineRunning, runMatchingPipeline } from '../scanner/services/pipeline.js';
|
||||
import {
|
||||
scrapeFreelancermap,
|
||||
scrapeFreelanceDe,
|
||||
scrapeHays,
|
||||
scrapeEtengo,
|
||||
scrapeSThreePortal,
|
||||
SThreePortals,
|
||||
} from '../scanner/services/scraper.js';
|
||||
|
||||
/*
|
||||
* Portierte Scanner-API. Inhaltlich identisch zur urspruenglichen
|
||||
* routes/api.js des Projekt Scanners, mit drei Unterschieden:
|
||||
* - haengt unter /api/scanner und hinter der CRM-Anmeldung (requireAuth),
|
||||
* die HTTP-Basic-Auth des Originals entfaellt damit
|
||||
* - eigene Datenbank, eigener Schluessel, eigenes Upload-Verzeichnis: diese
|
||||
* Instanz laeuft parallel zur alten App, ohne deren Daten anzufassen
|
||||
* - CV-PDFs werden ueber GET /agents/:id/cv ausgeliefert statt als statisches
|
||||
* Verzeichnis: ein Link im neuen Tab kann kein Bearer-Token mitschicken,
|
||||
* das Frontend holt die Datei deshalb per fetch und oeffnet sie als Blob
|
||||
*/
|
||||
const router = express.Router();
|
||||
|
||||
const UPLOAD_DIR = ensureUploadDir();
|
||||
|
||||
const storage = multer.diskStorage({
|
||||
destination: (req, file, cb) => cb(null, UPLOAD_DIR),
|
||||
filename: (req, file, cb) => cb(null, `${Date.now()}-${file.originalname}`),
|
||||
});
|
||||
const upload = multer({ storage, limits: { fileSize: 20 * 1024 * 1024 } });
|
||||
|
||||
// ==========================================
|
||||
// AGENTEN
|
||||
// ==========================================
|
||||
|
||||
router.get('/agents', async (req, res) => {
|
||||
try {
|
||||
// cvText ist mehrere Hundert KB pro Agent und wird in der Liste nicht
|
||||
// gebraucht - nur der Hinweis, ob ueberhaupt ein CV-Text vorliegt.
|
||||
const agents = await Agent.findAll({
|
||||
attributes: [
|
||||
'id',
|
||||
'name',
|
||||
'email',
|
||||
'keywords',
|
||||
'excludeKeywords',
|
||||
'cvFilePath',
|
||||
'isActive',
|
||||
'createdAt',
|
||||
'updatedAt',
|
||||
],
|
||||
order: [['createdAt', 'DESC']],
|
||||
});
|
||||
res.json(agents);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/agents', upload.single('cv'), async (req, res) => {
|
||||
try {
|
||||
const { name, email, keywords, excludeKeywords } = req.body;
|
||||
let cvText = '';
|
||||
let cvFilePath = '';
|
||||
|
||||
if (req.file) {
|
||||
cvFilePath = req.file.path;
|
||||
const fileBuffer = fs.readFileSync(cvFilePath);
|
||||
cvText = await parsePdfCv(fileBuffer);
|
||||
}
|
||||
|
||||
const agent = await Agent.create({
|
||||
name,
|
||||
email: email || null,
|
||||
keywords,
|
||||
excludeKeywords: excludeKeywords || null,
|
||||
cvText,
|
||||
cvFilePath,
|
||||
isActive: req.body.isActive !== 'false',
|
||||
});
|
||||
|
||||
res.status(201).json(agent);
|
||||
} catch (error) {
|
||||
console.error('[Scanner] Agent konnte nicht angelegt werden:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/agents/:id', upload.single('cv'), async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const { name, email, keywords, excludeKeywords, isActive } = req.body;
|
||||
const agent = await Agent.findByPk(id);
|
||||
|
||||
if (!agent) {
|
||||
return res.status(404).json({ error: 'Agent nicht gefunden.' });
|
||||
}
|
||||
|
||||
agent.name = name || agent.name;
|
||||
agent.email = email || agent.email;
|
||||
agent.keywords = keywords || agent.keywords;
|
||||
if (excludeKeywords !== undefined) {
|
||||
agent.excludeKeywords = excludeKeywords || null;
|
||||
}
|
||||
if (isActive !== undefined) {
|
||||
agent.isActive = isActive === 'true' || isActive === true;
|
||||
}
|
||||
|
||||
if (req.file) {
|
||||
const old = resolveCvPath(agent.cvFilePath);
|
||||
if (old) {
|
||||
try {
|
||||
fs.unlinkSync(old);
|
||||
} catch (err) {
|
||||
console.warn('[Scanner] Altes CV konnte nicht geloescht werden:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
agent.cvFilePath = req.file.path;
|
||||
const fileBuffer = fs.readFileSync(req.file.path);
|
||||
agent.cvText = await parsePdfCv(fileBuffer);
|
||||
}
|
||||
|
||||
await agent.save();
|
||||
res.json(agent);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/agents/:id', async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const agent = await Agent.findByPk(id);
|
||||
|
||||
if (!agent) {
|
||||
return res.status(404).json({ error: 'Agent nicht gefunden.' });
|
||||
}
|
||||
|
||||
const stored = resolveCvPath(agent.cvFilePath);
|
||||
if (stored) {
|
||||
try {
|
||||
fs.unlinkSync(stored);
|
||||
} catch (err) {
|
||||
console.warn('[Scanner] CV konnte nicht geloescht werden:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
await agent.destroy();
|
||||
res.json({ message: 'Agent gelöscht.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/** CV als PDF ausliefern - inline, damit der Browser es direkt anzeigt. */
|
||||
router.get('/agents/:id/cv', async (req, res) => {
|
||||
try {
|
||||
const agent = await Agent.findByPk(req.params.id, { attributes: ['id', 'cvFilePath'] });
|
||||
if (!agent) return res.status(404).json({ error: 'Agent nicht gefunden.' });
|
||||
|
||||
const file = resolveCvPath(agent.cvFilePath);
|
||||
if (!file) return res.status(404).json({ error: 'Kein CV hinterlegt.' });
|
||||
|
||||
res.type('application/pdf');
|
||||
res.setHeader('Content-Disposition', `inline; filename="${path.basename(file)}"`);
|
||||
fs.createReadStream(file).pipe(res);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ==========================================
|
||||
// PORTALE
|
||||
// ==========================================
|
||||
|
||||
router.get('/portals', async (req, res) => {
|
||||
try {
|
||||
const portals = await Portal.findAll();
|
||||
const sanitisedPortals = portals.map((p) => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
username: p.username,
|
||||
hasPassword: !!p.passwordEncrypted,
|
||||
hasSession: !!p.sessionState,
|
||||
connectionStatus: p.connectionStatus,
|
||||
lastLoginAttempt: p.lastLoginAttempt,
|
||||
updatedAt: p.updatedAt,
|
||||
}));
|
||||
res.json(sanitisedPortals);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/portals/credentials', async (req, res) => {
|
||||
try {
|
||||
const { name, username, password } = req.body;
|
||||
const portal = await Portal.findOne({ where: { name } });
|
||||
|
||||
if (!portal) {
|
||||
return res.status(404).json({ error: `Portal ${name} nicht gefunden.` });
|
||||
}
|
||||
|
||||
portal.username = username;
|
||||
if (password) {
|
||||
portal.passwordEncrypted = encrypt(password);
|
||||
}
|
||||
await portal.save();
|
||||
|
||||
res.json({ message: 'Zugangsdaten gespeichert.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Cookie-Objekt auf die von Playwright erwartete Form bringen. Exporte aus
|
||||
// Cookie-Editor & Co. enthalten Felder und sameSite-Werte, die Playwright
|
||||
// ablehnt (z. B. "no_restriction", "expirationDate", "hostOnly").
|
||||
const normalizeCookie = (c) => {
|
||||
if (!c || typeof c !== 'object') return null;
|
||||
|
||||
let sameSite = c.sameSite;
|
||||
if (typeof sameSite === 'string') {
|
||||
const s = sameSite.toLowerCase();
|
||||
if (s === 'strict') sameSite = 'Strict';
|
||||
else if (s === 'lax') sameSite = 'Lax';
|
||||
else if (s === 'none' || s === 'no_restriction') sameSite = 'None';
|
||||
else sameSite = 'Lax';
|
||||
} else {
|
||||
sameSite = 'Lax';
|
||||
}
|
||||
|
||||
let expires = c.expires;
|
||||
if (expires === undefined && c.expirationDate !== undefined) {
|
||||
expires = c.expirationDate;
|
||||
}
|
||||
if (typeof expires !== 'number' || !isFinite(expires)) expires = -1;
|
||||
|
||||
if (!c.name || c.value === undefined) return null;
|
||||
|
||||
return {
|
||||
name: String(c.name),
|
||||
value: String(c.value),
|
||||
domain: c.domain || '',
|
||||
path: c.path || '/',
|
||||
expires,
|
||||
httpOnly: !!c.httpOnly,
|
||||
secure: !!c.secure,
|
||||
sameSite,
|
||||
};
|
||||
};
|
||||
|
||||
router.post('/portals/sync-session', async (req, res) => {
|
||||
try {
|
||||
const { name, sessionState } = req.body;
|
||||
const portal = await Portal.findOne({ where: { name } });
|
||||
|
||||
if (!portal) {
|
||||
return res.status(404).json({ error: `Portal ${name} nicht gefunden.` });
|
||||
}
|
||||
|
||||
let parsedState;
|
||||
try {
|
||||
parsedState = typeof sessionState === 'string' ? JSON.parse(sessionState) : sessionState;
|
||||
} catch {
|
||||
return res.status(400).json({ error: 'Ungueltiges Format - es muss JSON sein.' });
|
||||
}
|
||||
|
||||
// Drei Formen sind erlaubt: Playwright storageState, reines Cookie-Array
|
||||
// (Cookie-Editor) und die Nutzlast des Bookmarklets.
|
||||
let rawCookies = [];
|
||||
let origins = [];
|
||||
if (Array.isArray(parsedState)) {
|
||||
rawCookies = parsedState;
|
||||
} else if (parsedState && typeof parsedState === 'object') {
|
||||
rawCookies = Array.isArray(parsedState.cookies) ? parsedState.cookies : [];
|
||||
origins = Array.isArray(parsedState.origins) ? parsedState.origins : [];
|
||||
}
|
||||
|
||||
const cookies = rawCookies.map(normalizeCookie).filter(Boolean);
|
||||
if (cookies.length === 0) {
|
||||
return res.status(400).json({ error: 'Keine gueltigen Cookies gefunden.' });
|
||||
}
|
||||
|
||||
portal.sessionState = JSON.stringify({ cookies, origins });
|
||||
portal.connectionStatus = 'connected';
|
||||
portal.lastLoginAttempt = new Date();
|
||||
await portal.save();
|
||||
|
||||
res.json({
|
||||
message: `Session für ${name} importiert. ${cookies.length} Cookies gespeichert.`,
|
||||
status: 'connected',
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/portals/test-connection/:id', async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const portal = await Portal.findByPk(id);
|
||||
|
||||
if (!portal) {
|
||||
return res.status(404).json({ error: 'Portal nicht gefunden.' });
|
||||
}
|
||||
|
||||
const sThreeMatch = SThreePortals.find((p) => p.name === portal.name);
|
||||
const publicPortals = ['hays.de', 'etengo.de'];
|
||||
if (publicPortals.includes(portal.name) || sThreeMatch) {
|
||||
// Oeffentliche Portale brauchen keine Session. Ein minimaler Scrape
|
||||
// zeigt, ob das Portal antwortet und die Selektoren noch passen.
|
||||
let result;
|
||||
try {
|
||||
if (portal.name === 'hays.de') result = await scrapeHays(['test']);
|
||||
else if (portal.name === 'etengo.de') result = await scrapeEtengo(['test']);
|
||||
else result = await scrapeSThreePortal(sThreeMatch);
|
||||
portal.connectionStatus = 'connected';
|
||||
portal.lastLoginAttempt = new Date();
|
||||
await portal.save();
|
||||
return res.json({
|
||||
status: 'connected',
|
||||
message: `Öffentliches Portal erreichbar. ${result.projects.length} Treffer im Listing.`,
|
||||
});
|
||||
} catch (err) {
|
||||
portal.connectionStatus = 'error';
|
||||
await portal.save();
|
||||
return res.json({ status: 'error', message: 'Portal nicht erreichbar: ' + err.message });
|
||||
}
|
||||
}
|
||||
|
||||
if (!portal.sessionState) {
|
||||
portal.connectionStatus = 'disconnected';
|
||||
await portal.save();
|
||||
return res.json({
|
||||
status: 'disconnected',
|
||||
message: 'Keine Session hinterlegt. Bitte zuerst Cookies importieren.',
|
||||
});
|
||||
}
|
||||
|
||||
let isValid = false;
|
||||
if (portal.name === 'freelancermap.de') {
|
||||
const result = await scrapeFreelancermap(['test'], portal.sessionState);
|
||||
isValid = result.sessionValid;
|
||||
} else if (portal.name === 'freelance.de') {
|
||||
const result = await scrapeFreelanceDe(['test'], portal.sessionState);
|
||||
isValid = result.sessionValid;
|
||||
}
|
||||
|
||||
portal.connectionStatus = isValid ? 'connected' : 'expired';
|
||||
await portal.save();
|
||||
|
||||
res.json({
|
||||
status: portal.connectionStatus,
|
||||
message: isValid ? 'Session ist gültig.' : 'Session ist abgelaufen. Bitte neu anmelden.',
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ==========================================
|
||||
// MATCHES
|
||||
// ==========================================
|
||||
|
||||
router.get('/matches', async (req, res) => {
|
||||
try {
|
||||
const { agentId, portal, status, minPercentage, limit } = req.query;
|
||||
const where = {};
|
||||
const projectWhere = {};
|
||||
|
||||
if (agentId) where.agentId = agentId;
|
||||
if (status) where.status = status;
|
||||
if (minPercentage) {
|
||||
where.matchPercentage = { [Op.gte]: parseInt(minPercentage, 10) };
|
||||
}
|
||||
if (portal) projectWhere.portal = portal;
|
||||
|
||||
const parsedLimit = parseInt(limit, 10);
|
||||
const finalLimit =
|
||||
Number.isFinite(parsedLimit) && parsedLimit > 0 ? Math.min(parsedLimit, 2000) : 500;
|
||||
|
||||
const matches = await Match.findAll({
|
||||
where,
|
||||
include: [
|
||||
{ model: Agent, attributes: ['id', 'name', 'email', 'isActive'] },
|
||||
{ model: Project, where: projectWhere },
|
||||
],
|
||||
order: [
|
||||
['matchPercentage', 'DESC'],
|
||||
['createdAt', 'DESC'],
|
||||
],
|
||||
limit: finalLimit,
|
||||
});
|
||||
|
||||
res.json(matches);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/matches/:id', async (req, res) => {
|
||||
try {
|
||||
const match = await Match.findByPk(req.params.id, {
|
||||
include: [{ model: Agent, attributes: ['id', 'name', 'email', 'isActive'] }, { model: Project }],
|
||||
});
|
||||
if (!match) return res.status(404).json({ error: 'Match nicht gefunden.' });
|
||||
res.json(match);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/matches/:id', async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const { status } = req.body;
|
||||
const match = await Match.findByPk(id);
|
||||
|
||||
if (!match) {
|
||||
return res.status(404).json({ error: 'Match nicht gefunden.' });
|
||||
}
|
||||
|
||||
match.status = status;
|
||||
await match.save();
|
||||
|
||||
res.json(match);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/matches/:id', async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const match = await Match.findByPk(id);
|
||||
|
||||
if (!match) {
|
||||
return res.status(404).json({ error: 'Match nicht gefunden.' });
|
||||
}
|
||||
|
||||
await match.destroy();
|
||||
res.json({ message: 'Match gelöscht.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ==========================================
|
||||
// EINSTELLUNGEN
|
||||
// ==========================================
|
||||
|
||||
// Geheimnisse werden in GET-Antworten maskiert und beim Speichern
|
||||
// uebersprungen, solange die Maske noch drinsteht - so bleibt ein gesetzter
|
||||
// Wert erhalten, wenn das Feld leer gelassen wird.
|
||||
const SECRET_KEYS = ['smtp_pass', 'openai_api_key'];
|
||||
const MASK = '********';
|
||||
|
||||
router.get('/settings', async (req, res) => {
|
||||
try {
|
||||
const settingsList = await Setting.findAll();
|
||||
const settingsMap = {};
|
||||
settingsList.forEach((s) => {
|
||||
if (SECRET_KEYS.includes(s.keyName) && s.value) {
|
||||
settingsMap[s.keyName] = MASK;
|
||||
} else {
|
||||
settingsMap[s.keyName] = s.value;
|
||||
}
|
||||
});
|
||||
res.json(settingsMap);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/settings', async (req, res) => {
|
||||
try {
|
||||
const settings = req.body;
|
||||
for (const key of Object.keys(settings)) {
|
||||
if (SECRET_KEYS.includes(key) && (settings[key] === MASK || settings[key] === '')) {
|
||||
continue;
|
||||
}
|
||||
await Setting.setVal(key, settings[key]);
|
||||
}
|
||||
res.json({ message: 'Einstellungen gespeichert.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ==========================================
|
||||
// MANUELLER SCAN
|
||||
// ==========================================
|
||||
|
||||
router.post('/scan', async (req, res) => {
|
||||
try {
|
||||
const scanningActive = await Setting.getVal('scanning_active', 'true');
|
||||
if (scanningActive === 'false') {
|
||||
return res
|
||||
.status(409)
|
||||
.json({ error: 'Scanner ist deaktiviert. Aktiviere ihn in den Einstellungen.' });
|
||||
}
|
||||
|
||||
if (isPipelineRunning()) {
|
||||
return res.status(409).json({ error: 'Ein Suchlauf läuft bereits. Bitte kurz warten.' });
|
||||
}
|
||||
|
||||
// Pipeline laeuft im Hintergrund weiter, damit die Anfrage nicht in einen
|
||||
// Timeout laeuft.
|
||||
runMatchingPipeline()
|
||||
.then(() => console.log('[Scanner] Manueller Scan abgeschlossen.'))
|
||||
.catch((err) => console.error('[Scanner] Fehler im manuellen Scan:', err));
|
||||
|
||||
res.json({
|
||||
message:
|
||||
'Scraping und Matching gestartet. Das kann einige Minuten dauern - die Ansicht aktualisiert sich automatisch.',
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/*
|
||||
* Betriebszustand fuer das Dashboard: laeuft gerade ein Suchlauf und ist der
|
||||
* Hintergrund-Scheduler in dieser Instanz aktiv. Beim Parallelbetrieb mit der
|
||||
* alten App ist Letzteres die entscheidende Information.
|
||||
*/
|
||||
router.get('/status', async (req, res) => {
|
||||
try {
|
||||
const scanningActive = (await Setting.getVal('scanning_active', 'true')) !== 'false';
|
||||
res.json({
|
||||
scanRunning: isPipelineRunning(),
|
||||
schedulerEnabled: process.env.SCANNER_SCHEDULER !== 'false',
|
||||
scanningActive,
|
||||
database: process.env.SCANNER_DB_NAME || 'perinco_scanner',
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,237 @@
|
||||
import express from 'express';
|
||||
import bcrypt from 'bcryptjs';
|
||||
import { pool } from '../db.js';
|
||||
import { requireAuth, requireAdmin } from '../middleware/auth.js';
|
||||
import { publicUser } from './auth.js';
|
||||
import { normalizeAvatar, normalizeColor } from '../userFields.js';
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireAuth);
|
||||
|
||||
const ROLES = new Set(['admin', 'user']);
|
||||
|
||||
/** Felder, die Admins optional mitschicken duerfen; Pruefung im Modul. */
|
||||
const OPTIONAL_FIELDS = [
|
||||
['color', normalizeColor],
|
||||
['avatar', normalizeAvatar],
|
||||
];
|
||||
|
||||
function normalizeEmail(value) {
|
||||
const email = String(value ?? '').trim().toLowerCase();
|
||||
return email === '' ? null : email;
|
||||
}
|
||||
|
||||
// GET /api/users/directory
|
||||
// Benutzerliste ohne Adminrechte - jeder angemeldete Benutzer braucht sie fuer
|
||||
// das Netzwerkdiagramm, Zustaendigkeiten, @-Mentions, Teilnehmerauswahl und
|
||||
// Farben im Kalender. Bewusst vor dem Admin-Guard registriert; Passwort-Hashes
|
||||
// tauchen hier nie auf. Deaktivierte Konten bleiben enthalten, damit alte
|
||||
// Zuordnungen und Mentions weiter aufloesbar sind - die Auswahlfelder im
|
||||
// Frontend filtern sie heraus.
|
||||
router.get('/directory', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, username, email, name, role, color, avatar, is_active, created_at, updated_at
|
||||
FROM crm_users ORDER BY name ASC`
|
||||
);
|
||||
res.json(rows.map(publicUser));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Ab hier ist alles Admin-only - die Rolle "user" sieht die Einstellungen im
|
||||
// Frontend gar nicht und wird hier zusaetzlich abgewiesen.
|
||||
router.use(requireAdmin);
|
||||
|
||||
/** Verhindert, dass der letzte aktive Admin verschwindet. */
|
||||
async function countOtherActiveAdmins(excludeId) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT COUNT(*) AS count FROM crm_users
|
||||
WHERE role = 'admin' AND is_active = 1 AND id <> ?`,
|
||||
[excludeId]
|
||||
);
|
||||
return rows[0].count;
|
||||
}
|
||||
|
||||
function duplicateMessage(err) {
|
||||
if (err?.code !== 'ER_DUP_ENTRY') return null;
|
||||
return err.message?.includes('uq_crm_users_email')
|
||||
? 'Diese E-Mail-Adresse ist bereits vergeben.'
|
||||
: 'Dieser Benutzername ist bereits vergeben.';
|
||||
}
|
||||
|
||||
// GET /api/users
|
||||
router.get('/', async (req, res, next) => {
|
||||
try {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, username, email, name, role, color, avatar, is_active, created_at, updated_at
|
||||
FROM crm_users ORDER BY role DESC, name ASC`
|
||||
);
|
||||
res.json(rows.map(publicUser));
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/users
|
||||
router.post('/', async (req, res, next) => {
|
||||
const username = String(req.body?.username || '').trim();
|
||||
const name = String(req.body?.name || '').trim();
|
||||
const email = normalizeEmail(req.body?.email);
|
||||
const password = String(req.body?.password || '');
|
||||
const role = ROLES.has(req.body?.role) ? req.body.role : 'user';
|
||||
const isActive = req.body?.is_active === false ? 0 : 1;
|
||||
|
||||
if (!username || !name || !password) {
|
||||
return res.status(400).json({ error: 'Benutzername, Name und Passwort sind erforderlich.' });
|
||||
}
|
||||
if (password.length < 8) {
|
||||
return res.status(400).json({ error: 'Das Passwort muss mindestens 8 Zeichen haben.' });
|
||||
}
|
||||
|
||||
const optional = {};
|
||||
for (const [field, normalize] of OPTIONAL_FIELDS) {
|
||||
const result = normalize(req.body?.[field]);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
optional[field] = result.value;
|
||||
}
|
||||
|
||||
try {
|
||||
const hash = await bcrypt.hash(password, 12);
|
||||
const [result] = await pool.query(
|
||||
`INSERT INTO crm_users (username, email, name, role, color, avatar, password_hash, is_active)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[username, email, name, role, optional.color, optional.avatar, hash, isActive]
|
||||
);
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, username, email, name, role, color, avatar, is_active, created_at, updated_at
|
||||
FROM crm_users WHERE id = ?`,
|
||||
[result.insertId]
|
||||
);
|
||||
res.status(201).json(publicUser(rows[0]));
|
||||
} catch (err) {
|
||||
const message = duplicateMessage(err);
|
||||
if (message) return res.status(409).json({ error: message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// PATCH /api/users/:id - bearbeitet Benutzer inkl. Admin
|
||||
router.patch('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Benutzer-ID.' });
|
||||
|
||||
try {
|
||||
const [existingRows] = await pool.query('SELECT * FROM crm_users WHERE id = ? LIMIT 1', [id]);
|
||||
const existing = existingRows[0];
|
||||
if (!existing) return res.status(404).json({ error: 'Benutzer nicht gefunden.' });
|
||||
|
||||
const fields = [];
|
||||
const values = [];
|
||||
|
||||
if (req.body?.username !== undefined) {
|
||||
const username = String(req.body.username).trim();
|
||||
if (!username) return res.status(400).json({ error: 'Benutzername darf nicht leer sein.' });
|
||||
fields.push('username = ?');
|
||||
values.push(username);
|
||||
}
|
||||
if (req.body?.name !== undefined) {
|
||||
const name = String(req.body.name).trim();
|
||||
if (!name) return res.status(400).json({ error: 'Name darf nicht leer sein.' });
|
||||
fields.push('name = ?');
|
||||
values.push(name);
|
||||
}
|
||||
if (req.body?.email !== undefined) {
|
||||
fields.push('email = ?');
|
||||
values.push(normalizeEmail(req.body.email));
|
||||
}
|
||||
for (const [field, normalize] of OPTIONAL_FIELDS) {
|
||||
if (req.body?.[field] === undefined) continue;
|
||||
const result = normalize(req.body[field]);
|
||||
if (result.error) return res.status(400).json({ error: result.error });
|
||||
fields.push(`${field} = ?`);
|
||||
values.push(result.value);
|
||||
}
|
||||
|
||||
const nextRole = req.body?.role !== undefined ? String(req.body.role) : existing.role;
|
||||
if (req.body?.role !== undefined) {
|
||||
if (!ROLES.has(nextRole)) return res.status(400).json({ error: 'Ungueltige Rolle.' });
|
||||
fields.push('role = ?');
|
||||
values.push(nextRole);
|
||||
}
|
||||
|
||||
const nextActive =
|
||||
req.body?.is_active !== undefined ? (req.body.is_active ? 1 : 0) : existing.is_active ? 1 : 0;
|
||||
if (req.body?.is_active !== undefined) {
|
||||
fields.push('is_active = ?');
|
||||
values.push(nextActive);
|
||||
}
|
||||
|
||||
// Sicherheitsnetz: es muss immer mindestens ein aktiver Admin uebrig bleiben.
|
||||
const losesAdminRights =
|
||||
existing.role === 'admin' && existing.is_active && (nextRole !== 'admin' || nextActive === 0);
|
||||
if (losesAdminRights && (await countOtherActiveAdmins(id)) === 0) {
|
||||
return res.status(409).json({
|
||||
error: 'Das ist der letzte aktive Administrator. Lege zuerst einen weiteren Admin an.',
|
||||
});
|
||||
}
|
||||
|
||||
if (req.body?.password) {
|
||||
const password = String(req.body.password);
|
||||
if (password.length < 8) {
|
||||
return res.status(400).json({ error: 'Das Passwort muss mindestens 8 Zeichen haben.' });
|
||||
}
|
||||
fields.push('password_hash = ?');
|
||||
values.push(await bcrypt.hash(password, 12));
|
||||
}
|
||||
|
||||
if (fields.length === 0) {
|
||||
return res.status(400).json({ error: 'Keine Aenderungen uebergeben.' });
|
||||
}
|
||||
|
||||
values.push(id);
|
||||
await pool.query(`UPDATE crm_users SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, username, email, name, role, color, avatar, is_active, created_at, updated_at
|
||||
FROM crm_users WHERE id = ?`,
|
||||
[id]
|
||||
);
|
||||
res.json(publicUser(rows[0]));
|
||||
} catch (err) {
|
||||
const message = duplicateMessage(err);
|
||||
if (message) return res.status(409).json({ error: message });
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/users/:id
|
||||
router.delete('/:id', async (req, res, next) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Ungueltige Benutzer-ID.' });
|
||||
if (id === req.user.id) {
|
||||
return res.status(400).json({ error: 'Das eigene Konto kann nicht geloescht werden.' });
|
||||
}
|
||||
|
||||
try {
|
||||
const [rows] = await pool.query('SELECT role, is_active FROM crm_users WHERE id = ? LIMIT 1', [
|
||||
id,
|
||||
]);
|
||||
const target = rows[0];
|
||||
if (!target) return res.status(404).json({ error: 'Benutzer nicht gefunden.' });
|
||||
|
||||
if (target.role === 'admin' && target.is_active && (await countOtherActiveAdmins(id)) === 0) {
|
||||
return res
|
||||
.status(409)
|
||||
.json({ error: 'Der letzte aktive Administrator kann nicht geloescht werden.' });
|
||||
}
|
||||
|
||||
await pool.query('DELETE FROM crm_users WHERE id = ?', [id]);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Sequelize } from 'sequelize';
|
||||
|
||||
/*
|
||||
* Der Scanner-Teil bringt seine eigene Sequelize-Verbindung mit - das CRM
|
||||
* selbst arbeitet mit mysql2 direkt (siehe ../../db.js).
|
||||
*
|
||||
* Die Datenbank ist bewusst eine EIGENE (SCANNER_DB_NAME, Default
|
||||
* "perinco_scanner") und nicht die "project_scanner" der alten App: beide
|
||||
* Scanner laufen parallel und duerfen sich nicht in die Daten schreiben. Die
|
||||
* Tabellennamen bleiben identisch (agents, portals, projects, matches,
|
||||
* settings), weil sie in einer getrennten Datenbank liegen. Bestehende Agenten,
|
||||
* Portale und Einstellungen holt `npm run scanner:import` einmalig herueber.
|
||||
*
|
||||
* Zugangsdaten fallen auf die CRM-Werte zurueck, wenn nichts eigenes gesetzt
|
||||
* ist. dotenv wird nicht hier geladen - das erledigt server.js.
|
||||
*/
|
||||
const sequelize = new Sequelize(
|
||||
process.env.SCANNER_DB_NAME || 'perinco_scanner',
|
||||
process.env.SCANNER_DB_USER || process.env.DB_USER || 'root',
|
||||
process.env.SCANNER_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
{
|
||||
host: process.env.SCANNER_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.SCANNER_DB_PORT || process.env.DB_PORT || 3306),
|
||||
dialect: 'mysql',
|
||||
logging: false,
|
||||
define: {
|
||||
underscored: true,
|
||||
timestamps: true,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
export default sequelize;
|
||||
@@ -0,0 +1,35 @@
|
||||
import mysql from 'mysql2/promise';
|
||||
|
||||
/*
|
||||
* Legt die Scanner-Datenbank an, falls sie noch fehlt. Sequelize kann das nicht
|
||||
* selbst - `sync()` erzeugt Tabellen, aber keine Datenbank. Da der Scanner hier
|
||||
* bewusst eine eigene Datenbank benutzt (nicht die der alten App), soll die
|
||||
* erste Einrichtung ohne Handarbeit an der MariaDB laufen.
|
||||
*/
|
||||
export async function ensureScannerDatabase() {
|
||||
const name = process.env.SCANNER_DB_NAME || 'perinco_scanner';
|
||||
|
||||
// Bezeichner lassen sich nicht als Platzhalter binden - deshalb hart pruefen,
|
||||
// welche Zeichen im Datenbanknamen erlaubt sind.
|
||||
if (!/^[A-Za-z0-9_]+$/.test(name)) {
|
||||
throw new Error(`Ungueltiger SCANNER_DB_NAME: "${name}" (nur A-Z, a-z, 0-9, _).`);
|
||||
}
|
||||
|
||||
const connection = await mysql.createConnection({
|
||||
host: process.env.SCANNER_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.SCANNER_DB_PORT || process.env.DB_PORT || 3306),
|
||||
user: process.env.SCANNER_DB_USER || process.env.DB_USER || 'root',
|
||||
password: process.env.SCANNER_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
});
|
||||
|
||||
try {
|
||||
await connection.query(
|
||||
`CREATE DATABASE IF NOT EXISTS \`${name}\`
|
||||
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
|
||||
);
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* Einmalige Uebernahme aus der alten Projekt-Scanner-Datenbank in die eigene
|
||||
* Datenbank dieser Instanz. Beide Scanner laufen danach parallel und getrennt -
|
||||
* dieses Skript kopiert nur, es veraendert die Quelle nicht.
|
||||
*
|
||||
* Aufruf (aus server/):
|
||||
* npm run scanner:import
|
||||
* npm run scanner:import -- --with-matches auch Projekte und Treffer
|
||||
* npm run scanner:import -- --force vorhandene Zieldaten ersetzen
|
||||
* npm run scanner:import -- --dry-run nur anzeigen, nichts schreiben
|
||||
*
|
||||
* Quelle konfigurieren (server/.env):
|
||||
* LEGACY_DB_NAME=project_scanner
|
||||
* LEGACY_DB_HOST / _PORT / _USER / _PASSWORD (leer = CRM-Werte)
|
||||
* LEGACY_UPLOAD_DIR=/opt/project-scanner/backend/uploads
|
||||
* LEGACY_ENCRYPTION_KEY=... Portal-Passwoerter mit uebernehmen
|
||||
*
|
||||
* Was uebernommen wird:
|
||||
* - Einstellungen (inkl. OpenAI-Key, SMTP-Zugang, Schwellenwerte)
|
||||
* - Portale: Benutzername, Session-Cookies, Status; Passwoerter nur mit
|
||||
* LEGACY_ENCRYPTION_KEY, weil sie mit dem alten Schluessel verschluesselt
|
||||
* sind und hier neu verschluesselt werden muessen
|
||||
* - Suchagenten inkl. CV-Text und CV-PDF (Datei wird kopiert)
|
||||
* - mit --with-matches zusaetzlich Projekte und Treffer
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import mysql from 'mysql2/promise';
|
||||
import {
|
||||
Agent,
|
||||
Match,
|
||||
Portal,
|
||||
Project,
|
||||
Setting,
|
||||
initScannerDatabase,
|
||||
} from './models/index.js';
|
||||
import { encrypt, decryptWith } from './utils/crypto.js';
|
||||
import { ensureUploadDir } from './uploadDir.js';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const withMatches = args.includes('--with-matches');
|
||||
const force = args.includes('--force');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
const LEGACY_UPLOAD_DIR = process.env.LEGACY_UPLOAD_DIR || '';
|
||||
const LEGACY_KEY = process.env.LEGACY_ENCRYPTION_KEY || '';
|
||||
|
||||
const log = (message) => console.log(`[Import] ${message}`);
|
||||
|
||||
async function openLegacy() {
|
||||
const database = process.env.LEGACY_DB_NAME || 'project_scanner';
|
||||
const connection = await mysql.createConnection({
|
||||
host: process.env.LEGACY_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.LEGACY_DB_PORT || process.env.DB_PORT || 3306),
|
||||
user: process.env.LEGACY_DB_USER || process.env.DB_USER || 'root',
|
||||
password: process.env.LEGACY_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
database,
|
||||
dateStrings: false,
|
||||
});
|
||||
log(`Quelle: ${database}`);
|
||||
return connection;
|
||||
}
|
||||
|
||||
/** Kopiert das CV-PDF in das eigene Upload-Verzeichnis. */
|
||||
function copyCv(storedPath, uploadDir) {
|
||||
if (!storedPath) return '';
|
||||
|
||||
const fileName = path.basename(storedPath);
|
||||
const candidates = [
|
||||
path.isAbsolute(storedPath) ? storedPath : null,
|
||||
LEGACY_UPLOAD_DIR ? path.join(LEGACY_UPLOAD_DIR, fileName) : null,
|
||||
path.resolve(storedPath),
|
||||
].filter(Boolean);
|
||||
|
||||
const source = candidates.find((candidate) => fs.existsSync(candidate));
|
||||
if (!source) {
|
||||
log(` CV-Datei nicht gefunden: ${storedPath} (LEGACY_UPLOAD_DIR setzen?)`);
|
||||
return '';
|
||||
}
|
||||
|
||||
const target = path.join(uploadDir, fileName);
|
||||
if (!dryRun) fs.copyFileSync(source, target);
|
||||
return target;
|
||||
}
|
||||
|
||||
async function importSettings(legacy) {
|
||||
const [rows] = await legacy.query('SELECT key_name, value FROM settings');
|
||||
for (const row of rows) {
|
||||
if (!dryRun) await Setting.setVal(row.key_name, row.value ?? '');
|
||||
}
|
||||
log(`Einstellungen: ${rows.length} Schluessel uebernommen.`);
|
||||
}
|
||||
|
||||
async function importPortals(legacy) {
|
||||
const [rows] = await legacy.query(
|
||||
'SELECT name, username, password_encrypted, session_state, connection_status, last_login_attempt FROM portals'
|
||||
);
|
||||
|
||||
let withSession = 0;
|
||||
let withPassword = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
// Portale sind bei der Initialisierung schon angelegt - hier wird nur der
|
||||
// Inhalt nachgezogen, ueber den Namen zugeordnet.
|
||||
const portal = await Portal.findOne({ where: { name: row.name } });
|
||||
if (!portal) {
|
||||
log(` Portal ${row.name} gibt es hier nicht - uebersprungen.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
portal.username = row.username || portal.username;
|
||||
if (row.session_state) {
|
||||
portal.sessionState = row.session_state;
|
||||
portal.connectionStatus = row.connection_status || 'connected';
|
||||
portal.lastLoginAttempt = row.last_login_attempt || new Date();
|
||||
withSession++;
|
||||
}
|
||||
|
||||
if (row.password_encrypted && LEGACY_KEY) {
|
||||
const plain = decryptWith(row.password_encrypted, LEGACY_KEY);
|
||||
if (plain) {
|
||||
// Neu verschluesseln - diese Instanz hat ihren eigenen Schluessel.
|
||||
portal.passwordEncrypted = encrypt(plain);
|
||||
withPassword++;
|
||||
} else {
|
||||
log(` Passwort von ${row.name} liess sich mit LEGACY_ENCRYPTION_KEY nicht lesen.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun) await portal.save();
|
||||
}
|
||||
|
||||
log(`Portale: ${rows.length} geprueft, ${withSession} mit Session, ${withPassword} mit Passwort.`);
|
||||
if (!LEGACY_KEY) {
|
||||
log(' Hinweis: ohne LEGACY_ENCRYPTION_KEY bleiben die Portal-Passwoerter leer.');
|
||||
}
|
||||
}
|
||||
|
||||
async function importAgents(legacy, uploadDir) {
|
||||
const [rows] = await legacy.query(
|
||||
`SELECT id, name, email, cv_text, cv_file_path, keywords, exclude_keywords, is_active,
|
||||
created_at, updated_at
|
||||
FROM agents ORDER BY id`
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
const cvFilePath = copyCv(row.cv_file_path, uploadDir);
|
||||
if (dryRun) {
|
||||
log(` wuerde Agent ${row.id} "${row.name}" anlegen`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ids bleiben erhalten, damit uebernommene Treffer weiter passen.
|
||||
await Agent.create(
|
||||
{
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
email: row.email,
|
||||
cvText: row.cv_text,
|
||||
cvFilePath,
|
||||
keywords: row.keywords,
|
||||
excludeKeywords: row.exclude_keywords,
|
||||
isActive: Boolean(row.is_active),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
|
||||
log(`Suchagenten: ${rows.length} uebernommen.`);
|
||||
}
|
||||
|
||||
async function importProjectsAndMatches(legacy) {
|
||||
const [projects] = await legacy.query(
|
||||
`SELECT id, portal, portal_project_id, title, description, company, location, rate, url,
|
||||
published_at, created_at
|
||||
FROM projects ORDER BY id`
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of projects) {
|
||||
await Project.create(
|
||||
{
|
||||
id: row.id,
|
||||
portal: row.portal,
|
||||
portalProjectId: row.portal_project_id,
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
company: row.company,
|
||||
location: row.location,
|
||||
rate: row.rate,
|
||||
url: row.url,
|
||||
publishedAt: row.published_at,
|
||||
createdAt: row.created_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Projekte: ${projects.length} uebernommen.`);
|
||||
|
||||
const [matches] = await legacy.query(
|
||||
`SELECT id, agent_id, project_id, match_percentage, pros, cons, status, notification_sent,
|
||||
created_at, updated_at
|
||||
FROM matches ORDER BY id`
|
||||
);
|
||||
|
||||
const parseList = (value) => {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of matches) {
|
||||
await Match.create(
|
||||
{
|
||||
id: row.id,
|
||||
agentId: row.agent_id,
|
||||
projectId: row.project_id,
|
||||
matchPercentage: row.match_percentage,
|
||||
pros: parseList(row.pros),
|
||||
cons: parseList(row.cons),
|
||||
status: row.status,
|
||||
notificationSent: Boolean(row.notification_sent),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Treffer: ${matches.length} uebernommen.`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (dryRun) log('Probelauf - es wird nichts geschrieben.');
|
||||
|
||||
await initScannerDatabase();
|
||||
const uploadDir = ensureUploadDir();
|
||||
log(`Ziel-Upload-Verzeichnis: ${uploadDir}`);
|
||||
|
||||
const existingAgents = await Agent.count();
|
||||
if (existingAgents > 0 && !force) {
|
||||
log(
|
||||
`Abbruch: es gibt hier schon ${existingAgents} Suchagenten. Mit --force werden Agenten, ` +
|
||||
'Projekte und Treffer vorher geloescht.'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const legacy = await openLegacy();
|
||||
try {
|
||||
if (force && !dryRun) {
|
||||
// Reihenfolge wegen der Fremdschluessel: Treffer, dann Projekte, dann Agenten.
|
||||
await Match.destroy({ where: {}, truncate: false });
|
||||
await Project.destroy({ where: {}, truncate: false });
|
||||
await Agent.destroy({ where: {}, truncate: false });
|
||||
log('Vorhandene Agenten, Projekte und Treffer geloescht.');
|
||||
}
|
||||
|
||||
await importSettings(legacy);
|
||||
await importPortals(legacy);
|
||||
await importAgents(legacy, uploadDir);
|
||||
if (withMatches) {
|
||||
await importProjectsAndMatches(legacy);
|
||||
} else {
|
||||
log('Projekte und Treffer uebersprungen (--with-matches aktiviert sie).');
|
||||
}
|
||||
} finally {
|
||||
await legacy.end();
|
||||
}
|
||||
|
||||
log('Fertig.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[Import] Fehlgeschlagen:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Agent = sequelize.define('Agent', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
},
|
||||
email: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
validate: {
|
||||
isEmail: true,
|
||||
},
|
||||
},
|
||||
cvText: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
field: 'cv_text',
|
||||
},
|
||||
cvFilePath: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
field: 'cv_file_path',
|
||||
},
|
||||
keywords: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
comment: 'Comma-separated keywords for search',
|
||||
},
|
||||
excludeKeywords: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
field: 'exclude_keywords',
|
||||
comment: 'Comma-separated terms; projects containing any are ignored',
|
||||
},
|
||||
isActive: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: true,
|
||||
field: 'is_active',
|
||||
},
|
||||
}, {
|
||||
tableName: 'agents',
|
||||
});
|
||||
|
||||
export default Agent;
|
||||
@@ -0,0 +1,85 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
import Agent from './Agent.js';
|
||||
import Project from './Project.js';
|
||||
|
||||
const Match = sequelize.define('Match', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
agentId: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
field: 'agent_id',
|
||||
references: {
|
||||
model: Agent,
|
||||
key: 'id',
|
||||
},
|
||||
},
|
||||
projectId: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
field: 'project_id',
|
||||
references: {
|
||||
model: Project,
|
||||
key: 'id',
|
||||
},
|
||||
},
|
||||
matchPercentage: {
|
||||
type: DataTypes.INTEGER,
|
||||
defaultValue: 0,
|
||||
field: 'match_percentage',
|
||||
},
|
||||
pros: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
comment: 'JSON array of strings',
|
||||
get() {
|
||||
const rawValue = this.getDataValue('pros');
|
||||
return rawValue ? JSON.parse(rawValue) : [];
|
||||
},
|
||||
set(value) {
|
||||
this.setDataValue('pros', JSON.stringify(value));
|
||||
},
|
||||
},
|
||||
cons: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
comment: 'JSON array of strings',
|
||||
get() {
|
||||
const rawValue = this.getDataValue('cons');
|
||||
return rawValue ? JSON.parse(rawValue) : [];
|
||||
},
|
||||
set(value) {
|
||||
this.setDataValue('cons', JSON.stringify(value));
|
||||
},
|
||||
},
|
||||
status: {
|
||||
type: DataTypes.STRING(50),
|
||||
defaultValue: 'new', // 'new', 'reviewed', 'applied', 'rejected'
|
||||
},
|
||||
notificationSent: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: false,
|
||||
field: 'notification_sent',
|
||||
},
|
||||
}, {
|
||||
tableName: 'matches',
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['agent_id', 'project_id'],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Associations
|
||||
Agent.hasMany(Match, { foreignKey: 'agent_id', onDelete: 'CASCADE' });
|
||||
Match.belongsTo(Agent, { foreignKey: 'agent_id' });
|
||||
|
||||
Project.hasMany(Match, { foreignKey: 'project_id', onDelete: 'CASCADE' });
|
||||
Match.belongsTo(Project, { foreignKey: 'project_id' });
|
||||
|
||||
export default Match;
|
||||
@@ -0,0 +1,44 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Portal = sequelize.define('Portal', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
type: DataTypes.STRING(50),
|
||||
allowNull: false,
|
||||
unique: true,
|
||||
},
|
||||
username: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
passwordEncrypted: {
|
||||
type: DataTypes.STRING(500),
|
||||
allowNull: true,
|
||||
field: 'password_encrypted',
|
||||
},
|
||||
sessionState: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
field: 'session_state',
|
||||
comment: 'JSON string of cookies and localStorage',
|
||||
},
|
||||
connectionStatus: {
|
||||
type: DataTypes.STRING(50),
|
||||
defaultValue: 'disconnected',
|
||||
field: 'connection_status',
|
||||
},
|
||||
lastLoginAttempt: {
|
||||
type: DataTypes.DATE,
|
||||
allowNull: true,
|
||||
field: 'last_login_attempt',
|
||||
},
|
||||
}, {
|
||||
tableName: 'portals',
|
||||
});
|
||||
|
||||
export default Portal;
|
||||
@@ -0,0 +1,61 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Project = sequelize.define('Project', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
portal: {
|
||||
type: DataTypes.STRING(50),
|
||||
allowNull: false,
|
||||
},
|
||||
portalProjectId: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: false,
|
||||
field: 'portal_project_id',
|
||||
},
|
||||
title: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
},
|
||||
description: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
},
|
||||
company: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
location: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
rate: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: true,
|
||||
},
|
||||
url: {
|
||||
type: DataTypes.STRING(500),
|
||||
allowNull: false,
|
||||
},
|
||||
publishedAt: {
|
||||
type: DataTypes.DATE,
|
||||
allowNull: true,
|
||||
field: 'published_at',
|
||||
},
|
||||
}, {
|
||||
tableName: 'projects',
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['portal', 'portal_project_id'],
|
||||
},
|
||||
],
|
||||
timestamps: true,
|
||||
updatedAt: false, // projects are immutable once scraped, we only track when they were created
|
||||
createdAt: 'created_at',
|
||||
});
|
||||
|
||||
export default Project;
|
||||
@@ -0,0 +1,50 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Setting = sequelize.define('Setting', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
keyName: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: false,
|
||||
unique: true,
|
||||
field: 'key_name',
|
||||
},
|
||||
value: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
},
|
||||
}, {
|
||||
tableName: 'settings',
|
||||
});
|
||||
|
||||
// Helper static methods to get and set settings easily
|
||||
Setting.getVal = async function(key, defaultValue = null) {
|
||||
try {
|
||||
const setting = await this.findOne({ where: { keyName: key } });
|
||||
return setting ? setting.value : defaultValue;
|
||||
} catch (err) {
|
||||
console.error(`Error getting setting ${key}:`, err);
|
||||
return defaultValue;
|
||||
}
|
||||
};
|
||||
|
||||
Setting.setVal = async function(key, value) {
|
||||
try {
|
||||
const [setting] = await this.findOrCreate({
|
||||
where: { keyName: key },
|
||||
defaults: { value: String(value) }
|
||||
});
|
||||
setting.value = String(value);
|
||||
await setting.save();
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(`Error setting ${key}:`, err);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export default Setting;
|
||||
@@ -0,0 +1,86 @@
|
||||
import sequelize from '../db/config.js';
|
||||
import { ensureScannerDatabase } from '../db/ensureDatabase.js';
|
||||
import Agent from './Agent.js';
|
||||
import Portal from './Portal.js';
|
||||
import Project from './Project.js';
|
||||
import Match from './Match.js';
|
||||
import Setting from './Setting.js';
|
||||
|
||||
const initScannerDatabase = async () => {
|
||||
try {
|
||||
// Eigene Datenbank, deshalb erst anlegen und dann verbinden.
|
||||
const dbName = await ensureScannerDatabase();
|
||||
await sequelize.authenticate();
|
||||
console.log(`[Scanner] Datenbankverbindung steht (${dbName}).`);
|
||||
|
||||
// Sync database tables. Plain sync() — `alter: true` repeatedly re-creates
|
||||
// UNIQUE indexes on each boot and eventually exhausts MySQL's 64-key limit.
|
||||
// Schema changes are managed via SQL migrations / schema.sql instead.
|
||||
await sequelize.sync();
|
||||
console.log('[Scanner] Tabellen synchronisiert.');
|
||||
|
||||
// Ensure every supported portal exists. Per-row findOrCreate so newly added
|
||||
// portals appear after upgrade without manual SQL.
|
||||
const defaultPortals = [
|
||||
{ name: 'freelance.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'freelancermap.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'hays.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'etengo.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'huxley.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'computerfutures.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'progressiverecruitment.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'realstaffing.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'globalenterprisepartners.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'madisonblack.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'jpgray.nl', connectionStatus: 'disconnected' }
|
||||
];
|
||||
for (const p of defaultPortals) {
|
||||
const [, created] = await Portal.findOrCreate({
|
||||
where: { name: p.name },
|
||||
defaults: p
|
||||
});
|
||||
if (created) console.log(`[Scanner] Portal angelegt: ${p.name}`);
|
||||
}
|
||||
|
||||
// Default settings
|
||||
const defaultSettings = [
|
||||
{ keyName: 'scanning_active', value: 'true' },
|
||||
{ keyName: 'search_interval', value: '60' },
|
||||
{ keyName: 'openai_api_key', value: '' },
|
||||
{ keyName: 'email_notifications_active', value: 'false' },
|
||||
{ keyName: 'sound_notifications_active', value: 'true' },
|
||||
{ keyName: 'smtp_host', value: '' },
|
||||
{ keyName: 'smtp_port', value: '587' },
|
||||
{ keyName: 'smtp_user', value: '' },
|
||||
{ keyName: 'smtp_pass', value: '' },
|
||||
{ keyName: 'smtp_from', value: 'scanner@perinco.de' },
|
||||
{ keyName: 'notification_email_recipient', value: '' },
|
||||
{ keyName: 'match_threshold', value: '70' },
|
||||
{ keyName: 'display_threshold', value: '50' }
|
||||
];
|
||||
|
||||
for (const s of defaultSettings) {
|
||||
const exists = await Setting.findOne({ where: { keyName: s.keyName } });
|
||||
if (!exists) {
|
||||
await Setting.create(s);
|
||||
}
|
||||
}
|
||||
console.log('[Scanner] Standardeinstellungen geprueft.');
|
||||
|
||||
} catch (error) {
|
||||
// Ein fehlender Scanner-Teil darf das CRM nicht am Start hindern - die
|
||||
// Scanner-Endpunkte antworten dann mit einem Fehler, alles andere laeuft.
|
||||
console.error('[Scanner] Datenbank nicht erreichbar oder Sync fehlgeschlagen:', error.message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export {
|
||||
sequelize,
|
||||
Agent,
|
||||
Portal,
|
||||
Project,
|
||||
Match,
|
||||
Setting,
|
||||
initScannerDatabase
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import sequelize from '../db/config.js';
|
||||
import Project from '../models/Project.js';
|
||||
|
||||
// Retention window for scraped projects. Matches cascade-delete via FK.
|
||||
const RETENTION_DAYS = 7;
|
||||
|
||||
// Project model defines `createdAt: 'created_at'` for the column rename, but
|
||||
// Sequelize's attribute-to-column mapping in `where` does not pick it up here
|
||||
// (likely because of the underscored global default + explicit override
|
||||
// interaction). Using sequelize.col('created_at') sidesteps it.
|
||||
export const cleanupOldProjects = async () => {
|
||||
const cutoff = new Date(Date.now() - RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
||||
try {
|
||||
const deleted = await Project.destroy({
|
||||
where: sequelize.where(sequelize.col('created_at'), '<', cutoff)
|
||||
});
|
||||
console.log(`[Cleanup] Removed ${deleted} projects older than ${RETENTION_DAYS} days (cutoff ${cutoff.toISOString()}).`);
|
||||
return deleted;
|
||||
} catch (err) {
|
||||
console.error('[Cleanup] Failed to delete old projects:', err);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,79 @@
|
||||
import { OpenAI } from 'openai';
|
||||
import Setting from '../models/Setting.js';
|
||||
|
||||
/**
|
||||
* Evaluates the match between an agent's CV and a scraped project description using OpenAI.
|
||||
* @param {string} cvText - Plain text extracted from CV PDF.
|
||||
* @param {string} projectDescription - Plain text of the project description.
|
||||
* @returns {Promise<{matchPercentage: number, pros: string[], cons: string[]}>}
|
||||
*/
|
||||
export const evaluateMatch = async (cvText, projectDescription) => {
|
||||
if (!cvText || !projectDescription) {
|
||||
return {
|
||||
matchPercentage: 0,
|
||||
pros: ['Fehlende CV-Daten oder Projektbeschreibung für den Abgleich.'],
|
||||
cons: []
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch API key from DB, fallback to environment variable
|
||||
const apiKey = await Setting.getVal('openai_api_key') || process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API Key is not configured. Please set it in Settings.');
|
||||
}
|
||||
|
||||
const openai = new OpenAI({ apiKey });
|
||||
|
||||
const systemPrompt = `Du bist ein hochentwickeltes IT-Recruiting-Matching-System.
|
||||
Deine Aufgabe ist es, den Lebenslauf (CV) eines IT-Beraters mit einer Projektbeschreibung abzugleichen.
|
||||
Berechne eine realistische Übereinstimmungsquote (Trefferquote) in Prozent (Zahl von 0 bis 100).
|
||||
Analysiere Gemeinsamkeiten und Lücken.
|
||||
|
||||
Erstelle eine Liste von Stichpunkten für:
|
||||
1. Pros: Warum der Berater für das Projekt geeignet ist (Erfahrung, Kernkompetenzen, passende Programmiersprachen/Frameworks).
|
||||
2. Cons: Was gegen eine Eignung spricht oder welche Risiken/Lücken es gibt (fehlende Technologien, Sprachbarrieren, Reisebereitschaft, zu geringe Seniorität).
|
||||
|
||||
Antworte ausschließlich im JSON-Format mit folgendem Schema:
|
||||
{
|
||||
"matchPercentage": 75,
|
||||
"pros": [
|
||||
"Sehr gute Übereinstimmung bei React und Node.js-Kenntnissen.",
|
||||
"Erfahrung im geforderten Finanzsektor vorhanden."
|
||||
],
|
||||
"cons": [
|
||||
"Keine Erwähnung der geforderten Docker/Kubernetes-Erfahrung im CV.",
|
||||
"Projektsprache ist Deutsch (fließend), der Berater listet Deutsch nur auf B2-Niveau."
|
||||
]
|
||||
}
|
||||
Schreibe keinen zusätzlichen Text vor oder nach dem JSON-Objekt.`;
|
||||
|
||||
const userContent = `### LEBENSLAUF (CV):
|
||||
${cvText}
|
||||
|
||||
### PROJEKTBESCHREIBUNG:
|
||||
${projectDescription}`;
|
||||
|
||||
try {
|
||||
const response = await openai.chat.completions.create({
|
||||
model: 'gpt-4o-mini',
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userContent }
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
temperature: 0.1 // Low temperature for more deterministic analysis
|
||||
});
|
||||
|
||||
const rawContent = response.choices[0].message.content.trim();
|
||||
const result = JSON.parse(rawContent);
|
||||
|
||||
return {
|
||||
matchPercentage: Number(result.matchPercentage) || 0,
|
||||
pros: Array.isArray(result.pros) ? result.pros : [],
|
||||
cons: Array.isArray(result.cons) ? result.cons : []
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error during OpenAI project matching:', error);
|
||||
throw new Error('LLM Evaluation failed: ' + error.message);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
import pdf from 'pdf-parse';
|
||||
|
||||
/**
|
||||
* Extracts plain text from a CV PDF file buffer.
|
||||
* @param {Buffer} fileBuffer - The PDF file content as a buffer.
|
||||
* @returns {Promise<string>} - The extracted plain text.
|
||||
*/
|
||||
export const parsePdfCv = async (fileBuffer) => {
|
||||
try {
|
||||
const data = await pdf(fileBuffer);
|
||||
|
||||
// Clean up the text: remove double newlines and trim whitespace
|
||||
let text = data.text;
|
||||
text = text.replace(/\n\s*\n/g, '\n');
|
||||
text = text.trim();
|
||||
|
||||
return text;
|
||||
} catch (error) {
|
||||
console.error('Error parsing PDF CV:', error);
|
||||
throw new Error('Failed to parse PDF: ' + error.message);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,286 @@
|
||||
import { Agent, Portal, Project, Match, Setting } from '../models/index.js';
|
||||
import { scrapeFreelancermap, scrapeFreelanceDe, scrapeHays, scrapeEtengo, scrapeSThreePortal, SThreePortals } from './scraper.js';
|
||||
import { evaluateMatch } from './matchingEngine.js';
|
||||
import nodemailer from 'nodemailer';
|
||||
|
||||
// Helper to encrypt/decrypt credentials if needed, or send email notifications
|
||||
const sendEmailNotification = async (agent, project, matchPercentage, pros, cons) => {
|
||||
const active = await Setting.getVal('email_notifications_active') === 'true';
|
||||
if (!active) return;
|
||||
|
||||
const smtpHost = await Setting.getVal('smtp_host');
|
||||
const smtpPort = parseInt(await Setting.getVal('smtp_port', '587'), 10);
|
||||
const smtpUser = await Setting.getVal('smtp_user');
|
||||
const smtpPass = await Setting.getVal('smtp_pass');
|
||||
const smtpFrom = await Setting.getVal('smtp_from', 'scanner@perinco.de');
|
||||
const recipient = await Setting.getVal('notification_email_recipient') || agent.email;
|
||||
|
||||
if (!smtpHost || !smtpUser || !smtpPass || !recipient) {
|
||||
console.warn('SMTP settings are incomplete. Skipping email notification.');
|
||||
return;
|
||||
}
|
||||
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: smtpHost,
|
||||
port: smtpPort,
|
||||
secure: smtpPort === 465,
|
||||
auth: {
|
||||
user: smtpUser,
|
||||
pass: smtpPass
|
||||
}
|
||||
});
|
||||
|
||||
const prosList = pros.map(p => `<li>✅ ${p}</li>`).join('');
|
||||
const consList = cons.map(c => `<li>❌ ${c}</li>`).join('');
|
||||
|
||||
const mailOptions = {
|
||||
from: smtpFrom,
|
||||
to: recipient,
|
||||
subject: `[PROJEKT-MATCHING ${matchPercentage}%] Neue Chance für ${agent.name}`,
|
||||
html: `
|
||||
<div style="font-family: sans-serif; max-width: 600px; margin: 0 auto; padding: 20px; border: 1px solid #e0e0e0; border-radius: 8px;">
|
||||
<h2 style="color: #2A5B41;">Neues Projekt-Match gefunden</h2>
|
||||
<p>Hallo,</p>
|
||||
<p>es gibt ein neues passendes IT-Projekt auf <strong>${project.portal}</strong> für <strong>${agent.name}</strong>.</p>
|
||||
|
||||
<div style="background-color: #F4F7F5; padding: 15px; border-radius: 6px; margin: 20px 0;">
|
||||
<h3 style="margin-top: 0; color: #2A5B41;">${project.title}</h3>
|
||||
<p><strong>Portal:</strong> ${project.portal}<br>
|
||||
<strong>Ort:</strong> ${project.location}<br>
|
||||
<strong>Rate/Budget:</strong> ${project.rate}</p>
|
||||
<p><a href="${project.url}" target="_blank" style="background-color: #6A8DC3; color: white; padding: 10px 15px; text-decoration: none; border-radius: 4px; display: inline-block;">Projekt aufrufen</a></p>
|
||||
</div>
|
||||
|
||||
<div style="margin: 20px 0;">
|
||||
<h3 style="color: #2A5B41;">Eignung: <span style="color: ${matchPercentage >= 75 ? '#2A5B41' : '#E67E22'};">${matchPercentage}% Match</span></h3>
|
||||
|
||||
<h4 style="margin-bottom: 5px;">Vorteile (Warum es passt):</h4>
|
||||
<ul style="padding-left: 20px; margin-top: 0;">${prosList || '<li>Keine Angaben</li>'}</ul>
|
||||
|
||||
<h4 style="margin-bottom: 5px;">Herausforderungen (Gaps):</h4>
|
||||
<ul style="padding-left: 20px; margin-top: 0;">${consList || '<li>Keine Angaben</li>'}</ul>
|
||||
</div>
|
||||
|
||||
<p style="font-size: 12px; color: #7f8c8d; margin-top: 30px;">Diese E-Mail wurde automatisch vom Perinco Projekt Scanner generiert.</p>
|
||||
</div>
|
||||
`
|
||||
};
|
||||
|
||||
try {
|
||||
await transporter.sendMail(mailOptions);
|
||||
console.log(`Notification email sent for Agent ${agent.name} and Project ${project.title}`);
|
||||
} catch (error) {
|
||||
console.error('Failed to send email notification:', error);
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* Ein Lauf zur Zeit. Scheduler und der Knopf "Jetzt scannen" koennen sich sonst
|
||||
* ueberholen - zwei parallele Laeufe starten doppelt Chromium und bezahlen jede
|
||||
* Bewertung zweimal bei OpenAI.
|
||||
*/
|
||||
let pipelineRunning = false;
|
||||
|
||||
export const isPipelineRunning = () => pipelineRunning;
|
||||
|
||||
/**
|
||||
* Scrapen, Speichern und Bewerten in einem Durchlauf.
|
||||
* @returns {Promise<boolean>} false, wenn schon ein Lauf aktiv war.
|
||||
*/
|
||||
export const runMatchingPipeline = async () => {
|
||||
if (pipelineRunning) {
|
||||
console.log('[Pipeline] Lauf laeuft schon - neuer Aufruf uebersprungen.');
|
||||
return false;
|
||||
}
|
||||
pipelineRunning = true;
|
||||
try {
|
||||
return await executePipeline();
|
||||
} finally {
|
||||
pipelineRunning = false;
|
||||
}
|
||||
};
|
||||
|
||||
const executePipeline = async () => {
|
||||
console.log('--- STARTING MATCHING PIPELINE ---');
|
||||
|
||||
// 1. Fetch active agents
|
||||
const activeAgents = await Agent.findAll({ where: { isActive: true } });
|
||||
if (activeAgents.length === 0) {
|
||||
console.log('No active agents found. Pipeline execution stopped.');
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. Extract unique keywords from active agents
|
||||
const keywordSet = new Set();
|
||||
activeAgents.forEach(agent => {
|
||||
if (agent.keywords) {
|
||||
agent.keywords.split(',').forEach(k => {
|
||||
const trimmed = k.trim();
|
||||
if (trimmed) keywordSet.add(trimmed);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const keywords = Array.from(keywordSet);
|
||||
if (keywords.length === 0) {
|
||||
console.log('No active search keywords found. Pipeline execution stopped.');
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log(`Keywords to scrape: ${keywords.join(', ')}`);
|
||||
|
||||
// 3. Fetch portal configurations
|
||||
const freelancermapPortal = await Portal.findOne({ where: { name: 'freelancermap.de' } });
|
||||
const freelanceDePortal = await Portal.findOne({ where: { name: 'freelance.de' } });
|
||||
|
||||
// 4. Run scrapers
|
||||
let scrapedProjects = [];
|
||||
|
||||
try {
|
||||
const mapResult = await scrapeFreelancermap(keywords, freelancermapPortal?.sessionState);
|
||||
scrapedProjects = scrapedProjects.concat(mapResult.projects);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Freelancermap:', err);
|
||||
}
|
||||
|
||||
try {
|
||||
const freelanceResult = await scrapeFreelanceDe(keywords, freelanceDePortal?.sessionState);
|
||||
scrapedProjects = scrapedProjects.concat(freelanceResult.projects);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Freelance.de:', err);
|
||||
}
|
||||
|
||||
try {
|
||||
const haysResult = await scrapeHays(keywords);
|
||||
scrapedProjects = scrapedProjects.concat(haysResult.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: 'hays.de' } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Hays.de:', err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: 'hays.de' } });
|
||||
}
|
||||
|
||||
try {
|
||||
const etengoResult = await scrapeEtengo(keywords);
|
||||
scrapedProjects = scrapedProjects.concat(etengoResult.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: 'etengo.de' } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Etengo.de:', err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: 'etengo.de' } });
|
||||
}
|
||||
|
||||
// SThree-family staffing portals (shared template — generic scraper).
|
||||
for (const cfg of SThreePortals) {
|
||||
try {
|
||||
const result = await scrapeSThreePortal(cfg);
|
||||
scrapedProjects = scrapedProjects.concat(result.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: cfg.name } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(`Error scraping ${cfg.name}:`, err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: cfg.name } });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Total projects scraped: ${scrapedProjects.length}`);
|
||||
|
||||
// 5. Store projects and run matching analysis
|
||||
let newProjectsStoredCount = 0;
|
||||
let matchesEvaluatedCount = 0;
|
||||
|
||||
const thresholdStr = await Setting.getVal('match_threshold', '70');
|
||||
const matchThreshold = parseInt(thresholdStr, 10) || 70;
|
||||
|
||||
for (const scrapedProj of scrapedProjects) {
|
||||
try {
|
||||
// Find or create project in DB
|
||||
const [project, created] = await Project.findOrCreate({
|
||||
where: {
|
||||
portal: scrapedProj.portal,
|
||||
portalProjectId: scrapedProj.portalProjectId
|
||||
},
|
||||
defaults: scrapedProj
|
||||
});
|
||||
|
||||
if (created) {
|
||||
newProjectsStoredCount++;
|
||||
}
|
||||
|
||||
// Perform matching analysis with active agents
|
||||
for (const agent of activeAgents) {
|
||||
// Skip if match already exists
|
||||
const existingMatch = await Match.findOne({
|
||||
where: {
|
||||
agentId: agent.id,
|
||||
projectId: project.id
|
||||
}
|
||||
});
|
||||
|
||||
if (existingMatch) continue;
|
||||
|
||||
// Exclusion list: skip this project for the agent if its title or
|
||||
// description contains any excluded term (case-insensitive). Runs
|
||||
// before the OpenAI call so excluded projects cost nothing.
|
||||
if (agent.excludeKeywords) {
|
||||
const haystack = `${project.title || ''} ${project.description || ''}`.toLowerCase();
|
||||
const excluded = agent.excludeKeywords
|
||||
.split(',')
|
||||
.map(t => t.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.find(term => haystack.includes(term));
|
||||
if (excluded) {
|
||||
console.log(`Skipping project [${project.title}] for Agent [${agent.name}] — matched exclusion term "${excluded}".`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Skip matching if CV text is empty
|
||||
if (!agent.cvText) {
|
||||
console.log(`Skipping match for Agent ${agent.name} (ID: ${agent.id}) - No CV text uploaded.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`Evaluating match: Agent [${agent.name}] <-> Project [${project.title}]`);
|
||||
|
||||
try {
|
||||
const matchResult = await evaluateMatch(agent.cvText, project.description);
|
||||
matchesEvaluatedCount++;
|
||||
|
||||
const match = await Match.create({
|
||||
agentId: agent.id,
|
||||
projectId: project.id,
|
||||
matchPercentage: matchResult.matchPercentage,
|
||||
pros: matchResult.pros,
|
||||
cons: matchResult.cons,
|
||||
status: 'new',
|
||||
notificationSent: false
|
||||
});
|
||||
|
||||
// Trigger email notification if threshold reached
|
||||
if (match.matchPercentage >= matchThreshold) {
|
||||
await sendEmailNotification(agent, project, match.matchPercentage, match.pros, match.cons);
|
||||
match.notificationSent = true;
|
||||
await match.save();
|
||||
}
|
||||
} catch (matchErr) {
|
||||
console.error(`Failed to evaluate match for Agent ${agent.id} and Project ${project.id}:`, matchErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (dbErr) {
|
||||
console.error('Error inserting project or creating match:', dbErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`--- PIPELINE COMPLETED ---`);
|
||||
console.log(`Stored ${newProjectsStoredCount} new projects.`);
|
||||
console.log(`Evaluated ${matchesEvaluatedCount} new matches.`);
|
||||
return true;
|
||||
};
|
||||
@@ -0,0 +1,125 @@
|
||||
import cron from 'node-cron';
|
||||
import Setting from '../models/Setting.js';
|
||||
import { runMatchingPipeline } from './pipeline.js';
|
||||
import { cleanupOldProjects } from './cleanup.js';
|
||||
|
||||
let currentJob = null;
|
||||
let cleanupJob = null;
|
||||
let currentIntervalMinutes = null;
|
||||
|
||||
/**
|
||||
* Startet den Hintergrund-Crawler und plant sich bei geaenderten Intervallen
|
||||
* selbst neu.
|
||||
*/
|
||||
export const startScheduler = async () => {
|
||||
try {
|
||||
const intervalStr = await Setting.getVal('search_interval', '60');
|
||||
const intervalMinutes = parseInt(intervalStr, 10) || 60;
|
||||
|
||||
currentIntervalMinutes = intervalMinutes;
|
||||
console.log('[Scheduler] Cron-Scheduler wird eingerichtet.');
|
||||
console.log(`[Scheduler] Suchintervall: ${intervalMinutes} Minuten.`);
|
||||
|
||||
// If a job is already running, stop it
|
||||
if (currentJob) {
|
||||
currentJob.stop();
|
||||
console.log('[Scheduler] Vorherigen Cron-Job gestoppt.');
|
||||
}
|
||||
|
||||
/*
|
||||
* Zeitversatz zum parallel laufenden alten Scanner. Ohne ihn starten beide
|
||||
* Instanzen direkt nach einem Neustart gleichzeitig und greifen mit
|
||||
* denselben Portal-Sessions zeitgleich auf die Boersen zu - das provoziert
|
||||
* Rate-Limits und CAPTCHAs. Mit Versatz laufen die Laeufe verschraenkt.
|
||||
*/
|
||||
const offsetMinutes = Number(process.env.SCANNER_SCHEDULER_OFFSET ?? 30);
|
||||
const useOffset = Number.isFinite(offsetMinutes) && offsetMinutes > 0;
|
||||
|
||||
/*
|
||||
* Eigene Sperrfrist statt eines vorgealterten lastRunTime: ein Zeitstempel
|
||||
* kann einen Lauf nur vorziehen, nie verzoegern. Sobald der Versatz groesser
|
||||
* als das Suchintervall ist - bei 15 Minuten Intervall der Normalfall -
|
||||
* waere der erste Lauf sonst sofort faellig gewesen.
|
||||
*/
|
||||
const notBefore = useOffset ? Date.now() + offsetMinutes * 60 * 1000 : 0;
|
||||
let lastRunTime = null;
|
||||
|
||||
if (useOffset) {
|
||||
console.log(`[Scheduler] Erster Lauf in ${offsetMinutes} Minuten (Versatz zum alten Scanner).`);
|
||||
}
|
||||
|
||||
// We check every minute if we are due for a run.
|
||||
// This allows exact dynamic changes without having to manually restart the node process.
|
||||
currentJob = cron.schedule('* * * * *', async () => {
|
||||
// Master on/off switch (toggled from Settings UI). When off, the
|
||||
// background scanner stays idle but the backend/UI keep running.
|
||||
const scanningActive = await Setting.getVal('scanning_active', 'true');
|
||||
if (scanningActive === 'false') {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeIntervalStr = await Setting.getVal('search_interval', '60');
|
||||
const activeIntervalMinutes = parseInt(activeIntervalStr, 10) || 60;
|
||||
|
||||
// Geaendertes Intervall im Betrieb uebernehmen
|
||||
if (activeIntervalMinutes !== currentIntervalMinutes) {
|
||||
console.log(`[Scheduler] Intervall von ${currentIntervalMinutes} auf ${activeIntervalMinutes} Minuten geaendert - Neuplanung.`);
|
||||
startScheduler();
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Sperrfrist nach dem Start (Versatz zum parallel laufenden Scanner)
|
||||
if (notBefore && now.getTime() < notBefore) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Weekday gate: skip Saturday (6) and Sunday (0). Scraper runs Mon-Fri only.
|
||||
const day = now.getDay();
|
||||
if (day === 0 || day === 6) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Time-of-day gate: only scan between 07:00 and 21:59 server-local time
|
||||
// (Europe/Berlin). getHours() returns local hour on the host.
|
||||
const hour = now.getHours();
|
||||
if (hour < 7 || hour >= 22) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Run if never run before, or if enough time has passed
|
||||
const isDue = !lastRunTime || (now - lastRunTime) >= activeIntervalMinutes * 60 * 1000;
|
||||
|
||||
if (isDue) {
|
||||
lastRunTime = now;
|
||||
console.log(`[Scheduler] Geplanter Lauf startet: ${now.toLocaleTimeString('de-DE')}`);
|
||||
try {
|
||||
await runMatchingPipeline();
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Fehler im Suchlauf:', err);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
currentJob.start();
|
||||
console.log('[Scheduler] Cron-Job laeuft, Pruefung jede Minute.');
|
||||
|
||||
// Taegliches Aufraeumen um 03:30 - halbe Stunde nach dem alten Scanner,
|
||||
// damit nicht beide gleichzeitig grosse Loeschlaeufe auf derselben MariaDB
|
||||
// fahren. Laeuft nur einmal, auch wenn der Scheduler neu geplant wird.
|
||||
if (!cleanupJob) {
|
||||
cleanupJob = cron.schedule('30 3 * * *', () => {
|
||||
console.log('[Scheduler] Taegliches Aufraeumen laeuft.');
|
||||
cleanupOldProjects().catch(err => console.error('[Scheduler] Fehler beim Aufraeumen:', err));
|
||||
});
|
||||
cleanupJob.start();
|
||||
console.log('[Scheduler] Aufraeum-Job registriert (03:30).');
|
||||
|
||||
// Also run once on boot so retention applies immediately after restart.
|
||||
cleanupOldProjects().catch(() => {});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Scheduler] Failed to initialize scheduler:', error);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,720 @@
|
||||
import { chromium } from 'playwright';
|
||||
import Portal from '../models/Portal.js';
|
||||
|
||||
// Stealth-like launch args — no need for playwright-extra/puppeteer-extra-plugin-stealth
|
||||
// We pass anti-detection flags and a realistic user agent directly.
|
||||
const STEALTH_ARGS = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars',
|
||||
];
|
||||
|
||||
// Defensive parse: storage state stored in DB may originate from older sync
|
||||
// versions or from Cookie-Editor exports with non-Playwright cookie shapes.
|
||||
// Normalize before passing to browser.newContext().
|
||||
const parseStorageState = (sessionState) => {
|
||||
if (!sessionState) return null;
|
||||
let raw;
|
||||
try {
|
||||
raw = typeof sessionState === 'string' ? JSON.parse(sessionState) : sessionState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
let rawCookies = [];
|
||||
let origins = [];
|
||||
if (Array.isArray(raw)) {
|
||||
rawCookies = raw;
|
||||
} else if (raw && typeof raw === 'object') {
|
||||
rawCookies = Array.isArray(raw.cookies) ? raw.cookies : [];
|
||||
origins = Array.isArray(raw.origins) ? raw.origins : [];
|
||||
}
|
||||
|
||||
const cookies = rawCookies.map(c => {
|
||||
if (!c || !c.name || c.value === undefined) return null;
|
||||
let sameSite = c.sameSite;
|
||||
if (typeof sameSite === 'string') {
|
||||
const s = sameSite.toLowerCase();
|
||||
if (s === 'strict') sameSite = 'Strict';
|
||||
else if (s === 'lax') sameSite = 'Lax';
|
||||
else if (s === 'none' || s === 'no_restriction') sameSite = 'None';
|
||||
else sameSite = 'Lax';
|
||||
} else {
|
||||
sameSite = 'Lax';
|
||||
}
|
||||
let expires = c.expires;
|
||||
if (expires === undefined && c.expirationDate !== undefined) expires = c.expirationDate;
|
||||
if (typeof expires !== 'number' || !isFinite(expires)) expires = -1;
|
||||
return {
|
||||
name: String(c.name),
|
||||
value: String(c.value),
|
||||
domain: c.domain || '',
|
||||
path: c.path || '/',
|
||||
expires,
|
||||
httpOnly: !!c.httpOnly,
|
||||
secure: !!c.secure,
|
||||
sameSite,
|
||||
};
|
||||
}).filter(Boolean);
|
||||
|
||||
if (cookies.length === 0) return null;
|
||||
return { cookies, origins };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelancermap.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelancermap = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelancermap scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
// Initialize context with cookies if available
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelancermap session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelancermap session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelancermap.de/dashboard.html', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="/login"]');
|
||||
const userProfile = await page.$('.user-avatar, .user-name, a[href*="/logout"]');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelancermap session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelancermap session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelancermap session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Loop through each keyword to search
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.freelancermap.de/projekte?q=${encodeURIComponent(keyword)}&sort=created_desc`;
|
||||
console.log(`Navigating to Freelancermap search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Freelancermap project cards use stable class `.project-card`. Detail
|
||||
// attributes use data-testid which is reliable across rerenders.
|
||||
const projectElements = await page.$$('.project-card');
|
||||
console.log(`Found ${projectElements.length} project cards for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
const linkEl = await el.$('a[data-testid="title"], a[data-id="project-card-title"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = (await linkEl.innerText()).trim();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelancermap.de${rawUrl}`;
|
||||
|
||||
// Project ID: trailing numeric segment if present, else full slug.
|
||||
const slugMatch = url.match(/\/projekt\/([^/?#]+)/);
|
||||
const slug = slugMatch ? slugMatch[1] : null;
|
||||
const idMatch = slug ? slug.match(/-(\d+)$/) : null;
|
||||
const portalProjectId = idMatch ? idMatch[1] : (slug || url);
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Company: first div in .project-info (no class on text node)
|
||||
const company = await el.evaluate(node => {
|
||||
const info = node.querySelector('.project-info');
|
||||
if (!info) return '';
|
||||
const first = info.querySelector('div');
|
||||
return first ? first.innerText.trim() : '';
|
||||
});
|
||||
|
||||
// Location: data-testid="city" container
|
||||
const location = await el.evaluate(node => {
|
||||
const c = node.querySelector('[data-testid="city"]');
|
||||
return c ? c.innerText.replace(/\s+/g, ' ').trim() : '';
|
||||
});
|
||||
|
||||
// Type / duration / start → concatenated into rate as listing card has no €
|
||||
const meta = await el.evaluate(node => {
|
||||
const parts = [];
|
||||
['type', 'duration', 'beginningText', 'remoteInPercent'].forEach(t => {
|
||||
const e2 = node.querySelector(`[data-testid="${t}"]`);
|
||||
if (e2) parts.push(e2.innerText.replace(/\s+/g, ' ').trim());
|
||||
});
|
||||
return parts.join(' | ');
|
||||
});
|
||||
|
||||
// Listing card has no description text — fetch detail page lazily would
|
||||
// be expensive; leave description empty here. Matching engine handles
|
||||
// empty descriptions by returning low score, but title + meta still
|
||||
// give signal once we add a detail fetch step later.
|
||||
const description = `${title}\n${meta}`.trim();
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelancermap.de',
|
||||
portalProjectId,
|
||||
title,
|
||||
description,
|
||||
company: company || 'N/A',
|
||||
location: location || 'Remote / N/A',
|
||||
rate: meta || 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Freelancermap card:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid (cookies might rotate)
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
// Session was provided but check failed
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelancermap scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes job listings from Hays.de. Public portal, no login required.
|
||||
* Search URL: https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=<keyword>&e=false
|
||||
* joblevel=3 (Senior/Lead-Bereich, vom Nutzer vorgegeben), mrew=1, e=false (alle Vertragsarten).
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeHays = async (keywords) => {
|
||||
console.log(`Starting Hays.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=${encodeURIComponent(keyword)}&e=false`;
|
||||
console.log(`Navigating to Hays search: ${searchUrl}`);
|
||||
|
||||
try {
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
// Wait for actual job-detail anchors (JS hydrates them).
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/jobsuche/stellenangebote-jobs-detail-"]', { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`Hays: no job-detail anchors visible for "${keyword}"`);
|
||||
continue;
|
||||
}
|
||||
await page.waitForTimeout(800 + Math.random() * 1000);
|
||||
|
||||
// Only real job-detail URLs. Pattern: /jobsuche/stellenangebote-jobs-detail-<slug>-<id>/<page>
|
||||
const linkEls = await page.$$('a[href*="/jobsuche/stellenangebote-jobs-detail-"]');
|
||||
console.log(`Found ${linkEls.length} Hays job-detail anchors for keyword: ${keyword}`);
|
||||
|
||||
for (const linkEl of linkEls) {
|
||||
try {
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!rawUrl) continue;
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.hays.de${rawUrl}`;
|
||||
|
||||
// Hays job-id: numeric segment at end before trailing /N
|
||||
const idMatch = url.match(/-(\d{5,})(?:\/\d+)?(?:[?#].*)?$/);
|
||||
if (!idMatch) continue;
|
||||
const portalProjectId = idMatch[1];
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Title: link inner text first, fall back to nearest heading.
|
||||
let title = ((await linkEl.innerText()) || '').replace(/\s+/g, ' ').trim();
|
||||
if (!title) {
|
||||
title = await linkEl.evaluate(node => {
|
||||
let n = node;
|
||||
for (let i = 0; i < 6 && n; i++) {
|
||||
n = n.parentElement;
|
||||
if (!n) break;
|
||||
const h = n.querySelector('h2, h3, [class*="title"], [class*="Title"]');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
return '';
|
||||
}).catch(() => '');
|
||||
}
|
||||
// Derive title from URL slug if still missing.
|
||||
if (!title) {
|
||||
const slugMatch = url.match(/stellenangebote-jobs-detail-(.+?)-\d{5,}/);
|
||||
if (slugMatch) {
|
||||
title = slugMatch[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
if (!title) continue;
|
||||
|
||||
// Card text: closest ancestor that contains structured info (max 6 levels up).
|
||||
const meta = await linkEl.evaluate(node => {
|
||||
let card = node;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (!card.parentElement) break;
|
||||
card = card.parentElement;
|
||||
const txt = (card.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
if (txt.length > 80 && txt.length < 1500) return txt.slice(0, 1200);
|
||||
}
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||||
}).catch(() => '');
|
||||
|
||||
// Location: try common DE cities or fall back to scanning "Standort:" pattern.
|
||||
const locMatch = meta.match(/Standort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| meta.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Dortmund|Essen|Bremen|Dresden|Hannover|Nürnberg|Mannheim|Karlsruhe|Wiesbaden|Bonn|Münster|Augsburg|Aachen|Erfurt|Mainz|Kiel|Halle|Magdeburg|Freiburg|Lübeck|Oberhausen|Rostock|Kassel|Saarbrücken|Mönchengladbach|Krefeld|Chemnitz|Braunschweig|Wuppertal|Bielefeld|Duisburg|remote|Remote)\b/);
|
||||
const location = locMatch ? locMatch[1] || locMatch[0] : 'Remote / N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'hays.de',
|
||||
portalProjectId,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${meta}`.trim().slice(0, 1500),
|
||||
company: 'Hays (Vermittlung)',
|
||||
location: location.slice(0, 200),
|
||||
rate: (meta.match(/(?:€|EUR)[^|·]{0,40}/) || ['N/A'])[0].slice(0, 90),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Hays card:', err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping Hays for "${keyword}":`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Hays scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Generic scraper for the SThree-family staffing portals (Huxley, Computer
|
||||
* Futures, Progressive, Real Staffing, Global Enterprise Partners, Madison
|
||||
* Black, JP Gray, etc.). They all share the same template — job detail URLs
|
||||
* follow `<base>/<locale>/job/<slug>/<numeric-id>/`. Public, no login.
|
||||
*
|
||||
* @param {object} cfg
|
||||
* @param {string} cfg.name - DB portal name (e.g. 'huxley.com')
|
||||
* @param {string} cfg.listingUrl - Search-result listing URL
|
||||
* @param {string} cfg.localePath - e.g. '/de-de/' or '/en-nl/'
|
||||
*/
|
||||
const scrapeSThreeStyle = async ({ name, listingUrl, localePath }) => {
|
||||
console.log(`Starting ${name} scrape (listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(listingUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
const jobLinkSelector = `a[href*="${localePath}job/"]`;
|
||||
try {
|
||||
await page.waitForSelector(jobLinkSelector, { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`${name}: no job anchors visible after wait.`);
|
||||
}
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
const links = await page.$$eval(jobLinkSelector, (els, locale) => {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
// Escape regex special chars in locale (keeps slashes as literal).
|
||||
const esc = locale.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const re = new RegExp(esc + 'job/([^/]+)/(\\d{4,})/?');
|
||||
for (const a of els) {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (href.includes('{{')) continue;
|
||||
const m = href.match(re);
|
||||
if (!m) continue;
|
||||
const id = m[2];
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
out.push({ id, slug: m[1], href });
|
||||
}
|
||||
return out;
|
||||
}, localePath).catch(() => []);
|
||||
console.log(`Found ${links.length} ${name} job URLs.`);
|
||||
|
||||
// Detail-page fetches were triggering Cloudflare/WAF on the SThree
|
||||
// family. Title is derived from the URL slug; description = title only.
|
||||
// No detail requests → no bot signature, no rate spikes.
|
||||
for (const link of links) {
|
||||
try {
|
||||
const url = link.href.startsWith('http') ? link.href : new URL(link.href, listingUrl).toString();
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === link.id)) continue;
|
||||
|
||||
let title = '';
|
||||
try {
|
||||
title = decodeURIComponent(link.slug);
|
||||
} catch {
|
||||
title = link.slug;
|
||||
}
|
||||
title = title.replace(/-/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
// Title-case for the slug pieces (German diacritics kept).
|
||||
title = title.replace(/\b(\w)([\w'’äöüß]*)/g, (_, a, b) => a.toUpperCase() + b);
|
||||
if (!title) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: name,
|
||||
portalProjectId: link.id,
|
||||
title: title.slice(0, 250),
|
||||
description: title.slice(0, 250),
|
||||
company: `${name} (Vermittlung)`,
|
||||
location: 'Siehe Anzeige',
|
||||
rate: 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error building ${name} project entry:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping ${name} listing:`, err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error during ${name} scrape:`, error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`${name} scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
// Per-portal wrappers used by pipeline.js / api.js
|
||||
export const SThreePortals = [
|
||||
{ name: 'huxley.com', listingUrl: 'https://www.huxley.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'computerfutures.com', listingUrl: 'https://www.computerfutures.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'progressiverecruitment.com', listingUrl: 'https://www.progressiverecruitment.com/de-de/jobs-projekte-suchen/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'realstaffing.com', listingUrl: 'https://www.realstaffing.com/de-de/job-suche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'globalenterprisepartners.com', listingUrl: 'https://www.globalenterprisepartners.com/de-de/job-search/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'madisonblack.com', listingUrl: 'https://www.madisonblack.com/de-de/job-search/?searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'jpgray.nl', listingUrl: 'https://www.jpgray.nl/en-nl/job-search/?country=Netherlands&searchRadius=50km', localePath: '/en-nl/' }
|
||||
];
|
||||
|
||||
export const scrapeSThreePortal = (cfg) => scrapeSThreeStyle(cfg);
|
||||
|
||||
/**
|
||||
* Scrapes IT projects from Etengo.de. Public portal, no login required.
|
||||
* Search URL: https://www.etengo.de/it-projektsuche/?q=<keyword>
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeEtengo = async (/* keywords */) => {
|
||||
// Etengo's search filter is JS-side and the listing URL ignores ?q. Load the
|
||||
// full project list once and return all entries; matching engine handles
|
||||
// keyword relevance per agent CV.
|
||||
console.log(`Starting Etengo.de scrape (full listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto('https://www.etengo.de/it-projektsuche/', { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/it-projektsuche/"][href*="/"]', { timeout: 10000 });
|
||||
} catch {
|
||||
console.warn('Etengo: project anchors not visible.');
|
||||
}
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Collect detail-page URLs: /it-projektsuche/<numeric-id>/
|
||||
const anchors = await page.$$('a[href*="/it-projektsuche/"]');
|
||||
const seen = new Set();
|
||||
const detailUrls = [];
|
||||
for (const a of anchors) {
|
||||
const href = await a.getAttribute('href');
|
||||
if (!href) continue;
|
||||
const m = href.match(/\/it-projektsuche\/(\d{3,})\/?$/);
|
||||
if (!m) continue;
|
||||
const url = href.startsWith('http') ? href : `https://www.etengo.de${href}`;
|
||||
if (seen.has(m[1])) continue;
|
||||
seen.add(m[1]);
|
||||
detailUrls.push({ id: m[1], url });
|
||||
}
|
||||
console.log(`Found ${detailUrls.length} Etengo project URLs.`);
|
||||
|
||||
// Fetch each detail page in same context for title + description.
|
||||
for (const { id, url } of detailUrls) {
|
||||
try {
|
||||
const detail = await context.newPage();
|
||||
await detail.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
||||
await detail.waitForTimeout(500);
|
||||
|
||||
const title = await detail.evaluate(() => {
|
||||
const h = document.querySelector('h1');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
const t = document.querySelector('title');
|
||||
return t ? t.innerText.replace(/ - Etengo.*$/i, '').trim() : '';
|
||||
}).catch(() => '');
|
||||
|
||||
const description = await detail.evaluate(() => {
|
||||
// Look for main article / project description container.
|
||||
const cand = document.querySelector('main, article, .entry-content, .project-detail, [class*="detail"]');
|
||||
const node = cand || document.body;
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 1500);
|
||||
}).catch(() => '');
|
||||
|
||||
const location = (description.match(/Einsatzort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| description.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Hannover|Nürnberg|Bonn|Münster|Karlsruhe|Mannheim|Bremen|Dresden|Erfurt|Mainz|remote|Remote)\b/) || ['Remote / N/A'])[0];
|
||||
|
||||
const rateMatch = description.match(/(?:€|EUR)[^|·]{0,40}/);
|
||||
|
||||
await detail.close();
|
||||
|
||||
if (!title) continue;
|
||||
if (projectsList.some(p => p.portalProjectId === id)) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: 'etengo.de',
|
||||
portalProjectId: id,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${description}`.trim().slice(0, 1500),
|
||||
company: 'Etengo (Vermittlung)',
|
||||
location: String(location).slice(0, 200),
|
||||
rate: rateMatch ? rateMatch[0].slice(0, 90) : 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error fetching Etengo detail ${url}:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error scraping Etengo listing:', err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Etengo scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`Etengo scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelance.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelanceDe = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelance.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelance.de session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelance.de session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelance.de/users/profile.php', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="login"], a[href*="sign-in"]');
|
||||
const userProfile = await page.$('a[href*="logout"], .user-info, .profile-settings');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelance.de session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelance.de session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelance.de session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
for (const keyword of keywords) {
|
||||
// freelance.de search url: search_sort_by=2 means date descending (newest first)
|
||||
const searchUrl = `https://www.freelance.de/search/project.php?__search_sort_by=2&query=${encodeURIComponent(keyword)}`;
|
||||
console.log(`Navigating to Freelance.de search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Select project list items
|
||||
const projectElements = await page.$$('.project-list-item, .project-row, .list-item-main, div[class*="project-list-item"]');
|
||||
console.log(`Found ${projectElements.length} potential project elements for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
// Find the link to the project detail page
|
||||
// Format e.g., /Projekte/Projekt-Details-12345.html
|
||||
const linkEl = await el.$('a[href*="/Projekte/"], a[href*="/Projekt/"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = await linkEl.innerText();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelance.de${rawUrl}`;
|
||||
|
||||
// Parse project ID from URL
|
||||
// Format e.g., /Projekte/123456-React-Entwickler or /Projekt/123456
|
||||
const matchId = url.match(/\/(\d+)-/) || url.match(/\/(\d+)\.html/) || url.match(/\/(\d+)(?:\/|\?|$)/);
|
||||
const portalProjectId = matchId ? matchId[1] : url;
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Scrape details from listing card
|
||||
const descEl = await el.$('.project-description, .description, p, .project-details');
|
||||
const description = descEl ? await descEl.innerText() : '';
|
||||
|
||||
const locEl = await el.$('.location, .project-location, span[title*="Ort"], .meta-location');
|
||||
const location = locEl ? await locEl.innerText() : 'Remote / N/A';
|
||||
|
||||
const rateEl = await el.$('.rate, .project-rate, .meta-rate, span[title*="Budget"]');
|
||||
const rate = rateEl ? await rateEl.innerText() : 'N/A';
|
||||
|
||||
const compEl = await el.$('.company, .project-company, .meta-company');
|
||||
const company = compEl ? await compEl.innerText() : 'N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelance.de',
|
||||
portalProjectId,
|
||||
title: title.trim(),
|
||||
description: description.trim(),
|
||||
company: company.trim(),
|
||||
location: location.trim(),
|
||||
rate: rate.trim(),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing individual Freelance.de element:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelance.de scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
@@ -0,0 +1,29 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
/*
|
||||
* Verzeichnis der CV-PDFs dieser Scanner-Instanz. Absolut aufgeloest, damit es
|
||||
* nicht am Arbeitsverzeichnis des Prozesses haengt.
|
||||
*
|
||||
* Bewusst ein EIGENES Verzeichnis und nicht das der alten App: beide Instanzen
|
||||
* verwalten ihre Agenten getrennt, und ein Loeschen hier darf dort keine Datei
|
||||
* mitreissen. Bestehende CVs holt `npm run scanner:import` herueber.
|
||||
*/
|
||||
export const UPLOAD_DIR = path.resolve(
|
||||
process.env.SCANNER_UPLOAD_DIR || path.join(process.cwd(), 'uploads')
|
||||
);
|
||||
|
||||
export function ensureUploadDir() {
|
||||
if (!fs.existsSync(UPLOAD_DIR)) {
|
||||
fs.mkdirSync(UPLOAD_DIR, { recursive: true });
|
||||
}
|
||||
return UPLOAD_DIR;
|
||||
}
|
||||
|
||||
/** Loest einen in der DB gespeicherten CV-Pfad gegen UPLOAD_DIR auf. */
|
||||
export function resolveCvPath(stored) {
|
||||
if (!stored) return null;
|
||||
if (path.isAbsolute(stored) && fs.existsSync(stored)) return stored;
|
||||
const candidate = path.join(UPLOAD_DIR, path.basename(stored));
|
||||
return fs.existsSync(candidate) ? candidate : null;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import CryptoJS from 'crypto-js';
|
||||
|
||||
/*
|
||||
* Eigener Schluessel, unabhaengig vom alten Scanner: die Portal-Passwoerter
|
||||
* liegen in einer eigenen Datenbank und werden hier neu verschluesselt. Der
|
||||
* Schluessel der alten App wird absichtlich NICHT als Fallback gelesen - beide
|
||||
* Instanzen sollen sich nicht ueber gemeinsame Geheimnisse verkoppeln.
|
||||
*
|
||||
* Ohne SCANNER_ENCRYPTION_KEY laeuft alles weiter, aber mit einem bekannten
|
||||
* Ersatzschluessel - das ist nur fuer die lokale Entwicklung gedacht, deshalb
|
||||
* die Warnung.
|
||||
*/
|
||||
const FALLBACK_KEY = 'perinco-scanner-dev-key-nicht-fuer-produktion';
|
||||
|
||||
export const SCANNER_KEY_CONFIGURED = Boolean(process.env.SCANNER_ENCRYPTION_KEY);
|
||||
|
||||
const SECRET_KEY = process.env.SCANNER_ENCRYPTION_KEY || FALLBACK_KEY;
|
||||
|
||||
/** Verschluesselt ein Klartext-Passwort. */
|
||||
export const encrypt = (text) => {
|
||||
if (!text) return '';
|
||||
return CryptoJS.AES.encrypt(text, SECRET_KEY).toString();
|
||||
};
|
||||
|
||||
/** Entschluesselt einen Cipher-String; leerer String, wenn es nicht passt. */
|
||||
export const decrypt = (cipherText) => {
|
||||
if (!cipherText) return '';
|
||||
try {
|
||||
const bytes = CryptoJS.AES.decrypt(cipherText, SECRET_KEY);
|
||||
return bytes.toString(CryptoJS.enc.Utf8);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Entschluesselt mit einem fremden Schluessel - gebraucht beim Uebernehmen der
|
||||
* Portal-Zugangsdaten aus der alten Scanner-Datenbank.
|
||||
*/
|
||||
export const decryptWith = (cipherText, key) => {
|
||||
if (!cipherText || !key) return '';
|
||||
try {
|
||||
const bytes = CryptoJS.AES.decrypt(cipherText, key);
|
||||
return bytes.toString(CryptoJS.enc.Utf8);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
-- perinco CRM - Schema fuer MariaDB-Datenbank perinco_CRM
|
||||
-- Idempotent: kann jederzeit erneut ausgefuehrt werden.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS crm_users (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
username VARCHAR(64) NOT NULL,
|
||||
email VARCHAR(190) NULL,
|
||||
name VARCHAR(190) NOT NULL,
|
||||
role ENUM('admin','user') NOT NULL DEFAULT 'user',
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uq_crm_users_username (username),
|
||||
UNIQUE KEY uq_crm_users_email (email)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Persoenliche Farbe fuer die Kalenderansicht. NULL = automatisch aus der
|
||||
-- Palette (siehe userColor() im Frontend). ADD COLUMN IF NOT EXISTS haelt das
|
||||
-- Statement idempotent (MariaDB-Erweiterung).
|
||||
ALTER TABLE crm_users ADD COLUMN IF NOT EXISTS color VARCHAR(9) NULL AFTER role;
|
||||
|
||||
-- Benutzerfoto als Data-URL oder externe URL - wie bei crm_leads.avatar.
|
||||
ALTER TABLE crm_users ADD COLUMN IF NOT EXISTS avatar LONGTEXT NULL AFTER color;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS crm_leads (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
salutation VARCHAR(32) NULL,
|
||||
first_name VARCHAR(120) NOT NULL,
|
||||
last_name VARCHAR(120) NOT NULL,
|
||||
company VARCHAR(190) NULL,
|
||||
position VARCHAR(190) NULL,
|
||||
email VARCHAR(190) NULL,
|
||||
phone VARCHAR(64) NULL,
|
||||
mobile VARCHAR(64) NULL,
|
||||
linkedin VARCHAR(255) NULL,
|
||||
website VARCHAR(255) NULL,
|
||||
street VARCHAR(190) NULL,
|
||||
zip VARCHAR(24) NULL,
|
||||
city VARCHAR(120) NULL,
|
||||
country VARCHAR(120) NULL,
|
||||
status ENUM('Neu','Kontaktiert','Qualifiziert','Angebot','Gewonnen','Verloren')
|
||||
NOT NULL DEFAULT 'Neu',
|
||||
source VARCHAR(120) NULL,
|
||||
tags LONGTEXT NULL, -- JSON-Array als Text (MariaDB-kompatibel)
|
||||
notes TEXT NULL,
|
||||
avatar LONGTEXT NULL, -- Data-URL oder externe URL
|
||||
owner_id INT UNSIGNED NULL, -- zustaendiger Benutzer
|
||||
created_by INT UNSIGNED NULL, -- wer den Kontakt angelegt hat
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
KEY idx_crm_leads_name (last_name, first_name),
|
||||
KEY idx_crm_leads_company (company),
|
||||
KEY idx_crm_leads_status (status),
|
||||
KEY idx_crm_leads_owner (owner_id),
|
||||
KEY idx_crm_leads_creator (created_by),
|
||||
CONSTRAINT fk_crm_leads_owner FOREIGN KEY (owner_id) REFERENCES crm_users (id) ON DELETE SET NULL,
|
||||
CONSTRAINT fk_crm_leads_creator FOREIGN KEY (created_by) REFERENCES crm_users (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Wer steht mit wem in Kontakt. Ungerichtet gedacht; Duplikate werden ueber
|
||||
-- den Unique-Key auf dem sortierten Paar verhindert (siehe Backend).
|
||||
CREATE TABLE IF NOT EXISTS crm_lead_connections (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
from_lead_id INT UNSIGNED NOT NULL,
|
||||
to_lead_id INT UNSIGNED NOT NULL,
|
||||
type ENUM('Kollege','Vorgesetzter','Kunde','Partner','Empfehlung','Netzwerk','Sonstige')
|
||||
NOT NULL DEFAULT 'Netzwerk',
|
||||
label VARCHAR(190) NULL,
|
||||
strength TINYINT UNSIGNED NOT NULL DEFAULT 2, -- 1 = lose, 2 = normal, 3 = eng
|
||||
created_by INT UNSIGNED NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uq_crm_conn_pair (from_lead_id, to_lead_id),
|
||||
KEY idx_crm_conn_to (to_lead_id),
|
||||
CONSTRAINT fk_crm_conn_from FOREIGN KEY (from_lead_id) REFERENCES crm_leads (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_crm_conn_to FOREIGN KEY (to_lead_id) REFERENCES crm_leads (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_crm_conn_user FOREIGN KEY (created_by) REFERENCES crm_users (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Vertriebschancen. Ansprechpartner haengen als n:m an den Kontakten, damit ein
|
||||
-- Kontakt in mehreren Opportunities auftauchen kann.
|
||||
CREATE TABLE IF NOT EXISTS crm_opportunities (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
title VARCHAR(190) NOT NULL,
|
||||
company VARCHAR(190) NULL,
|
||||
description TEXT NULL,
|
||||
stage ENUM('Neu','Qualifizierung','Angebot','Verhandlung','Gewonnen','Verloren')
|
||||
NOT NULL DEFAULT 'Neu',
|
||||
probability TINYINT UNSIGNED NOT NULL DEFAULT 10, -- 0-100 %
|
||||
daily_rate DECIMAL(10,2) NULL, -- Tagessatz in Euro
|
||||
duration_days DECIMAL(6,1) NULL, -- Personentage, halbe moeglich
|
||||
start_date DATE NULL,
|
||||
end_date DATE NULL,
|
||||
last_interaction_at DATE NULL,
|
||||
last_interaction_note VARCHAR(500) NULL,
|
||||
next_step VARCHAR(500) NULL,
|
||||
tags LONGTEXT NULL, -- JSON-Array als Text, wie bei crm_leads
|
||||
notes TEXT NULL,
|
||||
owner_id INT UNSIGNED NULL,
|
||||
created_by INT UNSIGNED NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
KEY idx_crm_opp_stage (stage),
|
||||
KEY idx_crm_opp_owner (owner_id),
|
||||
KEY idx_crm_opp_interaction (last_interaction_at),
|
||||
CONSTRAINT fk_crm_opp_owner FOREIGN KEY (owner_id) REFERENCES crm_users (id) ON DELETE SET NULL,
|
||||
CONSTRAINT fk_crm_opp_creator FOREIGN KEY (created_by) REFERENCES crm_users (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS crm_opportunity_contacts (
|
||||
opportunity_id INT UNSIGNED NOT NULL,
|
||||
lead_id INT UNSIGNED NOT NULL,
|
||||
role VARCHAR(120) NULL, -- z. B. "Entscheider", "Fachbereich"
|
||||
PRIMARY KEY (opportunity_id, lead_id),
|
||||
KEY idx_crm_opp_contacts_lead (lead_id),
|
||||
CONSTRAINT fk_crm_oc_opp FOREIGN KEY (opportunity_id) REFERENCES crm_opportunities (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_crm_oc_lead FOREIGN KEY (lead_id) REFERENCES crm_leads (id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Gemeinsamer Teamkalender: alle Benutzer sehen alle Termine.
|
||||
-- description darf Mention-Tokens der Form @[Anzeigename](lead:12) bzw.
|
||||
-- @[Anzeigename](user:3) enthalten - das Frontend rendert sie klickbar.
|
||||
CREATE TABLE IF NOT EXISTS crm_events (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
title VARCHAR(190) NOT NULL,
|
||||
description TEXT NULL,
|
||||
location VARCHAR(190) NULL,
|
||||
starts_at DATETIME NOT NULL,
|
||||
ends_at DATETIME NOT NULL,
|
||||
all_day TINYINT(1) NOT NULL DEFAULT 0,
|
||||
created_by INT UNSIGNED NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
KEY idx_crm_events_start (starts_at),
|
||||
KEY idx_crm_events_creator (created_by),
|
||||
CONSTRAINT fk_crm_events_creator FOREIGN KEY (created_by) REFERENCES crm_users (id) ON DELETE SET NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Ein Termin kann mehrere Teilnehmer haben; der Primaerschluessel verhindert
|
||||
-- Doppeleintraege.
|
||||
CREATE TABLE IF NOT EXISTS crm_event_participants (
|
||||
event_id INT UNSIGNED NOT NULL,
|
||||
user_id INT UNSIGNED NOT NULL,
|
||||
PRIMARY KEY (event_id, user_id),
|
||||
KEY idx_crm_event_participants_user (user_id),
|
||||
CONSTRAINT fk_crm_ep_event FOREIGN KEY (event_id) REFERENCES crm_events (id) ON DELETE CASCADE,
|
||||
CONSTRAINT fk_crm_ep_user FOREIGN KEY (user_id) REFERENCES crm_users (id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -0,0 +1,119 @@
|
||||
import 'dotenv/config';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import { initDatabase, pool } from './db.js';
|
||||
import authRoutes from './routes/auth.js';
|
||||
import userRoutes from './routes/users.js';
|
||||
import leadRoutes from './routes/leads.js';
|
||||
import connectionRoutes from './routes/connections.js';
|
||||
import eventRoutes from './routes/events.js';
|
||||
import opportunityRoutes from './routes/opportunities.js';
|
||||
import scannerRoutes from './routes/scanner.js';
|
||||
import { requireAuth } from './middleware/auth.js';
|
||||
import { initScannerDatabase } from './scanner/models/index.js';
|
||||
import { startScheduler } from './scanner/services/scheduler.js';
|
||||
import { SCANNER_KEY_CONFIGURED } from './scanner/utils/crypto.js';
|
||||
|
||||
const app = express();
|
||||
const PORT = Number(process.env.PORT || 5006);
|
||||
const HOST = process.env.HOST || '127.0.0.1';
|
||||
|
||||
// Avatare kommen als Data-URL - deshalb ein groesseres Limit als der Default.
|
||||
app.use(express.json({ limit: '6mb' }));
|
||||
|
||||
// Im Produktivbetrieb liefert Apache Frontend und API unter derselben Origin
|
||||
// aus, CORS ist dann irrelevant. Fuer den lokalen Vite-Dev-Server auf einem
|
||||
// anderen Port braucht es die Freigabe.
|
||||
app.use(
|
||||
cors({
|
||||
origin: (process.env.CORS_ORIGINS || 'http://localhost:3100,http://127.0.0.1:3100').split(','),
|
||||
})
|
||||
);
|
||||
|
||||
app.get('/api/health', async (req, res) => {
|
||||
try {
|
||||
await pool.query('SELECT 1');
|
||||
res.json({ ok: true, db: 'up' });
|
||||
} catch {
|
||||
res.status(503).json({ ok: false, db: 'down' });
|
||||
}
|
||||
});
|
||||
|
||||
app.use('/api/auth', authRoutes);
|
||||
app.use('/api/users', userRoutes);
|
||||
app.use('/api/leads', leadRoutes);
|
||||
app.use('/api/connections', connectionRoutes);
|
||||
app.use('/api/events', eventRoutes);
|
||||
app.use('/api/opportunities', opportunityRoutes);
|
||||
|
||||
// Projekt Scanner. Komplett hinter der CRM-Anmeldung - der urspruengliche
|
||||
// HTTP-Basic-Auth-Zugang des Scanners entfaellt damit.
|
||||
app.use('/api/scanner', requireAuth, scannerRoutes);
|
||||
|
||||
// Das Diktat laeuft komplett im Browser (Whisper ueber transformers.js) - das
|
||||
// Backend ist daran nicht beteiligt und sieht keine Audiodaten.
|
||||
|
||||
app.use('/api', (req, res) => res.status(404).json({ error: 'Endpunkt nicht gefunden.' }));
|
||||
|
||||
// Zentrale Fehlerbehandlung: Details ins Log, generische Meldung nach aussen.
|
||||
app.use((err, req, res, next) => {
|
||||
if (res.headersSent) return next(err);
|
||||
|
||||
// Kaputtes JSON o.ae. ist ein Client-Fehler - kein 500, und es muss auch
|
||||
// nicht als unbehandelter Serverfehler im Log landen.
|
||||
const status = err?.status || err?.statusCode;
|
||||
if (err?.type === 'entity.parse.failed' || status === 400) {
|
||||
return res.status(400).json({ error: 'Ungueltige Anfrage (JSON konnte nicht gelesen werden).' });
|
||||
}
|
||||
if (err?.type === 'entity.too.large' || status === 413) {
|
||||
return res.status(413).json({ error: 'Die Anfrage ist zu gross (max. 6 MB).' });
|
||||
}
|
||||
|
||||
console.error('[perinco CRM] Unbehandelter Fehler:', err);
|
||||
res.status(500).json({ error: 'Interner Serverfehler.' });
|
||||
});
|
||||
|
||||
try {
|
||||
await initDatabase();
|
||||
|
||||
/*
|
||||
* Der Scanner ist eine vollstaendig eigenstaendige Instanz: eigene Datenbank,
|
||||
* eigener Verschluesselungsschluessel, eigenes Upload-Verzeichnis, eigener
|
||||
* Scheduler. Er laeuft damit parallel zur alten App, ohne ihre Daten zu
|
||||
* beruehren. Fehlt die Datenbank, startet das CRM trotzdem - dann antworten
|
||||
* nur die Scanner-Endpunkte mit einem Fehler.
|
||||
*/
|
||||
let scannerReady = true;
|
||||
try {
|
||||
await initScannerDatabase();
|
||||
} catch {
|
||||
scannerReady = false;
|
||||
console.warn('[Scanner] Bereich ist nicht verfuegbar - CRM startet ohne Scanner.');
|
||||
}
|
||||
|
||||
if (scannerReady && !SCANNER_KEY_CONFIGURED) {
|
||||
console.warn(
|
||||
'[Scanner] SCANNER_ENCRYPTION_KEY ist nicht gesetzt - Portal-Passwoerter werden mit ' +
|
||||
'einem bekannten Ersatzschluessel verschluesselt. Fuer den Produktivbetrieb setzen.'
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* Hintergrund-Scheduler (Scraping + OpenAI-Bewertung). Standard: AN, denn der
|
||||
* Parallelbetrieb soll funktional sein. Beide Scanner bewerten dieselben
|
||||
* Projekte unabhaengig voneinander - die OpenAI-Kosten fallen also doppelt an,
|
||||
* solange die alte App laeuft. Zum Abschalten SCANNER_SCHEDULER=false setzen.
|
||||
*/
|
||||
if (scannerReady && process.env.SCANNER_SCHEDULER !== 'false') {
|
||||
await startScheduler();
|
||||
} else if (scannerReady) {
|
||||
console.log('[Scanner] Scheduler deaktiviert (SCANNER_SCHEDULER=false).');
|
||||
}
|
||||
|
||||
app.listen(PORT, HOST, () => {
|
||||
console.log(`[perinco CRM] Backend laeuft auf http://${HOST}:${PORT}`);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[perinco CRM] Start fehlgeschlagen:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Validierung der Benutzerfelder, die sowohl die Nutzerverwaltung (Admin) als
|
||||
// auch "Mein Konto" (jeder fuer sich selbst) setzen darf. Bewusst in einem
|
||||
// eigenen Modul, damit auth.js und users.js dieselbe Pruefung nutzen, ohne sich
|
||||
// gegenseitig zu importieren.
|
||||
|
||||
const COLOR_RE = /^#[0-9a-f]{6}$/;
|
||||
|
||||
/** Data-URL eines Bildes oder externe http(s)-URL. */
|
||||
const AVATAR_RE = /^(data:image\/(png|jpe?g|gif|webp|avif|svg\+xml);base64,[A-Za-z0-9+/=\s]+|https?:\/\/\S+)$/i;
|
||||
|
||||
/**
|
||||
* Ein Foto liegt als Data-URL in der Datenbank. Das JSON-Limit der Express-App
|
||||
* ist 6 MB; hier bleibt bewusst Luft fuer den Rest des Requests.
|
||||
*/
|
||||
const AVATAR_MAX_LENGTH = 3_000_000;
|
||||
|
||||
/** Prueft eine Hex-Farbe; null bzw. '' bedeutet "automatisch aus der Palette". */
|
||||
export function normalizeColor(value) {
|
||||
if (value === null || value === undefined || String(value).trim() === '') return { value: null };
|
||||
const color = String(value).trim().toLowerCase();
|
||||
if (!COLOR_RE.test(color)) return { error: 'Ungueltige Farbe (erwartet z. B. #6ba04a).' };
|
||||
return { value: color };
|
||||
}
|
||||
|
||||
/** Prueft ein Benutzerfoto; null bzw. '' entfernt es. */
|
||||
export function normalizeAvatar(value) {
|
||||
if (value === null || value === undefined || String(value).trim() === '') return { value: null };
|
||||
const avatar = String(value).trim();
|
||||
if (avatar.length > AVATAR_MAX_LENGTH) {
|
||||
return { error: 'Das Foto ist zu gross. Bitte ein kleineres Bild verwenden.' };
|
||||
}
|
||||
if (!AVATAR_RE.test(avatar)) {
|
||||
return { error: 'Ungueltiges Foto (erwartet eine Bild-Data-URL oder eine http(s)-Adresse).' };
|
||||
}
|
||||
return { value: avatar };
|
||||
}
|
||||
Reference in New Issue
Block a user