feat: Projekt Scanner integrieren, Whisper in die Einstellungen
Der eigenstaendige Projekt Scanner zieht komplett ins CRM ein und laeuft parallel zur bestehenden App weiter - jede geteilte Ressource ist verdoppelt statt geteilt, damit die alte Installation unberuehrt bleibt: - eigene Datenbank perinco_scanner, legt sich beim Start selbst an - eigener SCANNER_ENCRYPTION_KEY, eigenes Upload-Verzeichnis - eigener Scheduler mit Zeitversatz, damit nicht beide Instanzen gleichzeitig mit denselben Portal-Cookies scrapen (Rate-Limits, CAPTCHA, doppelter RAM) - Sperre gegen ueberlappende Laeufe: Scheduler und "Jetzt scannen" teilen sie, ein zweiter Aufruf bekommt 409 statt doppelter OpenAI-Kosten Backend unter /api/scanner hinter der CRM-Anmeldung; die HTTP-Basic-Auth des Originals entfaellt. CV-PDFs kommen als Stream mit Token statt aus einem statischen Verzeichnis. Bestandsdaten holt "npm run scanner:import" aus der alten Datenbank, ohne sie zu veraendern. Oberflaeche in Tailwind und CI-Farben neu gebaut: Menuepunkt Scanner mit aufklappbaren Unterpunkten, Einstellungen als eigener Reiter. Whisper hat keinen eigenen Menuepunkt mehr, sondern ist ein Reiter in den Einstellungen. Damit diese Geraeteeinstellung jeder Rolle offensteht, sind die Einstellungen jetzt fuer alle sichtbar - welche Reiter erscheinen, entscheidet die Rolle. Ausserdem: kein festes Admin-Passwort mehr im Quelltext, playwright exakt auf 1.60.0 gepinnt (nutzt den vorhandenen Chromium-Build des Servers), technische Dokumentation mit Funktionszeichnungen unter docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sascha Wolf
@@ -0,0 +1,33 @@
|
||||
import { Sequelize } from 'sequelize';
|
||||
|
||||
/*
|
||||
* Der Scanner-Teil bringt seine eigene Sequelize-Verbindung mit - das CRM
|
||||
* selbst arbeitet mit mysql2 direkt (siehe ../../db.js).
|
||||
*
|
||||
* Die Datenbank ist bewusst eine EIGENE (SCANNER_DB_NAME, Default
|
||||
* "perinco_scanner") und nicht die "project_scanner" der alten App: beide
|
||||
* Scanner laufen parallel und duerfen sich nicht in die Daten schreiben. Die
|
||||
* Tabellennamen bleiben identisch (agents, portals, projects, matches,
|
||||
* settings), weil sie in einer getrennten Datenbank liegen. Bestehende Agenten,
|
||||
* Portale und Einstellungen holt `npm run scanner:import` einmalig herueber.
|
||||
*
|
||||
* Zugangsdaten fallen auf die CRM-Werte zurueck, wenn nichts eigenes gesetzt
|
||||
* ist. dotenv wird nicht hier geladen - das erledigt server.js.
|
||||
*/
|
||||
const sequelize = new Sequelize(
|
||||
process.env.SCANNER_DB_NAME || 'perinco_scanner',
|
||||
process.env.SCANNER_DB_USER || process.env.DB_USER || 'root',
|
||||
process.env.SCANNER_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
{
|
||||
host: process.env.SCANNER_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.SCANNER_DB_PORT || process.env.DB_PORT || 3306),
|
||||
dialect: 'mysql',
|
||||
logging: false,
|
||||
define: {
|
||||
underscored: true,
|
||||
timestamps: true,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
export default sequelize;
|
||||
@@ -0,0 +1,35 @@
|
||||
import mysql from 'mysql2/promise';
|
||||
|
||||
/*
|
||||
* Legt die Scanner-Datenbank an, falls sie noch fehlt. Sequelize kann das nicht
|
||||
* selbst - `sync()` erzeugt Tabellen, aber keine Datenbank. Da der Scanner hier
|
||||
* bewusst eine eigene Datenbank benutzt (nicht die der alten App), soll die
|
||||
* erste Einrichtung ohne Handarbeit an der MariaDB laufen.
|
||||
*/
|
||||
export async function ensureScannerDatabase() {
|
||||
const name = process.env.SCANNER_DB_NAME || 'perinco_scanner';
|
||||
|
||||
// Bezeichner lassen sich nicht als Platzhalter binden - deshalb hart pruefen,
|
||||
// welche Zeichen im Datenbanknamen erlaubt sind.
|
||||
if (!/^[A-Za-z0-9_]+$/.test(name)) {
|
||||
throw new Error(`Ungueltiger SCANNER_DB_NAME: "${name}" (nur A-Z, a-z, 0-9, _).`);
|
||||
}
|
||||
|
||||
const connection = await mysql.createConnection({
|
||||
host: process.env.SCANNER_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.SCANNER_DB_PORT || process.env.DB_PORT || 3306),
|
||||
user: process.env.SCANNER_DB_USER || process.env.DB_USER || 'root',
|
||||
password: process.env.SCANNER_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
});
|
||||
|
||||
try {
|
||||
await connection.query(
|
||||
`CREATE DATABASE IF NOT EXISTS \`${name}\`
|
||||
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
|
||||
);
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* Einmalige Uebernahme aus der alten Projekt-Scanner-Datenbank in die eigene
|
||||
* Datenbank dieser Instanz. Beide Scanner laufen danach parallel und getrennt -
|
||||
* dieses Skript kopiert nur, es veraendert die Quelle nicht.
|
||||
*
|
||||
* Aufruf (aus server/):
|
||||
* npm run scanner:import
|
||||
* npm run scanner:import -- --with-matches auch Projekte und Treffer
|
||||
* npm run scanner:import -- --force vorhandene Zieldaten ersetzen
|
||||
* npm run scanner:import -- --dry-run nur anzeigen, nichts schreiben
|
||||
*
|
||||
* Quelle konfigurieren (server/.env):
|
||||
* LEGACY_DB_NAME=project_scanner
|
||||
* LEGACY_DB_HOST / _PORT / _USER / _PASSWORD (leer = CRM-Werte)
|
||||
* LEGACY_UPLOAD_DIR=/opt/project-scanner/backend/uploads
|
||||
* LEGACY_ENCRYPTION_KEY=... Portal-Passwoerter mit uebernehmen
|
||||
*
|
||||
* Was uebernommen wird:
|
||||
* - Einstellungen (inkl. OpenAI-Key, SMTP-Zugang, Schwellenwerte)
|
||||
* - Portale: Benutzername, Session-Cookies, Status; Passwoerter nur mit
|
||||
* LEGACY_ENCRYPTION_KEY, weil sie mit dem alten Schluessel verschluesselt
|
||||
* sind und hier neu verschluesselt werden muessen
|
||||
* - Suchagenten inkl. CV-Text und CV-PDF (Datei wird kopiert)
|
||||
* - mit --with-matches zusaetzlich Projekte und Treffer
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import mysql from 'mysql2/promise';
|
||||
import {
|
||||
Agent,
|
||||
Match,
|
||||
Portal,
|
||||
Project,
|
||||
Setting,
|
||||
initScannerDatabase,
|
||||
} from './models/index.js';
|
||||
import { encrypt, decryptWith } from './utils/crypto.js';
|
||||
import { ensureUploadDir } from './uploadDir.js';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const withMatches = args.includes('--with-matches');
|
||||
const force = args.includes('--force');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
const LEGACY_UPLOAD_DIR = process.env.LEGACY_UPLOAD_DIR || '';
|
||||
const LEGACY_KEY = process.env.LEGACY_ENCRYPTION_KEY || '';
|
||||
|
||||
const log = (message) => console.log(`[Import] ${message}`);
|
||||
|
||||
async function openLegacy() {
|
||||
const database = process.env.LEGACY_DB_NAME || 'project_scanner';
|
||||
const connection = await mysql.createConnection({
|
||||
host: process.env.LEGACY_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.LEGACY_DB_PORT || process.env.DB_PORT || 3306),
|
||||
user: process.env.LEGACY_DB_USER || process.env.DB_USER || 'root',
|
||||
password: process.env.LEGACY_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
database,
|
||||
dateStrings: false,
|
||||
});
|
||||
log(`Quelle: ${database}`);
|
||||
return connection;
|
||||
}
|
||||
|
||||
/** Kopiert das CV-PDF in das eigene Upload-Verzeichnis. */
|
||||
function copyCv(storedPath, uploadDir) {
|
||||
if (!storedPath) return '';
|
||||
|
||||
const fileName = path.basename(storedPath);
|
||||
const candidates = [
|
||||
path.isAbsolute(storedPath) ? storedPath : null,
|
||||
LEGACY_UPLOAD_DIR ? path.join(LEGACY_UPLOAD_DIR, fileName) : null,
|
||||
path.resolve(storedPath),
|
||||
].filter(Boolean);
|
||||
|
||||
const source = candidates.find((candidate) => fs.existsSync(candidate));
|
||||
if (!source) {
|
||||
log(` CV-Datei nicht gefunden: ${storedPath} (LEGACY_UPLOAD_DIR setzen?)`);
|
||||
return '';
|
||||
}
|
||||
|
||||
const target = path.join(uploadDir, fileName);
|
||||
if (!dryRun) fs.copyFileSync(source, target);
|
||||
return target;
|
||||
}
|
||||
|
||||
async function importSettings(legacy) {
|
||||
const [rows] = await legacy.query('SELECT key_name, value FROM settings');
|
||||
for (const row of rows) {
|
||||
if (!dryRun) await Setting.setVal(row.key_name, row.value ?? '');
|
||||
}
|
||||
log(`Einstellungen: ${rows.length} Schluessel uebernommen.`);
|
||||
}
|
||||
|
||||
async function importPortals(legacy) {
|
||||
const [rows] = await legacy.query(
|
||||
'SELECT name, username, password_encrypted, session_state, connection_status, last_login_attempt FROM portals'
|
||||
);
|
||||
|
||||
let withSession = 0;
|
||||
let withPassword = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
// Portale sind bei der Initialisierung schon angelegt - hier wird nur der
|
||||
// Inhalt nachgezogen, ueber den Namen zugeordnet.
|
||||
const portal = await Portal.findOne({ where: { name: row.name } });
|
||||
if (!portal) {
|
||||
log(` Portal ${row.name} gibt es hier nicht - uebersprungen.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
portal.username = row.username || portal.username;
|
||||
if (row.session_state) {
|
||||
portal.sessionState = row.session_state;
|
||||
portal.connectionStatus = row.connection_status || 'connected';
|
||||
portal.lastLoginAttempt = row.last_login_attempt || new Date();
|
||||
withSession++;
|
||||
}
|
||||
|
||||
if (row.password_encrypted && LEGACY_KEY) {
|
||||
const plain = decryptWith(row.password_encrypted, LEGACY_KEY);
|
||||
if (plain) {
|
||||
// Neu verschluesseln - diese Instanz hat ihren eigenen Schluessel.
|
||||
portal.passwordEncrypted = encrypt(plain);
|
||||
withPassword++;
|
||||
} else {
|
||||
log(` Passwort von ${row.name} liess sich mit LEGACY_ENCRYPTION_KEY nicht lesen.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun) await portal.save();
|
||||
}
|
||||
|
||||
log(`Portale: ${rows.length} geprueft, ${withSession} mit Session, ${withPassword} mit Passwort.`);
|
||||
if (!LEGACY_KEY) {
|
||||
log(' Hinweis: ohne LEGACY_ENCRYPTION_KEY bleiben die Portal-Passwoerter leer.');
|
||||
}
|
||||
}
|
||||
|
||||
async function importAgents(legacy, uploadDir) {
|
||||
const [rows] = await legacy.query(
|
||||
`SELECT id, name, email, cv_text, cv_file_path, keywords, exclude_keywords, is_active,
|
||||
created_at, updated_at
|
||||
FROM agents ORDER BY id`
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
const cvFilePath = copyCv(row.cv_file_path, uploadDir);
|
||||
if (dryRun) {
|
||||
log(` wuerde Agent ${row.id} "${row.name}" anlegen`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ids bleiben erhalten, damit uebernommene Treffer weiter passen.
|
||||
await Agent.create(
|
||||
{
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
email: row.email,
|
||||
cvText: row.cv_text,
|
||||
cvFilePath,
|
||||
keywords: row.keywords,
|
||||
excludeKeywords: row.exclude_keywords,
|
||||
isActive: Boolean(row.is_active),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
|
||||
log(`Suchagenten: ${rows.length} uebernommen.`);
|
||||
}
|
||||
|
||||
async function importProjectsAndMatches(legacy) {
|
||||
const [projects] = await legacy.query(
|
||||
`SELECT id, portal, portal_project_id, title, description, company, location, rate, url,
|
||||
published_at, created_at
|
||||
FROM projects ORDER BY id`
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of projects) {
|
||||
await Project.create(
|
||||
{
|
||||
id: row.id,
|
||||
portal: row.portal,
|
||||
portalProjectId: row.portal_project_id,
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
company: row.company,
|
||||
location: row.location,
|
||||
rate: row.rate,
|
||||
url: row.url,
|
||||
publishedAt: row.published_at,
|
||||
createdAt: row.created_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Projekte: ${projects.length} uebernommen.`);
|
||||
|
||||
const [matches] = await legacy.query(
|
||||
`SELECT id, agent_id, project_id, match_percentage, pros, cons, status, notification_sent,
|
||||
created_at, updated_at
|
||||
FROM matches ORDER BY id`
|
||||
);
|
||||
|
||||
const parseList = (value) => {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of matches) {
|
||||
await Match.create(
|
||||
{
|
||||
id: row.id,
|
||||
agentId: row.agent_id,
|
||||
projectId: row.project_id,
|
||||
matchPercentage: row.match_percentage,
|
||||
pros: parseList(row.pros),
|
||||
cons: parseList(row.cons),
|
||||
status: row.status,
|
||||
notificationSent: Boolean(row.notification_sent),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Treffer: ${matches.length} uebernommen.`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (dryRun) log('Probelauf - es wird nichts geschrieben.');
|
||||
|
||||
await initScannerDatabase();
|
||||
const uploadDir = ensureUploadDir();
|
||||
log(`Ziel-Upload-Verzeichnis: ${uploadDir}`);
|
||||
|
||||
const existingAgents = await Agent.count();
|
||||
if (existingAgents > 0 && !force) {
|
||||
log(
|
||||
`Abbruch: es gibt hier schon ${existingAgents} Suchagenten. Mit --force werden Agenten, ` +
|
||||
'Projekte und Treffer vorher geloescht.'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const legacy = await openLegacy();
|
||||
try {
|
||||
if (force && !dryRun) {
|
||||
// Reihenfolge wegen der Fremdschluessel: Treffer, dann Projekte, dann Agenten.
|
||||
await Match.destroy({ where: {}, truncate: false });
|
||||
await Project.destroy({ where: {}, truncate: false });
|
||||
await Agent.destroy({ where: {}, truncate: false });
|
||||
log('Vorhandene Agenten, Projekte und Treffer geloescht.');
|
||||
}
|
||||
|
||||
await importSettings(legacy);
|
||||
await importPortals(legacy);
|
||||
await importAgents(legacy, uploadDir);
|
||||
if (withMatches) {
|
||||
await importProjectsAndMatches(legacy);
|
||||
} else {
|
||||
log('Projekte und Treffer uebersprungen (--with-matches aktiviert sie).');
|
||||
}
|
||||
} finally {
|
||||
await legacy.end();
|
||||
}
|
||||
|
||||
log('Fertig.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[Import] Fehlgeschlagen:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Agent = sequelize.define('Agent', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
},
|
||||
email: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
validate: {
|
||||
isEmail: true,
|
||||
},
|
||||
},
|
||||
cvText: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
field: 'cv_text',
|
||||
},
|
||||
cvFilePath: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
field: 'cv_file_path',
|
||||
},
|
||||
keywords: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
comment: 'Comma-separated keywords for search',
|
||||
},
|
||||
excludeKeywords: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
field: 'exclude_keywords',
|
||||
comment: 'Comma-separated terms; projects containing any are ignored',
|
||||
},
|
||||
isActive: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: true,
|
||||
field: 'is_active',
|
||||
},
|
||||
}, {
|
||||
tableName: 'agents',
|
||||
});
|
||||
|
||||
export default Agent;
|
||||
@@ -0,0 +1,85 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
import Agent from './Agent.js';
|
||||
import Project from './Project.js';
|
||||
|
||||
const Match = sequelize.define('Match', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
agentId: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
field: 'agent_id',
|
||||
references: {
|
||||
model: Agent,
|
||||
key: 'id',
|
||||
},
|
||||
},
|
||||
projectId: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
field: 'project_id',
|
||||
references: {
|
||||
model: Project,
|
||||
key: 'id',
|
||||
},
|
||||
},
|
||||
matchPercentage: {
|
||||
type: DataTypes.INTEGER,
|
||||
defaultValue: 0,
|
||||
field: 'match_percentage',
|
||||
},
|
||||
pros: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
comment: 'JSON array of strings',
|
||||
get() {
|
||||
const rawValue = this.getDataValue('pros');
|
||||
return rawValue ? JSON.parse(rawValue) : [];
|
||||
},
|
||||
set(value) {
|
||||
this.setDataValue('pros', JSON.stringify(value));
|
||||
},
|
||||
},
|
||||
cons: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
comment: 'JSON array of strings',
|
||||
get() {
|
||||
const rawValue = this.getDataValue('cons');
|
||||
return rawValue ? JSON.parse(rawValue) : [];
|
||||
},
|
||||
set(value) {
|
||||
this.setDataValue('cons', JSON.stringify(value));
|
||||
},
|
||||
},
|
||||
status: {
|
||||
type: DataTypes.STRING(50),
|
||||
defaultValue: 'new', // 'new', 'reviewed', 'applied', 'rejected'
|
||||
},
|
||||
notificationSent: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: false,
|
||||
field: 'notification_sent',
|
||||
},
|
||||
}, {
|
||||
tableName: 'matches',
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['agent_id', 'project_id'],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Associations
|
||||
Agent.hasMany(Match, { foreignKey: 'agent_id', onDelete: 'CASCADE' });
|
||||
Match.belongsTo(Agent, { foreignKey: 'agent_id' });
|
||||
|
||||
Project.hasMany(Match, { foreignKey: 'project_id', onDelete: 'CASCADE' });
|
||||
Match.belongsTo(Project, { foreignKey: 'project_id' });
|
||||
|
||||
export default Match;
|
||||
@@ -0,0 +1,44 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Portal = sequelize.define('Portal', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
type: DataTypes.STRING(50),
|
||||
allowNull: false,
|
||||
unique: true,
|
||||
},
|
||||
username: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
passwordEncrypted: {
|
||||
type: DataTypes.STRING(500),
|
||||
allowNull: true,
|
||||
field: 'password_encrypted',
|
||||
},
|
||||
sessionState: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
field: 'session_state',
|
||||
comment: 'JSON string of cookies and localStorage',
|
||||
},
|
||||
connectionStatus: {
|
||||
type: DataTypes.STRING(50),
|
||||
defaultValue: 'disconnected',
|
||||
field: 'connection_status',
|
||||
},
|
||||
lastLoginAttempt: {
|
||||
type: DataTypes.DATE,
|
||||
allowNull: true,
|
||||
field: 'last_login_attempt',
|
||||
},
|
||||
}, {
|
||||
tableName: 'portals',
|
||||
});
|
||||
|
||||
export default Portal;
|
||||
@@ -0,0 +1,61 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Project = sequelize.define('Project', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
portal: {
|
||||
type: DataTypes.STRING(50),
|
||||
allowNull: false,
|
||||
},
|
||||
portalProjectId: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: false,
|
||||
field: 'portal_project_id',
|
||||
},
|
||||
title: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
},
|
||||
description: {
|
||||
type: DataTypes.TEXT('long'),
|
||||
allowNull: true,
|
||||
},
|
||||
company: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
location: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
rate: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: true,
|
||||
},
|
||||
url: {
|
||||
type: DataTypes.STRING(500),
|
||||
allowNull: false,
|
||||
},
|
||||
publishedAt: {
|
||||
type: DataTypes.DATE,
|
||||
allowNull: true,
|
||||
field: 'published_at',
|
||||
},
|
||||
}, {
|
||||
tableName: 'projects',
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['portal', 'portal_project_id'],
|
||||
},
|
||||
],
|
||||
timestamps: true,
|
||||
updatedAt: false, // projects are immutable once scraped, we only track when they were created
|
||||
createdAt: 'created_at',
|
||||
});
|
||||
|
||||
export default Project;
|
||||
@@ -0,0 +1,50 @@
|
||||
import { DataTypes } from 'sequelize';
|
||||
import sequelize from '../db/config.js';
|
||||
|
||||
const Setting = sequelize.define('Setting', {
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
},
|
||||
keyName: {
|
||||
type: DataTypes.STRING(100),
|
||||
allowNull: false,
|
||||
unique: true,
|
||||
field: 'key_name',
|
||||
},
|
||||
value: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
},
|
||||
}, {
|
||||
tableName: 'settings',
|
||||
});
|
||||
|
||||
// Helper static methods to get and set settings easily
|
||||
Setting.getVal = async function(key, defaultValue = null) {
|
||||
try {
|
||||
const setting = await this.findOne({ where: { keyName: key } });
|
||||
return setting ? setting.value : defaultValue;
|
||||
} catch (err) {
|
||||
console.error(`Error getting setting ${key}:`, err);
|
||||
return defaultValue;
|
||||
}
|
||||
};
|
||||
|
||||
Setting.setVal = async function(key, value) {
|
||||
try {
|
||||
const [setting] = await this.findOrCreate({
|
||||
where: { keyName: key },
|
||||
defaults: { value: String(value) }
|
||||
});
|
||||
setting.value = String(value);
|
||||
await setting.save();
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(`Error setting ${key}:`, err);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export default Setting;
|
||||
@@ -0,0 +1,86 @@
|
||||
import sequelize from '../db/config.js';
|
||||
import { ensureScannerDatabase } from '../db/ensureDatabase.js';
|
||||
import Agent from './Agent.js';
|
||||
import Portal from './Portal.js';
|
||||
import Project from './Project.js';
|
||||
import Match from './Match.js';
|
||||
import Setting from './Setting.js';
|
||||
|
||||
const initScannerDatabase = async () => {
|
||||
try {
|
||||
// Eigene Datenbank, deshalb erst anlegen und dann verbinden.
|
||||
const dbName = await ensureScannerDatabase();
|
||||
await sequelize.authenticate();
|
||||
console.log(`[Scanner] Datenbankverbindung steht (${dbName}).`);
|
||||
|
||||
// Sync database tables. Plain sync() — `alter: true` repeatedly re-creates
|
||||
// UNIQUE indexes on each boot and eventually exhausts MySQL's 64-key limit.
|
||||
// Schema changes are managed via SQL migrations / schema.sql instead.
|
||||
await sequelize.sync();
|
||||
console.log('[Scanner] Tabellen synchronisiert.');
|
||||
|
||||
// Ensure every supported portal exists. Per-row findOrCreate so newly added
|
||||
// portals appear after upgrade without manual SQL.
|
||||
const defaultPortals = [
|
||||
{ name: 'freelance.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'freelancermap.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'hays.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'etengo.de', connectionStatus: 'disconnected' },
|
||||
{ name: 'huxley.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'computerfutures.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'progressiverecruitment.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'realstaffing.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'globalenterprisepartners.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'madisonblack.com', connectionStatus: 'disconnected' },
|
||||
{ name: 'jpgray.nl', connectionStatus: 'disconnected' }
|
||||
];
|
||||
for (const p of defaultPortals) {
|
||||
const [, created] = await Portal.findOrCreate({
|
||||
where: { name: p.name },
|
||||
defaults: p
|
||||
});
|
||||
if (created) console.log(`[Scanner] Portal angelegt: ${p.name}`);
|
||||
}
|
||||
|
||||
// Default settings
|
||||
const defaultSettings = [
|
||||
{ keyName: 'scanning_active', value: 'true' },
|
||||
{ keyName: 'search_interval', value: '60' },
|
||||
{ keyName: 'openai_api_key', value: '' },
|
||||
{ keyName: 'email_notifications_active', value: 'false' },
|
||||
{ keyName: 'sound_notifications_active', value: 'true' },
|
||||
{ keyName: 'smtp_host', value: '' },
|
||||
{ keyName: 'smtp_port', value: '587' },
|
||||
{ keyName: 'smtp_user', value: '' },
|
||||
{ keyName: 'smtp_pass', value: '' },
|
||||
{ keyName: 'smtp_from', value: 'scanner@perinco.de' },
|
||||
{ keyName: 'notification_email_recipient', value: '' },
|
||||
{ keyName: 'match_threshold', value: '70' },
|
||||
{ keyName: 'display_threshold', value: '50' }
|
||||
];
|
||||
|
||||
for (const s of defaultSettings) {
|
||||
const exists = await Setting.findOne({ where: { keyName: s.keyName } });
|
||||
if (!exists) {
|
||||
await Setting.create(s);
|
||||
}
|
||||
}
|
||||
console.log('[Scanner] Standardeinstellungen geprueft.');
|
||||
|
||||
} catch (error) {
|
||||
// Ein fehlender Scanner-Teil darf das CRM nicht am Start hindern - die
|
||||
// Scanner-Endpunkte antworten dann mit einem Fehler, alles andere laeuft.
|
||||
console.error('[Scanner] Datenbank nicht erreichbar oder Sync fehlgeschlagen:', error.message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export {
|
||||
sequelize,
|
||||
Agent,
|
||||
Portal,
|
||||
Project,
|
||||
Match,
|
||||
Setting,
|
||||
initScannerDatabase
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import sequelize from '../db/config.js';
|
||||
import Project from '../models/Project.js';
|
||||
|
||||
// Retention window for scraped projects. Matches cascade-delete via FK.
|
||||
const RETENTION_DAYS = 7;
|
||||
|
||||
// Project model defines `createdAt: 'created_at'` for the column rename, but
|
||||
// Sequelize's attribute-to-column mapping in `where` does not pick it up here
|
||||
// (likely because of the underscored global default + explicit override
|
||||
// interaction). Using sequelize.col('created_at') sidesteps it.
|
||||
export const cleanupOldProjects = async () => {
|
||||
const cutoff = new Date(Date.now() - RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
||||
try {
|
||||
const deleted = await Project.destroy({
|
||||
where: sequelize.where(sequelize.col('created_at'), '<', cutoff)
|
||||
});
|
||||
console.log(`[Cleanup] Removed ${deleted} projects older than ${RETENTION_DAYS} days (cutoff ${cutoff.toISOString()}).`);
|
||||
return deleted;
|
||||
} catch (err) {
|
||||
console.error('[Cleanup] Failed to delete old projects:', err);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,79 @@
|
||||
import { OpenAI } from 'openai';
|
||||
import Setting from '../models/Setting.js';
|
||||
|
||||
/**
|
||||
* Evaluates the match between an agent's CV and a scraped project description using OpenAI.
|
||||
* @param {string} cvText - Plain text extracted from CV PDF.
|
||||
* @param {string} projectDescription - Plain text of the project description.
|
||||
* @returns {Promise<{matchPercentage: number, pros: string[], cons: string[]}>}
|
||||
*/
|
||||
export const evaluateMatch = async (cvText, projectDescription) => {
|
||||
if (!cvText || !projectDescription) {
|
||||
return {
|
||||
matchPercentage: 0,
|
||||
pros: ['Fehlende CV-Daten oder Projektbeschreibung für den Abgleich.'],
|
||||
cons: []
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch API key from DB, fallback to environment variable
|
||||
const apiKey = await Setting.getVal('openai_api_key') || process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API Key is not configured. Please set it in Settings.');
|
||||
}
|
||||
|
||||
const openai = new OpenAI({ apiKey });
|
||||
|
||||
const systemPrompt = `Du bist ein hochentwickeltes IT-Recruiting-Matching-System.
|
||||
Deine Aufgabe ist es, den Lebenslauf (CV) eines IT-Beraters mit einer Projektbeschreibung abzugleichen.
|
||||
Berechne eine realistische Übereinstimmungsquote (Trefferquote) in Prozent (Zahl von 0 bis 100).
|
||||
Analysiere Gemeinsamkeiten und Lücken.
|
||||
|
||||
Erstelle eine Liste von Stichpunkten für:
|
||||
1. Pros: Warum der Berater für das Projekt geeignet ist (Erfahrung, Kernkompetenzen, passende Programmiersprachen/Frameworks).
|
||||
2. Cons: Was gegen eine Eignung spricht oder welche Risiken/Lücken es gibt (fehlende Technologien, Sprachbarrieren, Reisebereitschaft, zu geringe Seniorität).
|
||||
|
||||
Antworte ausschließlich im JSON-Format mit folgendem Schema:
|
||||
{
|
||||
"matchPercentage": 75,
|
||||
"pros": [
|
||||
"Sehr gute Übereinstimmung bei React und Node.js-Kenntnissen.",
|
||||
"Erfahrung im geforderten Finanzsektor vorhanden."
|
||||
],
|
||||
"cons": [
|
||||
"Keine Erwähnung der geforderten Docker/Kubernetes-Erfahrung im CV.",
|
||||
"Projektsprache ist Deutsch (fließend), der Berater listet Deutsch nur auf B2-Niveau."
|
||||
]
|
||||
}
|
||||
Schreibe keinen zusätzlichen Text vor oder nach dem JSON-Objekt.`;
|
||||
|
||||
const userContent = `### LEBENSLAUF (CV):
|
||||
${cvText}
|
||||
|
||||
### PROJEKTBESCHREIBUNG:
|
||||
${projectDescription}`;
|
||||
|
||||
try {
|
||||
const response = await openai.chat.completions.create({
|
||||
model: 'gpt-4o-mini',
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userContent }
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
temperature: 0.1 // Low temperature for more deterministic analysis
|
||||
});
|
||||
|
||||
const rawContent = response.choices[0].message.content.trim();
|
||||
const result = JSON.parse(rawContent);
|
||||
|
||||
return {
|
||||
matchPercentage: Number(result.matchPercentage) || 0,
|
||||
pros: Array.isArray(result.pros) ? result.pros : [],
|
||||
cons: Array.isArray(result.cons) ? result.cons : []
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error during OpenAI project matching:', error);
|
||||
throw new Error('LLM Evaluation failed: ' + error.message);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
import pdf from 'pdf-parse';
|
||||
|
||||
/**
|
||||
* Extracts plain text from a CV PDF file buffer.
|
||||
* @param {Buffer} fileBuffer - The PDF file content as a buffer.
|
||||
* @returns {Promise<string>} - The extracted plain text.
|
||||
*/
|
||||
export const parsePdfCv = async (fileBuffer) => {
|
||||
try {
|
||||
const data = await pdf(fileBuffer);
|
||||
|
||||
// Clean up the text: remove double newlines and trim whitespace
|
||||
let text = data.text;
|
||||
text = text.replace(/\n\s*\n/g, '\n');
|
||||
text = text.trim();
|
||||
|
||||
return text;
|
||||
} catch (error) {
|
||||
console.error('Error parsing PDF CV:', error);
|
||||
throw new Error('Failed to parse PDF: ' + error.message);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,286 @@
|
||||
import { Agent, Portal, Project, Match, Setting } from '../models/index.js';
|
||||
import { scrapeFreelancermap, scrapeFreelanceDe, scrapeHays, scrapeEtengo, scrapeSThreePortal, SThreePortals } from './scraper.js';
|
||||
import { evaluateMatch } from './matchingEngine.js';
|
||||
import nodemailer from 'nodemailer';
|
||||
|
||||
// Helper to encrypt/decrypt credentials if needed, or send email notifications
|
||||
const sendEmailNotification = async (agent, project, matchPercentage, pros, cons) => {
|
||||
const active = await Setting.getVal('email_notifications_active') === 'true';
|
||||
if (!active) return;
|
||||
|
||||
const smtpHost = await Setting.getVal('smtp_host');
|
||||
const smtpPort = parseInt(await Setting.getVal('smtp_port', '587'), 10);
|
||||
const smtpUser = await Setting.getVal('smtp_user');
|
||||
const smtpPass = await Setting.getVal('smtp_pass');
|
||||
const smtpFrom = await Setting.getVal('smtp_from', 'scanner@perinco.de');
|
||||
const recipient = await Setting.getVal('notification_email_recipient') || agent.email;
|
||||
|
||||
if (!smtpHost || !smtpUser || !smtpPass || !recipient) {
|
||||
console.warn('SMTP settings are incomplete. Skipping email notification.');
|
||||
return;
|
||||
}
|
||||
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: smtpHost,
|
||||
port: smtpPort,
|
||||
secure: smtpPort === 465,
|
||||
auth: {
|
||||
user: smtpUser,
|
||||
pass: smtpPass
|
||||
}
|
||||
});
|
||||
|
||||
const prosList = pros.map(p => `<li>✅ ${p}</li>`).join('');
|
||||
const consList = cons.map(c => `<li>❌ ${c}</li>`).join('');
|
||||
|
||||
const mailOptions = {
|
||||
from: smtpFrom,
|
||||
to: recipient,
|
||||
subject: `[PROJEKT-MATCHING ${matchPercentage}%] Neue Chance für ${agent.name}`,
|
||||
html: `
|
||||
<div style="font-family: sans-serif; max-width: 600px; margin: 0 auto; padding: 20px; border: 1px solid #e0e0e0; border-radius: 8px;">
|
||||
<h2 style="color: #2A5B41;">Neues Projekt-Match gefunden</h2>
|
||||
<p>Hallo,</p>
|
||||
<p>es gibt ein neues passendes IT-Projekt auf <strong>${project.portal}</strong> für <strong>${agent.name}</strong>.</p>
|
||||
|
||||
<div style="background-color: #F4F7F5; padding: 15px; border-radius: 6px; margin: 20px 0;">
|
||||
<h3 style="margin-top: 0; color: #2A5B41;">${project.title}</h3>
|
||||
<p><strong>Portal:</strong> ${project.portal}<br>
|
||||
<strong>Ort:</strong> ${project.location}<br>
|
||||
<strong>Rate/Budget:</strong> ${project.rate}</p>
|
||||
<p><a href="${project.url}" target="_blank" style="background-color: #6A8DC3; color: white; padding: 10px 15px; text-decoration: none; border-radius: 4px; display: inline-block;">Projekt aufrufen</a></p>
|
||||
</div>
|
||||
|
||||
<div style="margin: 20px 0;">
|
||||
<h3 style="color: #2A5B41;">Eignung: <span style="color: ${matchPercentage >= 75 ? '#2A5B41' : '#E67E22'};">${matchPercentage}% Match</span></h3>
|
||||
|
||||
<h4 style="margin-bottom: 5px;">Vorteile (Warum es passt):</h4>
|
||||
<ul style="padding-left: 20px; margin-top: 0;">${prosList || '<li>Keine Angaben</li>'}</ul>
|
||||
|
||||
<h4 style="margin-bottom: 5px;">Herausforderungen (Gaps):</h4>
|
||||
<ul style="padding-left: 20px; margin-top: 0;">${consList || '<li>Keine Angaben</li>'}</ul>
|
||||
</div>
|
||||
|
||||
<p style="font-size: 12px; color: #7f8c8d; margin-top: 30px;">Diese E-Mail wurde automatisch vom Perinco Projekt Scanner generiert.</p>
|
||||
</div>
|
||||
`
|
||||
};
|
||||
|
||||
try {
|
||||
await transporter.sendMail(mailOptions);
|
||||
console.log(`Notification email sent for Agent ${agent.name} and Project ${project.title}`);
|
||||
} catch (error) {
|
||||
console.error('Failed to send email notification:', error);
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* Ein Lauf zur Zeit. Scheduler und der Knopf "Jetzt scannen" koennen sich sonst
|
||||
* ueberholen - zwei parallele Laeufe starten doppelt Chromium und bezahlen jede
|
||||
* Bewertung zweimal bei OpenAI.
|
||||
*/
|
||||
let pipelineRunning = false;
|
||||
|
||||
export const isPipelineRunning = () => pipelineRunning;
|
||||
|
||||
/**
|
||||
* Scrapen, Speichern und Bewerten in einem Durchlauf.
|
||||
* @returns {Promise<boolean>} false, wenn schon ein Lauf aktiv war.
|
||||
*/
|
||||
export const runMatchingPipeline = async () => {
|
||||
if (pipelineRunning) {
|
||||
console.log('[Pipeline] Lauf laeuft schon - neuer Aufruf uebersprungen.');
|
||||
return false;
|
||||
}
|
||||
pipelineRunning = true;
|
||||
try {
|
||||
return await executePipeline();
|
||||
} finally {
|
||||
pipelineRunning = false;
|
||||
}
|
||||
};
|
||||
|
||||
const executePipeline = async () => {
|
||||
console.log('--- STARTING MATCHING PIPELINE ---');
|
||||
|
||||
// 1. Fetch active agents
|
||||
const activeAgents = await Agent.findAll({ where: { isActive: true } });
|
||||
if (activeAgents.length === 0) {
|
||||
console.log('No active agents found. Pipeline execution stopped.');
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. Extract unique keywords from active agents
|
||||
const keywordSet = new Set();
|
||||
activeAgents.forEach(agent => {
|
||||
if (agent.keywords) {
|
||||
agent.keywords.split(',').forEach(k => {
|
||||
const trimmed = k.trim();
|
||||
if (trimmed) keywordSet.add(trimmed);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const keywords = Array.from(keywordSet);
|
||||
if (keywords.length === 0) {
|
||||
console.log('No active search keywords found. Pipeline execution stopped.');
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log(`Keywords to scrape: ${keywords.join(', ')}`);
|
||||
|
||||
// 3. Fetch portal configurations
|
||||
const freelancermapPortal = await Portal.findOne({ where: { name: 'freelancermap.de' } });
|
||||
const freelanceDePortal = await Portal.findOne({ where: { name: 'freelance.de' } });
|
||||
|
||||
// 4. Run scrapers
|
||||
let scrapedProjects = [];
|
||||
|
||||
try {
|
||||
const mapResult = await scrapeFreelancermap(keywords, freelancermapPortal?.sessionState);
|
||||
scrapedProjects = scrapedProjects.concat(mapResult.projects);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Freelancermap:', err);
|
||||
}
|
||||
|
||||
try {
|
||||
const freelanceResult = await scrapeFreelanceDe(keywords, freelanceDePortal?.sessionState);
|
||||
scrapedProjects = scrapedProjects.concat(freelanceResult.projects);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Freelance.de:', err);
|
||||
}
|
||||
|
||||
try {
|
||||
const haysResult = await scrapeHays(keywords);
|
||||
scrapedProjects = scrapedProjects.concat(haysResult.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: 'hays.de' } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Hays.de:', err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: 'hays.de' } });
|
||||
}
|
||||
|
||||
try {
|
||||
const etengoResult = await scrapeEtengo(keywords);
|
||||
scrapedProjects = scrapedProjects.concat(etengoResult.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: 'etengo.de' } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('Error scraping Etengo.de:', err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: 'etengo.de' } });
|
||||
}
|
||||
|
||||
// SThree-family staffing portals (shared template — generic scraper).
|
||||
for (const cfg of SThreePortals) {
|
||||
try {
|
||||
const result = await scrapeSThreePortal(cfg);
|
||||
scrapedProjects = scrapedProjects.concat(result.projects);
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'connected', lastLoginAttempt: new Date() },
|
||||
{ where: { name: cfg.name } }
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(`Error scraping ${cfg.name}:`, err);
|
||||
await Portal.update({ connectionStatus: 'error' }, { where: { name: cfg.name } });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Total projects scraped: ${scrapedProjects.length}`);
|
||||
|
||||
// 5. Store projects and run matching analysis
|
||||
let newProjectsStoredCount = 0;
|
||||
let matchesEvaluatedCount = 0;
|
||||
|
||||
const thresholdStr = await Setting.getVal('match_threshold', '70');
|
||||
const matchThreshold = parseInt(thresholdStr, 10) || 70;
|
||||
|
||||
for (const scrapedProj of scrapedProjects) {
|
||||
try {
|
||||
// Find or create project in DB
|
||||
const [project, created] = await Project.findOrCreate({
|
||||
where: {
|
||||
portal: scrapedProj.portal,
|
||||
portalProjectId: scrapedProj.portalProjectId
|
||||
},
|
||||
defaults: scrapedProj
|
||||
});
|
||||
|
||||
if (created) {
|
||||
newProjectsStoredCount++;
|
||||
}
|
||||
|
||||
// Perform matching analysis with active agents
|
||||
for (const agent of activeAgents) {
|
||||
// Skip if match already exists
|
||||
const existingMatch = await Match.findOne({
|
||||
where: {
|
||||
agentId: agent.id,
|
||||
projectId: project.id
|
||||
}
|
||||
});
|
||||
|
||||
if (existingMatch) continue;
|
||||
|
||||
// Exclusion list: skip this project for the agent if its title or
|
||||
// description contains any excluded term (case-insensitive). Runs
|
||||
// before the OpenAI call so excluded projects cost nothing.
|
||||
if (agent.excludeKeywords) {
|
||||
const haystack = `${project.title || ''} ${project.description || ''}`.toLowerCase();
|
||||
const excluded = agent.excludeKeywords
|
||||
.split(',')
|
||||
.map(t => t.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.find(term => haystack.includes(term));
|
||||
if (excluded) {
|
||||
console.log(`Skipping project [${project.title}] for Agent [${agent.name}] — matched exclusion term "${excluded}".`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Skip matching if CV text is empty
|
||||
if (!agent.cvText) {
|
||||
console.log(`Skipping match for Agent ${agent.name} (ID: ${agent.id}) - No CV text uploaded.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`Evaluating match: Agent [${agent.name}] <-> Project [${project.title}]`);
|
||||
|
||||
try {
|
||||
const matchResult = await evaluateMatch(agent.cvText, project.description);
|
||||
matchesEvaluatedCount++;
|
||||
|
||||
const match = await Match.create({
|
||||
agentId: agent.id,
|
||||
projectId: project.id,
|
||||
matchPercentage: matchResult.matchPercentage,
|
||||
pros: matchResult.pros,
|
||||
cons: matchResult.cons,
|
||||
status: 'new',
|
||||
notificationSent: false
|
||||
});
|
||||
|
||||
// Trigger email notification if threshold reached
|
||||
if (match.matchPercentage >= matchThreshold) {
|
||||
await sendEmailNotification(agent, project, match.matchPercentage, match.pros, match.cons);
|
||||
match.notificationSent = true;
|
||||
await match.save();
|
||||
}
|
||||
} catch (matchErr) {
|
||||
console.error(`Failed to evaluate match for Agent ${agent.id} and Project ${project.id}:`, matchErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (dbErr) {
|
||||
console.error('Error inserting project or creating match:', dbErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`--- PIPELINE COMPLETED ---`);
|
||||
console.log(`Stored ${newProjectsStoredCount} new projects.`);
|
||||
console.log(`Evaluated ${matchesEvaluatedCount} new matches.`);
|
||||
return true;
|
||||
};
|
||||
@@ -0,0 +1,125 @@
|
||||
import cron from 'node-cron';
|
||||
import Setting from '../models/Setting.js';
|
||||
import { runMatchingPipeline } from './pipeline.js';
|
||||
import { cleanupOldProjects } from './cleanup.js';
|
||||
|
||||
let currentJob = null;
|
||||
let cleanupJob = null;
|
||||
let currentIntervalMinutes = null;
|
||||
|
||||
/**
|
||||
* Startet den Hintergrund-Crawler und plant sich bei geaenderten Intervallen
|
||||
* selbst neu.
|
||||
*/
|
||||
export const startScheduler = async () => {
|
||||
try {
|
||||
const intervalStr = await Setting.getVal('search_interval', '60');
|
||||
const intervalMinutes = parseInt(intervalStr, 10) || 60;
|
||||
|
||||
currentIntervalMinutes = intervalMinutes;
|
||||
console.log('[Scheduler] Cron-Scheduler wird eingerichtet.');
|
||||
console.log(`[Scheduler] Suchintervall: ${intervalMinutes} Minuten.`);
|
||||
|
||||
// If a job is already running, stop it
|
||||
if (currentJob) {
|
||||
currentJob.stop();
|
||||
console.log('[Scheduler] Vorherigen Cron-Job gestoppt.');
|
||||
}
|
||||
|
||||
/*
|
||||
* Zeitversatz zum parallel laufenden alten Scanner. Ohne ihn starten beide
|
||||
* Instanzen direkt nach einem Neustart gleichzeitig und greifen mit
|
||||
* denselben Portal-Sessions zeitgleich auf die Boersen zu - das provoziert
|
||||
* Rate-Limits und CAPTCHAs. Mit Versatz laufen die Laeufe verschraenkt.
|
||||
*/
|
||||
const offsetMinutes = Number(process.env.SCANNER_SCHEDULER_OFFSET ?? 30);
|
||||
const useOffset = Number.isFinite(offsetMinutes) && offsetMinutes > 0;
|
||||
|
||||
/*
|
||||
* Eigene Sperrfrist statt eines vorgealterten lastRunTime: ein Zeitstempel
|
||||
* kann einen Lauf nur vorziehen, nie verzoegern. Sobald der Versatz groesser
|
||||
* als das Suchintervall ist - bei 15 Minuten Intervall der Normalfall -
|
||||
* waere der erste Lauf sonst sofort faellig gewesen.
|
||||
*/
|
||||
const notBefore = useOffset ? Date.now() + offsetMinutes * 60 * 1000 : 0;
|
||||
let lastRunTime = null;
|
||||
|
||||
if (useOffset) {
|
||||
console.log(`[Scheduler] Erster Lauf in ${offsetMinutes} Minuten (Versatz zum alten Scanner).`);
|
||||
}
|
||||
|
||||
// We check every minute if we are due for a run.
|
||||
// This allows exact dynamic changes without having to manually restart the node process.
|
||||
currentJob = cron.schedule('* * * * *', async () => {
|
||||
// Master on/off switch (toggled from Settings UI). When off, the
|
||||
// background scanner stays idle but the backend/UI keep running.
|
||||
const scanningActive = await Setting.getVal('scanning_active', 'true');
|
||||
if (scanningActive === 'false') {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeIntervalStr = await Setting.getVal('search_interval', '60');
|
||||
const activeIntervalMinutes = parseInt(activeIntervalStr, 10) || 60;
|
||||
|
||||
// Geaendertes Intervall im Betrieb uebernehmen
|
||||
if (activeIntervalMinutes !== currentIntervalMinutes) {
|
||||
console.log(`[Scheduler] Intervall von ${currentIntervalMinutes} auf ${activeIntervalMinutes} Minuten geaendert - Neuplanung.`);
|
||||
startScheduler();
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Sperrfrist nach dem Start (Versatz zum parallel laufenden Scanner)
|
||||
if (notBefore && now.getTime() < notBefore) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Weekday gate: skip Saturday (6) and Sunday (0). Scraper runs Mon-Fri only.
|
||||
const day = now.getDay();
|
||||
if (day === 0 || day === 6) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Time-of-day gate: only scan between 07:00 and 21:59 server-local time
|
||||
// (Europe/Berlin). getHours() returns local hour on the host.
|
||||
const hour = now.getHours();
|
||||
if (hour < 7 || hour >= 22) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Run if never run before, or if enough time has passed
|
||||
const isDue = !lastRunTime || (now - lastRunTime) >= activeIntervalMinutes * 60 * 1000;
|
||||
|
||||
if (isDue) {
|
||||
lastRunTime = now;
|
||||
console.log(`[Scheduler] Geplanter Lauf startet: ${now.toLocaleTimeString('de-DE')}`);
|
||||
try {
|
||||
await runMatchingPipeline();
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Fehler im Suchlauf:', err);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
currentJob.start();
|
||||
console.log('[Scheduler] Cron-Job laeuft, Pruefung jede Minute.');
|
||||
|
||||
// Taegliches Aufraeumen um 03:30 - halbe Stunde nach dem alten Scanner,
|
||||
// damit nicht beide gleichzeitig grosse Loeschlaeufe auf derselben MariaDB
|
||||
// fahren. Laeuft nur einmal, auch wenn der Scheduler neu geplant wird.
|
||||
if (!cleanupJob) {
|
||||
cleanupJob = cron.schedule('30 3 * * *', () => {
|
||||
console.log('[Scheduler] Taegliches Aufraeumen laeuft.');
|
||||
cleanupOldProjects().catch(err => console.error('[Scheduler] Fehler beim Aufraeumen:', err));
|
||||
});
|
||||
cleanupJob.start();
|
||||
console.log('[Scheduler] Aufraeum-Job registriert (03:30).');
|
||||
|
||||
// Also run once on boot so retention applies immediately after restart.
|
||||
cleanupOldProjects().catch(() => {});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Scheduler] Failed to initialize scheduler:', error);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,720 @@
|
||||
import { chromium } from 'playwright';
|
||||
import Portal from '../models/Portal.js';
|
||||
|
||||
// Stealth-like launch args — no need for playwright-extra/puppeteer-extra-plugin-stealth
|
||||
// We pass anti-detection flags and a realistic user agent directly.
|
||||
const STEALTH_ARGS = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars',
|
||||
];
|
||||
|
||||
// Defensive parse: storage state stored in DB may originate from older sync
|
||||
// versions or from Cookie-Editor exports with non-Playwright cookie shapes.
|
||||
// Normalize before passing to browser.newContext().
|
||||
const parseStorageState = (sessionState) => {
|
||||
if (!sessionState) return null;
|
||||
let raw;
|
||||
try {
|
||||
raw = typeof sessionState === 'string' ? JSON.parse(sessionState) : sessionState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
let rawCookies = [];
|
||||
let origins = [];
|
||||
if (Array.isArray(raw)) {
|
||||
rawCookies = raw;
|
||||
} else if (raw && typeof raw === 'object') {
|
||||
rawCookies = Array.isArray(raw.cookies) ? raw.cookies : [];
|
||||
origins = Array.isArray(raw.origins) ? raw.origins : [];
|
||||
}
|
||||
|
||||
const cookies = rawCookies.map(c => {
|
||||
if (!c || !c.name || c.value === undefined) return null;
|
||||
let sameSite = c.sameSite;
|
||||
if (typeof sameSite === 'string') {
|
||||
const s = sameSite.toLowerCase();
|
||||
if (s === 'strict') sameSite = 'Strict';
|
||||
else if (s === 'lax') sameSite = 'Lax';
|
||||
else if (s === 'none' || s === 'no_restriction') sameSite = 'None';
|
||||
else sameSite = 'Lax';
|
||||
} else {
|
||||
sameSite = 'Lax';
|
||||
}
|
||||
let expires = c.expires;
|
||||
if (expires === undefined && c.expirationDate !== undefined) expires = c.expirationDate;
|
||||
if (typeof expires !== 'number' || !isFinite(expires)) expires = -1;
|
||||
return {
|
||||
name: String(c.name),
|
||||
value: String(c.value),
|
||||
domain: c.domain || '',
|
||||
path: c.path || '/',
|
||||
expires,
|
||||
httpOnly: !!c.httpOnly,
|
||||
secure: !!c.secure,
|
||||
sameSite,
|
||||
};
|
||||
}).filter(Boolean);
|
||||
|
||||
if (cookies.length === 0) return null;
|
||||
return { cookies, origins };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelancermap.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelancermap = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelancermap scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
// Initialize context with cookies if available
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelancermap session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelancermap session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelancermap.de/dashboard.html', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="/login"]');
|
||||
const userProfile = await page.$('.user-avatar, .user-name, a[href*="/logout"]');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelancermap session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelancermap session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelancermap session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Loop through each keyword to search
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.freelancermap.de/projekte?q=${encodeURIComponent(keyword)}&sort=created_desc`;
|
||||
console.log(`Navigating to Freelancermap search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Freelancermap project cards use stable class `.project-card`. Detail
|
||||
// attributes use data-testid which is reliable across rerenders.
|
||||
const projectElements = await page.$$('.project-card');
|
||||
console.log(`Found ${projectElements.length} project cards for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
const linkEl = await el.$('a[data-testid="title"], a[data-id="project-card-title"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = (await linkEl.innerText()).trim();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelancermap.de${rawUrl}`;
|
||||
|
||||
// Project ID: trailing numeric segment if present, else full slug.
|
||||
const slugMatch = url.match(/\/projekt\/([^/?#]+)/);
|
||||
const slug = slugMatch ? slugMatch[1] : null;
|
||||
const idMatch = slug ? slug.match(/-(\d+)$/) : null;
|
||||
const portalProjectId = idMatch ? idMatch[1] : (slug || url);
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Company: first div in .project-info (no class on text node)
|
||||
const company = await el.evaluate(node => {
|
||||
const info = node.querySelector('.project-info');
|
||||
if (!info) return '';
|
||||
const first = info.querySelector('div');
|
||||
return first ? first.innerText.trim() : '';
|
||||
});
|
||||
|
||||
// Location: data-testid="city" container
|
||||
const location = await el.evaluate(node => {
|
||||
const c = node.querySelector('[data-testid="city"]');
|
||||
return c ? c.innerText.replace(/\s+/g, ' ').trim() : '';
|
||||
});
|
||||
|
||||
// Type / duration / start → concatenated into rate as listing card has no €
|
||||
const meta = await el.evaluate(node => {
|
||||
const parts = [];
|
||||
['type', 'duration', 'beginningText', 'remoteInPercent'].forEach(t => {
|
||||
const e2 = node.querySelector(`[data-testid="${t}"]`);
|
||||
if (e2) parts.push(e2.innerText.replace(/\s+/g, ' ').trim());
|
||||
});
|
||||
return parts.join(' | ');
|
||||
});
|
||||
|
||||
// Listing card has no description text — fetch detail page lazily would
|
||||
// be expensive; leave description empty here. Matching engine handles
|
||||
// empty descriptions by returning low score, but title + meta still
|
||||
// give signal once we add a detail fetch step later.
|
||||
const description = `${title}\n${meta}`.trim();
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelancermap.de',
|
||||
portalProjectId,
|
||||
title,
|
||||
description,
|
||||
company: company || 'N/A',
|
||||
location: location || 'Remote / N/A',
|
||||
rate: meta || 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Freelancermap card:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid (cookies might rotate)
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
// Session was provided but check failed
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelancermap scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes job listings from Hays.de. Public portal, no login required.
|
||||
* Search URL: https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=<keyword>&e=false
|
||||
* joblevel=3 (Senior/Lead-Bereich, vom Nutzer vorgegeben), mrew=1, e=false (alle Vertragsarten).
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeHays = async (keywords) => {
|
||||
console.log(`Starting Hays.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=${encodeURIComponent(keyword)}&e=false`;
|
||||
console.log(`Navigating to Hays search: ${searchUrl}`);
|
||||
|
||||
try {
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
// Wait for actual job-detail anchors (JS hydrates them).
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/jobsuche/stellenangebote-jobs-detail-"]', { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`Hays: no job-detail anchors visible for "${keyword}"`);
|
||||
continue;
|
||||
}
|
||||
await page.waitForTimeout(800 + Math.random() * 1000);
|
||||
|
||||
// Only real job-detail URLs. Pattern: /jobsuche/stellenangebote-jobs-detail-<slug>-<id>/<page>
|
||||
const linkEls = await page.$$('a[href*="/jobsuche/stellenangebote-jobs-detail-"]');
|
||||
console.log(`Found ${linkEls.length} Hays job-detail anchors for keyword: ${keyword}`);
|
||||
|
||||
for (const linkEl of linkEls) {
|
||||
try {
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!rawUrl) continue;
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.hays.de${rawUrl}`;
|
||||
|
||||
// Hays job-id: numeric segment at end before trailing /N
|
||||
const idMatch = url.match(/-(\d{5,})(?:\/\d+)?(?:[?#].*)?$/);
|
||||
if (!idMatch) continue;
|
||||
const portalProjectId = idMatch[1];
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Title: link inner text first, fall back to nearest heading.
|
||||
let title = ((await linkEl.innerText()) || '').replace(/\s+/g, ' ').trim();
|
||||
if (!title) {
|
||||
title = await linkEl.evaluate(node => {
|
||||
let n = node;
|
||||
for (let i = 0; i < 6 && n; i++) {
|
||||
n = n.parentElement;
|
||||
if (!n) break;
|
||||
const h = n.querySelector('h2, h3, [class*="title"], [class*="Title"]');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
return '';
|
||||
}).catch(() => '');
|
||||
}
|
||||
// Derive title from URL slug if still missing.
|
||||
if (!title) {
|
||||
const slugMatch = url.match(/stellenangebote-jobs-detail-(.+?)-\d{5,}/);
|
||||
if (slugMatch) {
|
||||
title = slugMatch[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
if (!title) continue;
|
||||
|
||||
// Card text: closest ancestor that contains structured info (max 6 levels up).
|
||||
const meta = await linkEl.evaluate(node => {
|
||||
let card = node;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (!card.parentElement) break;
|
||||
card = card.parentElement;
|
||||
const txt = (card.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
if (txt.length > 80 && txt.length < 1500) return txt.slice(0, 1200);
|
||||
}
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||||
}).catch(() => '');
|
||||
|
||||
// Location: try common DE cities or fall back to scanning "Standort:" pattern.
|
||||
const locMatch = meta.match(/Standort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| meta.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Dortmund|Essen|Bremen|Dresden|Hannover|Nürnberg|Mannheim|Karlsruhe|Wiesbaden|Bonn|Münster|Augsburg|Aachen|Erfurt|Mainz|Kiel|Halle|Magdeburg|Freiburg|Lübeck|Oberhausen|Rostock|Kassel|Saarbrücken|Mönchengladbach|Krefeld|Chemnitz|Braunschweig|Wuppertal|Bielefeld|Duisburg|remote|Remote)\b/);
|
||||
const location = locMatch ? locMatch[1] || locMatch[0] : 'Remote / N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'hays.de',
|
||||
portalProjectId,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${meta}`.trim().slice(0, 1500),
|
||||
company: 'Hays (Vermittlung)',
|
||||
location: location.slice(0, 200),
|
||||
rate: (meta.match(/(?:€|EUR)[^|·]{0,40}/) || ['N/A'])[0].slice(0, 90),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Hays card:', err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping Hays for "${keyword}":`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Hays scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Generic scraper for the SThree-family staffing portals (Huxley, Computer
|
||||
* Futures, Progressive, Real Staffing, Global Enterprise Partners, Madison
|
||||
* Black, JP Gray, etc.). They all share the same template — job detail URLs
|
||||
* follow `<base>/<locale>/job/<slug>/<numeric-id>/`. Public, no login.
|
||||
*
|
||||
* @param {object} cfg
|
||||
* @param {string} cfg.name - DB portal name (e.g. 'huxley.com')
|
||||
* @param {string} cfg.listingUrl - Search-result listing URL
|
||||
* @param {string} cfg.localePath - e.g. '/de-de/' or '/en-nl/'
|
||||
*/
|
||||
const scrapeSThreeStyle = async ({ name, listingUrl, localePath }) => {
|
||||
console.log(`Starting ${name} scrape (listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(listingUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
const jobLinkSelector = `a[href*="${localePath}job/"]`;
|
||||
try {
|
||||
await page.waitForSelector(jobLinkSelector, { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`${name}: no job anchors visible after wait.`);
|
||||
}
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
const links = await page.$$eval(jobLinkSelector, (els, locale) => {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
// Escape regex special chars in locale (keeps slashes as literal).
|
||||
const esc = locale.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const re = new RegExp(esc + 'job/([^/]+)/(\\d{4,})/?');
|
||||
for (const a of els) {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (href.includes('{{')) continue;
|
||||
const m = href.match(re);
|
||||
if (!m) continue;
|
||||
const id = m[2];
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
out.push({ id, slug: m[1], href });
|
||||
}
|
||||
return out;
|
||||
}, localePath).catch(() => []);
|
||||
console.log(`Found ${links.length} ${name} job URLs.`);
|
||||
|
||||
// Detail-page fetches were triggering Cloudflare/WAF on the SThree
|
||||
// family. Title is derived from the URL slug; description = title only.
|
||||
// No detail requests → no bot signature, no rate spikes.
|
||||
for (const link of links) {
|
||||
try {
|
||||
const url = link.href.startsWith('http') ? link.href : new URL(link.href, listingUrl).toString();
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === link.id)) continue;
|
||||
|
||||
let title = '';
|
||||
try {
|
||||
title = decodeURIComponent(link.slug);
|
||||
} catch {
|
||||
title = link.slug;
|
||||
}
|
||||
title = title.replace(/-/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
// Title-case for the slug pieces (German diacritics kept).
|
||||
title = title.replace(/\b(\w)([\w'’äöüß]*)/g, (_, a, b) => a.toUpperCase() + b);
|
||||
if (!title) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: name,
|
||||
portalProjectId: link.id,
|
||||
title: title.slice(0, 250),
|
||||
description: title.slice(0, 250),
|
||||
company: `${name} (Vermittlung)`,
|
||||
location: 'Siehe Anzeige',
|
||||
rate: 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error building ${name} project entry:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping ${name} listing:`, err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error during ${name} scrape:`, error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`${name} scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
// Per-portal wrappers used by pipeline.js / api.js
|
||||
export const SThreePortals = [
|
||||
{ name: 'huxley.com', listingUrl: 'https://www.huxley.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'computerfutures.com', listingUrl: 'https://www.computerfutures.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'progressiverecruitment.com', listingUrl: 'https://www.progressiverecruitment.com/de-de/jobs-projekte-suchen/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'realstaffing.com', listingUrl: 'https://www.realstaffing.com/de-de/job-suche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'globalenterprisepartners.com', listingUrl: 'https://www.globalenterprisepartners.com/de-de/job-search/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'madisonblack.com', listingUrl: 'https://www.madisonblack.com/de-de/job-search/?searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'jpgray.nl', listingUrl: 'https://www.jpgray.nl/en-nl/job-search/?country=Netherlands&searchRadius=50km', localePath: '/en-nl/' }
|
||||
];
|
||||
|
||||
export const scrapeSThreePortal = (cfg) => scrapeSThreeStyle(cfg);
|
||||
|
||||
/**
|
||||
* Scrapes IT projects from Etengo.de. Public portal, no login required.
|
||||
* Search URL: https://www.etengo.de/it-projektsuche/?q=<keyword>
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeEtengo = async (/* keywords */) => {
|
||||
// Etengo's search filter is JS-side and the listing URL ignores ?q. Load the
|
||||
// full project list once and return all entries; matching engine handles
|
||||
// keyword relevance per agent CV.
|
||||
console.log(`Starting Etengo.de scrape (full listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto('https://www.etengo.de/it-projektsuche/', { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/it-projektsuche/"][href*="/"]', { timeout: 10000 });
|
||||
} catch {
|
||||
console.warn('Etengo: project anchors not visible.');
|
||||
}
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Collect detail-page URLs: /it-projektsuche/<numeric-id>/
|
||||
const anchors = await page.$$('a[href*="/it-projektsuche/"]');
|
||||
const seen = new Set();
|
||||
const detailUrls = [];
|
||||
for (const a of anchors) {
|
||||
const href = await a.getAttribute('href');
|
||||
if (!href) continue;
|
||||
const m = href.match(/\/it-projektsuche\/(\d{3,})\/?$/);
|
||||
if (!m) continue;
|
||||
const url = href.startsWith('http') ? href : `https://www.etengo.de${href}`;
|
||||
if (seen.has(m[1])) continue;
|
||||
seen.add(m[1]);
|
||||
detailUrls.push({ id: m[1], url });
|
||||
}
|
||||
console.log(`Found ${detailUrls.length} Etengo project URLs.`);
|
||||
|
||||
// Fetch each detail page in same context for title + description.
|
||||
for (const { id, url } of detailUrls) {
|
||||
try {
|
||||
const detail = await context.newPage();
|
||||
await detail.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
||||
await detail.waitForTimeout(500);
|
||||
|
||||
const title = await detail.evaluate(() => {
|
||||
const h = document.querySelector('h1');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
const t = document.querySelector('title');
|
||||
return t ? t.innerText.replace(/ - Etengo.*$/i, '').trim() : '';
|
||||
}).catch(() => '');
|
||||
|
||||
const description = await detail.evaluate(() => {
|
||||
// Look for main article / project description container.
|
||||
const cand = document.querySelector('main, article, .entry-content, .project-detail, [class*="detail"]');
|
||||
const node = cand || document.body;
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 1500);
|
||||
}).catch(() => '');
|
||||
|
||||
const location = (description.match(/Einsatzort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| description.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Hannover|Nürnberg|Bonn|Münster|Karlsruhe|Mannheim|Bremen|Dresden|Erfurt|Mainz|remote|Remote)\b/) || ['Remote / N/A'])[0];
|
||||
|
||||
const rateMatch = description.match(/(?:€|EUR)[^|·]{0,40}/);
|
||||
|
||||
await detail.close();
|
||||
|
||||
if (!title) continue;
|
||||
if (projectsList.some(p => p.portalProjectId === id)) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: 'etengo.de',
|
||||
portalProjectId: id,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${description}`.trim().slice(0, 1500),
|
||||
company: 'Etengo (Vermittlung)',
|
||||
location: String(location).slice(0, 200),
|
||||
rate: rateMatch ? rateMatch[0].slice(0, 90) : 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error fetching Etengo detail ${url}:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error scraping Etengo listing:', err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Etengo scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`Etengo scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelance.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelanceDe = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelance.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelance.de session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelance.de session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelance.de/users/profile.php', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="login"], a[href*="sign-in"]');
|
||||
const userProfile = await page.$('a[href*="logout"], .user-info, .profile-settings');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelance.de session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelance.de session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelance.de session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
for (const keyword of keywords) {
|
||||
// freelance.de search url: search_sort_by=2 means date descending (newest first)
|
||||
const searchUrl = `https://www.freelance.de/search/project.php?__search_sort_by=2&query=${encodeURIComponent(keyword)}`;
|
||||
console.log(`Navigating to Freelance.de search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Select project list items
|
||||
const projectElements = await page.$$('.project-list-item, .project-row, .list-item-main, div[class*="project-list-item"]');
|
||||
console.log(`Found ${projectElements.length} potential project elements for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
// Find the link to the project detail page
|
||||
// Format e.g., /Projekte/Projekt-Details-12345.html
|
||||
const linkEl = await el.$('a[href*="/Projekte/"], a[href*="/Projekt/"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = await linkEl.innerText();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelance.de${rawUrl}`;
|
||||
|
||||
// Parse project ID from URL
|
||||
// Format e.g., /Projekte/123456-React-Entwickler or /Projekt/123456
|
||||
const matchId = url.match(/\/(\d+)-/) || url.match(/\/(\d+)\.html/) || url.match(/\/(\d+)(?:\/|\?|$)/);
|
||||
const portalProjectId = matchId ? matchId[1] : url;
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Scrape details from listing card
|
||||
const descEl = await el.$('.project-description, .description, p, .project-details');
|
||||
const description = descEl ? await descEl.innerText() : '';
|
||||
|
||||
const locEl = await el.$('.location, .project-location, span[title*="Ort"], .meta-location');
|
||||
const location = locEl ? await locEl.innerText() : 'Remote / N/A';
|
||||
|
||||
const rateEl = await el.$('.rate, .project-rate, .meta-rate, span[title*="Budget"]');
|
||||
const rate = rateEl ? await rateEl.innerText() : 'N/A';
|
||||
|
||||
const compEl = await el.$('.company, .project-company, .meta-company');
|
||||
const company = compEl ? await compEl.innerText() : 'N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelance.de',
|
||||
portalProjectId,
|
||||
title: title.trim(),
|
||||
description: description.trim(),
|
||||
company: company.trim(),
|
||||
location: location.trim(),
|
||||
rate: rate.trim(),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing individual Freelance.de element:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelance.de scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
@@ -0,0 +1,29 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
/*
|
||||
* Verzeichnis der CV-PDFs dieser Scanner-Instanz. Absolut aufgeloest, damit es
|
||||
* nicht am Arbeitsverzeichnis des Prozesses haengt.
|
||||
*
|
||||
* Bewusst ein EIGENES Verzeichnis und nicht das der alten App: beide Instanzen
|
||||
* verwalten ihre Agenten getrennt, und ein Loeschen hier darf dort keine Datei
|
||||
* mitreissen. Bestehende CVs holt `npm run scanner:import` herueber.
|
||||
*/
|
||||
export const UPLOAD_DIR = path.resolve(
|
||||
process.env.SCANNER_UPLOAD_DIR || path.join(process.cwd(), 'uploads')
|
||||
);
|
||||
|
||||
export function ensureUploadDir() {
|
||||
if (!fs.existsSync(UPLOAD_DIR)) {
|
||||
fs.mkdirSync(UPLOAD_DIR, { recursive: true });
|
||||
}
|
||||
return UPLOAD_DIR;
|
||||
}
|
||||
|
||||
/** Loest einen in der DB gespeicherten CV-Pfad gegen UPLOAD_DIR auf. */
|
||||
export function resolveCvPath(stored) {
|
||||
if (!stored) return null;
|
||||
if (path.isAbsolute(stored) && fs.existsSync(stored)) return stored;
|
||||
const candidate = path.join(UPLOAD_DIR, path.basename(stored));
|
||||
return fs.existsSync(candidate) ? candidate : null;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import CryptoJS from 'crypto-js';
|
||||
|
||||
/*
|
||||
* Eigener Schluessel, unabhaengig vom alten Scanner: die Portal-Passwoerter
|
||||
* liegen in einer eigenen Datenbank und werden hier neu verschluesselt. Der
|
||||
* Schluessel der alten App wird absichtlich NICHT als Fallback gelesen - beide
|
||||
* Instanzen sollen sich nicht ueber gemeinsame Geheimnisse verkoppeln.
|
||||
*
|
||||
* Ohne SCANNER_ENCRYPTION_KEY laeuft alles weiter, aber mit einem bekannten
|
||||
* Ersatzschluessel - das ist nur fuer die lokale Entwicklung gedacht, deshalb
|
||||
* die Warnung.
|
||||
*/
|
||||
const FALLBACK_KEY = 'perinco-scanner-dev-key-nicht-fuer-produktion';
|
||||
|
||||
export const SCANNER_KEY_CONFIGURED = Boolean(process.env.SCANNER_ENCRYPTION_KEY);
|
||||
|
||||
const SECRET_KEY = process.env.SCANNER_ENCRYPTION_KEY || FALLBACK_KEY;
|
||||
|
||||
/** Verschluesselt ein Klartext-Passwort. */
|
||||
export const encrypt = (text) => {
|
||||
if (!text) return '';
|
||||
return CryptoJS.AES.encrypt(text, SECRET_KEY).toString();
|
||||
};
|
||||
|
||||
/** Entschluesselt einen Cipher-String; leerer String, wenn es nicht passt. */
|
||||
export const decrypt = (cipherText) => {
|
||||
if (!cipherText) return '';
|
||||
try {
|
||||
const bytes = CryptoJS.AES.decrypt(cipherText, SECRET_KEY);
|
||||
return bytes.toString(CryptoJS.enc.Utf8);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Entschluesselt mit einem fremden Schluessel - gebraucht beim Uebernehmen der
|
||||
* Portal-Zugangsdaten aus der alten Scanner-Datenbank.
|
||||
*/
|
||||
export const decryptWith = (cipherText, key) => {
|
||||
if (!cipherText || !key) return '';
|
||||
try {
|
||||
const bytes = CryptoJS.AES.decrypt(cipherText, key);
|
||||
return bytes.toString(CryptoJS.enc.Utf8);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user