chore(scanner): alte Scanner-App zurueckgebaut
Die eigenstaendige Vorgaenger-App unter scanner.perinco.de ist auf dem Server entfernt: Unit project-scanner-backend (Port 5005), beide Apache-Vhosts, das Let's-Encrypt-Zertifikat samt Renewal-Config, /var/www/Projekt Scanner (486 MB) und die Datenbank project_scanner (Dump unter /root/backups). Damit entfallen die LEGACY_*-Variablen und das Importskript, dessen Quelle es nicht mehr gibt. /root/.cache/ms-playwright bleibt bewusst stehen: der Cache stammt aus der Installation der alten App, liegt aber ausserhalb beider Verzeichnisse und wird vom CRM-Scanner benutzt - playwright ist auf 1.60.0 gepinnt, damit es genau dieser Build (chromium-1223) bleibt. Der Hinweis steht jetzt in README und docs/06-betrieb.md, damit ihn beim naechsten Aufraeumen niemand mitnimmt. SCANNER_SCHEDULER_OFFSET geht auf 0: der Versatz sollte verhindern, dass beide Instanzen mit denselben Portal-Cookies gleichzeitig losziehen. Ohne zweite Instanz hat er keinen Zweck mehr, die Mechanik bleibt fuer kuenftige Erststarts im Code. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sascha Wolf
co-authored by
Claude Opus 5
Claude Opus 5
parent
ab6b9f2372
commit
3c9d83161e
@@ -1,339 +0,0 @@
|
||||
/*
|
||||
* Einmalige Uebernahme aus der alten Projekt-Scanner-Datenbank in die eigene
|
||||
* Datenbank dieser Instanz. Beide Scanner laufen danach parallel und getrennt -
|
||||
* dieses Skript kopiert nur, es veraendert die Quelle nicht.
|
||||
*
|
||||
* Aufruf (aus server/):
|
||||
* npm run scanner:import
|
||||
* npm run scanner:import -- --with-matches auch Projekte und Treffer
|
||||
* npm run scanner:import -- --force vorhandene Zieldaten ersetzen
|
||||
* npm run scanner:import -- --dry-run nur anzeigen, nichts schreiben
|
||||
* npm run scanner:import -- --agents-only nur Suchagenten, sonst nichts
|
||||
* npm run scanner:import -- --skip=Bilal,Nico Agenten nach Namen auslassen
|
||||
*
|
||||
* Quelle konfigurieren (server/.env):
|
||||
* LEGACY_DB_NAME=project_scanner
|
||||
* LEGACY_DB_HOST / _PORT / _USER / _PASSWORD (leer = CRM-Werte)
|
||||
* LEGACY_UPLOAD_DIR=/opt/project-scanner/backend/uploads
|
||||
* LEGACY_ENCRYPTION_KEY=... Portal-Passwoerter mit uebernehmen
|
||||
*
|
||||
* Was uebernommen wird:
|
||||
* - Einstellungen (inkl. OpenAI-Key, SMTP-Zugang, Schwellenwerte)
|
||||
* - Portale: Benutzername, Session-Cookies, Status; Passwoerter nur mit
|
||||
* LEGACY_ENCRYPTION_KEY, weil sie mit dem alten Schluessel verschluesselt
|
||||
* sind und hier neu verschluesselt werden muessen
|
||||
* - Suchagenten inkl. CV-Text und CV-PDF (Datei wird kopiert)
|
||||
* - mit --with-matches zusaetzlich Projekte und Treffer
|
||||
*
|
||||
* --agents-only laesst Einstellungen und Portale unberuehrt und laeuft
|
||||
* nachtraeglich: vorhandene Agenten blockieren nicht mehr, sondern werden am
|
||||
* Namen erkannt und uebersprungen. Die Ids der Quelle werden nur uebernommen,
|
||||
* solange sie hier frei sind - sonst vergibt die Datenbank eine neue.
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import mysql from 'mysql2/promise';
|
||||
import {
|
||||
Agent,
|
||||
Match,
|
||||
Portal,
|
||||
Project,
|
||||
Setting,
|
||||
initScannerDatabase,
|
||||
} from './models/index.js';
|
||||
import { encrypt, decryptWith } from './utils/crypto.js';
|
||||
import { ensureUploadDir } from './uploadDir.js';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const withMatches = args.includes('--with-matches');
|
||||
const force = args.includes('--force');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const agentsOnly = args.includes('--agents-only');
|
||||
|
||||
// --skip=Bilal oder --skip=Bilal,Nico - Vergleich ohne Gross-/Kleinschreibung.
|
||||
const skipNames = args
|
||||
.filter((arg) => arg.startsWith('--skip='))
|
||||
.flatMap((arg) => arg.slice('--skip='.length).split(','))
|
||||
.map((name) => name.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
|
||||
const LEGACY_UPLOAD_DIR = process.env.LEGACY_UPLOAD_DIR || '';
|
||||
const LEGACY_KEY = process.env.LEGACY_ENCRYPTION_KEY || '';
|
||||
|
||||
const log = (message) => console.log(`[Import] ${message}`);
|
||||
|
||||
async function openLegacy() {
|
||||
const database = process.env.LEGACY_DB_NAME || 'project_scanner';
|
||||
const connection = await mysql.createConnection({
|
||||
host: process.env.LEGACY_DB_HOST || process.env.DB_HOST || 'localhost',
|
||||
port: Number(process.env.LEGACY_DB_PORT || process.env.DB_PORT || 3306),
|
||||
user: process.env.LEGACY_DB_USER || process.env.DB_USER || 'root',
|
||||
password: process.env.LEGACY_DB_PASSWORD || process.env.DB_PASSWORD || '',
|
||||
database,
|
||||
dateStrings: false,
|
||||
});
|
||||
log(`Quelle: ${database}`);
|
||||
return connection;
|
||||
}
|
||||
|
||||
/** Kopiert das CV-PDF in das eigene Upload-Verzeichnis. */
|
||||
function copyCv(storedPath, uploadDir) {
|
||||
if (!storedPath) return '';
|
||||
|
||||
const fileName = path.basename(storedPath);
|
||||
const candidates = [
|
||||
path.isAbsolute(storedPath) ? storedPath : null,
|
||||
LEGACY_UPLOAD_DIR ? path.join(LEGACY_UPLOAD_DIR, fileName) : null,
|
||||
path.resolve(storedPath),
|
||||
].filter(Boolean);
|
||||
|
||||
const source = candidates.find((candidate) => fs.existsSync(candidate));
|
||||
if (!source) {
|
||||
log(` CV-Datei nicht gefunden: ${storedPath} (LEGACY_UPLOAD_DIR setzen?)`);
|
||||
return '';
|
||||
}
|
||||
|
||||
const target = path.join(uploadDir, fileName);
|
||||
if (!dryRun) fs.copyFileSync(source, target);
|
||||
return target;
|
||||
}
|
||||
|
||||
async function importSettings(legacy) {
|
||||
const [rows] = await legacy.query('SELECT key_name, value FROM settings');
|
||||
for (const row of rows) {
|
||||
if (!dryRun) await Setting.setVal(row.key_name, row.value ?? '');
|
||||
}
|
||||
log(`Einstellungen: ${rows.length} Schluessel uebernommen.`);
|
||||
}
|
||||
|
||||
async function importPortals(legacy) {
|
||||
const [rows] = await legacy.query(
|
||||
'SELECT name, username, password_encrypted, session_state, connection_status, last_login_attempt FROM portals'
|
||||
);
|
||||
|
||||
let withSession = 0;
|
||||
let withPassword = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
// Portale sind bei der Initialisierung schon angelegt - hier wird nur der
|
||||
// Inhalt nachgezogen, ueber den Namen zugeordnet.
|
||||
const portal = await Portal.findOne({ where: { name: row.name } });
|
||||
if (!portal) {
|
||||
log(` Portal ${row.name} gibt es hier nicht - uebersprungen.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
portal.username = row.username || portal.username;
|
||||
if (row.session_state) {
|
||||
portal.sessionState = row.session_state;
|
||||
portal.connectionStatus = row.connection_status || 'connected';
|
||||
portal.lastLoginAttempt = row.last_login_attempt || new Date();
|
||||
withSession++;
|
||||
}
|
||||
|
||||
if (row.password_encrypted && LEGACY_KEY) {
|
||||
const plain = decryptWith(row.password_encrypted, LEGACY_KEY);
|
||||
if (plain) {
|
||||
// Neu verschluesseln - diese Instanz hat ihren eigenen Schluessel.
|
||||
portal.passwordEncrypted = encrypt(plain);
|
||||
withPassword++;
|
||||
} else {
|
||||
log(` Passwort von ${row.name} liess sich mit LEGACY_ENCRYPTION_KEY nicht lesen.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dryRun) await portal.save();
|
||||
}
|
||||
|
||||
log(`Portale: ${rows.length} geprueft, ${withSession} mit Session, ${withPassword} mit Passwort.`);
|
||||
if (!LEGACY_KEY) {
|
||||
log(' Hinweis: ohne LEGACY_ENCRYPTION_KEY bleiben die Portal-Passwoerter leer.');
|
||||
}
|
||||
}
|
||||
|
||||
async function importAgents(legacy, uploadDir) {
|
||||
const [rows] = await legacy.query(
|
||||
`SELECT id, name, email, cv_text, cv_file_path, keywords, exclude_keywords, is_active,
|
||||
created_at, updated_at
|
||||
FROM agents ORDER BY id`
|
||||
);
|
||||
|
||||
let created = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
if (skipNames.includes((row.name || '').trim().toLowerCase())) {
|
||||
log(` Agent "${row.name}" per --skip ausgelassen.`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Nachtraegliche Uebernahme: was hier schon unter dem Namen steht, bleibt
|
||||
// unangetastet - der Agent koennte hier inzwischen bearbeitet worden sein.
|
||||
const existing = await Agent.findOne({ where: { name: row.name } });
|
||||
if (existing) {
|
||||
log(` Agent "${row.name}" gibt es hier schon (Id ${existing.id}) - uebersprungen.`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
log(` wuerde Agent ${row.id} "${row.name}" anlegen`);
|
||||
created++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const cvFilePath = copyCv(row.cv_file_path, uploadDir);
|
||||
|
||||
// Die Id der Quelle bleibt erhalten, damit uebernommene Treffer weiter
|
||||
// passen - aber nur, wenn sie hier noch frei ist.
|
||||
const idTaken = await Agent.findByPk(row.id);
|
||||
await Agent.create(
|
||||
{
|
||||
...(idTaken ? {} : { id: row.id }),
|
||||
name: row.name,
|
||||
email: row.email,
|
||||
cvText: row.cv_text,
|
||||
cvFilePath,
|
||||
keywords: row.keywords,
|
||||
excludeKeywords: row.exclude_keywords,
|
||||
isActive: Boolean(row.is_active),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
created++;
|
||||
}
|
||||
|
||||
log(`Suchagenten: ${created} uebernommen, ${skipped} uebersprungen (${rows.length} in der Quelle).`);
|
||||
}
|
||||
|
||||
async function importProjectsAndMatches(legacy) {
|
||||
const [projects] = await legacy.query(
|
||||
`SELECT id, portal, portal_project_id, title, description, company, location, rate, url,
|
||||
published_at, created_at
|
||||
FROM projects ORDER BY id`
|
||||
);
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of projects) {
|
||||
await Project.create(
|
||||
{
|
||||
id: row.id,
|
||||
portal: row.portal,
|
||||
portalProjectId: row.portal_project_id,
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
company: row.company,
|
||||
location: row.location,
|
||||
rate: row.rate,
|
||||
url: row.url,
|
||||
publishedAt: row.published_at,
|
||||
createdAt: row.created_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Projekte: ${projects.length} uebernommen.`);
|
||||
|
||||
const [matches] = await legacy.query(
|
||||
`SELECT id, agent_id, project_id, match_percentage, pros, cons, status, notification_sent,
|
||||
created_at, updated_at
|
||||
FROM matches ORDER BY id`
|
||||
);
|
||||
|
||||
const parseList = (value) => {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value);
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
if (!dryRun) {
|
||||
for (const row of matches) {
|
||||
await Match.create(
|
||||
{
|
||||
id: row.id,
|
||||
agentId: row.agent_id,
|
||||
projectId: row.project_id,
|
||||
matchPercentage: row.match_percentage,
|
||||
pros: parseList(row.pros),
|
||||
cons: parseList(row.cons),
|
||||
status: row.status,
|
||||
notificationSent: Boolean(row.notification_sent),
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
},
|
||||
{ silent: true }
|
||||
);
|
||||
}
|
||||
}
|
||||
log(`Treffer: ${matches.length} uebernommen.`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (dryRun) log('Probelauf - es wird nichts geschrieben.');
|
||||
|
||||
await initScannerDatabase();
|
||||
const uploadDir = ensureUploadDir();
|
||||
log(`Ziel-Upload-Verzeichnis: ${uploadDir}`);
|
||||
|
||||
// Bei --agents-only ist der Lauf nachtraeglich gedacht und laeuft additiv:
|
||||
// vorhandene Agenten sind kein Grund zum Abbruch, sie werden in
|
||||
// importAgents am Namen erkannt.
|
||||
const existingAgents = await Agent.count();
|
||||
if (existingAgents > 0 && !force && !agentsOnly) {
|
||||
log(
|
||||
`Abbruch: es gibt hier schon ${existingAgents} Suchagenten. Mit --force werden Agenten, ` +
|
||||
'Projekte und Treffer vorher geloescht, mit --agents-only kommen nur die fehlenden dazu.'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const legacy = await openLegacy();
|
||||
try {
|
||||
if (force && agentsOnly) {
|
||||
log('Abbruch: --force und --agents-only zusammen ergeben keinen Sinn.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (force && !dryRun) {
|
||||
// Reihenfolge wegen der Fremdschluessel: Treffer, dann Projekte, dann Agenten.
|
||||
await Match.destroy({ where: {}, truncate: false });
|
||||
await Project.destroy({ where: {}, truncate: false });
|
||||
await Agent.destroy({ where: {}, truncate: false });
|
||||
log('Vorhandene Agenten, Projekte und Treffer geloescht.');
|
||||
}
|
||||
|
||||
if (agentsOnly) {
|
||||
log('Nur Suchagenten - Einstellungen und Portale bleiben unberuehrt.');
|
||||
} else {
|
||||
await importSettings(legacy);
|
||||
await importPortals(legacy);
|
||||
}
|
||||
|
||||
await importAgents(legacy, uploadDir);
|
||||
|
||||
if (withMatches && !agentsOnly) {
|
||||
await importProjectsAndMatches(legacy);
|
||||
} else {
|
||||
log('Projekte und Treffer uebersprungen (--with-matches aktiviert sie).');
|
||||
}
|
||||
} finally {
|
||||
await legacy.end();
|
||||
}
|
||||
|
||||
log('Fertig.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[Import] Fehlgeschlagen:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user