feat: Projekt Scanner integrieren, Whisper in die Einstellungen

Der eigenstaendige Projekt Scanner zieht komplett ins CRM ein und laeuft parallel zur bestehenden App weiter - jede geteilte Ressource ist verdoppelt statt geteilt, damit die alte Installation unberuehrt bleibt: - eigene Datenbank perinco_scanner, legt sich beim Start selbst an - eigener SCANNER_ENCRYPTION_KEY, eigenes Upload-Verzeichnis - eigener Scheduler mit Zeitversatz, damit nicht beide Instanzen gleichzeitig mit denselben Portal-Cookies scrapen (Rate-Limits, CAPTCHA, doppelter RAM) - Sperre gegen ueberlappende Laeufe: Scheduler und "Jetzt scannen" teilen sie, ein zweiter Aufruf bekommt 409 statt doppelter OpenAI-Kosten Backend unter /api/scanner hinter der CRM-Anmeldung; die HTTP-Basic-Auth des Originals entfaellt. CV-PDFs kommen als Stream mit Token statt aus einem statischen Verzeichnis. Bestandsdaten holt "npm run scanner:import" aus der alten Datenbank, ohne sie zu veraendern. Oberflaeche in Tailwind und CI-Farben neu gebaut: Menuepunkt Scanner mit aufklappbaren Unterpunkten, Einstellungen als eigener Reiter. Whisper hat keinen eigenen Menuepunkt mehr, sondern ist ein Reiter in den Einstellungen. Damit diese Geraeteeinstellung jeder Rolle offensteht, sind die Einstellungen jetzt fuer alle sichtbar - welche Reiter erscheinen, entscheidet die Rolle. Ausserdem: kein festes Admin-Passwort mehr im Quelltext, playwright exakt auf 1.60.0 gepinnt (nutzt den vorhandenen Chromium-Build des Servers), technische Dokumentation mit Funktionszeichnungen unter docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-06 15:08:59 +02:00
commit 82d1b7eab4
106 changed files with 24133 additions and 0 deletions
@@ -0,0 +1,286 @@
+import { Agent, Portal, Project, Match, Setting } from '../models/index.js';
+import { scrapeFreelancermap, scrapeFreelanceDe, scrapeHays, scrapeEtengo, scrapeSThreePortal, SThreePortals } from './scraper.js';
+import { evaluateMatch } from './matchingEngine.js';
+import nodemailer from 'nodemailer';
+
+// Helper to encrypt/decrypt credentials if needed, or send email notifications
+const sendEmailNotification = async (agent, project, matchPercentage, pros, cons) => {
+  const active = await Setting.getVal('email_notifications_active') === 'true';
+  if (!active) return;
+
+  const smtpHost = await Setting.getVal('smtp_host');
+  const smtpPort = parseInt(await Setting.getVal('smtp_port', '587'), 10);
+  const smtpUser = await Setting.getVal('smtp_user');
+  const smtpPass = await Setting.getVal('smtp_pass');
+  const smtpFrom = await Setting.getVal('smtp_from', 'scanner@perinco.de');
+  const recipient = await Setting.getVal('notification_email_recipient') || agent.email;
+
+  if (!smtpHost || !smtpUser || !smtpPass || !recipient) {
+    console.warn('SMTP settings are incomplete. Skipping email notification.');
+    return;
+  }
+
+  const transporter = nodemailer.createTransport({
+    host: smtpHost,
+    port: smtpPort,
+    secure: smtpPort === 465,
+    auth: {
+      user: smtpUser,
+      pass: smtpPass
+    }
+  });
+
+  const prosList = pros.map(p => `<li>✅ ${p}</li>`).join('');
+  const consList = cons.map(c => `<li>❌ ${c}</li>`).join('');
+
+  const mailOptions = {
+    from: smtpFrom,
+    to: recipient,
+    subject: `[PROJEKT-MATCHING ${matchPercentage}%] Neue Chance für ${agent.name}`,
+    html: `
+      <div style="font-family: sans-serif; max-width: 600px; margin: 0 auto; padding: 20px; border: 1px solid #e0e0e0; border-radius: 8px;">
+        <h2 style="color: #2A5B41;">Neues Projekt-Match gefunden</h2>
+        <p>Hallo,</p>
+        <p>es gibt ein neues passendes IT-Projekt auf <strong>${project.portal}</strong> für <strong>${agent.name}</strong>.</p>
+        
+        <div style="background-color: #F4F7F5; padding: 15px; border-radius: 6px; margin: 20px 0;">
+          <h3 style="margin-top: 0; color: #2A5B41;">${project.title}</h3>
+          <p><strong>Portal:</strong> ${project.portal}<br>
+          <strong>Ort:</strong> ${project.location}<br>
+          <strong>Rate/Budget:</strong> ${project.rate}</p>
+          <p><a href="${project.url}" target="_blank" style="background-color: #6A8DC3; color: white; padding: 10px 15px; text-decoration: none; border-radius: 4px; display: inline-block;">Projekt aufrufen</a></p>
+        </div>
+
+        <div style="margin: 20px 0;">
+          <h3 style="color: #2A5B41;">Eignung: <span style="color: ${matchPercentage >= 75 ? '#2A5B41' : '#E67E22'};">${matchPercentage}% Match</span></h3>
+          
+          <h4 style="margin-bottom: 5px;">Vorteile (Warum es passt):</h4>
+          <ul style="padding-left: 20px; margin-top: 0;">${prosList || '<li>Keine Angaben</li>'}</ul>
+
+          <h4 style="margin-bottom: 5px;">Herausforderungen (Gaps):</h4>
+          <ul style="padding-left: 20px; margin-top: 0;">${consList || '<li>Keine Angaben</li>'}</ul>
+        </div>
+        
+        <p style="font-size: 12px; color: #7f8c8d; margin-top: 30px;">Diese E-Mail wurde automatisch vom Perinco Projekt Scanner generiert.</p>
+      </div>
+    `
+  };
+
+  try {
+    await transporter.sendMail(mailOptions);
+    console.log(`Notification email sent for Agent ${agent.name} and Project ${project.title}`);
+  } catch (error) {
+    console.error('Failed to send email notification:', error);
+  }
+};
+
+/*
+ * Ein Lauf zur Zeit. Scheduler und der Knopf "Jetzt scannen" koennen sich sonst
+ * ueberholen - zwei parallele Laeufe starten doppelt Chromium und bezahlen jede
+ * Bewertung zweimal bei OpenAI.
+ */
+let pipelineRunning = false;
+
+export const isPipelineRunning = () => pipelineRunning;
+
+/**
+ * Scrapen, Speichern und Bewerten in einem Durchlauf.
+ * @returns {Promise<boolean>} false, wenn schon ein Lauf aktiv war.
+ */
+export const runMatchingPipeline = async () => {
+  if (pipelineRunning) {
+    console.log('[Pipeline] Lauf laeuft schon - neuer Aufruf uebersprungen.');
+    return false;
+  }
+  pipelineRunning = true;
+  try {
+    return await executePipeline();
+  } finally {
+    pipelineRunning = false;
+  }
+};
+
+const executePipeline = async () => {
+  console.log('--- STARTING MATCHING PIPELINE ---');
+
+  // 1. Fetch active agents
+  const activeAgents = await Agent.findAll({ where: { isActive: true } });
+  if (activeAgents.length === 0) {
+    console.log('No active agents found. Pipeline execution stopped.');
+    return true;
+  }
+
+  // 2. Extract unique keywords from active agents
+  const keywordSet = new Set();
+  activeAgents.forEach(agent => {
+    if (agent.keywords) {
+      agent.keywords.split(',').forEach(k => {
+        const trimmed = k.trim();
+        if (trimmed) keywordSet.add(trimmed);
+      });
+    }
+  });
+
+  const keywords = Array.from(keywordSet);
+  if (keywords.length === 0) {
+    console.log('No active search keywords found. Pipeline execution stopped.');
+    return true;
+  }
+
+  console.log(`Keywords to scrape: ${keywords.join(', ')}`);
+
+  // 3. Fetch portal configurations
+  const freelancermapPortal = await Portal.findOne({ where: { name: 'freelancermap.de' } });
+  const freelanceDePortal = await Portal.findOne({ where: { name: 'freelance.de' } });
+
+  // 4. Run scrapers
+  let scrapedProjects = [];
+
+  try {
+    const mapResult = await scrapeFreelancermap(keywords, freelancermapPortal?.sessionState);
+    scrapedProjects = scrapedProjects.concat(mapResult.projects);
+  } catch (err) {
+    console.error('Error scraping Freelancermap:', err);
+  }
+
+  try {
+    const freelanceResult = await scrapeFreelanceDe(keywords, freelanceDePortal?.sessionState);
+    scrapedProjects = scrapedProjects.concat(freelanceResult.projects);
+  } catch (err) {
+    console.error('Error scraping Freelance.de:', err);
+  }
+
+  try {
+    const haysResult = await scrapeHays(keywords);
+    scrapedProjects = scrapedProjects.concat(haysResult.projects);
+    await Portal.update(
+      { connectionStatus: 'connected', lastLoginAttempt: new Date() },
+      { where: { name: 'hays.de' } }
+    );
+  } catch (err) {
+    console.error('Error scraping Hays.de:', err);
+    await Portal.update({ connectionStatus: 'error' }, { where: { name: 'hays.de' } });
+  }
+
+  try {
+    const etengoResult = await scrapeEtengo(keywords);
+    scrapedProjects = scrapedProjects.concat(etengoResult.projects);
+    await Portal.update(
+      { connectionStatus: 'connected', lastLoginAttempt: new Date() },
+      { where: { name: 'etengo.de' } }
+    );
+  } catch (err) {
+    console.error('Error scraping Etengo.de:', err);
+    await Portal.update({ connectionStatus: 'error' }, { where: { name: 'etengo.de' } });
+  }
+
+  // SThree-family staffing portals (shared template — generic scraper).
+  for (const cfg of SThreePortals) {
+    try {
+      const result = await scrapeSThreePortal(cfg);
+      scrapedProjects = scrapedProjects.concat(result.projects);
+      await Portal.update(
+        { connectionStatus: 'connected', lastLoginAttempt: new Date() },
+        { where: { name: cfg.name } }
+      );
+    } catch (err) {
+      console.error(`Error scraping ${cfg.name}:`, err);
+      await Portal.update({ connectionStatus: 'error' }, { where: { name: cfg.name } });
+    }
+  }
+
+  console.log(`Total projects scraped: ${scrapedProjects.length}`);
+
+  // 5. Store projects and run matching analysis
+  let newProjectsStoredCount = 0;
+  let matchesEvaluatedCount = 0;
+
+  const thresholdStr = await Setting.getVal('match_threshold', '70');
+  const matchThreshold = parseInt(thresholdStr, 10) || 70;
+
+  for (const scrapedProj of scrapedProjects) {
+    try {
+      // Find or create project in DB
+      const [project, created] = await Project.findOrCreate({
+        where: {
+          portal: scrapedProj.portal,
+          portalProjectId: scrapedProj.portalProjectId
+        },
+        defaults: scrapedProj
+      });
+
+      if (created) {
+        newProjectsStoredCount++;
+      }
+
+      // Perform matching analysis with active agents
+      for (const agent of activeAgents) {
+        // Skip if match already exists
+        const existingMatch = await Match.findOne({
+          where: {
+            agentId: agent.id,
+            projectId: project.id
+          }
+        });
+
+        if (existingMatch) continue;
+
+        // Exclusion list: skip this project for the agent if its title or
+        // description contains any excluded term (case-insensitive). Runs
+        // before the OpenAI call so excluded projects cost nothing.
+        if (agent.excludeKeywords) {
+          const haystack = `${project.title || ''} ${project.description || ''}`.toLowerCase();
+          const excluded = agent.excludeKeywords
+            .split(',')
+            .map(t => t.trim().toLowerCase())
+            .filter(Boolean)
+            .find(term => haystack.includes(term));
+          if (excluded) {
+            console.log(`Skipping project [${project.title}] for Agent [${agent.name}] — matched exclusion term "${excluded}".`);
+            continue;
+          }
+        }
+
+        // Skip matching if CV text is empty
+        if (!agent.cvText) {
+          console.log(`Skipping match for Agent ${agent.name} (ID: ${agent.id}) - No CV text uploaded.`);
+          continue;
+        }
+
+        console.log(`Evaluating match: Agent [${agent.name}] <-> Project [${project.title}]`);
+        
+        try {
+          const matchResult = await evaluateMatch(agent.cvText, project.description);
+          matchesEvaluatedCount++;
+
+          const match = await Match.create({
+            agentId: agent.id,
+            projectId: project.id,
+            matchPercentage: matchResult.matchPercentage,
+            pros: matchResult.pros,
+            cons: matchResult.cons,
+            status: 'new',
+            notificationSent: false
+          });
+
+          // Trigger email notification if threshold reached
+          if (match.matchPercentage >= matchThreshold) {
+            await sendEmailNotification(agent, project, match.matchPercentage, match.pros, match.cons);
+            match.notificationSent = true;
+            await match.save();
+          }
+        } catch (matchErr) {
+          console.error(`Failed to evaluate match for Agent ${agent.id} and Project ${project.id}:`, matchErr.message);
+        }
+      }
+
+    } catch (dbErr) {
+      console.error('Error inserting project or creating match:', dbErr.message);
+    }
+  }
+
+  console.log(`--- PIPELINE COMPLETED ---`);
+  console.log(`Stored ${newProjectsStoredCount} new projects.`);
+  console.log(`Evaluated ${matchesEvaluatedCount} new matches.`);
+  return true;
+};