perinco-CRM/server/src/scanner/services/pipeline.js

import { Agent, Portal, Project, Match, Setting } from '../models/index.js';
import { scrapeFreelancermap, scrapeFreelanceDe, scrapeHays, scrapeEtengo, scrapeSThreePortal, SThreePortals } from './scraper.js';
import { evaluateMatch } from './matchingEngine.js';
import nodemailer from 'nodemailer';

// Helper to encrypt/decrypt credentials if needed, or send email notifications
const sendEmailNotification = async (agent, project, matchPercentage, pros, cons) => {
  const active = await Setting.getVal('email_notifications_active') === 'true';
  if (!active) return;

  const smtpHost = await Setting.getVal('smtp_host');
  const smtpPort = parseInt(await Setting.getVal('smtp_port', '587'), 10);
  const smtpUser = await Setting.getVal('smtp_user');
  const smtpPass = await Setting.getVal('smtp_pass');
  const smtpFrom = await Setting.getVal('smtp_from', 'scanner@perinco.de');
  const recipient = await Setting.getVal('notification_email_recipient') || agent.email;

  if (!smtpHost || !smtpUser || !smtpPass || !recipient) {
    console.warn('SMTP settings are incomplete. Skipping email notification.');
    return;
  }

  const transporter = nodemailer.createTransport({
    host: smtpHost,
    port: smtpPort,
    secure: smtpPort === 465,
    auth: {
      user: smtpUser,
      pass: smtpPass
    }
  });

  const prosList = pros.map(p => `<li>✅ ${p}</li>`).join('');
  const consList = cons.map(c => `<li>❌ ${c}</li>`).join('');

  const mailOptions = {
    from: smtpFrom,
    to: recipient,
    subject: `[PROJEKT-MATCHING ${matchPercentage}%] Neue Chance für ${agent.name}`,
    html: `
      <div style="font-family: sans-serif; max-width: 600px; margin: 0 auto; padding: 20px; border: 1px solid #e0e0e0; border-radius: 8px;">
        <h2 style="color: #2A5B41;">Neues Projekt-Match gefunden</h2>
        <p>Hallo,</p>
        <p>es gibt ein neues passendes IT-Projekt auf <strong>${project.portal}</strong> für <strong>${agent.name}</strong>.</p>
        
        <div style="background-color: #F4F7F5; padding: 15px; border-radius: 6px; margin: 20px 0;">
          <h3 style="margin-top: 0; color: #2A5B41;">${project.title}</h3>
          <p><strong>Portal:</strong> ${project.portal}<br>
          <strong>Ort:</strong> ${project.location}<br>
          <strong>Rate/Budget:</strong> ${project.rate}</p>
          <p><a href="${project.url}" target="_blank" style="background-color: #6A8DC3; color: white; padding: 10px 15px; text-decoration: none; border-radius: 4px; display: inline-block;">Projekt aufrufen</a></p>
        </div>

        <div style="margin: 20px 0;">
          <h3 style="color: #2A5B41;">Eignung: <span style="color: ${matchPercentage >= 75 ? '#2A5B41' : '#E67E22'};">${matchPercentage}% Match</span></h3>
          
          <h4 style="margin-bottom: 5px;">Vorteile (Warum es passt):</h4>
          <ul style="padding-left: 20px; margin-top: 0;">${prosList || '<li>Keine Angaben</li>'}</ul>

          <h4 style="margin-bottom: 5px;">Herausforderungen (Gaps):</h4>
          <ul style="padding-left: 20px; margin-top: 0;">${consList || '<li>Keine Angaben</li>'}</ul>
        </div>
        
        <p style="font-size: 12px; color: #7f8c8d; margin-top: 30px;">Diese E-Mail wurde automatisch vom Perinco Projekt Scanner generiert.</p>
      </div>
    `
  };

  try {
    await transporter.sendMail(mailOptions);
    console.log(`Notification email sent for Agent ${agent.name} and Project ${project.title}`);
  } catch (error) {
    console.error('Failed to send email notification:', error);
  }
};

/*
 * Ein Lauf zur Zeit. Scheduler und der Knopf "Jetzt scannen" koennen sich sonst
 * ueberholen - zwei parallele Laeufe starten doppelt Chromium und bezahlen jede
 * Bewertung zweimal bei OpenAI.
 */
let pipelineRunning = false;

export const isPipelineRunning = () => pipelineRunning;

/**
 * Scrapen, Speichern und Bewerten in einem Durchlauf.
 * @returns {Promise<boolean>} false, wenn schon ein Lauf aktiv war.
 */
export const runMatchingPipeline = async () => {
  if (pipelineRunning) {
    console.log('[Pipeline] Lauf laeuft schon - neuer Aufruf uebersprungen.');
    return false;
  }
  pipelineRunning = true;
  try {
    return await executePipeline();
  } finally {
    pipelineRunning = false;
    /*
     * Zeitpunkt festhalten, damit der Zeitplan einen Neustart uebersteht - sonst
     * beginnt nach jedem Deployment die Wartezeit von vorn. Auch ein manueller
     * Scan zaehlt: er hat die Boersen ja gerade abgefragt.
     *
     * Im finally, also auch nach einem Abbruch: ein gescheiterter Lauf soll sich
     * nicht sofort wiederholen.
     */
    await Setting.setVal('last_scan_at', new Date().toISOString()).catch(() => {});
  }
};

const executePipeline = async () => {
  console.log('--- STARTING MATCHING PIPELINE ---');

  // 1. Fetch active agents
  const activeAgents = await Agent.findAll({ where: { isActive: true } });
  if (activeAgents.length === 0) {
    console.log('No active agents found. Pipeline execution stopped.');
    return true;
  }

  // 2. Extract unique keywords from active agents
  const keywordSet = new Set();
  activeAgents.forEach(agent => {
    if (agent.keywords) {
      agent.keywords.split(',').forEach(k => {
        const trimmed = k.trim();
        if (trimmed) keywordSet.add(trimmed);
      });
    }
  });

  const keywords = Array.from(keywordSet);
  if (keywords.length === 0) {
    console.log('No active search keywords found. Pipeline execution stopped.');
    return true;
  }

  console.log(`Keywords to scrape: ${keywords.join(', ')}`);

  // 3. Fetch portal configurations
  const freelancermapPortal = await Portal.findOne({ where: { name: 'freelancermap.de' } });
  const freelanceDePortal = await Portal.findOne({ where: { name: 'freelance.de' } });

  // 4. Run scrapers
  let scrapedProjects = [];

  try {
    const mapResult = await scrapeFreelancermap(keywords, freelancermapPortal?.sessionState);
    scrapedProjects = scrapedProjects.concat(mapResult.projects);
  } catch (err) {
    console.error('Error scraping Freelancermap:', err);
  }

  try {
    const freelanceResult = await scrapeFreelanceDe(keywords, freelanceDePortal?.sessionState);
    scrapedProjects = scrapedProjects.concat(freelanceResult.projects);
  } catch (err) {
    console.error('Error scraping Freelance.de:', err);
  }

  try {
    const haysResult = await scrapeHays(keywords);
    scrapedProjects = scrapedProjects.concat(haysResult.projects);
    await Portal.update(
      { connectionStatus: 'connected', lastLoginAttempt: new Date() },
      { where: { name: 'hays.de' } }
    );
  } catch (err) {
    console.error('Error scraping Hays.de:', err);
    await Portal.update({ connectionStatus: 'error' }, { where: { name: 'hays.de' } });
  }

  try {
    const etengoResult = await scrapeEtengo(keywords);
    scrapedProjects = scrapedProjects.concat(etengoResult.projects);
    await Portal.update(
      { connectionStatus: 'connected', lastLoginAttempt: new Date() },
      { where: { name: 'etengo.de' } }
    );
  } catch (err) {
    console.error('Error scraping Etengo.de:', err);
    await Portal.update({ connectionStatus: 'error' }, { where: { name: 'etengo.de' } });
  }

  // SThree-family staffing portals (shared template — generic scraper).
  for (const cfg of SThreePortals) {
    try {
      const result = await scrapeSThreePortal(cfg);
      scrapedProjects = scrapedProjects.concat(result.projects);
      await Portal.update(
        { connectionStatus: 'connected', lastLoginAttempt: new Date() },
        { where: { name: cfg.name } }
      );
    } catch (err) {
      console.error(`Error scraping ${cfg.name}:`, err);
      await Portal.update({ connectionStatus: 'error' }, { where: { name: cfg.name } });
    }
  }

  console.log(`Total projects scraped: ${scrapedProjects.length}`);

  // 5. Store projects and run matching analysis
  let newProjectsStoredCount = 0;
  let matchesEvaluatedCount = 0;

  const thresholdStr = await Setting.getVal('match_threshold', '70');
  const matchThreshold = parseInt(thresholdStr, 10) || 70;

  for (const scrapedProj of scrapedProjects) {
    try {
      // Find or create project in DB
      const [project, created] = await Project.findOrCreate({
        where: {
          portal: scrapedProj.portal,
          portalProjectId: scrapedProj.portalProjectId
        },
        defaults: scrapedProj
      });

      if (created) {
        newProjectsStoredCount++;
      }

      // Perform matching analysis with active agents
      for (const agent of activeAgents) {
        // Skip if match already exists
        const existingMatch = await Match.findOne({
          where: {
            agentId: agent.id,
            projectId: project.id
          }
        });

        if (existingMatch) continue;

        // Exclusion list: skip this project for the agent if its title or
        // description contains any excluded term (case-insensitive). Runs
        // before the OpenAI call so excluded projects cost nothing.
        if (agent.excludeKeywords) {
          const haystack = `${project.title || ''} ${project.description || ''}`.toLowerCase();
          const excluded = agent.excludeKeywords
            .split(',')
            .map(t => t.trim().toLowerCase())
            .filter(Boolean)
            .find(term => haystack.includes(term));
          if (excluded) {
            console.log(`Skipping project [${project.title}] for Agent [${agent.name}] — matched exclusion term "${excluded}".`);
            continue;
          }
        }

        // Skip matching if CV text is empty
        if (!agent.cvText) {
          console.log(`Skipping match for Agent ${agent.name} (ID: ${agent.id}) - No CV text uploaded.`);
          continue;
        }

        console.log(`Evaluating match: Agent [${agent.name}] <-> Project [${project.title}]`);
        
        try {
          const matchResult = await evaluateMatch(agent.cvText, project.description);
          matchesEvaluatedCount++;

          const match = await Match.create({
            agentId: agent.id,
            projectId: project.id,
            matchPercentage: matchResult.matchPercentage,
            pros: matchResult.pros,
            cons: matchResult.cons,
            status: 'new',
            notificationSent: false
          });

          // Trigger email notification if threshold reached
          if (match.matchPercentage >= matchThreshold) {
            await sendEmailNotification(agent, project, match.matchPercentage, match.pros, match.cons);
            match.notificationSent = true;
            await match.save();
          }
        } catch (matchErr) {
          console.error(`Failed to evaluate match for Agent ${agent.id} and Project ${project.id}:`, matchErr.message);
        }
      }

    } catch (dbErr) {
      console.error('Error inserting project or creating match:', dbErr.message);
    }
  }

  console.log(`--- PIPELINE COMPLETED ---`);
  console.log(`Stored ${newProjectsStoredCount} new projects.`);
  console.log(`Evaluated ${matchesEvaluatedCount} new matches.`);
  return true;
};