import { chromium } from 'playwright';
import Portal from '../models/Portal.js';

// Stealth-like launch args — no need for playwright-extra/puppeteer-extra-plugin-stealth
// We pass anti-detection flags and a realistic user agent directly.
const STEALTH_ARGS = [
  '--no-sandbox',
  '--disable-setuid-sandbox',
  '--disable-blink-features=AutomationControlled',
  '--disable-infobars',
];

// Defensive parse: storage state stored in DB may originate from older sync
// versions or from Cookie-Editor exports with non-Playwright cookie shapes.
// Normalize before passing to browser.newContext().
const parseStorageState = (sessionState) => {
  if (!sessionState) return null;
  let raw;
  try {
    raw = typeof sessionState === 'string' ? JSON.parse(sessionState) : sessionState;
  } catch {
    return null;
  }

  let rawCookies = [];
  let origins = [];
  if (Array.isArray(raw)) {
    rawCookies = raw;
  } else if (raw && typeof raw === 'object') {
    rawCookies = Array.isArray(raw.cookies) ? raw.cookies : [];
    origins = Array.isArray(raw.origins) ? raw.origins : [];
  }

  const cookies = rawCookies.map(c => {
    if (!c || !c.name || c.value === undefined) return null;
    let sameSite = c.sameSite;
    if (typeof sameSite === 'string') {
      const s = sameSite.toLowerCase();
      if (s === 'strict') sameSite = 'Strict';
      else if (s === 'lax') sameSite = 'Lax';
      else if (s === 'none' || s === 'no_restriction') sameSite = 'None';
      else sameSite = 'Lax';
    } else {
      sameSite = 'Lax';
    }
    let expires = c.expires;
    if (expires === undefined && c.expirationDate !== undefined) expires = c.expirationDate;
    if (typeof expires !== 'number' || !isFinite(expires)) expires = -1;
    return {
      name: String(c.name),
      value: String(c.value),
      domain: c.domain || '',
      path: c.path || '/',
      expires,
      httpOnly: !!c.httpOnly,
      secure: !!c.secure,
      sameSite,
    };
  }).filter(Boolean);

  if (cookies.length === 0) return null;
  return { cookies, origins };
};

/**
 * Scrapes projects from Freelancermap.de based on keywords.
 * @param {string[]} keywords - List of search terms.
 * @param {string|null} sessionState - JSON string of cookies & storage.
 * @returns {Promise<{projects: Object[], sessionValid: boolean}>}
 */
export const scrapeFreelancermap = async (keywords, sessionState = null) => {
  console.log(`Starting Freelancermap scrape for keywords: ${keywords.join(', ')}`);
  const projectsList = [];
  let sessionValid = false;

  const browser = await chromium.launch({
    headless: true,
    args: STEALTH_ARGS
  });

  try {
    // Initialize context with cookies if available
    let contextOptions = {
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 }
    };

    if (sessionState) {
      const parsed = parseStorageState(sessionState);
      if (parsed) {
        contextOptions.storageState = parsed;
        console.log(`Loaded Freelancermap session: ${parsed.cookies.length} cookies.`);
      } else {
        console.warn('Freelancermap session state empty or invalid after normalize.');
      }
    }

    const context = await browser.newContext(contextOptions);
    const page = await context.newPage();

    // Anti-detection: override navigator.webdriver
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    });

    // Check if session is valid by visiting dashboard/profile page
    if (sessionState) {
      try {
        await page.goto('https://www.freelancermap.de/dashboard.html', { waitUntil: 'domcontentloaded', timeout: 15000 });
        const loginButton = await page.$('a[href*="/login"]');
        const userProfile = await page.$('.user-avatar, .user-name, a[href*="/logout"]');
        
        if (userProfile && !loginButton) {
          sessionValid = true;
          console.log('Freelancermap session is valid (Logged In).');
        } else {
          console.log('Freelancermap session has expired.');
        }
      } catch (err) {
        console.warn('Error checking Freelancermap session validity:', err.message);
      }
    }

    // Loop through each keyword to search
    for (const keyword of keywords) {
      const searchUrl = `https://www.freelancermap.de/projekte?q=${encodeURIComponent(keyword)}&sort=created_desc`;
      console.log(`Navigating to Freelancermap search: ${searchUrl}`);

      await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await page.waitForTimeout(1000 + Math.random() * 2000);

      // Freelancermap project cards use stable class `.project-card`. Detail
      // attributes use data-testid which is reliable across rerenders.
      const projectElements = await page.$$('.project-card');
      console.log(`Found ${projectElements.length} project cards for keyword: ${keyword}`);

      for (const el of projectElements) {
        try {
          const linkEl = await el.$('a[data-testid="title"], a[data-id="project-card-title"]');
          if (!linkEl) continue;

          const title = (await linkEl.innerText()).trim();
          const rawUrl = await linkEl.getAttribute('href');
          if (!title || !rawUrl) continue;

          const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelancermap.de${rawUrl}`;

          // Project ID: trailing numeric segment if present, else full slug.
          const slugMatch = url.match(/\/projekt\/([^/?#]+)/);
          const slug = slugMatch ? slugMatch[1] : null;
          const idMatch = slug ? slug.match(/-(\d+)$/) : null;
          const portalProjectId = idMatch ? idMatch[1] : (slug || url);

          if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;

          // Company: first div in .project-info (no class on text node)
          const company = await el.evaluate(node => {
            const info = node.querySelector('.project-info');
            if (!info) return '';
            const first = info.querySelector('div');
            return first ? first.innerText.trim() : '';
          });

          // Location: data-testid="city" container
          const location = await el.evaluate(node => {
            const c = node.querySelector('[data-testid="city"]');
            return c ? c.innerText.replace(/\s+/g, ' ').trim() : '';
          });

          // Type / duration / start → concatenated into rate as listing card has no €
          const meta = await el.evaluate(node => {
            const parts = [];
            ['type', 'duration', 'beginningText', 'remoteInPercent'].forEach(t => {
              const e2 = node.querySelector(`[data-testid="${t}"]`);
              if (e2) parts.push(e2.innerText.replace(/\s+/g, ' ').trim());
            });
            return parts.join(' | ');
          });

          // Listing card has no description text — fetch detail page lazily would
          // be expensive; leave description empty here. Matching engine handles
          // empty descriptions by returning low score, but title + meta still
          // give signal once we add a detail fetch step later.
          const description = `${title}\n${meta}`.trim();

          projectsList.push({
            portal: 'freelancermap.de',
            portalProjectId,
            title,
            description,
            company: company || 'N/A',
            location: location || 'Remote / N/A',
            rate: meta || 'N/A',
            url,
            publishedAt: new Date()
          });
        } catch (err) {
          console.error('Error parsing Freelancermap card:', err.message);
        }
      }
    }

    // Save updated storage state if session was valid (cookies might rotate)
    if (sessionValid) {
      const updatedState = await context.storageState();
      sessionValid = true;
      await Portal.update(
        { sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
        { where: { name: 'freelancermap.de' } }
      );
    } else if (sessionState) {
      // Session was provided but check failed
      await Portal.update(
        { connectionStatus: 'expired' },
        { where: { name: 'freelancermap.de' } }
      );
    }

  } catch (error) {
    console.error('Error during Freelancermap scrape:', error);
  } finally {
    await browser.close();
  }

  return { projects: projectsList, sessionValid };
};

/**
 * Scrapes job listings from Hays.de. Public portal, no login required.
 * Search URL: https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=<keyword>&e=false
 * joblevel=3 (Senior/Lead-Bereich, vom Nutzer vorgegeben), mrew=1, e=false (alle Vertragsarten).
 * @param {string[]} keywords
 * @returns {Promise<{projects: Object[], sessionValid: boolean}>}
 */
export const scrapeHays = async (keywords) => {
  console.log(`Starting Hays.de scrape for keywords: ${keywords.join(', ')}`);
  const projectsList = [];

  const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });

  try {
    const context = await browser.newContext({
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 }
    });
    const page = await context.newPage();
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    });

    for (const keyword of keywords) {
      const searchUrl = `https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=${encodeURIComponent(keyword)}&e=false`;
      console.log(`Navigating to Hays search: ${searchUrl}`);

      try {
        await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
        // Wait for actual job-detail anchors (JS hydrates them).
        try {
          await page.waitForSelector('a[href*="/jobsuche/stellenangebote-jobs-detail-"]', { timeout: 12000 });
        } catch {
          console.warn(`Hays: no job-detail anchors visible for "${keyword}"`);
          continue;
        }
        await page.waitForTimeout(800 + Math.random() * 1000);

        // Only real job-detail URLs. Pattern: /jobsuche/stellenangebote-jobs-detail-<slug>-<id>/<page>
        const linkEls = await page.$$('a[href*="/jobsuche/stellenangebote-jobs-detail-"]');
        console.log(`Found ${linkEls.length} Hays job-detail anchors for keyword: ${keyword}`);

        for (const linkEl of linkEls) {
          try {
            const rawUrl = await linkEl.getAttribute('href');
            if (!rawUrl) continue;
            const url = rawUrl.startsWith('http') ? rawUrl : `https://www.hays.de${rawUrl}`;

            // Hays job-id: numeric segment at end before trailing /N
            const idMatch = url.match(/-(\d{5,})(?:\/\d+)?(?:[?#].*)?$/);
            if (!idMatch) continue;
            const portalProjectId = idMatch[1];

            if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;

            // Title: link inner text first, fall back to nearest heading.
            let title = ((await linkEl.innerText()) || '').replace(/\s+/g, ' ').trim();
            if (!title) {
              title = await linkEl.evaluate(node => {
                let n = node;
                for (let i = 0; i < 6 && n; i++) {
                  n = n.parentElement;
                  if (!n) break;
                  const h = n.querySelector('h2, h3, [class*="title"], [class*="Title"]');
                  if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
                }
                return '';
              }).catch(() => '');
            }
            // Derive title from URL slug if still missing.
            if (!title) {
              const slugMatch = url.match(/stellenangebote-jobs-detail-(.+?)-\d{5,}/);
              if (slugMatch) {
                title = slugMatch[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
              }
            }
            if (!title) continue;

            // Card text: closest ancestor that contains structured info (max 6 levels up).
            const meta = await linkEl.evaluate(node => {
              let card = node;
              for (let i = 0; i < 6; i++) {
                if (!card.parentElement) break;
                card = card.parentElement;
                const txt = (card.innerText || '').replace(/\s+/g, ' ').trim();
                if (txt.length > 80 && txt.length < 1500) return txt.slice(0, 1200);
              }
              return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 600);
            }).catch(() => '');

            // Location: try common DE cities or fall back to scanning "Standort:" pattern.
            const locMatch = meta.match(/Standort[:\s]+([^|·\n]{2,40})/i)
              || meta.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Dortmund|Essen|Bremen|Dresden|Hannover|Nürnberg|Mannheim|Karlsruhe|Wiesbaden|Bonn|Münster|Augsburg|Aachen|Erfurt|Mainz|Kiel|Halle|Magdeburg|Freiburg|Lübeck|Oberhausen|Rostock|Kassel|Saarbrücken|Mönchengladbach|Krefeld|Chemnitz|Braunschweig|Wuppertal|Bielefeld|Duisburg|remote|Remote)\b/);
            const location = locMatch ? locMatch[1] || locMatch[0] : 'Remote / N/A';

            projectsList.push({
              portal: 'hays.de',
              portalProjectId,
              title: title.slice(0, 250),
              description: `${title}\n${meta}`.trim().slice(0, 1500),
              company: 'Hays (Vermittlung)',
              location: location.slice(0, 200),
              rate: (meta.match(/(?:€|EUR)[^|·]{0,40}/) || ['N/A'])[0].slice(0, 90),
              url,
              publishedAt: new Date()
            });
          } catch (err) {
            console.error('Error parsing Hays card:', err.message);
          }
        }
      } catch (err) {
        console.error(`Error scraping Hays for "${keyword}":`, err.message);
      }
    }
  } catch (error) {
    console.error('Error during Hays scrape:', error);
  } finally {
    await browser.close();
  }

  return { projects: projectsList, sessionValid: true };
};

/**
 * Generic scraper for the SThree-family staffing portals (Huxley, Computer
 * Futures, Progressive, Real Staffing, Global Enterprise Partners, Madison
 * Black, JP Gray, etc.). They all share the same template — job detail URLs
 * follow `<base>/<locale>/job/<slug>/<numeric-id>/`. Public, no login.
 *
 * @param {object} cfg
 * @param {string} cfg.name - DB portal name (e.g. 'huxley.com')
 * @param {string} cfg.listingUrl - Search-result listing URL
 * @param {string} cfg.localePath - e.g. '/de-de/' or '/en-nl/'
 */
const scrapeSThreeStyle = async ({ name, listingUrl, localePath }) => {
  console.log(`Starting ${name} scrape (listing).`);
  const projectsList = [];

  const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });

  try {
    const context = await browser.newContext({
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 }
    });
    const page = await context.newPage();
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    });

    try {
      await page.goto(listingUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
      const jobLinkSelector = `a[href*="${localePath}job/"]`;
      try {
        await page.waitForSelector(jobLinkSelector, { timeout: 12000 });
      } catch {
        console.warn(`${name}: no job anchors visible after wait.`);
      }
      await page.waitForTimeout(1500);

      const links = await page.$$eval(jobLinkSelector, (els, locale) => {
        const seen = new Set();
        const out = [];
        // Escape regex special chars in locale (keeps slashes as literal).
        const esc = locale.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
        const re = new RegExp(esc + 'job/([^/]+)/(\\d{4,})/?');
        for (const a of els) {
          const href = a.getAttribute('href') || '';
          if (href.includes('{{')) continue;
          const m = href.match(re);
          if (!m) continue;
          const id = m[2];
          if (seen.has(id)) continue;
          seen.add(id);
          out.push({ id, slug: m[1], href });
        }
        return out;
      }, localePath).catch(() => []);
      console.log(`Found ${links.length} ${name} job URLs.`);

      // Detail-page fetches were triggering Cloudflare/WAF on the SThree
      // family. Title is derived from the URL slug; description = title only.
      // No detail requests → no bot signature, no rate spikes.
      for (const link of links) {
        try {
          const url = link.href.startsWith('http') ? link.href : new URL(link.href, listingUrl).toString();

          if (projectsList.some(p => p.portalProjectId === link.id)) continue;

          let title = '';
          try {
            title = decodeURIComponent(link.slug);
          } catch {
            title = link.slug;
          }
          title = title.replace(/-/g, ' ').replace(/\s+/g, ' ').trim();
          // Title-case for the slug pieces (German diacritics kept).
          title = title.replace(/\b(\w)([\w'’äöüß]*)/g, (_, a, b) => a.toUpperCase() + b);
          if (!title) continue;

          projectsList.push({
            portal: name,
            portalProjectId: link.id,
            title: title.slice(0, 250),
            description: title.slice(0, 250),
            company: `${name} (Vermittlung)`,
            location: 'Siehe Anzeige',
            rate: 'N/A',
            url,
            publishedAt: new Date()
          });
        } catch (err) {
          console.error(`Error building ${name} project entry:`, err.message);
        }
      }
    } catch (err) {
      console.error(`Error scraping ${name} listing:`, err.message);
    }
  } catch (error) {
    console.error(`Error during ${name} scrape:`, error);
  } finally {
    await browser.close();
  }

  console.log(`${name} scrape collected ${projectsList.length} projects.`);
  return { projects: projectsList, sessionValid: true };
};

// Per-portal wrappers used by pipeline.js / api.js
export const SThreePortals = [
  { name: 'huxley.com', listingUrl: 'https://www.huxley.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
  { name: 'computerfutures.com', listingUrl: 'https://www.computerfutures.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
  { name: 'progressiverecruitment.com', listingUrl: 'https://www.progressiverecruitment.com/de-de/jobs-projekte-suchen/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
  { name: 'realstaffing.com', listingUrl: 'https://www.realstaffing.com/de-de/job-suche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
  { name: 'globalenterprisepartners.com', listingUrl: 'https://www.globalenterprisepartners.com/de-de/job-search/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
  { name: 'madisonblack.com', listingUrl: 'https://www.madisonblack.com/de-de/job-search/?searchRadius=50km', localePath: '/de-de/' },
  { name: 'jpgray.nl', listingUrl: 'https://www.jpgray.nl/en-nl/job-search/?country=Netherlands&searchRadius=50km', localePath: '/en-nl/' }
];

export const scrapeSThreePortal = (cfg) => scrapeSThreeStyle(cfg);

/**
 * Scrapes IT projects from Etengo.de. Public portal, no login required.
 * Search URL: https://www.etengo.de/it-projektsuche/?q=<keyword>
 * @param {string[]} keywords
 * @returns {Promise<{projects: Object[], sessionValid: boolean}>}
 */
export const scrapeEtengo = async (/* keywords */) => {
  // Etengo's search filter is JS-side and the listing URL ignores ?q. Load the
  // full project list once and return all entries; matching engine handles
  // keyword relevance per agent CV.
  console.log(`Starting Etengo.de scrape (full listing).`);
  const projectsList = [];

  const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });

  try {
    const context = await browser.newContext({
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 }
    });
    const page = await context.newPage();
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    });

    try {
      await page.goto('https://www.etengo.de/it-projektsuche/', { waitUntil: 'domcontentloaded', timeout: 30000 });
      try {
        await page.waitForSelector('a[href*="/it-projektsuche/"][href*="/"]', { timeout: 10000 });
      } catch {
        console.warn('Etengo: project anchors not visible.');
      }
      await page.waitForTimeout(1000);

      // Collect detail-page URLs: /it-projektsuche/<numeric-id>/
      const anchors = await page.$$('a[href*="/it-projektsuche/"]');
      const seen = new Set();
      const detailUrls = [];
      for (const a of anchors) {
        const href = await a.getAttribute('href');
        if (!href) continue;
        const m = href.match(/\/it-projektsuche\/(\d{3,})\/?$/);
        if (!m) continue;
        const url = href.startsWith('http') ? href : `https://www.etengo.de${href}`;
        if (seen.has(m[1])) continue;
        seen.add(m[1]);
        detailUrls.push({ id: m[1], url });
      }
      console.log(`Found ${detailUrls.length} Etengo project URLs.`);

      // Fetch each detail page in same context for title + description.
      for (const { id, url } of detailUrls) {
        try {
          const detail = await context.newPage();
          await detail.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
          await detail.waitForTimeout(500);

          const title = await detail.evaluate(() => {
            const h = document.querySelector('h1');
            if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
            const t = document.querySelector('title');
            return t ? t.innerText.replace(/ - Etengo.*$/i, '').trim() : '';
          }).catch(() => '');

          const description = await detail.evaluate(() => {
            // Look for main article / project description container.
            const cand = document.querySelector('main, article, .entry-content, .project-detail, [class*="detail"]');
            const node = cand || document.body;
            return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 1500);
          }).catch(() => '');

          const location = (description.match(/Einsatzort[:\s]+([^|·\n]{2,40})/i)
            || description.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Hannover|Nürnberg|Bonn|Münster|Karlsruhe|Mannheim|Bremen|Dresden|Erfurt|Mainz|remote|Remote)\b/) || ['Remote / N/A'])[0];

          const rateMatch = description.match(/(?:€|EUR)[^|·]{0,40}/);

          await detail.close();

          if (!title) continue;
          if (projectsList.some(p => p.portalProjectId === id)) continue;

          projectsList.push({
            portal: 'etengo.de',
            portalProjectId: id,
            title: title.slice(0, 250),
            description: `${title}\n${description}`.trim().slice(0, 1500),
            company: 'Etengo (Vermittlung)',
            location: String(location).slice(0, 200),
            rate: rateMatch ? rateMatch[0].slice(0, 90) : 'N/A',
            url,
            publishedAt: new Date()
          });
        } catch (err) {
          console.error(`Error fetching Etengo detail ${url}:`, err.message);
        }
      }
    } catch (err) {
      console.error('Error scraping Etengo listing:', err.message);
    }
  } catch (error) {
    console.error('Error during Etengo scrape:', error);
  } finally {
    await browser.close();
  }

  console.log(`Etengo scrape collected ${projectsList.length} projects.`);
  return { projects: projectsList, sessionValid: true };
};

/**
 * Scrapes projects from Freelance.de based on keywords.
 * @param {string[]} keywords - List of search terms.
 * @param {string|null} sessionState - JSON string of cookies & storage.
 * @returns {Promise<{projects: Object[], sessionValid: boolean}>}
 */
export const scrapeFreelanceDe = async (keywords, sessionState = null) => {
  console.log(`Starting Freelance.de scrape for keywords: ${keywords.join(', ')}`);
  const projectsList = [];
  let sessionValid = false;

  const browser = await chromium.launch({
    headless: true,
    args: STEALTH_ARGS
  });

  try {
    let contextOptions = {
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 }
    };

    if (sessionState) {
      const parsed = parseStorageState(sessionState);
      if (parsed) {
        contextOptions.storageState = parsed;
        console.log(`Loaded Freelance.de session: ${parsed.cookies.length} cookies.`);
      } else {
        console.warn('Freelance.de session state empty or invalid after normalize.');
      }
    }

    const context = await browser.newContext(contextOptions);
    const page = await context.newPage();

    // Anti-detection: override navigator.webdriver
    await page.addInitScript(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    });

    // Check if session is valid by visiting dashboard/profile page
    if (sessionState) {
      try {
        await page.goto('https://www.freelance.de/users/profile.php', { waitUntil: 'domcontentloaded', timeout: 15000 });
        const loginButton = await page.$('a[href*="login"], a[href*="sign-in"]');
        const userProfile = await page.$('a[href*="logout"], .user-info, .profile-settings');
        
        if (userProfile && !loginButton) {
          sessionValid = true;
          console.log('Freelance.de session is valid (Logged In).');
        } else {
          console.log('Freelance.de session has expired.');
        }
      } catch (err) {
        console.warn('Error checking Freelance.de session validity:', err.message);
      }
    }

    for (const keyword of keywords) {
      // freelance.de search url: search_sort_by=2 means date descending (newest first)
      const searchUrl = `https://www.freelance.de/search/project.php?__search_sort_by=2&query=${encodeURIComponent(keyword)}`;
      console.log(`Navigating to Freelance.de search: ${searchUrl}`);
      
      await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await page.waitForTimeout(1000 + Math.random() * 2000);

      // Select project list items
      const projectElements = await page.$$('.project-list-item, .project-row, .list-item-main, div[class*="project-list-item"]');
      console.log(`Found ${projectElements.length} potential project elements for keyword: ${keyword}`);

      for (const el of projectElements) {
        try {
          // Find the link to the project detail page
          // Format e.g., /Projekte/Projekt-Details-12345.html
          const linkEl = await el.$('a[href*="/Projekte/"], a[href*="/Projekt/"]');
          if (!linkEl) continue;

          const title = await linkEl.innerText();
          const rawUrl = await linkEl.getAttribute('href');
          if (!title || !rawUrl) continue;

          const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelance.de${rawUrl}`;
          
          // Parse project ID from URL
          // Format e.g., /Projekte/123456-React-Entwickler or /Projekt/123456
          const matchId = url.match(/\/(\d+)-/) || url.match(/\/(\d+)\.html/) || url.match(/\/(\d+)(?:\/|\?|$)/);
          const portalProjectId = matchId ? matchId[1] : url;

          if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;

          // Scrape details from listing card
          const descEl = await el.$('.project-description, .description, p, .project-details');
          const description = descEl ? await descEl.innerText() : '';

          const locEl = await el.$('.location, .project-location, span[title*="Ort"], .meta-location');
          const location = locEl ? await locEl.innerText() : 'Remote / N/A';

          const rateEl = await el.$('.rate, .project-rate, .meta-rate, span[title*="Budget"]');
          const rate = rateEl ? await rateEl.innerText() : 'N/A';

          const compEl = await el.$('.company, .project-company, .meta-company');
          const company = compEl ? await compEl.innerText() : 'N/A';

          projectsList.push({
            portal: 'freelance.de',
            portalProjectId,
            title: title.trim(),
            description: description.trim(),
            company: company.trim(),
            location: location.trim(),
            rate: rate.trim(),
            url,
            publishedAt: new Date()
          });
        } catch (err) {
          console.error('Error parsing individual Freelance.de element:', err.message);
        }
      }
    }

    // Save updated storage state if session was valid
    if (sessionValid) {
      const updatedState = await context.storageState();
      sessionValid = true;
      await Portal.update(
        { sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
        { where: { name: 'freelance.de' } }
      );
    } else if (sessionState) {
      await Portal.update(
        { connectionStatus: 'expired' },
        { where: { name: 'freelance.de' } }
      );
    }

  } catch (error) {
    console.error('Error during Freelance.de scrape:', error);
  } finally {
    await browser.close();
  }

  return { projects: projectsList, sessionValid };
};