feat: Projekt Scanner integrieren, Whisper in die Einstellungen
Der eigenstaendige Projekt Scanner zieht komplett ins CRM ein und laeuft parallel zur bestehenden App weiter - jede geteilte Ressource ist verdoppelt statt geteilt, damit die alte Installation unberuehrt bleibt: - eigene Datenbank perinco_scanner, legt sich beim Start selbst an - eigener SCANNER_ENCRYPTION_KEY, eigenes Upload-Verzeichnis - eigener Scheduler mit Zeitversatz, damit nicht beide Instanzen gleichzeitig mit denselben Portal-Cookies scrapen (Rate-Limits, CAPTCHA, doppelter RAM) - Sperre gegen ueberlappende Laeufe: Scheduler und "Jetzt scannen" teilen sie, ein zweiter Aufruf bekommt 409 statt doppelter OpenAI-Kosten Backend unter /api/scanner hinter der CRM-Anmeldung; die HTTP-Basic-Auth des Originals entfaellt. CV-PDFs kommen als Stream mit Token statt aus einem statischen Verzeichnis. Bestandsdaten holt "npm run scanner:import" aus der alten Datenbank, ohne sie zu veraendern. Oberflaeche in Tailwind und CI-Farben neu gebaut: Menuepunkt Scanner mit aufklappbaren Unterpunkten, Einstellungen als eigener Reiter. Whisper hat keinen eigenen Menuepunkt mehr, sondern ist ein Reiter in den Einstellungen. Damit diese Geraeteeinstellung jeder Rolle offensteht, sind die Einstellungen jetzt fuer alle sichtbar - welche Reiter erscheinen, entscheidet die Rolle. Ausserdem: kein festes Admin-Passwort mehr im Quelltext, playwright exakt auf 1.60.0 gepinnt (nutzt den vorhandenen Chromium-Build des Servers), technische Dokumentation mit Funktionszeichnungen unter docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sascha Wolf
@@ -0,0 +1,720 @@
|
||||
import { chromium } from 'playwright';
|
||||
import Portal from '../models/Portal.js';
|
||||
|
||||
// Stealth-like launch args — no need for playwright-extra/puppeteer-extra-plugin-stealth
|
||||
// We pass anti-detection flags and a realistic user agent directly.
|
||||
const STEALTH_ARGS = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars',
|
||||
];
|
||||
|
||||
// Defensive parse: storage state stored in DB may originate from older sync
|
||||
// versions or from Cookie-Editor exports with non-Playwright cookie shapes.
|
||||
// Normalize before passing to browser.newContext().
|
||||
const parseStorageState = (sessionState) => {
|
||||
if (!sessionState) return null;
|
||||
let raw;
|
||||
try {
|
||||
raw = typeof sessionState === 'string' ? JSON.parse(sessionState) : sessionState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
let rawCookies = [];
|
||||
let origins = [];
|
||||
if (Array.isArray(raw)) {
|
||||
rawCookies = raw;
|
||||
} else if (raw && typeof raw === 'object') {
|
||||
rawCookies = Array.isArray(raw.cookies) ? raw.cookies : [];
|
||||
origins = Array.isArray(raw.origins) ? raw.origins : [];
|
||||
}
|
||||
|
||||
const cookies = rawCookies.map(c => {
|
||||
if (!c || !c.name || c.value === undefined) return null;
|
||||
let sameSite = c.sameSite;
|
||||
if (typeof sameSite === 'string') {
|
||||
const s = sameSite.toLowerCase();
|
||||
if (s === 'strict') sameSite = 'Strict';
|
||||
else if (s === 'lax') sameSite = 'Lax';
|
||||
else if (s === 'none' || s === 'no_restriction') sameSite = 'None';
|
||||
else sameSite = 'Lax';
|
||||
} else {
|
||||
sameSite = 'Lax';
|
||||
}
|
||||
let expires = c.expires;
|
||||
if (expires === undefined && c.expirationDate !== undefined) expires = c.expirationDate;
|
||||
if (typeof expires !== 'number' || !isFinite(expires)) expires = -1;
|
||||
return {
|
||||
name: String(c.name),
|
||||
value: String(c.value),
|
||||
domain: c.domain || '',
|
||||
path: c.path || '/',
|
||||
expires,
|
||||
httpOnly: !!c.httpOnly,
|
||||
secure: !!c.secure,
|
||||
sameSite,
|
||||
};
|
||||
}).filter(Boolean);
|
||||
|
||||
if (cookies.length === 0) return null;
|
||||
return { cookies, origins };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelancermap.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelancermap = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelancermap scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
// Initialize context with cookies if available
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelancermap session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelancermap session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelancermap.de/dashboard.html', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="/login"]');
|
||||
const userProfile = await page.$('.user-avatar, .user-name, a[href*="/logout"]');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelancermap session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelancermap session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelancermap session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Loop through each keyword to search
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.freelancermap.de/projekte?q=${encodeURIComponent(keyword)}&sort=created_desc`;
|
||||
console.log(`Navigating to Freelancermap search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Freelancermap project cards use stable class `.project-card`. Detail
|
||||
// attributes use data-testid which is reliable across rerenders.
|
||||
const projectElements = await page.$$('.project-card');
|
||||
console.log(`Found ${projectElements.length} project cards for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
const linkEl = await el.$('a[data-testid="title"], a[data-id="project-card-title"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = (await linkEl.innerText()).trim();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelancermap.de${rawUrl}`;
|
||||
|
||||
// Project ID: trailing numeric segment if present, else full slug.
|
||||
const slugMatch = url.match(/\/projekt\/([^/?#]+)/);
|
||||
const slug = slugMatch ? slugMatch[1] : null;
|
||||
const idMatch = slug ? slug.match(/-(\d+)$/) : null;
|
||||
const portalProjectId = idMatch ? idMatch[1] : (slug || url);
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Company: first div in .project-info (no class on text node)
|
||||
const company = await el.evaluate(node => {
|
||||
const info = node.querySelector('.project-info');
|
||||
if (!info) return '';
|
||||
const first = info.querySelector('div');
|
||||
return first ? first.innerText.trim() : '';
|
||||
});
|
||||
|
||||
// Location: data-testid="city" container
|
||||
const location = await el.evaluate(node => {
|
||||
const c = node.querySelector('[data-testid="city"]');
|
||||
return c ? c.innerText.replace(/\s+/g, ' ').trim() : '';
|
||||
});
|
||||
|
||||
// Type / duration / start → concatenated into rate as listing card has no €
|
||||
const meta = await el.evaluate(node => {
|
||||
const parts = [];
|
||||
['type', 'duration', 'beginningText', 'remoteInPercent'].forEach(t => {
|
||||
const e2 = node.querySelector(`[data-testid="${t}"]`);
|
||||
if (e2) parts.push(e2.innerText.replace(/\s+/g, ' ').trim());
|
||||
});
|
||||
return parts.join(' | ');
|
||||
});
|
||||
|
||||
// Listing card has no description text — fetch detail page lazily would
|
||||
// be expensive; leave description empty here. Matching engine handles
|
||||
// empty descriptions by returning low score, but title + meta still
|
||||
// give signal once we add a detail fetch step later.
|
||||
const description = `${title}\n${meta}`.trim();
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelancermap.de',
|
||||
portalProjectId,
|
||||
title,
|
||||
description,
|
||||
company: company || 'N/A',
|
||||
location: location || 'Remote / N/A',
|
||||
rate: meta || 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Freelancermap card:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid (cookies might rotate)
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
// Session was provided but check failed
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelancermap.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelancermap scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes job listings from Hays.de. Public portal, no login required.
|
||||
* Search URL: https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=<keyword>&e=false
|
||||
* joblevel=3 (Senior/Lead-Bereich, vom Nutzer vorgegeben), mrew=1, e=false (alle Vertragsarten).
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeHays = async (keywords) => {
|
||||
console.log(`Starting Hays.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
for (const keyword of keywords) {
|
||||
const searchUrl = `https://www.hays.de/jobsuche/stellenangebote-jobs?page=1&mrew=1&joblevel=3&q=${encodeURIComponent(keyword)}&e=false`;
|
||||
console.log(`Navigating to Hays search: ${searchUrl}`);
|
||||
|
||||
try {
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
// Wait for actual job-detail anchors (JS hydrates them).
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/jobsuche/stellenangebote-jobs-detail-"]', { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`Hays: no job-detail anchors visible for "${keyword}"`);
|
||||
continue;
|
||||
}
|
||||
await page.waitForTimeout(800 + Math.random() * 1000);
|
||||
|
||||
// Only real job-detail URLs. Pattern: /jobsuche/stellenangebote-jobs-detail-<slug>-<id>/<page>
|
||||
const linkEls = await page.$$('a[href*="/jobsuche/stellenangebote-jobs-detail-"]');
|
||||
console.log(`Found ${linkEls.length} Hays job-detail anchors for keyword: ${keyword}`);
|
||||
|
||||
for (const linkEl of linkEls) {
|
||||
try {
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!rawUrl) continue;
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.hays.de${rawUrl}`;
|
||||
|
||||
// Hays job-id: numeric segment at end before trailing /N
|
||||
const idMatch = url.match(/-(\d{5,})(?:\/\d+)?(?:[?#].*)?$/);
|
||||
if (!idMatch) continue;
|
||||
const portalProjectId = idMatch[1];
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Title: link inner text first, fall back to nearest heading.
|
||||
let title = ((await linkEl.innerText()) || '').replace(/\s+/g, ' ').trim();
|
||||
if (!title) {
|
||||
title = await linkEl.evaluate(node => {
|
||||
let n = node;
|
||||
for (let i = 0; i < 6 && n; i++) {
|
||||
n = n.parentElement;
|
||||
if (!n) break;
|
||||
const h = n.querySelector('h2, h3, [class*="title"], [class*="Title"]');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
return '';
|
||||
}).catch(() => '');
|
||||
}
|
||||
// Derive title from URL slug if still missing.
|
||||
if (!title) {
|
||||
const slugMatch = url.match(/stellenangebote-jobs-detail-(.+?)-\d{5,}/);
|
||||
if (slugMatch) {
|
||||
title = slugMatch[1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
if (!title) continue;
|
||||
|
||||
// Card text: closest ancestor that contains structured info (max 6 levels up).
|
||||
const meta = await linkEl.evaluate(node => {
|
||||
let card = node;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (!card.parentElement) break;
|
||||
card = card.parentElement;
|
||||
const txt = (card.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
if (txt.length > 80 && txt.length < 1500) return txt.slice(0, 1200);
|
||||
}
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||||
}).catch(() => '');
|
||||
|
||||
// Location: try common DE cities or fall back to scanning "Standort:" pattern.
|
||||
const locMatch = meta.match(/Standort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| meta.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Dortmund|Essen|Bremen|Dresden|Hannover|Nürnberg|Mannheim|Karlsruhe|Wiesbaden|Bonn|Münster|Augsburg|Aachen|Erfurt|Mainz|Kiel|Halle|Magdeburg|Freiburg|Lübeck|Oberhausen|Rostock|Kassel|Saarbrücken|Mönchengladbach|Krefeld|Chemnitz|Braunschweig|Wuppertal|Bielefeld|Duisburg|remote|Remote)\b/);
|
||||
const location = locMatch ? locMatch[1] || locMatch[0] : 'Remote / N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'hays.de',
|
||||
portalProjectId,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${meta}`.trim().slice(0, 1500),
|
||||
company: 'Hays (Vermittlung)',
|
||||
location: location.slice(0, 200),
|
||||
rate: (meta.match(/(?:€|EUR)[^|·]{0,40}/) || ['N/A'])[0].slice(0, 90),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing Hays card:', err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping Hays for "${keyword}":`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Hays scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Generic scraper for the SThree-family staffing portals (Huxley, Computer
|
||||
* Futures, Progressive, Real Staffing, Global Enterprise Partners, Madison
|
||||
* Black, JP Gray, etc.). They all share the same template — job detail URLs
|
||||
* follow `<base>/<locale>/job/<slug>/<numeric-id>/`. Public, no login.
|
||||
*
|
||||
* @param {object} cfg
|
||||
* @param {string} cfg.name - DB portal name (e.g. 'huxley.com')
|
||||
* @param {string} cfg.listingUrl - Search-result listing URL
|
||||
* @param {string} cfg.localePath - e.g. '/de-de/' or '/en-nl/'
|
||||
*/
|
||||
const scrapeSThreeStyle = async ({ name, listingUrl, localePath }) => {
|
||||
console.log(`Starting ${name} scrape (listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(listingUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
const jobLinkSelector = `a[href*="${localePath}job/"]`;
|
||||
try {
|
||||
await page.waitForSelector(jobLinkSelector, { timeout: 12000 });
|
||||
} catch {
|
||||
console.warn(`${name}: no job anchors visible after wait.`);
|
||||
}
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
const links = await page.$$eval(jobLinkSelector, (els, locale) => {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
// Escape regex special chars in locale (keeps slashes as literal).
|
||||
const esc = locale.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const re = new RegExp(esc + 'job/([^/]+)/(\\d{4,})/?');
|
||||
for (const a of els) {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (href.includes('{{')) continue;
|
||||
const m = href.match(re);
|
||||
if (!m) continue;
|
||||
const id = m[2];
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
out.push({ id, slug: m[1], href });
|
||||
}
|
||||
return out;
|
||||
}, localePath).catch(() => []);
|
||||
console.log(`Found ${links.length} ${name} job URLs.`);
|
||||
|
||||
// Detail-page fetches were triggering Cloudflare/WAF on the SThree
|
||||
// family. Title is derived from the URL slug; description = title only.
|
||||
// No detail requests → no bot signature, no rate spikes.
|
||||
for (const link of links) {
|
||||
try {
|
||||
const url = link.href.startsWith('http') ? link.href : new URL(link.href, listingUrl).toString();
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === link.id)) continue;
|
||||
|
||||
let title = '';
|
||||
try {
|
||||
title = decodeURIComponent(link.slug);
|
||||
} catch {
|
||||
title = link.slug;
|
||||
}
|
||||
title = title.replace(/-/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
// Title-case for the slug pieces (German diacritics kept).
|
||||
title = title.replace(/\b(\w)([\w'’äöüß]*)/g, (_, a, b) => a.toUpperCase() + b);
|
||||
if (!title) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: name,
|
||||
portalProjectId: link.id,
|
||||
title: title.slice(0, 250),
|
||||
description: title.slice(0, 250),
|
||||
company: `${name} (Vermittlung)`,
|
||||
location: 'Siehe Anzeige',
|
||||
rate: 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error building ${name} project entry:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error scraping ${name} listing:`, err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error during ${name} scrape:`, error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`${name} scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
// Per-portal wrappers used by pipeline.js / api.js
|
||||
export const SThreePortals = [
|
||||
{ name: 'huxley.com', listingUrl: 'https://www.huxley.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'computerfutures.com', listingUrl: 'https://www.computerfutures.com/de-de/stellensuche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'progressiverecruitment.com', listingUrl: 'https://www.progressiverecruitment.com/de-de/jobs-projekte-suchen/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'realstaffing.com', listingUrl: 'https://www.realstaffing.com/de-de/job-suche/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'globalenterprisepartners.com', listingUrl: 'https://www.globalenterprisepartners.com/de-de/job-search/?country=Deutschland&searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'madisonblack.com', listingUrl: 'https://www.madisonblack.com/de-de/job-search/?searchRadius=50km', localePath: '/de-de/' },
|
||||
{ name: 'jpgray.nl', listingUrl: 'https://www.jpgray.nl/en-nl/job-search/?country=Netherlands&searchRadius=50km', localePath: '/en-nl/' }
|
||||
];
|
||||
|
||||
export const scrapeSThreePortal = (cfg) => scrapeSThreeStyle(cfg);
|
||||
|
||||
/**
|
||||
* Scrapes IT projects from Etengo.de. Public portal, no login required.
|
||||
* Search URL: https://www.etengo.de/it-projektsuche/?q=<keyword>
|
||||
* @param {string[]} keywords
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeEtengo = async (/* keywords */) => {
|
||||
// Etengo's search filter is JS-side and the listing URL ignores ?q. Load the
|
||||
// full project list once and return all entries; matching engine handles
|
||||
// keyword relevance per agent CV.
|
||||
console.log(`Starting Etengo.de scrape (full listing).`);
|
||||
const projectsList = [];
|
||||
|
||||
const browser = await chromium.launch({ headless: true, args: STEALTH_ARGS });
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto('https://www.etengo.de/it-projektsuche/', { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/it-projektsuche/"][href*="/"]', { timeout: 10000 });
|
||||
} catch {
|
||||
console.warn('Etengo: project anchors not visible.');
|
||||
}
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Collect detail-page URLs: /it-projektsuche/<numeric-id>/
|
||||
const anchors = await page.$$('a[href*="/it-projektsuche/"]');
|
||||
const seen = new Set();
|
||||
const detailUrls = [];
|
||||
for (const a of anchors) {
|
||||
const href = await a.getAttribute('href');
|
||||
if (!href) continue;
|
||||
const m = href.match(/\/it-projektsuche\/(\d{3,})\/?$/);
|
||||
if (!m) continue;
|
||||
const url = href.startsWith('http') ? href : `https://www.etengo.de${href}`;
|
||||
if (seen.has(m[1])) continue;
|
||||
seen.add(m[1]);
|
||||
detailUrls.push({ id: m[1], url });
|
||||
}
|
||||
console.log(`Found ${detailUrls.length} Etengo project URLs.`);
|
||||
|
||||
// Fetch each detail page in same context for title + description.
|
||||
for (const { id, url } of detailUrls) {
|
||||
try {
|
||||
const detail = await context.newPage();
|
||||
await detail.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
||||
await detail.waitForTimeout(500);
|
||||
|
||||
const title = await detail.evaluate(() => {
|
||||
const h = document.querySelector('h1');
|
||||
if (h && h.innerText) return h.innerText.replace(/\s+/g, ' ').trim();
|
||||
const t = document.querySelector('title');
|
||||
return t ? t.innerText.replace(/ - Etengo.*$/i, '').trim() : '';
|
||||
}).catch(() => '');
|
||||
|
||||
const description = await detail.evaluate(() => {
|
||||
// Look for main article / project description container.
|
||||
const cand = document.querySelector('main, article, .entry-content, .project-detail, [class*="detail"]');
|
||||
const node = cand || document.body;
|
||||
return (node.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 1500);
|
||||
}).catch(() => '');
|
||||
|
||||
const location = (description.match(/Einsatzort[:\s]+([^|·\n]{2,40})/i)
|
||||
|| description.match(/\b(Berlin|Hamburg|München|Köln|Frankfurt|Stuttgart|Düsseldorf|Leipzig|Hannover|Nürnberg|Bonn|Münster|Karlsruhe|Mannheim|Bremen|Dresden|Erfurt|Mainz|remote|Remote)\b/) || ['Remote / N/A'])[0];
|
||||
|
||||
const rateMatch = description.match(/(?:€|EUR)[^|·]{0,40}/);
|
||||
|
||||
await detail.close();
|
||||
|
||||
if (!title) continue;
|
||||
if (projectsList.some(p => p.portalProjectId === id)) continue;
|
||||
|
||||
projectsList.push({
|
||||
portal: 'etengo.de',
|
||||
portalProjectId: id,
|
||||
title: title.slice(0, 250),
|
||||
description: `${title}\n${description}`.trim().slice(0, 1500),
|
||||
company: 'Etengo (Vermittlung)',
|
||||
location: String(location).slice(0, 200),
|
||||
rate: rateMatch ? rateMatch[0].slice(0, 90) : 'N/A',
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`Error fetching Etengo detail ${url}:`, err.message);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error scraping Etengo listing:', err.message);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during Etengo scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log(`Etengo scrape collected ${projectsList.length} projects.`);
|
||||
return { projects: projectsList, sessionValid: true };
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrapes projects from Freelance.de based on keywords.
|
||||
* @param {string[]} keywords - List of search terms.
|
||||
* @param {string|null} sessionState - JSON string of cookies & storage.
|
||||
* @returns {Promise<{projects: Object[], sessionValid: boolean}>}
|
||||
*/
|
||||
export const scrapeFreelanceDe = async (keywords, sessionState = null) => {
|
||||
console.log(`Starting Freelance.de scrape for keywords: ${keywords.join(', ')}`);
|
||||
const projectsList = [];
|
||||
let sessionValid = false;
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: STEALTH_ARGS
|
||||
});
|
||||
|
||||
try {
|
||||
let contextOptions = {
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 800 }
|
||||
};
|
||||
|
||||
if (sessionState) {
|
||||
const parsed = parseStorageState(sessionState);
|
||||
if (parsed) {
|
||||
contextOptions.storageState = parsed;
|
||||
console.log(`Loaded Freelance.de session: ${parsed.cookies.length} cookies.`);
|
||||
} else {
|
||||
console.warn('Freelance.de session state empty or invalid after normalize.');
|
||||
}
|
||||
}
|
||||
|
||||
const context = await browser.newContext(contextOptions);
|
||||
const page = await context.newPage();
|
||||
|
||||
// Anti-detection: override navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
// Check if session is valid by visiting dashboard/profile page
|
||||
if (sessionState) {
|
||||
try {
|
||||
await page.goto('https://www.freelance.de/users/profile.php', { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
const loginButton = await page.$('a[href*="login"], a[href*="sign-in"]');
|
||||
const userProfile = await page.$('a[href*="logout"], .user-info, .profile-settings');
|
||||
|
||||
if (userProfile && !loginButton) {
|
||||
sessionValid = true;
|
||||
console.log('Freelance.de session is valid (Logged In).');
|
||||
} else {
|
||||
console.log('Freelance.de session has expired.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Error checking Freelance.de session validity:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
for (const keyword of keywords) {
|
||||
// freelance.de search url: search_sort_by=2 means date descending (newest first)
|
||||
const searchUrl = `https://www.freelance.de/search/project.php?__search_sort_by=2&query=${encodeURIComponent(keyword)}`;
|
||||
console.log(`Navigating to Freelance.de search: ${searchUrl}`);
|
||||
|
||||
await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await page.waitForTimeout(1000 + Math.random() * 2000);
|
||||
|
||||
// Select project list items
|
||||
const projectElements = await page.$$('.project-list-item, .project-row, .list-item-main, div[class*="project-list-item"]');
|
||||
console.log(`Found ${projectElements.length} potential project elements for keyword: ${keyword}`);
|
||||
|
||||
for (const el of projectElements) {
|
||||
try {
|
||||
// Find the link to the project detail page
|
||||
// Format e.g., /Projekte/Projekt-Details-12345.html
|
||||
const linkEl = await el.$('a[href*="/Projekte/"], a[href*="/Projekt/"]');
|
||||
if (!linkEl) continue;
|
||||
|
||||
const title = await linkEl.innerText();
|
||||
const rawUrl = await linkEl.getAttribute('href');
|
||||
if (!title || !rawUrl) continue;
|
||||
|
||||
const url = rawUrl.startsWith('http') ? rawUrl : `https://www.freelance.de${rawUrl}`;
|
||||
|
||||
// Parse project ID from URL
|
||||
// Format e.g., /Projekte/123456-React-Entwickler or /Projekt/123456
|
||||
const matchId = url.match(/\/(\d+)-/) || url.match(/\/(\d+)\.html/) || url.match(/\/(\d+)(?:\/|\?|$)/);
|
||||
const portalProjectId = matchId ? matchId[1] : url;
|
||||
|
||||
if (projectsList.some(p => p.portalProjectId === portalProjectId)) continue;
|
||||
|
||||
// Scrape details from listing card
|
||||
const descEl = await el.$('.project-description, .description, p, .project-details');
|
||||
const description = descEl ? await descEl.innerText() : '';
|
||||
|
||||
const locEl = await el.$('.location, .project-location, span[title*="Ort"], .meta-location');
|
||||
const location = locEl ? await locEl.innerText() : 'Remote / N/A';
|
||||
|
||||
const rateEl = await el.$('.rate, .project-rate, .meta-rate, span[title*="Budget"]');
|
||||
const rate = rateEl ? await rateEl.innerText() : 'N/A';
|
||||
|
||||
const compEl = await el.$('.company, .project-company, .meta-company');
|
||||
const company = compEl ? await compEl.innerText() : 'N/A';
|
||||
|
||||
projectsList.push({
|
||||
portal: 'freelance.de',
|
||||
portalProjectId,
|
||||
title: title.trim(),
|
||||
description: description.trim(),
|
||||
company: company.trim(),
|
||||
location: location.trim(),
|
||||
rate: rate.trim(),
|
||||
url,
|
||||
publishedAt: new Date()
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Error parsing individual Freelance.de element:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated storage state if session was valid
|
||||
if (sessionValid) {
|
||||
const updatedState = await context.storageState();
|
||||
sessionValid = true;
|
||||
await Portal.update(
|
||||
{ sessionState: JSON.stringify(updatedState), connectionStatus: 'connected' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
} else if (sessionState) {
|
||||
await Portal.update(
|
||||
{ connectionStatus: 'expired' },
|
||||
{ where: { name: 'freelance.de' } }
|
||||
);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during Freelance.de scrape:', error);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
return { projects: projectsList, sessionValid };
|
||||
};
|
||||
Reference in New Issue
Block a user