diff --git a/config.yaml b/config.yaml index ed2da7d..7dd1d1d 100644 --- a/config.yaml +++ b/config.yaml @@ -47,7 +47,8 @@ scraper: - "Cloud Network Engineer" - "Systems Engineer" location: "Canada" - work_types: ["remote", "hybrid"] + work_types: ["remote"] + remote_only: true # also drop jobs whose location/description reveals hybrid or on-site pages_per_keyword: 3 delay_between_pages: [3, 6] delay_between_keywords: [5, 10] diff --git a/daemon.py b/daemon.py index c40209a..0858c9e 100644 --- a/daemon.py +++ b/daemon.py @@ -226,6 +226,41 @@ class LinkedInJobScraper: return jobs +# LinkedIn's f_WT=2 (remote) URL filter is leaky — hybrid/on-site posts slip +# through. These patterns catch them from the card location and description. +# "hybrid" alone is NOT matched in descriptions: networking JDs legitimately +# say "hybrid cloud" / "hybrid infrastructure". +NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I) +NON_REMOTE_DESC_RE = re.compile( + r"hybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup)" + r"|\b(work|be)\s+(from|in)\s+(the\s+)?office" + r"|\bin[- ]office\b" + r"|\b\d+\s*(?:-\s*\d+\s*)?days?\s+(?:per\s+week\s+)?(?:in|at)\s+(?:the\s+)?office" + r"|this\s+(?:is\s+an?|role\s+is)\s+(?:hybrid|on[- ]?site)" + r"|must\s+(?:be\s+able\s+to\s+)?(?:commute|relocate)", + re.I, +) + + +def filter_remote_only(jobs, check_descriptions=False): + """Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter.""" + kept = [] + for job in jobs: + if NON_REMOTE_LOCATION_RE.search(job.get("location") or ""): + logger.info(f" 🚫 Not remote (location): {job['title']} @ {job['company']} [{job['location']}]") + continue + if check_descriptions and job.get("description"): + hit = NON_REMOTE_DESC_RE.search(job["description"]) + if hit: + logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {job['title']} @ {job['company']}") + continue + kept.append(job) + dropped = len(jobs) - len(kept) + if dropped: + logger.info(f"Remote-only filter: {dropped} hybrid/on-site jobs dropped, {len(kept)} remaining.") + return kept + + def pre_filter_jobs(jobs, config): title_blacklist = [t.lower() for t in config.get("title_blacklist", [])] company_blacklist = [c.lower() for c in config.get("company_blacklist", [])] @@ -478,9 +513,16 @@ def run_once(config, profile): jobs = pre_filter_jobs(jobs, config) + remote_only = scraper_cfg.get("remote_only", False) + if remote_only: + jobs = filter_remote_only(jobs) + logger.info("📄 Fetching job descriptions...") jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details) + if remote_only: + jobs = filter_remote_only(jobs, check_descriptions=True) + desc_count = sum(1 for j in jobs if j.get("description")) logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")