diff --git a/config.yaml b/config.yaml index 7dd1d1d..55a05d8 100644 --- a/config.yaml +++ b/config.yaml @@ -49,6 +49,11 @@ scraper: location: "Canada" work_types: ["remote"] remote_only: true # also drop jobs whose location/description reveals hybrid or on-site + # LinkedIn's remote filter is unreliable for logged-out scraping, so on top of + # dropping visible hybrid/on-site jobs, require positive proof of remote + # (title/location/description must say remote). Strict: jobs that never state + # they're remote are dropped. Set false to keep unverified jobs. + require_remote_evidence: true pages_per_keyword: 3 delay_between_pages: [3, 6] delay_between_keywords: [5, 10] diff --git a/daemon.py b/daemon.py index 0858c9e..0f926cb 100644 --- a/daemon.py +++ b/daemon.py @@ -81,7 +81,13 @@ class LinkedInJobScraper: def scrape_jobs(self, keyword, location, pages=3): jobs = [] seen_links = set() - encoded_keyword = quote(keyword) + # The guest f_WT=2 filter barely works, but keyword matching does: + # searching " Remote" surfaces far more genuinely remote + # posts, and the evidence filter drops the rest. + search_keyword = keyword + if self.config.get("remote_only") and "remote" not in keyword.lower(): + search_keyword = f"{keyword} Remote" + encoded_keyword = quote(search_keyword) encoded_location = quote(location) work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"} work_types = self.config.get("work_types", []) @@ -226,38 +232,110 @@ class LinkedInJobScraper: return jobs -# LinkedIn's f_WT=2 (remote) URL filter is leaky — hybrid/on-site posts slip -# through. These patterns catch them from the card location and description. -# "hybrid" alone is NOT matched in descriptions: networking JDs legitimately -# say "hybrid cloud" / "hybrid infrastructure". +# LinkedIn's f_WT=2 (remote) URL filter is nearly ignored for guest (logged-out) +# scraping — hybrid/on-site posts routinely come back, and guest pages expose no +# structured workplace-type field (no jobLocationType in the JSON-LD, even for +# jobs titled "... - Remote"). So remote-only filtering is two-sided: +# 1. NEGATIVE: drop anything whose title/location/description reveals +# hybrid or on-site (e.g. Pet Valu's JD literally starts "Hybrid: Markham"). +# 2. POSITIVE (require_remote_evidence): keep only jobs whose title, location, +# or description affirmatively says remote. No evidence = drop, because +# the search filter itself can't be trusted. +# "hybrid" alone is NOT matched in titles/descriptions: networking JDs +# legitimately say "hybrid cloud" / "hybrid infrastructure". NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I) +NON_REMOTE_TITLE_RE = re.compile( + r"\bhybrid\b(?!\s+(cloud|infrastructure|network|environment|mesh))" + r"|\bon[- ]?site\b", + re.I, +) +_NUM = r"(?:\d+|one|two|three|four|five)" +_WEEK = r"(?:\s*/\s*week|\s+(?:per|a|each)\s+week)?" NON_REMOTE_DESC_RE = re.compile( - r"hybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup)" - r"|\b(work|be)\s+(from|in)\s+(the\s+)?office" + r"\bhybrid\s*[:\-–—]" # "Hybrid: Markham, Ontario" + r"|\bhybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup|environment)" + r"|\b(work(place)?|working)\s+(arrangement|model|type|location)\s*[:\-–—]?\s*(is\s+)?hybrid" + r"|\bthis\s+(is\s+an?|role\s+is|position\s+is)\s+(hybrid|on[- ]?site)" + r"|\b(work|be)\s+(from|in|at)\s+(the\s+|our\s+|an?\s+)?office" r"|\bin[- ]office\b" - r"|\b\d+\s*(?:-\s*\d+\s*)?days?\s+(?:per\s+week\s+)?(?:in|at)\s+(?:the\s+)?office" - r"|this\s+(?:is\s+an?|role\s+is)\s+(?:hybrid|on[- ]?site)" - r"|must\s+(?:be\s+able\s+to\s+)?(?:commute|relocate)", + rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+(in|at)\s+(the\s+|our\s+|an?\s+)?office" + rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+on[- ]?site" + rf"|\b{_NUM}\s+office\s+days" + r"|\bon[- ]?site\s+(presence|attendance|requirement|expectation)" + r"|\brequired\s+to\s+(work|be)\s+on[- ]?site" + r"|\breturn\s+to\s+office" + r"|\boffice[- ]based\b" + r"|\bcommuting\s+distance\b" + r"|\bmust\s+(be\s+able\s+to\s+)?(commute|relocate)" + r"|\boccasional(ly)?\s+remote\b", + re.I, +) +REMOTE_EVIDENCE_RE = re.compile( + r"\b(fully|100\s*%|completely|entirely)[- ]remote\b" + r"|\bremote[- ](first|only|position|role|opportunity|job|work(ing)?|environment|team|based)\b" + r"|\bwork(ing)?[- ](from|at)[- ]home\b" + r"|\bwork(ing)?\s+remotely\b" + r"|\btelecommut(e|ing|er)\b" + r"|\b(location|workplace|work\s+type)\s*[:\-–—]\s*remote\b" + r"|\(\s*remote\s*\)" + r"|\bremote\s+(within|in|across)\s+canada\b" + r"|[-–—|]\s*remote\s*$", re.I, ) -def filter_remote_only(jobs, check_descriptions=False): - """Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter.""" +def filter_remote_only(jobs, check_descriptions=False, require_evidence=False, + search_location=None): + """Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter. + + With require_evidence=True (used after descriptions are fetched), a job is + also dropped unless something affirmatively marks it remote: the title, the + description, or a card location that is just the search region (LinkedIn + shows a bare country/region like "Canada" for remote-tagged posts). + """ kept = [] + no_evidence = 0 + never_fetched = 0 for job in jobs: - if NON_REMOTE_LOCATION_RE.search(job.get("location") or ""): - logger.info(f" 🚫 Not remote (location): {job['title']} @ {job['company']} [{job['location']}]") + title = job.get("title") or "" + location = job.get("location") or "" + description = job.get("description") or "" + if NON_REMOTE_LOCATION_RE.search(location): + logger.info(f" 🚫 Not remote (location): {title} @ {job['company']} [{location}]") continue - if check_descriptions and job.get("description"): - hit = NON_REMOTE_DESC_RE.search(job["description"]) + if NON_REMOTE_TITLE_RE.search(title): + logger.info(f" 🚫 Not remote (title): {title} @ {job['company']}") + continue + if check_descriptions and description: + hit = NON_REMOTE_DESC_RE.search(description) if hit: - logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {job['title']} @ {job['company']}") + logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {title} @ {job['company']}") + continue + if require_evidence: + evidence = ( + REMOTE_EVIDENCE_RE.search(title) + or REMOTE_EVIDENCE_RE.search(location) + or (search_location and location.strip().lower() == search_location.strip().lower()) + or (description and REMOTE_EVIDENCE_RE.search(description)) + ) + if not evidence: + no_evidence += 1 + if job.get("description") is None: + never_fetched += 1 + logger.info(f" 🚫 No remote evidence: {title} @ {job['company']} [{location}]") continue kept.append(job) dropped = len(jobs) - len(kept) if dropped: - logger.info(f"Remote-only filter: {dropped} hybrid/on-site jobs dropped, {len(kept)} remaining.") + logger.info( + f"Remote-only filter: {dropped} dropped " + f"({no_evidence} lacked remote evidence), {len(kept)} remaining." + ) + if never_fetched: + logger.warning( + f"{never_fetched} jobs dropped without their description ever fetched " + f"(detail-fetch cap) — raise max_detail_fetches_per_run to give them a chance." + ) return kept @@ -514,6 +592,7 @@ def run_once(config, profile): jobs = pre_filter_jobs(jobs, config) remote_only = scraper_cfg.get("remote_only", False) + require_evidence = scraper_cfg.get("require_remote_evidence", True) if remote_only: jobs = filter_remote_only(jobs) @@ -521,7 +600,11 @@ def run_once(config, profile): jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details) if remote_only: - jobs = filter_remote_only(jobs, check_descriptions=True) + jobs = filter_remote_only( + jobs, check_descriptions=True, + require_evidence=require_evidence, + search_location=location, + ) desc_count = sum(1 for j in jobs if j.get("description")) logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")