feat: restrict search to 100% remote jobs with hybrid/on-site filter
LinkedIn's f_WT=2 remote filter is leaky, so besides requesting only remote work types, drop jobs whose card location says Hybrid/On-site before detail fetches, and drop jobs whose description reveals hybrid or in-office requirements after. Bare 'hybrid' is not matched in descriptions since networking JDs legitimately say 'hybrid cloud'. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+2
-1
@@ -47,7 +47,8 @@ scraper:
|
||||
- "Cloud Network Engineer"
|
||||
- "Systems Engineer"
|
||||
location: "Canada"
|
||||
work_types: ["remote", "hybrid"]
|
||||
work_types: ["remote"]
|
||||
remote_only: true # also drop jobs whose location/description reveals hybrid or on-site
|
||||
pages_per_keyword: 3
|
||||
delay_between_pages: [3, 6]
|
||||
delay_between_keywords: [5, 10]
|
||||
|
||||
@@ -226,6 +226,41 @@ class LinkedInJobScraper:
|
||||
return jobs
|
||||
|
||||
|
||||
# LinkedIn's f_WT=2 (remote) URL filter is leaky — hybrid/on-site posts slip
|
||||
# through. These patterns catch them from the card location and description.
|
||||
# "hybrid" alone is NOT matched in descriptions: networking JDs legitimately
|
||||
# say "hybrid cloud" / "hybrid infrastructure".
|
||||
NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I)
|
||||
NON_REMOTE_DESC_RE = re.compile(
|
||||
r"hybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup)"
|
||||
r"|\b(work|be)\s+(from|in)\s+(the\s+)?office"
|
||||
r"|\bin[- ]office\b"
|
||||
r"|\b\d+\s*(?:-\s*\d+\s*)?days?\s+(?:per\s+week\s+)?(?:in|at)\s+(?:the\s+)?office"
|
||||
r"|this\s+(?:is\s+an?|role\s+is)\s+(?:hybrid|on[- ]?site)"
|
||||
r"|must\s+(?:be\s+able\s+to\s+)?(?:commute|relocate)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def filter_remote_only(jobs, check_descriptions=False):
|
||||
"""Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter."""
|
||||
kept = []
|
||||
for job in jobs:
|
||||
if NON_REMOTE_LOCATION_RE.search(job.get("location") or ""):
|
||||
logger.info(f" 🚫 Not remote (location): {job['title']} @ {job['company']} [{job['location']}]")
|
||||
continue
|
||||
if check_descriptions and job.get("description"):
|
||||
hit = NON_REMOTE_DESC_RE.search(job["description"])
|
||||
if hit:
|
||||
logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {job['title']} @ {job['company']}")
|
||||
continue
|
||||
kept.append(job)
|
||||
dropped = len(jobs) - len(kept)
|
||||
if dropped:
|
||||
logger.info(f"Remote-only filter: {dropped} hybrid/on-site jobs dropped, {len(kept)} remaining.")
|
||||
return kept
|
||||
|
||||
|
||||
def pre_filter_jobs(jobs, config):
|
||||
title_blacklist = [t.lower() for t in config.get("title_blacklist", [])]
|
||||
company_blacklist = [c.lower() for c in config.get("company_blacklist", [])]
|
||||
@@ -478,9 +513,16 @@ def run_once(config, profile):
|
||||
|
||||
jobs = pre_filter_jobs(jobs, config)
|
||||
|
||||
remote_only = scraper_cfg.get("remote_only", False)
|
||||
if remote_only:
|
||||
jobs = filter_remote_only(jobs)
|
||||
|
||||
logger.info("📄 Fetching job descriptions...")
|
||||
jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details)
|
||||
|
||||
if remote_only:
|
||||
jobs = filter_remote_only(jobs, check_descriptions=True)
|
||||
|
||||
desc_count = sum(1 for j in jobs if j.get("description"))
|
||||
logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user