feat: restrict search to 100% remote jobs with hybrid/on-site filter
LinkedIn's f_WT=2 remote filter is leaky, so besides requesting only remote work types, drop jobs whose card location says Hybrid/On-site before detail fetches, and drop jobs whose description reveals hybrid or in-office requirements after. Bare 'hybrid' is not matched in descriptions since networking JDs legitimately say 'hybrid cloud'. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+2
-1
@@ -47,7 +47,8 @@ scraper:
|
|||||||
- "Cloud Network Engineer"
|
- "Cloud Network Engineer"
|
||||||
- "Systems Engineer"
|
- "Systems Engineer"
|
||||||
location: "Canada"
|
location: "Canada"
|
||||||
work_types: ["remote", "hybrid"]
|
work_types: ["remote"]
|
||||||
|
remote_only: true # also drop jobs whose location/description reveals hybrid or on-site
|
||||||
pages_per_keyword: 3
|
pages_per_keyword: 3
|
||||||
delay_between_pages: [3, 6]
|
delay_between_pages: [3, 6]
|
||||||
delay_between_keywords: [5, 10]
|
delay_between_keywords: [5, 10]
|
||||||
|
|||||||
@@ -226,6 +226,41 @@ class LinkedInJobScraper:
|
|||||||
return jobs
|
return jobs
|
||||||
|
|
||||||
|
|
||||||
|
# LinkedIn's f_WT=2 (remote) URL filter is leaky — hybrid/on-site posts slip
|
||||||
|
# through. These patterns catch them from the card location and description.
|
||||||
|
# "hybrid" alone is NOT matched in descriptions: networking JDs legitimately
|
||||||
|
# say "hybrid cloud" / "hybrid infrastructure".
|
||||||
|
NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I)
|
||||||
|
NON_REMOTE_DESC_RE = re.compile(
|
||||||
|
r"hybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup)"
|
||||||
|
r"|\b(work|be)\s+(from|in)\s+(the\s+)?office"
|
||||||
|
r"|\bin[- ]office\b"
|
||||||
|
r"|\b\d+\s*(?:-\s*\d+\s*)?days?\s+(?:per\s+week\s+)?(?:in|at)\s+(?:the\s+)?office"
|
||||||
|
r"|this\s+(?:is\s+an?|role\s+is)\s+(?:hybrid|on[- ]?site)"
|
||||||
|
r"|must\s+(?:be\s+able\s+to\s+)?(?:commute|relocate)",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def filter_remote_only(jobs, check_descriptions=False):
|
||||||
|
"""Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter."""
|
||||||
|
kept = []
|
||||||
|
for job in jobs:
|
||||||
|
if NON_REMOTE_LOCATION_RE.search(job.get("location") or ""):
|
||||||
|
logger.info(f" 🚫 Not remote (location): {job['title']} @ {job['company']} [{job['location']}]")
|
||||||
|
continue
|
||||||
|
if check_descriptions and job.get("description"):
|
||||||
|
hit = NON_REMOTE_DESC_RE.search(job["description"])
|
||||||
|
if hit:
|
||||||
|
logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {job['title']} @ {job['company']}")
|
||||||
|
continue
|
||||||
|
kept.append(job)
|
||||||
|
dropped = len(jobs) - len(kept)
|
||||||
|
if dropped:
|
||||||
|
logger.info(f"Remote-only filter: {dropped} hybrid/on-site jobs dropped, {len(kept)} remaining.")
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
def pre_filter_jobs(jobs, config):
|
def pre_filter_jobs(jobs, config):
|
||||||
title_blacklist = [t.lower() for t in config.get("title_blacklist", [])]
|
title_blacklist = [t.lower() for t in config.get("title_blacklist", [])]
|
||||||
company_blacklist = [c.lower() for c in config.get("company_blacklist", [])]
|
company_blacklist = [c.lower() for c in config.get("company_blacklist", [])]
|
||||||
@@ -478,9 +513,16 @@ def run_once(config, profile):
|
|||||||
|
|
||||||
jobs = pre_filter_jobs(jobs, config)
|
jobs = pre_filter_jobs(jobs, config)
|
||||||
|
|
||||||
|
remote_only = scraper_cfg.get("remote_only", False)
|
||||||
|
if remote_only:
|
||||||
|
jobs = filter_remote_only(jobs)
|
||||||
|
|
||||||
logger.info("📄 Fetching job descriptions...")
|
logger.info("📄 Fetching job descriptions...")
|
||||||
jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details)
|
jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details)
|
||||||
|
|
||||||
|
if remote_only:
|
||||||
|
jobs = filter_remote_only(jobs, check_descriptions=True)
|
||||||
|
|
||||||
desc_count = sum(1 for j in jobs if j.get("description"))
|
desc_count = sum(1 for j in jobs if j.get("description"))
|
||||||
logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")
|
logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user