feat: strict remote-only filtering with positive remote evidence

LinkedIn's f_WT=2 filter is nearly ignored for guest scraping and guest
pages expose no structured workplace-type field, so text filtering is
the only option:

- Require positive remote evidence (title/location/description must say
  remote) before keeping a job; silence now means drop, not keep.
  Configurable via scraper.require_remote_evidence.
- Catch more hybrid/on-site phrasings: "Hybrid: <city>", "office-based",
  "N days onsite/in office", "return to office", titles with
  hybrid/onsite (without tripping on "hybrid cloud").
- Append "Remote" to search keywords when remote_only is on, since
  keyword matching surfaces far more genuinely remote posts than the
  broken filter.

Verified against live LinkedIn postings: three known hybrid/on-site
leaks dropped, known remote post kept.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

config.yaml

@@ -49,6 +49,11 @@ scraper:
   location: "Canada"
   work_types: ["remote"]
   remote_only: true   # also drop jobs whose location/description reveals hybrid or on-site
+  # LinkedIn's remote filter is unreliable for logged-out scraping, so on top of
+  # dropping visible hybrid/on-site jobs, require positive proof of remote
+  # (title/location/description must say remote). Strict: jobs that never state
+  # they're remote are dropped. Set false to keep unverified jobs.
+  require_remote_evidence: true
   pages_per_keyword: 3
   delay_between_pages: [3, 6]
   delay_between_keywords: [5, 10]

daemon.py

@@ -81,7 +81,13 @@ class LinkedInJobScraper:
     def scrape_jobs(self, keyword, location, pages=3):
         jobs = []
         seen_links = set()
-        encoded_keyword = quote(keyword)
+        # The guest f_WT=2 filter barely works, but keyword matching does:
+        # searching "<keyword> Remote" surfaces far more genuinely remote
+        # posts, and the evidence filter drops the rest.
+        search_keyword = keyword
+        if self.config.get("remote_only") and "remote" not in keyword.lower():
+            search_keyword = f"{keyword} Remote"
+        encoded_keyword = quote(search_keyword)
         encoded_location = quote(location)
         work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"}
         work_types = self.config.get("work_types", [])
@@ -226,38 +232,110 @@ class LinkedInJobScraper:
         return jobs
 
 
-# LinkedIn's f_WT=2 (remote) URL filter is leaky — hybrid/on-site posts slip
+# LinkedIn's f_WT=2 (remote) URL filter is nearly ignored for guest (logged-out)
-# through. These patterns catch them from the card location and description.
+# scraping — hybrid/on-site posts routinely come back, and guest pages expose no
-# "hybrid" alone is NOT matched in descriptions: networking JDs legitimately
+# structured workplace-type field (no jobLocationType in the JSON-LD, even for
-# say "hybrid cloud" / "hybrid infrastructure".
+# jobs titled "... - Remote"). So remote-only filtering is two-sided:
+#   1. NEGATIVE: drop anything whose title/location/description reveals
+#      hybrid or on-site (e.g. Pet Valu's JD literally starts "Hybrid: Markham").
+#   2. POSITIVE (require_remote_evidence): keep only jobs whose title, location,
+#      or description affirmatively says remote. No evidence = drop, because
+#      the search filter itself can't be trusted.
+# "hybrid" alone is NOT matched in titles/descriptions: networking JDs
+# legitimately say "hybrid cloud" / "hybrid infrastructure".
 NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I)
+NON_REMOTE_TITLE_RE = re.compile(
+    r"\bhybrid\b(?!\s+(cloud|infrastructure|network|environment|mesh))"
+    r"|\bon[- ]?site\b",
+    re.I,
+)
+_NUM = r"(?:\d+|one|two|three|four|five)"
+_WEEK = r"(?:\s*/\s*week|\s+(?:per|a|each)\s+week)?"
 NON_REMOTE_DESC_RE = re.compile(
-    r"hybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup)"
+    r"\bhybrid\s*[:\-–—]"  # "Hybrid: Markham, Ontario"
-    r"|\b(work|be)\s+(from|in)\s+(the\s+)?office"
+    r"|\bhybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup|environment)"
+    r"|\b(work(place)?|working)\s+(arrangement|model|type|location)\s*[:\-–—]?\s*(is\s+)?hybrid"
+    r"|\bthis\s+(is\s+an?|role\s+is|position\s+is)\s+(hybrid|on[- ]?site)"
+    r"|\b(work|be)\s+(from|in|at)\s+(the\s+|our\s+|an?\s+)?office"
     r"|\bin[- ]office\b"
-    r"|\b\d+\s*(?:-\s*\d+\s*)?days?\s+(?:per\s+week\s+)?(?:in|at)\s+(?:the\s+)?office"
+    rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+(in|at)\s+(the\s+|our\s+|an?\s+)?office"
-    r"|this\s+(?:is\s+an?|role\s+is)\s+(?:hybrid|on[- ]?site)"
+    rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+on[- ]?site"
-    r"|must\s+(?:be\s+able\s+to\s+)?(?:commute|relocate)",
+    rf"|\b{_NUM}\s+office\s+days"
+    r"|\bon[- ]?site\s+(presence|attendance|requirement|expectation)"
+    r"|\brequired\s+to\s+(work|be)\s+on[- ]?site"
+    r"|\breturn\s+to\s+office"
+    r"|\boffice[- ]based\b"
+    r"|\bcommuting\s+distance\b"
+    r"|\bmust\s+(be\s+able\s+to\s+)?(commute|relocate)"
+    r"|\boccasional(ly)?\s+remote\b",
+    re.I,
+)
+REMOTE_EVIDENCE_RE = re.compile(
+    r"\b(fully|100\s*%|completely|entirely)[- ]remote\b"
+    r"|\bremote[- ](first|only|position|role|opportunity|job|work(ing)?|environment|team|based)\b"
+    r"|\bwork(ing)?[- ](from|at)[- ]home\b"
+    r"|\bwork(ing)?\s+remotely\b"
+    r"|\btelecommut(e|ing|er)\b"
+    r"|\b(location|workplace|work\s+type)\s*[:\-–—]\s*remote\b"
+    r"|\(\s*remote\s*\)"
+    r"|\bremote\s+(within|in|across)\s+canada\b"
+    r"|[-–—|]\s*remote\s*$",
     re.I,
 )
 
 
-def filter_remote_only(jobs, check_descriptions=False):
+def filter_remote_only(jobs, check_descriptions=False, require_evidence=False,
-    """Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter."""
+                       search_location=None):
+    """Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter.
+
+    With require_evidence=True (used after descriptions are fetched), a job is
+    also dropped unless something affirmatively marks it remote: the title, the
+    description, or a card location that is just the search region (LinkedIn
+    shows a bare country/region like "Canada" for remote-tagged posts).
+    """
     kept = []
+    no_evidence = 0
+    never_fetched = 0
     for job in jobs:
-        if NON_REMOTE_LOCATION_RE.search(job.get("location") or ""):
+        title = job.get("title") or ""
-            logger.info(f"  🚫 Not remote (location): {job['title']} @ {job['company']} [{job['location']}]")
+        location = job.get("location") or ""
+        description = job.get("description") or ""
+        if NON_REMOTE_LOCATION_RE.search(location):
+            logger.info(f"  🚫 Not remote (location): {title} @ {job['company']} [{location}]")
             continue
-        if check_descriptions and job.get("description"):
+        if NON_REMOTE_TITLE_RE.search(title):
-            hit = NON_REMOTE_DESC_RE.search(job["description"])
+            logger.info(f"  🚫 Not remote (title): {title} @ {job['company']}")
+            continue
+        if check_descriptions and description:
+            hit = NON_REMOTE_DESC_RE.search(description)
             if hit:
-                logger.info(f"  🚫 Not remote (description: '{hit.group(0)}'): {job['title']} @ {job['company']}")
+                logger.info(f"  🚫 Not remote (description: '{hit.group(0)}'): {title} @ {job['company']}")
+                continue
+        if require_evidence:
+            evidence = (
+                REMOTE_EVIDENCE_RE.search(title)
+                or REMOTE_EVIDENCE_RE.search(location)
+                or (search_location and location.strip().lower() == search_location.strip().lower())
+                or (description and REMOTE_EVIDENCE_RE.search(description))
+            )
+            if not evidence:
+                no_evidence += 1
+                if job.get("description") is None:
+                    never_fetched += 1
+                logger.info(f"  🚫 No remote evidence: {title} @ {job['company']} [{location}]")
                 continue
         kept.append(job)
     dropped = len(jobs) - len(kept)
     if dropped:
-        logger.info(f"Remote-only filter: {dropped} hybrid/on-site jobs dropped, {len(kept)} remaining.")
+        logger.info(
+            f"Remote-only filter: {dropped} dropped "
+            f"({no_evidence} lacked remote evidence), {len(kept)} remaining."
+        )
+    if never_fetched:
+        logger.warning(
+            f"{never_fetched} jobs dropped without their description ever fetched "
+            f"(detail-fetch cap) — raise max_detail_fetches_per_run to give them a chance."
+        )
     return kept
 
 
@@ -514,6 +592,7 @@ def run_once(config, profile):
     jobs = pre_filter_jobs(jobs, config)
 
     remote_only = scraper_cfg.get("remote_only", False)
+    require_evidence = scraper_cfg.get("require_remote_evidence", True)
     if remote_only:
         jobs = filter_remote_only(jobs)
 
@@ -521,7 +600,11 @@ def run_once(config, profile):
     jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details)
 
     if remote_only:
-        jobs = filter_remote_only(jobs, check_descriptions=True)
+        jobs = filter_remote_only(
+            jobs, check_descriptions=True,
+            require_evidence=require_evidence,
+            search_location=location,
+        )
 
     desc_count = sum(1 for j in jobs if j.get("description"))
     logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")