Harwinder Sekhon
37da3b8940
LinkedIn's f_WT=2 filter is nearly ignored for guest scraping and guest pages expose no structured workplace-type field, so text filtering is the only option: - Require positive remote evidence (title/location/description must say remote) before keeping a job; silence now means drop, not keep. Configurable via scraper.require_remote_evidence. - Catch more hybrid/on-site phrasings: "Hybrid: <city>", "office-based", "N days onsite/in office", "return to office", titles with hybrid/onsite (without tripping on "hybrid cloud"). - Append "Remote" to search keywords when remote_only is on, since keyword matching surfaces far more genuinely remote posts than the broken filter. Verified against live LinkedIn postings: three known hybrid/on-site leaks dropped, known remote post kept. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
656 lines
25 KiB
Python
656 lines
25 KiB
Python
"""
|
|
daemon.py
|
|
---------
|
|
Enhanced job scraper daemon with:
|
|
1. Job card scraping
|
|
2. Detail page fetching
|
|
3. Resume-based match scoring
|
|
4. Smart alerts - only 70%+ matches with breakdown
|
|
|
|
Usage:
|
|
python daemon.py # single run
|
|
python daemon.py --loop # run every 2 hours
|
|
python daemon.py --loop --interval 3600 # run every hour
|
|
"""
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import time
|
|
import random
|
|
import re
|
|
import json
|
|
import yaml
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from datetime import datetime
|
|
from urllib.parse import quote
|
|
import logging
|
|
import mysql.connector
|
|
from mysql.connector import Error
|
|
|
|
from resume_parser import parse_resume_text, parse_resume_file
|
|
from matcher import score_job
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
logger = logging.getLogger("job-matcher")
|
|
|
|
|
|
def load_config(config_path="config.yaml"):
|
|
with open(config_path, "r") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def load_profile(profile_path="profile.json", resume_path=None):
|
|
if os.path.exists(profile_path):
|
|
with open(profile_path, "r") as f:
|
|
profile = json.load(f)
|
|
if "full_text" not in profile and resume_path and os.path.exists(resume_path):
|
|
full_profile = parse_resume_file(resume_path)
|
|
profile["full_text"] = full_profile.get("full_text", "")
|
|
return profile
|
|
elif resume_path and os.path.exists(resume_path):
|
|
logger.info(f"No profile.json found. Parsing resume: {resume_path}")
|
|
profile = parse_resume_file(resume_path)
|
|
from resume_parser import save_profile
|
|
save_profile(profile, profile_path)
|
|
return profile
|
|
else:
|
|
logger.error("No profile.json or resume file found. Run resume_parser.py first.")
|
|
sys.exit(1)
|
|
|
|
|
|
class LinkedInJobScraper:
|
|
def __init__(self, config):
|
|
self.config = config.get("scraper", {})
|
|
self.session = requests.Session()
|
|
self.session.headers.update({
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
})
|
|
self.detail_delay = self.config.get("delay_between_details", [4, 8])
|
|
|
|
def scrape_jobs(self, keyword, location, pages=3):
|
|
jobs = []
|
|
seen_links = set()
|
|
# The guest f_WT=2 filter barely works, but keyword matching does:
|
|
# searching "<keyword> Remote" surfaces far more genuinely remote
|
|
# posts, and the evidence filter drops the rest.
|
|
search_keyword = keyword
|
|
if self.config.get("remote_only") and "remote" not in keyword.lower():
|
|
search_keyword = f"{keyword} Remote"
|
|
encoded_keyword = quote(search_keyword)
|
|
encoded_location = quote(location)
|
|
work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"}
|
|
work_types = self.config.get("work_types", [])
|
|
f_wt = ",".join(
|
|
work_type_map[wt.lower()]
|
|
for wt in work_types
|
|
if wt.lower() in work_type_map
|
|
)
|
|
delay_range = self.config.get("delay_between_pages", [3, 6])
|
|
|
|
for page in range(pages):
|
|
logger.info(f" Page {page + 1}/{pages} for '{keyword}'")
|
|
start_index = page * 25
|
|
url = (
|
|
f"https://www.linkedin.com/jobs/search?"
|
|
f"keywords={encoded_keyword}&location={encoded_location}"
|
|
f"&start={start_index}"
|
|
)
|
|
if f_wt:
|
|
url += f"&f_WT={f_wt}"
|
|
|
|
try:
|
|
response = self.session.get(url, timeout=15)
|
|
if response.status_code == 429:
|
|
wait = 60 * (page + 1)
|
|
logger.warning(f"Rate limited. Sleeping {wait}s...")
|
|
time.sleep(wait)
|
|
continue
|
|
if response.status_code != 200:
|
|
logger.error(f"Failed page {page + 1}: HTTP {response.status_code}")
|
|
continue
|
|
|
|
soup = BeautifulSoup(response.content, "html.parser")
|
|
job_cards = soup.find_all("div", class_="base-search-card")
|
|
|
|
if not job_cards:
|
|
logger.info("No job cards found (possibly blocked).")
|
|
break
|
|
|
|
for card in job_cards:
|
|
job = self._extract_card(card)
|
|
if not job or job["link"] in seen_links:
|
|
continue
|
|
seen_links.add(job["link"])
|
|
jobs.append(job)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scraping page {page + 1}: {e}")
|
|
|
|
time.sleep(random.uniform(*delay_range))
|
|
return jobs
|
|
|
|
def _extract_card(self, card):
|
|
try:
|
|
title_elem = card.find("h3", class_="base-search-card__title")
|
|
if not title_elem:
|
|
return None
|
|
title = title_elem.get_text(strip=True)
|
|
|
|
link_elem = card.find("a", class_="base-card__full-link")
|
|
if not link_elem or not link_elem.get("href"):
|
|
return None
|
|
link = link_elem["href"].split("?")[0]
|
|
|
|
company_elem = card.find("h4", class_="base-search-card__subtitle")
|
|
company = company_elem.get_text(strip=True) if company_elem else "Unknown"
|
|
|
|
location_elem = card.find("span", class_="job-search-card__location")
|
|
location = location_elem.get_text(strip=True) if location_elem else ""
|
|
|
|
date_elem = card.find("time")
|
|
posted_date = date_elem.get_text(strip=True) if date_elem else "Just posted"
|
|
|
|
benefit_elem = card.find("div", class_="job-posting-benefits")
|
|
actively_hiring = False
|
|
if benefit_elem:
|
|
actively_hiring = "actively hiring" in benefit_elem.get_text().lower()
|
|
|
|
return {
|
|
"title": title, "company": company, "location": location,
|
|
"posted_date": posted_date, "link": link,
|
|
"actively_hiring": actively_hiring, "description": None,
|
|
}
|
|
except Exception as e:
|
|
logger.debug(f"Card extraction error: {e}")
|
|
return None
|
|
|
|
def fetch_job_description(self, job_url):
|
|
try:
|
|
response = self.session.get(job_url, timeout=15)
|
|
if response.status_code != 200:
|
|
logger.debug(f"Detail fetch failed ({response.status_code}): {job_url}")
|
|
return None
|
|
|
|
soup = BeautifulSoup(response.content, "html.parser")
|
|
|
|
desc_div = soup.find("div", class_="show-more-less-html__markup")
|
|
if desc_div:
|
|
return desc_div.get_text(separator="\n", strip=True)
|
|
|
|
for selector in [
|
|
{"class_": "description__text"},
|
|
{"class_": "job-description"},
|
|
{"class_": "details"},
|
|
]:
|
|
fallback = soup.find("div", **selector)
|
|
if fallback:
|
|
text = fallback.get_text(separator="\n", strip=True)
|
|
if len(text) > 50:
|
|
return text
|
|
|
|
sections = soup.find_all("section", class_="show-more-less-html")
|
|
if sections:
|
|
return "\n".join(s.get_text(separator="\n", strip=True) for s in sections)
|
|
|
|
logger.debug(f"No description found on detail page: {job_url}")
|
|
return None
|
|
except Exception as e:
|
|
logger.debug(f"Detail fetch exception: {e}")
|
|
return None
|
|
|
|
def fetch_descriptions_batch(self, jobs, max_fetches=50):
|
|
fetched_count = 0
|
|
for job in jobs:
|
|
if job.get("description"):
|
|
continue
|
|
if fetched_count >= max_fetches:
|
|
logger.info(f"Hit max detail fetches ({max_fetches}).")
|
|
break
|
|
|
|
logger.info(f" Fetching details: {job['title']} @ {job['company']}")
|
|
description = self.fetch_job_description(job["link"])
|
|
if description:
|
|
job["description"] = description
|
|
fetched_count += 1
|
|
else:
|
|
job["description"] = ""
|
|
|
|
time.sleep(random.uniform(*self.detail_delay))
|
|
|
|
logger.info(f"Fetched {fetched_count} job descriptions.")
|
|
return jobs
|
|
|
|
|
|
# LinkedIn's f_WT=2 (remote) URL filter is nearly ignored for guest (logged-out)
|
|
# scraping — hybrid/on-site posts routinely come back, and guest pages expose no
|
|
# structured workplace-type field (no jobLocationType in the JSON-LD, even for
|
|
# jobs titled "... - Remote"). So remote-only filtering is two-sided:
|
|
# 1. NEGATIVE: drop anything whose title/location/description reveals
|
|
# hybrid or on-site (e.g. Pet Valu's JD literally starts "Hybrid: Markham").
|
|
# 2. POSITIVE (require_remote_evidence): keep only jobs whose title, location,
|
|
# or description affirmatively says remote. No evidence = drop, because
|
|
# the search filter itself can't be trusted.
|
|
# "hybrid" alone is NOT matched in titles/descriptions: networking JDs
|
|
# legitimately say "hybrid cloud" / "hybrid infrastructure".
|
|
NON_REMOTE_LOCATION_RE = re.compile(r"\b(hybrid|on[- ]?site)\b", re.I)
|
|
NON_REMOTE_TITLE_RE = re.compile(
|
|
r"\bhybrid\b(?!\s+(cloud|infrastructure|network|environment|mesh))"
|
|
r"|\bon[- ]?site\b",
|
|
re.I,
|
|
)
|
|
_NUM = r"(?:\d+|one|two|three|four|five)"
|
|
_WEEK = r"(?:\s*/\s*week|\s+(?:per|a|each)\s+week)?"
|
|
NON_REMOTE_DESC_RE = re.compile(
|
|
r"\bhybrid\s*[:\-–—]" # "Hybrid: Markham, Ontario"
|
|
r"|\bhybrid\s+(work(ing)?|model|role|schedule|position|arrangement|setup|environment)"
|
|
r"|\b(work(place)?|working)\s+(arrangement|model|type|location)\s*[:\-–—]?\s*(is\s+)?hybrid"
|
|
r"|\bthis\s+(is\s+an?|role\s+is|position\s+is)\s+(hybrid|on[- ]?site)"
|
|
r"|\b(work|be)\s+(from|in|at)\s+(the\s+|our\s+|an?\s+)?office"
|
|
r"|\bin[- ]office\b"
|
|
rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+(in|at)\s+(the\s+|our\s+|an?\s+)?office"
|
|
rf"|\b{_NUM}\s*(-\s*{_NUM}\s*)?days?{_WEEK}\s+on[- ]?site"
|
|
rf"|\b{_NUM}\s+office\s+days"
|
|
r"|\bon[- ]?site\s+(presence|attendance|requirement|expectation)"
|
|
r"|\brequired\s+to\s+(work|be)\s+on[- ]?site"
|
|
r"|\breturn\s+to\s+office"
|
|
r"|\boffice[- ]based\b"
|
|
r"|\bcommuting\s+distance\b"
|
|
r"|\bmust\s+(be\s+able\s+to\s+)?(commute|relocate)"
|
|
r"|\boccasional(ly)?\s+remote\b",
|
|
re.I,
|
|
)
|
|
REMOTE_EVIDENCE_RE = re.compile(
|
|
r"\b(fully|100\s*%|completely|entirely)[- ]remote\b"
|
|
r"|\bremote[- ](first|only|position|role|opportunity|job|work(ing)?|environment|team|based)\b"
|
|
r"|\bwork(ing)?[- ](from|at)[- ]home\b"
|
|
r"|\bwork(ing)?\s+remotely\b"
|
|
r"|\btelecommut(e|ing|er)\b"
|
|
r"|\b(location|workplace|work\s+type)\s*[:\-–—]\s*remote\b"
|
|
r"|\(\s*remote\s*\)"
|
|
r"|\bremote\s+(within|in|across)\s+canada\b"
|
|
r"|[-–—|]\s*remote\s*$",
|
|
re.I,
|
|
)
|
|
|
|
|
|
def filter_remote_only(jobs, check_descriptions=False, require_evidence=False,
|
|
search_location=None):
|
|
"""Drop jobs that are visibly hybrid/on-site despite LinkedIn's remote filter.
|
|
|
|
With require_evidence=True (used after descriptions are fetched), a job is
|
|
also dropped unless something affirmatively marks it remote: the title, the
|
|
description, or a card location that is just the search region (LinkedIn
|
|
shows a bare country/region like "Canada" for remote-tagged posts).
|
|
"""
|
|
kept = []
|
|
no_evidence = 0
|
|
never_fetched = 0
|
|
for job in jobs:
|
|
title = job.get("title") or ""
|
|
location = job.get("location") or ""
|
|
description = job.get("description") or ""
|
|
if NON_REMOTE_LOCATION_RE.search(location):
|
|
logger.info(f" 🚫 Not remote (location): {title} @ {job['company']} [{location}]")
|
|
continue
|
|
if NON_REMOTE_TITLE_RE.search(title):
|
|
logger.info(f" 🚫 Not remote (title): {title} @ {job['company']}")
|
|
continue
|
|
if check_descriptions and description:
|
|
hit = NON_REMOTE_DESC_RE.search(description)
|
|
if hit:
|
|
logger.info(f" 🚫 Not remote (description: '{hit.group(0)}'): {title} @ {job['company']}")
|
|
continue
|
|
if require_evidence:
|
|
evidence = (
|
|
REMOTE_EVIDENCE_RE.search(title)
|
|
or REMOTE_EVIDENCE_RE.search(location)
|
|
or (search_location and location.strip().lower() == search_location.strip().lower())
|
|
or (description and REMOTE_EVIDENCE_RE.search(description))
|
|
)
|
|
if not evidence:
|
|
no_evidence += 1
|
|
if job.get("description") is None:
|
|
never_fetched += 1
|
|
logger.info(f" 🚫 No remote evidence: {title} @ {job['company']} [{location}]")
|
|
continue
|
|
kept.append(job)
|
|
dropped = len(jobs) - len(kept)
|
|
if dropped:
|
|
logger.info(
|
|
f"Remote-only filter: {dropped} dropped "
|
|
f"({no_evidence} lacked remote evidence), {len(kept)} remaining."
|
|
)
|
|
if never_fetched:
|
|
logger.warning(
|
|
f"{never_fetched} jobs dropped without their description ever fetched "
|
|
f"(detail-fetch cap) — raise max_detail_fetches_per_run to give them a chance."
|
|
)
|
|
return kept
|
|
|
|
|
|
def pre_filter_jobs(jobs, config):
|
|
title_blacklist = [t.lower() for t in config.get("title_blacklist", [])]
|
|
company_blacklist = [c.lower() for c in config.get("company_blacklist", [])]
|
|
filtered = []
|
|
for job in jobs:
|
|
title_lower = job["title"].lower()
|
|
company_lower = job["company"].lower()
|
|
if any(bl in title_lower for bl in title_blacklist):
|
|
continue
|
|
if any(bl in company_lower for bl in company_blacklist):
|
|
continue
|
|
filtered.append(job)
|
|
skipped = len(jobs) - len(filtered)
|
|
if skipped:
|
|
logger.info(f"Pre-filter: {skipped} jobs skipped, {len(filtered)} remaining.")
|
|
return filtered
|
|
|
|
|
|
class JobAlertSystem:
|
|
def __init__(self, db_config):
|
|
self.db_config = db_config
|
|
|
|
def is_job_fresh(self, posted_date):
|
|
if not posted_date:
|
|
return False
|
|
text = posted_date.lower()
|
|
if any(x in text for x in ["just", "minute", "hour", "now", "today"]):
|
|
return True
|
|
match = re.search(r"(\d+)\s+day", text)
|
|
if match:
|
|
return int(match.group(1)) <= 4
|
|
return False
|
|
|
|
def send_telegram_alert(self, token, chat_id, job, match_result):
|
|
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
|
score = match_result["score"]
|
|
breakdown = match_result["breakdown"]
|
|
matched = match_result.get("matched_skills", [])
|
|
missing = match_result.get("missing_skills", [])
|
|
|
|
filled = score // 10
|
|
bar = "█" * filled + "░" * (10 - filled)
|
|
matched_str = ", ".join(matched[:8]) if matched else "—"
|
|
missing_str = ", ".join(missing[:5]) if missing else "None"
|
|
|
|
message = (
|
|
f"🎯 *{score}% Match — {match_result['verdict']}*\n"
|
|
f"{bar}\n\n"
|
|
f"*{job['title']}*\n"
|
|
f"🏢 {job['company']}\n"
|
|
f"📍 {job['location']}\n\n"
|
|
f"📊 *Breakdown:*\n"
|
|
f" Skills: {breakdown['skills']['score']}% | "
|
|
f"Title: {breakdown['title']['score']}% | "
|
|
f"Exp: {breakdown['experience']['score']}%\n"
|
|
f" Keywords: {breakdown['keywords']['score']}% | "
|
|
f"Certs: {breakdown['certifications']['score']}%\n\n"
|
|
f"✅ *Match:* {matched_str}\n"
|
|
f"❌ *Gap:* {missing_str}\n\n"
|
|
f"[View Job]({job['link']})"
|
|
)
|
|
|
|
try:
|
|
r = requests.post(url, json={
|
|
"chat_id": chat_id, "text": message,
|
|
"parse_mode": "Markdown", "disable_web_page_preview": True,
|
|
}, timeout=10)
|
|
if r.status_code != 200:
|
|
logger.error(f"Telegram error: {r.text}")
|
|
return False
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Telegram exception: {e}")
|
|
return False
|
|
|
|
def send_summary_alert(self, token, chat_id, stats):
|
|
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
|
message = (
|
|
f"✅ *Job Matcher Run Complete*\n"
|
|
f"⏰ {datetime.now().strftime('%H:%M %d-%b-%Y')}\n\n"
|
|
f"📊 *Stats:*\n"
|
|
f" Scanned: {stats['total_scanned']}\n"
|
|
f" Passed pre-filter: {stats['after_prefilter']}\n"
|
|
f" Descriptions fetched: {stats['descriptions_fetched']}\n"
|
|
f" Matches (70%+): {stats['matches']}\n"
|
|
f" Alerts sent: {stats['alerts_sent']}\n"
|
|
f" Top score: {stats.get('top_score', 'N/A')}%"
|
|
)
|
|
try:
|
|
requests.post(url, json={
|
|
"chat_id": chat_id, "text": message, "parse_mode": "Markdown",
|
|
}, timeout=10)
|
|
except Exception as e:
|
|
logger.error(f"Summary send error: {e}")
|
|
|
|
def process_jobs(self, jobs, profile, config):
|
|
conn = None
|
|
cursor = None
|
|
min_score = config.get("min_match_score", 70)
|
|
weights = config.get("weights", None)
|
|
aspirational = config.get("aspirational_skills", [])
|
|
|
|
stats = {
|
|
"total_scanned": len(jobs), "after_prefilter": len(jobs),
|
|
"descriptions_fetched": sum(1 for j in jobs if j.get("description")),
|
|
"matches": 0, "alerts_sent": 0, "top_score": 0,
|
|
}
|
|
|
|
try:
|
|
conn = mysql.connector.connect(**self.db_config)
|
|
cursor = conn.cursor(dictionary=True)
|
|
|
|
cursor.execute("SELECT * FROM users")
|
|
users = cursor.fetchall()
|
|
|
|
cursor.execute("SELECT user_id, job_id FROM alert_logs")
|
|
alerted = {(r["user_id"], r["job_id"]) for r in cursor.fetchall()}
|
|
|
|
job_sql = """
|
|
INSERT IGNORE INTO jobs
|
|
(title, company, location, posted_date_raw, job_link,
|
|
is_actively_hiring, description, match_score, match_breakdown,
|
|
matched_skills, missing_skills, verdict, detail_fetched_at, scored_at)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
|
|
for job in jobs:
|
|
if not self.is_job_fresh(job["posted_date"]):
|
|
continue
|
|
|
|
job_text = job.get("description") or ""
|
|
full_text = f"{job['title']}\n{job_text}"
|
|
|
|
match_result = score_job(
|
|
profile=profile, job_title=job["title"],
|
|
job_text=full_text, weights=weights,
|
|
aspirational_skills=aspirational,
|
|
)
|
|
|
|
score = match_result["score"]
|
|
stats["top_score"] = max(stats["top_score"], score)
|
|
if score >= min_score:
|
|
stats["matches"] += 1
|
|
|
|
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
cursor.execute(job_sql, (
|
|
job["title"], job["company"], job["location"],
|
|
job["posted_date"], job["link"], job["actively_hiring"],
|
|
job.get("description"), score,
|
|
json.dumps(match_result["breakdown"], default=str),
|
|
json.dumps(match_result["matched_skills"]),
|
|
json.dumps(match_result["missing_skills"]),
|
|
match_result["verdict"],
|
|
now if job.get("description") else None, now,
|
|
))
|
|
|
|
if cursor.rowcount == 1:
|
|
job_id = cursor.lastrowid
|
|
else:
|
|
cursor.execute(
|
|
"SELECT job_id FROM jobs WHERE job_link = %s", (job["link"],)
|
|
)
|
|
result = cursor.fetchone()
|
|
if not result:
|
|
continue
|
|
job_id = result["job_id"]
|
|
cursor.execute("""
|
|
UPDATE jobs SET match_score=%s, match_breakdown=%s,
|
|
matched_skills=%s, missing_skills=%s, verdict=%s,
|
|
description=%s, scored_at=%s WHERE job_id=%s
|
|
""", (
|
|
score,
|
|
json.dumps(match_result["breakdown"], default=str),
|
|
json.dumps(match_result["matched_skills"]),
|
|
json.dumps(match_result["missing_skills"]),
|
|
match_result["verdict"],
|
|
job.get("description"), now, job_id,
|
|
))
|
|
|
|
if score < min_score:
|
|
continue
|
|
|
|
for user in users:
|
|
user_min = user.get("min_match_score", min_score)
|
|
if score < user_min:
|
|
continue
|
|
key = (user["user_id"], job_id)
|
|
if key in alerted:
|
|
continue
|
|
if self.send_telegram_alert(
|
|
user["telegram_token"], user["telegram_chat_id"],
|
|
job, match_result,
|
|
):
|
|
cursor.execute(
|
|
"INSERT INTO alert_logs (user_id, job_id) VALUES (%s, %s)",
|
|
(user["user_id"], job_id)
|
|
)
|
|
alerted.add(key)
|
|
stats["alerts_sent"] += 1
|
|
time.sleep(1)
|
|
|
|
logger.info(
|
|
f" ✅ {score}% | {job['title']} @ {job['company']} "
|
|
f"[{', '.join(match_result['matched_skills'][:5])}]"
|
|
)
|
|
|
|
conn.commit()
|
|
|
|
for user in users:
|
|
self.send_summary_alert(
|
|
user["telegram_token"], user["telegram_chat_id"], stats,
|
|
)
|
|
|
|
except Error as e:
|
|
logger.error(f"Database error: {e}")
|
|
finally:
|
|
if cursor:
|
|
cursor.close()
|
|
if conn and conn.is_connected():
|
|
conn.close()
|
|
|
|
return stats
|
|
|
|
|
|
def run_once(config, profile):
|
|
scraper_cfg = config.get("scraper", {})
|
|
db_config = config.get("database", {})
|
|
scraper = LinkedInJobScraper(config)
|
|
alert_system = JobAlertSystem(db_config)
|
|
|
|
keywords = scraper_cfg.get("keywords", ["Network Engineer"])
|
|
location = scraper_cfg.get("location", "Toronto, ON")
|
|
pages = scraper_cfg.get("pages_per_keyword", 3)
|
|
keyword_delay = scraper_cfg.get("delay_between_keywords", [5, 10])
|
|
max_details = config.get("max_detail_fetches_per_run", 50)
|
|
|
|
all_jobs = []
|
|
for keyword in keywords:
|
|
logger.info(f"🔍 Scraping: {keyword}")
|
|
jobs = scraper.scrape_jobs(keyword=keyword, location=location, pages=pages)
|
|
all_jobs.extend(jobs)
|
|
time.sleep(random.uniform(*keyword_delay))
|
|
|
|
unique = {}
|
|
for job in all_jobs:
|
|
unique[job["link"]] = job
|
|
jobs = list(unique.values())
|
|
logger.info(f"📋 Total unique jobs: {len(jobs)}")
|
|
|
|
jobs = pre_filter_jobs(jobs, config)
|
|
|
|
remote_only = scraper_cfg.get("remote_only", False)
|
|
require_evidence = scraper_cfg.get("require_remote_evidence", True)
|
|
if remote_only:
|
|
jobs = filter_remote_only(jobs)
|
|
|
|
logger.info("📄 Fetching job descriptions...")
|
|
jobs = scraper.fetch_descriptions_batch(jobs, max_fetches=max_details)
|
|
|
|
if remote_only:
|
|
jobs = filter_remote_only(
|
|
jobs, check_descriptions=True,
|
|
require_evidence=require_evidence,
|
|
search_location=location,
|
|
)
|
|
|
|
desc_count = sum(1 for j in jobs if j.get("description"))
|
|
logger.info(f"📄 Jobs with descriptions: {desc_count}/{len(jobs)}")
|
|
|
|
logger.info("🎯 Scoring jobs against your profile...")
|
|
stats = alert_system.process_jobs(jobs, profile, config)
|
|
|
|
logger.info("=" * 50)
|
|
logger.info(f"📊 RUN COMPLETE")
|
|
logger.info(f" Scanned: {stats['total_scanned']}")
|
|
logger.info(f" Matches (70%+): {stats['matches']}")
|
|
logger.info(f" Alerts sent: {stats['alerts_sent']}")
|
|
logger.info(f" Top score: {stats['top_score']}%")
|
|
logger.info("=" * 50)
|
|
return stats
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Job Matcher Daemon")
|
|
parser.add_argument("--config", default="config.yaml", help="Config file path")
|
|
parser.add_argument("--profile", default="profile.json", help="Resume profile path")
|
|
parser.add_argument("--resume", default=None, help="Resume file (txt/pdf) to parse")
|
|
parser.add_argument("--loop", action="store_true", help="Run continuously")
|
|
parser.add_argument("--interval", type=int, default=7200, help="Loop interval in seconds (default: 2hrs)")
|
|
args = parser.parse_args()
|
|
|
|
config = load_config(args.config)
|
|
profile = load_profile(args.profile, args.resume)
|
|
|
|
logger.info(f"Profile loaded: {len(profile.get('skills', {}))} skills, "
|
|
f"{profile.get('years_experience', '?')} years exp, "
|
|
f"certs: {profile.get('certifications', [])}")
|
|
|
|
if args.loop:
|
|
logger.info(f"🔁 Daemon mode: running every {args.interval}s")
|
|
while True:
|
|
try:
|
|
run_once(config, profile)
|
|
except Exception as e:
|
|
logger.error(f"Run failed: {e}")
|
|
logger.info(f"💤 Sleeping {args.interval}s until next run...")
|
|
time.sleep(args.interval)
|
|
else:
|
|
run_once(config, profile)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|