diff --git a/config.yaml b/config.yaml index 86dc357..ed2da7d 100644 --- a/config.yaml +++ b/config.yaml @@ -47,7 +47,7 @@ scraper: - "Cloud Network Engineer" - "Systems Engineer" location: "Canada" - remote_only: true + work_types: ["remote", "hybrid"] pages_per_keyword: 3 delay_between_pages: [3, 6] delay_between_keywords: [5, 10] @@ -58,7 +58,7 @@ database: host: "localhost" user: "root" password: "" - database: "job_matcher_db" + database: "job_scraper_db" # --- Telegram --- # Fill these in with your bot token and chat ID diff --git a/daemon.py b/daemon.py index f193d4b..c40209a 100644 --- a/daemon.py +++ b/daemon.py @@ -83,7 +83,13 @@ class LinkedInJobScraper: seen_links = set() encoded_keyword = quote(keyword) encoded_location = quote(location) - remote_only = self.config.get("remote_only", False) + work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"} + work_types = self.config.get("work_types", []) + f_wt = ",".join( + work_type_map[wt.lower()] + for wt in work_types + if wt.lower() in work_type_map + ) delay_range = self.config.get("delay_between_pages", [3, 6]) for page in range(pages): @@ -94,8 +100,8 @@ class LinkedInJobScraper: f"keywords={encoded_keyword}&location={encoded_location}" f"&start={start_index}" ) - if remote_only: - url += "&f_WT=2" + if f_wt: + url += f"&f_WT={f_wt}" try: response = self.session.get(url, timeout=15) diff --git a/schema.sql b/schema.sql index c7af977..1f6528f 100644 --- a/schema.sql +++ b/schema.sql @@ -1,5 +1,5 @@ -CREATE DATABASE IF NOT EXISTS job_matcher_db; -USE job_matcher_db; +CREATE DATABASE IF NOT EXISTS job_scraper_db; +USE job_scraper_db; CREATE TABLE IF NOT EXISTS jobs ( job_id INT AUTO_INCREMENT PRIMARY KEY, @@ -39,3 +39,18 @@ CREATE TABLE IF NOT EXISTS alert_logs ( FOREIGN KEY (user_id) REFERENCES users(user_id), FOREIGN KEY (job_id) REFERENCES jobs(job_id) ); + +-- Migration: add columns required by daemon.py to an existing jobs table +ALTER TABLE jobs + ADD COLUMN IF NOT EXISTS description TEXT DEFAULT NULL, + ADD COLUMN IF NOT EXISTS match_score INT DEFAULT NULL, + ADD COLUMN IF NOT EXISTS match_breakdown JSON DEFAULT NULL, + ADD COLUMN IF NOT EXISTS matched_skills JSON DEFAULT NULL, + ADD COLUMN IF NOT EXISTS missing_skills JSON DEFAULT NULL, + ADD COLUMN IF NOT EXISTS verdict VARCHAR(50) DEFAULT NULL, + ADD COLUMN IF NOT EXISTS detail_fetched_at DATETIME DEFAULT NULL, + ADD COLUMN IF NOT EXISTS scored_at DATETIME DEFAULT NULL; + +ALTER TABLE jobs + ADD INDEX IF NOT EXISTS idx_match_score (match_score), + ADD INDEX IF NOT EXISTS idx_verdict (verdict);