feat: support hybrid+remote job search and use job_scraper_db

- Replace remote_only flag with configurable work_types list (remote, hybrid, onsite)
- Switch database from job_matcher_db to job_scraper_db
- Add ALTER TABLE migration for existing jobs table with scoring columns

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
harwinder
2026-05-27 19:47:22 -04:00
parent e608d367e2
commit 39a29a8e4e
3 changed files with 28 additions and 7 deletions
+2 -2
View File
@@ -47,7 +47,7 @@ scraper:
- "Cloud Network Engineer" - "Cloud Network Engineer"
- "Systems Engineer" - "Systems Engineer"
location: "Canada" location: "Canada"
remote_only: true work_types: ["remote", "hybrid"]
pages_per_keyword: 3 pages_per_keyword: 3
delay_between_pages: [3, 6] delay_between_pages: [3, 6]
delay_between_keywords: [5, 10] delay_between_keywords: [5, 10]
@@ -58,7 +58,7 @@ database:
host: "localhost" host: "localhost"
user: "root" user: "root"
password: "" password: ""
database: "job_matcher_db" database: "job_scraper_db"
# --- Telegram --- # --- Telegram ---
# Fill these in with your bot token and chat ID # Fill these in with your bot token and chat ID
+9 -3
View File
@@ -83,7 +83,13 @@ class LinkedInJobScraper:
seen_links = set() seen_links = set()
encoded_keyword = quote(keyword) encoded_keyword = quote(keyword)
encoded_location = quote(location) encoded_location = quote(location)
remote_only = self.config.get("remote_only", False) work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"}
work_types = self.config.get("work_types", [])
f_wt = ",".join(
work_type_map[wt.lower()]
for wt in work_types
if wt.lower() in work_type_map
)
delay_range = self.config.get("delay_between_pages", [3, 6]) delay_range = self.config.get("delay_between_pages", [3, 6])
for page in range(pages): for page in range(pages):
@@ -94,8 +100,8 @@ class LinkedInJobScraper:
f"keywords={encoded_keyword}&location={encoded_location}" f"keywords={encoded_keyword}&location={encoded_location}"
f"&start={start_index}" f"&start={start_index}"
) )
if remote_only: if f_wt:
url += "&f_WT=2" url += f"&f_WT={f_wt}"
try: try:
response = self.session.get(url, timeout=15) response = self.session.get(url, timeout=15)
+17 -2
View File
@@ -1,5 +1,5 @@
CREATE DATABASE IF NOT EXISTS job_matcher_db; CREATE DATABASE IF NOT EXISTS job_scraper_db;
USE job_matcher_db; USE job_scraper_db;
CREATE TABLE IF NOT EXISTS jobs ( CREATE TABLE IF NOT EXISTS jobs (
job_id INT AUTO_INCREMENT PRIMARY KEY, job_id INT AUTO_INCREMENT PRIMARY KEY,
@@ -39,3 +39,18 @@ CREATE TABLE IF NOT EXISTS alert_logs (
FOREIGN KEY (user_id) REFERENCES users(user_id), FOREIGN KEY (user_id) REFERENCES users(user_id),
FOREIGN KEY (job_id) REFERENCES jobs(job_id) FOREIGN KEY (job_id) REFERENCES jobs(job_id)
); );
-- Migration: add columns required by daemon.py to an existing jobs table
ALTER TABLE jobs
ADD COLUMN IF NOT EXISTS description TEXT DEFAULT NULL,
ADD COLUMN IF NOT EXISTS match_score INT DEFAULT NULL,
ADD COLUMN IF NOT EXISTS match_breakdown JSON DEFAULT NULL,
ADD COLUMN IF NOT EXISTS matched_skills JSON DEFAULT NULL,
ADD COLUMN IF NOT EXISTS missing_skills JSON DEFAULT NULL,
ADD COLUMN IF NOT EXISTS verdict VARCHAR(50) DEFAULT NULL,
ADD COLUMN IF NOT EXISTS detail_fetched_at DATETIME DEFAULT NULL,
ADD COLUMN IF NOT EXISTS scored_at DATETIME DEFAULT NULL;
ALTER TABLE jobs
ADD INDEX IF NOT EXISTS idx_match_score (match_score),
ADD INDEX IF NOT EXISTS idx_verdict (verdict);