feat: support hybrid+remote job search and use job_scraper_db
- Replace remote_only flag with configurable work_types list (remote, hybrid, onsite) - Switch database from job_matcher_db to job_scraper_db - Add ALTER TABLE migration for existing jobs table with scoring columns Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+2
-2
@@ -47,7 +47,7 @@ scraper:
|
||||
- "Cloud Network Engineer"
|
||||
- "Systems Engineer"
|
||||
location: "Canada"
|
||||
remote_only: true
|
||||
work_types: ["remote", "hybrid"]
|
||||
pages_per_keyword: 3
|
||||
delay_between_pages: [3, 6]
|
||||
delay_between_keywords: [5, 10]
|
||||
@@ -58,7 +58,7 @@ database:
|
||||
host: "localhost"
|
||||
user: "root"
|
||||
password: ""
|
||||
database: "job_matcher_db"
|
||||
database: "job_scraper_db"
|
||||
|
||||
# --- Telegram ---
|
||||
# Fill these in with your bot token and chat ID
|
||||
|
||||
@@ -83,7 +83,13 @@ class LinkedInJobScraper:
|
||||
seen_links = set()
|
||||
encoded_keyword = quote(keyword)
|
||||
encoded_location = quote(location)
|
||||
remote_only = self.config.get("remote_only", False)
|
||||
work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"}
|
||||
work_types = self.config.get("work_types", [])
|
||||
f_wt = ",".join(
|
||||
work_type_map[wt.lower()]
|
||||
for wt in work_types
|
||||
if wt.lower() in work_type_map
|
||||
)
|
||||
delay_range = self.config.get("delay_between_pages", [3, 6])
|
||||
|
||||
for page in range(pages):
|
||||
@@ -94,8 +100,8 @@ class LinkedInJobScraper:
|
||||
f"keywords={encoded_keyword}&location={encoded_location}"
|
||||
f"&start={start_index}"
|
||||
)
|
||||
if remote_only:
|
||||
url += "&f_WT=2"
|
||||
if f_wt:
|
||||
url += f"&f_WT={f_wt}"
|
||||
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
|
||||
+17
-2
@@ -1,5 +1,5 @@
|
||||
CREATE DATABASE IF NOT EXISTS job_matcher_db;
|
||||
USE job_matcher_db;
|
||||
CREATE DATABASE IF NOT EXISTS job_scraper_db;
|
||||
USE job_scraper_db;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
job_id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
@@ -39,3 +39,18 @@ CREATE TABLE IF NOT EXISTS alert_logs (
|
||||
FOREIGN KEY (user_id) REFERENCES users(user_id),
|
||||
FOREIGN KEY (job_id) REFERENCES jobs(job_id)
|
||||
);
|
||||
|
||||
-- Migration: add columns required by daemon.py to an existing jobs table
|
||||
ALTER TABLE jobs
|
||||
ADD COLUMN IF NOT EXISTS description TEXT DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS match_score INT DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS match_breakdown JSON DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS matched_skills JSON DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS missing_skills JSON DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS verdict VARCHAR(50) DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS detail_fetched_at DATETIME DEFAULT NULL,
|
||||
ADD COLUMN IF NOT EXISTS scored_at DATETIME DEFAULT NULL;
|
||||
|
||||
ALTER TABLE jobs
|
||||
ADD INDEX IF NOT EXISTS idx_match_score (match_score),
|
||||
ADD INDEX IF NOT EXISTS idx_verdict (verdict);
|
||||
|
||||
Reference in New Issue
Block a user