feat: support hybrid+remote job search and use job_scraper_db
- Replace remote_only flag with configurable work_types list (remote, hybrid, onsite) - Switch database from job_matcher_db to job_scraper_db - Add ALTER TABLE migration for existing jobs table with scoring columns Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+2
-2
@@ -47,7 +47,7 @@ scraper:
|
|||||||
- "Cloud Network Engineer"
|
- "Cloud Network Engineer"
|
||||||
- "Systems Engineer"
|
- "Systems Engineer"
|
||||||
location: "Canada"
|
location: "Canada"
|
||||||
remote_only: true
|
work_types: ["remote", "hybrid"]
|
||||||
pages_per_keyword: 3
|
pages_per_keyword: 3
|
||||||
delay_between_pages: [3, 6]
|
delay_between_pages: [3, 6]
|
||||||
delay_between_keywords: [5, 10]
|
delay_between_keywords: [5, 10]
|
||||||
@@ -58,7 +58,7 @@ database:
|
|||||||
host: "localhost"
|
host: "localhost"
|
||||||
user: "root"
|
user: "root"
|
||||||
password: ""
|
password: ""
|
||||||
database: "job_matcher_db"
|
database: "job_scraper_db"
|
||||||
|
|
||||||
# --- Telegram ---
|
# --- Telegram ---
|
||||||
# Fill these in with your bot token and chat ID
|
# Fill these in with your bot token and chat ID
|
||||||
|
|||||||
@@ -83,7 +83,13 @@ class LinkedInJobScraper:
|
|||||||
seen_links = set()
|
seen_links = set()
|
||||||
encoded_keyword = quote(keyword)
|
encoded_keyword = quote(keyword)
|
||||||
encoded_location = quote(location)
|
encoded_location = quote(location)
|
||||||
remote_only = self.config.get("remote_only", False)
|
work_type_map = {"onsite": "1", "remote": "2", "hybrid": "3"}
|
||||||
|
work_types = self.config.get("work_types", [])
|
||||||
|
f_wt = ",".join(
|
||||||
|
work_type_map[wt.lower()]
|
||||||
|
for wt in work_types
|
||||||
|
if wt.lower() in work_type_map
|
||||||
|
)
|
||||||
delay_range = self.config.get("delay_between_pages", [3, 6])
|
delay_range = self.config.get("delay_between_pages", [3, 6])
|
||||||
|
|
||||||
for page in range(pages):
|
for page in range(pages):
|
||||||
@@ -94,8 +100,8 @@ class LinkedInJobScraper:
|
|||||||
f"keywords={encoded_keyword}&location={encoded_location}"
|
f"keywords={encoded_keyword}&location={encoded_location}"
|
||||||
f"&start={start_index}"
|
f"&start={start_index}"
|
||||||
)
|
)
|
||||||
if remote_only:
|
if f_wt:
|
||||||
url += "&f_WT=2"
|
url += f"&f_WT={f_wt}"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = self.session.get(url, timeout=15)
|
response = self.session.get(url, timeout=15)
|
||||||
|
|||||||
+17
-2
@@ -1,5 +1,5 @@
|
|||||||
CREATE DATABASE IF NOT EXISTS job_matcher_db;
|
CREATE DATABASE IF NOT EXISTS job_scraper_db;
|
||||||
USE job_matcher_db;
|
USE job_scraper_db;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS jobs (
|
CREATE TABLE IF NOT EXISTS jobs (
|
||||||
job_id INT AUTO_INCREMENT PRIMARY KEY,
|
job_id INT AUTO_INCREMENT PRIMARY KEY,
|
||||||
@@ -39,3 +39,18 @@ CREATE TABLE IF NOT EXISTS alert_logs (
|
|||||||
FOREIGN KEY (user_id) REFERENCES users(user_id),
|
FOREIGN KEY (user_id) REFERENCES users(user_id),
|
||||||
FOREIGN KEY (job_id) REFERENCES jobs(job_id)
|
FOREIGN KEY (job_id) REFERENCES jobs(job_id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Migration: add columns required by daemon.py to an existing jobs table
|
||||||
|
ALTER TABLE jobs
|
||||||
|
ADD COLUMN IF NOT EXISTS description TEXT DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS match_score INT DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS match_breakdown JSON DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS matched_skills JSON DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS missing_skills JSON DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS verdict VARCHAR(50) DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS detail_fetched_at DATETIME DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS scored_at DATETIME DEFAULT NULL;
|
||||||
|
|
||||||
|
ALTER TABLE jobs
|
||||||
|
ADD INDEX IF NOT EXISTS idx_match_score (match_score),
|
||||||
|
ADD INDEX IF NOT EXISTS idx_verdict (verdict);
|
||||||
|
|||||||
Reference in New Issue
Block a user