kienan
/
haunter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Own the memory for url & selector in Job structs

This commit is contained in:
Kienan Stewart 2022-09-24 11:57:07 -04:00
parent b1dee1f5a8
commit 20870b6444
1 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/main.rs
View File

@ -1,6 +1,7 @@
use std::cmp::Ordering; use std::cmp::Ordering;
use std::env; use std::env;
use std::path::Path; use std::path::Path;
use std::str::FromStr;
use std::thread; use std::thread;
use std::time::Duration; use std::time::Duration;
use std::time::Instant; use std::time::Instant;
@ -10,15 +11,27 @@ use thirtyfour_sync::WebDriverCommands;
mod conf; mod conf;
use conf::Conf; use conf::Conf;
struct Job<'a> { struct Job {
url: &'a str, url: String,
selector: &'a str, selector: String,
every: Duration, every: Duration,
last_run: Option<Instant>, last_run: Option<Instant>,
} }
struct ThreadJob<'a> { impl Job {
job: Job<'a>, fn new(url: &str, selector: &str, conf: &Conf) -> Job {
let job = Job {
url: String::from_str(url).unwrap(),
selector: String::from_str(selector).unwrap(),
every: conf.check_interval,
last_run: None,
};
return job;
}
}
struct ThreadJob {
job: Job,
handle: Option< handle: Option<
std::thread::JoinHandle< std::thread::JoinHandle<
Result<String, &'static str> Result<String, &'static str>
@ -27,7 +40,7 @@ struct ThreadJob<'a> {
last_result: Option<String>, last_result: Option<String>,
} }
impl ThreadJob<'_> { impl ThreadJob {
fn lru_not_running(&self, b: &ThreadJob) -> Option<Ordering> { fn lru_not_running(&self, b: &ThreadJob) -> Option<Ordering> {
// the "greater" value is one that is running, but we // the "greater" value is one that is running, but we
// don't recheck the thread state every time. If there's // don't recheck the thread state every time. If there's
@ -87,18 +100,12 @@ fn main() {
} }
let mut jobs = Vec::new(); let mut jobs = Vec::new();
let some_job = Job { let some_job = Job::new("https://www.rust-lang.org", "a.download-link", &conf);
url: "https://www.rust-lang.org", let other_job = Job::new(
selector: "a.download-link", "https://arstechnica.com/",
every: Duration::new(60, 0), "li.split-feature:nth-child(1) > header:nth-child(4) > h2:nth-child(1) > a:nth-child(1)",
last_run: None, &conf
}; );
let other_job = Job {
url: "https://arstechnica.com/",
selector: "li.split-feature:nth-child(1) > header:nth-child(4) > h2:nth-child(1) > a:nth-child(1)",
every: Duration::new(120, 0),
last_run: None,
};
jobs.push(ThreadJob { jobs.push(ThreadJob {
job: some_job, job: some_job,
handle: None, handle: None,
@ -125,7 +132,7 @@ fn main() {
continue; continue;
} }
let fragment = scraper::Html::parse_document(source.unwrap().unwrap().as_str()); let fragment = scraper::Html::parse_document(source.unwrap().unwrap().as_str());
let selector = scraper::Selector::parse(tj.job.selector).expect("Failed to parse selector"); let selector = scraper::Selector::parse(&tj.job.selector.as_str()).expect("Failed to parse selector");
let mut result = String::from(""); let mut result = String::from("");
for element in fragment.select(&selector) { for element in fragment.select(&selector) {
result.push_str(element.inner_html().as_str()); result.push_str(element.inner_html().as_str());
@ -160,7 +167,7 @@ fn main() {
let driver = conf.driver_url.clone(); let driver = conf.driver_url.clone();
let url = tj.job.url.clone(); let url = tj.job.url.clone();
tj.handle = Some(thread::spawn(move || { tj.handle = Some(thread::spawn(move || {
return get_source(driver.as_str(), url); return get_source(driver.as_str(), url.as_str());
})); }));
println!("Started thread for '{}'", tj.job.url); println!("Started thread for '{}'", tj.job.url);
tj.job.last_run = Some(Instant::now()); tj.job.last_run = Some(Instant::now());