Compare commits
No commits in common. "20870b644482d7b930ea57f22aa2997d0248ab03" and "9cb06f8323fb5d534baf024f3bf26438b5979750" have entirely different histories.
20870b6444
...
9cb06f8323
|
@ -109,7 +109,7 @@ mod tests {
|
||||||
let mut conf = Conf::get_default_conf();
|
let mut conf = Conf::get_default_conf();
|
||||||
let tf = write_conf_to_temp_file(r#"
|
let tf = write_conf_to_temp_file(r#"
|
||||||
job_dir = /val=ue
|
job_dir = /val=ue
|
||||||
# this line is ignored, and the white space on the next line is intentional
|
# this line is ignored, and the white space on the next lien is intentional
|
||||||
output_dir=/var/lib/output
|
output_dir=/var/lib/output
|
||||||
check_interval=3600
|
check_interval=3600
|
||||||
driver_url = https://example.test:6666
|
driver_url = https://example.test:6666
|
||||||
|
|
47
src/main.rs
47
src/main.rs
|
@ -1,7 +1,6 @@
|
||||||
use std::cmp::Ordering;
|
use std::cmp::Ordering;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::str::FromStr;
|
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
@ -11,27 +10,15 @@ use thirtyfour_sync::WebDriverCommands;
|
||||||
mod conf;
|
mod conf;
|
||||||
use conf::Conf;
|
use conf::Conf;
|
||||||
|
|
||||||
struct Job {
|
struct Job<'a> {
|
||||||
url: String,
|
url: &'a str,
|
||||||
selector: String,
|
selector: &'a str,
|
||||||
every: Duration,
|
every: Duration,
|
||||||
last_run: Option<Instant>,
|
last_run: Option<Instant>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Job {
|
struct ThreadJob<'a> {
|
||||||
fn new(url: &str, selector: &str, conf: &Conf) -> Job {
|
job: Job<'a>,
|
||||||
let job = Job {
|
|
||||||
url: String::from_str(url).unwrap(),
|
|
||||||
selector: String::from_str(selector).unwrap(),
|
|
||||||
every: conf.check_interval,
|
|
||||||
last_run: None,
|
|
||||||
};
|
|
||||||
return job;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct ThreadJob {
|
|
||||||
job: Job,
|
|
||||||
handle: Option<
|
handle: Option<
|
||||||
std::thread::JoinHandle<
|
std::thread::JoinHandle<
|
||||||
Result<String, &'static str>
|
Result<String, &'static str>
|
||||||
|
@ -40,7 +27,7 @@ struct ThreadJob {
|
||||||
last_result: Option<String>,
|
last_result: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ThreadJob {
|
impl ThreadJob<'_> {
|
||||||
fn lru_not_running(&self, b: &ThreadJob) -> Option<Ordering> {
|
fn lru_not_running(&self, b: &ThreadJob) -> Option<Ordering> {
|
||||||
// the "greater" value is one that is running, but we
|
// the "greater" value is one that is running, but we
|
||||||
// don't recheck the thread state every time. If there's
|
// don't recheck the thread state every time. If there's
|
||||||
|
@ -100,12 +87,18 @@ fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut jobs = Vec::new();
|
let mut jobs = Vec::new();
|
||||||
let some_job = Job::new("https://www.rust-lang.org", "a.download-link", &conf);
|
let some_job = Job {
|
||||||
let other_job = Job::new(
|
url: "https://www.rust-lang.org",
|
||||||
"https://arstechnica.com/",
|
selector: "a.download-link",
|
||||||
"li.split-feature:nth-child(1) > header:nth-child(4) > h2:nth-child(1) > a:nth-child(1)",
|
every: Duration::new(60, 0),
|
||||||
&conf
|
last_run: None,
|
||||||
);
|
};
|
||||||
|
let other_job = Job {
|
||||||
|
url: "https://arstechnica.com/",
|
||||||
|
selector: "li.split-feature:nth-child(1) > header:nth-child(4) > h2:nth-child(1) > a:nth-child(1)",
|
||||||
|
every: Duration::new(120, 0),
|
||||||
|
last_run: None,
|
||||||
|
};
|
||||||
jobs.push(ThreadJob {
|
jobs.push(ThreadJob {
|
||||||
job: some_job,
|
job: some_job,
|
||||||
handle: None,
|
handle: None,
|
||||||
|
@ -132,7 +125,7 @@ fn main() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let fragment = scraper::Html::parse_document(source.unwrap().unwrap().as_str());
|
let fragment = scraper::Html::parse_document(source.unwrap().unwrap().as_str());
|
||||||
let selector = scraper::Selector::parse(&tj.job.selector.as_str()).expect("Failed to parse selector");
|
let selector = scraper::Selector::parse(tj.job.selector).expect("Failed to parse selector");
|
||||||
let mut result = String::from("");
|
let mut result = String::from("");
|
||||||
for element in fragment.select(&selector) {
|
for element in fragment.select(&selector) {
|
||||||
result.push_str(element.inner_html().as_str());
|
result.push_str(element.inner_html().as_str());
|
||||||
|
@ -167,7 +160,7 @@ fn main() {
|
||||||
let driver = conf.driver_url.clone();
|
let driver = conf.driver_url.clone();
|
||||||
let url = tj.job.url.clone();
|
let url = tj.job.url.clone();
|
||||||
tj.handle = Some(thread::spawn(move || {
|
tj.handle = Some(thread::spawn(move || {
|
||||||
return get_source(driver.as_str(), url.as_str());
|
return get_source(driver.as_str(), url);
|
||||||
}));
|
}));
|
||||||
println!("Started thread for '{}'", tj.job.url);
|
println!("Started thread for '{}'", tj.job.url);
|
||||||
tj.job.last_run = Some(Instant::now());
|
tj.job.last_run = Some(Instant::now());
|
||||||
|
|
Loading…
Reference in New Issue