kienan
/
haunter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch from chrome driver to firefox driver and increase concurrency

This commit is contained in:
Kienan Stewart 2022-09-24 10:13:51 -04:00
parent cf5ebbb457
commit 6fd8bf4304
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
container-compose.yml
View File

@ -3,7 +3,11 @@ version: '3'
services: services:
driver: driver:
image: docker.io/selenium/standalone-chrome image: docker.io/selenium/standalone-firefox
environment:
- "SE_START_XVFB=false"
- "SE_NODE_OVERRIDE_MAX_SESSIONS=true"
- "SE_NODE_MAX_SESSIONS=${HAUNTER_MAX_SESSIONS:-5}"
ports: ports:
- '4444:4444' - '4444:4444'
# watcher: # watcher:

12
src/main.rs
View File

@ -88,14 +88,7 @@ fn main() {
last_result: None, last_result: None,
}); });
// @BUG: It seems the selenium chrome driver can't handle concurrent sessions from let max_running_tasks = 5;
// multiple threads. When the threads attempt to run concurrently, there are crashes,
// eg.
//
// thread '<unnamed>' panicked at 'failed to get url: UnknownError(WebDriverErrorInfo { status: 500, error: "", value: WebDriverErrorValue { message: "unknown error: session deleted because of page crash\nfrom tab crashed\n (Session info: chrome=105.0.5195.52)
//
// This runs single jobs consecutively as a result.
let max_running_tasks = 1;
loop { loop {
let mut running_tasks = 0; let mut running_tasks = 0;
for tj in jobs.iter_mut().filter(|job| job.handle.is_some()) { for tj in jobs.iter_mut().filter(|job| job.handle.is_some()) {
@ -160,7 +153,8 @@ fn main() {
} }
fn get_source(driver: &str, url: &str) -> Result<String, &'static str> { fn get_source(driver: &str, url: &str) -> Result<String, &'static str> {
let caps = thirtyfour_sync::DesiredCapabilities::chrome(); // The firefox driver seems to crash less often than the chrome driver.
let caps = thirtyfour_sync::DesiredCapabilities::firefox();
let driver = thirtyfour_sync::WebDriver::new(driver, &caps).expect("failed to get driver"); let driver = thirtyfour_sync::WebDriver::new(driver, &caps).expect("failed to get driver");
driver.get(url).expect("failed to get url"); driver.get(url).expect("failed to get url");
let source = driver.page_source().expect("failed to get page source"); let source = driver.page_source().expect("failed to get page source");