diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e57e9c5..74ab5cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,7 @@ deploy_dev: - if [[ -n "$CONTAINER_NAME" ]]; then docker rm -f $CONTAINER_NAME; fi - - docker run -d --name $SERVICE_NAME -v $CACHE:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG + - docker run -d --name $SERVICE_NAME -v $CACHE_PATH:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG tags: - '001' only: diff --git a/src/web_parser.rs b/src/web_parser.rs index 6e2121c..4cf0c07 100644 --- a/src/web_parser.rs +++ b/src/web_parser.rs @@ -1,10 +1,12 @@ +use std::thread; +use std::time::Duration; use bytes::Bytes; use scraper::{Html, Selector}; use crate::file; fn get_response(url: &str) -> Result { - let response = reqwest::blocking::get(url).expect("Server error"); + let response = reqwest::blocking::get(url)?; let mut body = Bytes::new(); if response.status() == 200 { body = response.bytes().unwrap() @@ -12,40 +14,46 @@ fn get_response(url: &str) -> Result { Ok(body) } -pub(crate) fn get_html_title(body: String) -> String { +pub(crate) fn get_title(uid: i32) -> String { + let body; + let url = "https://www.tuiimg.com/meinv/"; + let temp_url = format!("{}{}/", url, uid); + let file_content = get_response(&temp_url); + match file_content { + Ok(value) => body = String::from_utf8(value.to_vec()).expect("Transforms error"), + Err(_e) => body = "".parse().unwrap(), + } + return get_html_title(body); +} + +fn get_html_title(body: String) -> String { let document = Html::parse_document(&body); let title_selector = Selector::parse("h1").unwrap(); let title_element = document.select(&title_selector).next().unwrap(); return title_element.text().next().unwrap().to_string(); } -pub(crate) fn download_image(url_path: &str, file_path: &str) -> bool { +fn download_image(url_path: &str, file_path: &str, retry_times: i8) -> i8 { let url = format!("https://i.tuiimg.net/{}", url_path); println!("{}", url); + let mut flag = 1; let response = get_response(&url); - let file_content; match response { - Ok(value) => file_content = value, - Err(_e) => return false, + Ok(value) => { + if value.is_empty() { + flag = 0 + } else { + let file_path = format!("images/{}", file_path); + file::write_file(&file_path, value).expect("Write file error"); + } + } + Err(_e) => flag = 2, } - let file_path = format!("images/{}", file_path); - file::write_file(&file_path, file_content).expect("Write file error"); - return true; -} - -pub(crate) fn get_title(uid: i32) -> String { - let body; - let url = "https://www.tuiimg.com/meinv/"; - let temp_url = format!("{}{}/", url, uid); - let response = reqwest::blocking::get(temp_url).expect("Could not load url."); - let status_code = response.status(); - return if status_code != 200 { - "".parse().unwrap() - } else { - body = response.text().unwrap(); - let name = get_html_title(body); - name - }; + if flag == 2 && retry_times > 1 { + thread::sleep(Duration::from_millis(5000)); + download_image(url_path, file_path, retry_times - 1) + } + return flag; } pub(crate) fn download_images(uid: i32, title: &str) { @@ -65,7 +73,8 @@ pub(crate) fn download_images(uid: i32, title: &str) { while continue_flag { image_path = format!("{}/{}/{}.jpg", page_str, uid, pic_num); file_path = format!("{}/{}.jpg", title, pic_num); - if !download_image(&image_path, &file_path) { + let status_code = download_image(&image_path, &file_path, 3); + if status_code == 0 { continue_flag = false; } pic_num += 1;