update get_response

.gitlab-ci.yml

@@ -32,7 +32,7 @@ deploy_dev:
     - if [[ -n "$CONTAINER_NAME" ]]; then
       docker rm -f $CONTAINER_NAME;
       fi
-    - docker run -d --name $SERVICE_NAME -v $CACHE:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG
+    - docker run -d --name $SERVICE_NAME -v $CACHE_PATH:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG
   tags:
     - '001'
   only:

src/web_parser.rs

@@ -1,10 +1,12 @@
+use std::thread;
+use std::time::Duration;
 use bytes::Bytes;
 use scraper::{Html, Selector};
 use crate::file;
 
 
 fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
-    let response = reqwest::blocking::get(url).expect("Server error");
+    let response = reqwest::blocking::get(url)?;
     let mut body = Bytes::new();
     if response.status() == 200 {
         body = response.bytes().unwrap()
@@ -12,40 +14,46 @@ fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
     Ok(body)
 }
 
-pub(crate) fn get_html_title(body: String) -> String {
+pub(crate) fn get_title(uid: i32) -> String {
+    let body;
+    let url = "https://www.tuiimg.com/meinv/";
+    let temp_url = format!("{}{}/", url, uid);
+    let file_content = get_response(&temp_url);
+    match file_content {
+        Ok(value) => body = String::from_utf8(value.to_vec()).expect("Transforms error"),
+        Err(_e) => body = "".parse().unwrap(),
+    }
+    return get_html_title(body);
+}
+
+fn get_html_title(body: String) -> String {
     let document = Html::parse_document(&body);
     let title_selector = Selector::parse("h1").unwrap();
     let title_element = document.select(&title_selector).next().unwrap();
     return title_element.text().next().unwrap().to_string();
 }
 
-pub(crate) fn download_image(url_path: &str, file_path: &str) -> bool {
+fn download_image(url_path: &str, file_path: &str, retry_times: i8) -> i8 {
     let url = format!("https://i.tuiimg.net/{}", url_path);
     println!("{}", url);
+    let mut flag = 1;
     let response = get_response(&url);
-    let file_content;
     match response {
-        Ok(value) => file_content = value,
-        Err(_e) => return false,
+        Ok(value) => {
+            if value.is_empty() {
+                flag = 0
+            } else {
+                let file_path = format!("images/{}", file_path);
+                file::write_file(&file_path, value).expect("Write file error");
+            }
+        }
+        Err(_e) => flag = 2,
     }
-    let file_path = format!("images/{}", file_path);
-    file::write_file(&file_path, file_content).expect("Write file error");
-    return true;
-}
-
-pub(crate) fn get_title(uid: i32) -> String {
-    let body;
-    let url = "https://www.tuiimg.com/meinv/";
-    let temp_url = format!("{}{}/", url, uid);
-    let response = reqwest::blocking::get(temp_url).expect("Could not load url.");
-    let status_code = response.status();
-    return if status_code != 200 {
-        "".parse().unwrap()
-    } else {
-        body = response.text().unwrap();
-        let name = get_html_title(body);
-        name
-    };
+    if flag == 2 && retry_times > 1 {
+        thread::sleep(Duration::from_millis(5000));
+        download_image(url_path, file_path, retry_times - 1)
+    }
+    return flag;
 }
 
 pub(crate) fn download_images(uid: i32, title: &str) {
@@ -65,7 +73,8 @@ pub(crate) fn download_images(uid: i32, title: &str) {
     while continue_flag {
         image_path = format!("{}/{}/{}.jpg", page_str, uid, pic_num);
         file_path = format!("{}/{}.jpg", title, pic_num);
-        if !download_image(&image_path, &file_path) {
+        let status_code = download_image(&image_path, &file_path, 3);
+        if status_code == 0 {
             continue_flag = false;
         }
         pic_num += 1;