From f8134d566cf61cac8b188d498dfcd69de6ef9a68 Mon Sep 17 00:00:00 2001 From: lychang <632945494@qq.com> Date: Fri, 13 Mar 2026 19:03:26 +0800 Subject: [PATCH] update get_response --- .gitignore | 1 + Cargo.toml | 2 +- src/file.rs | 9 ++- src/main.rs | 2 +- src/web_parser.rs | 194 ++++++++++++++++++++++++++++++++++------------ 5 files changed, 152 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index ea8c4bf..3a8cabc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.idea diff --git a/Cargo.toml b/Cargo.toml index 1125bc1..fa6800b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,4 @@ bytes = "1.5.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" flate2 = "1.0" -openssl = { version = "0.10", features = ["vendored"] } \ No newline at end of file +# openssl = { version = "0.10", features = ["vendored"] } \ No newline at end of file diff --git a/src/file.rs b/src/file.rs index 6e84683..32770d0 100644 --- a/src/file.rs +++ b/src/file.rs @@ -1,10 +1,9 @@ +use bytes; +use bytes::Bytes; use std::fs; use std::fs::File; use std::io::{Read, Write}; use std::path::Path; -use bytes::Bytes; -use bytes; - pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> { let path = Path::new(path); @@ -26,5 +25,7 @@ pub(crate) fn read_file_string(path: &str) -> Result { let mut file = File::open(path)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; + contents = contents.replace("\r\n", "\n"); + contents = contents.replace("\n", ""); Ok(contents) -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 6b7845e..e2a6323 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,6 @@ fn main() { if blank_count >= 10 { break; } - thread::sleep(Duration::from_millis(50)); + thread::sleep(Duration::from_millis(100)); } } \ No newline at end of file diff --git a/src/web_parser.rs b/src/web_parser.rs index 9d5f88f..7bbe67c 100644 --- a/src/web_parser.rs +++ b/src/web_parser.rs @@ -1,9 +1,11 @@ -use std::thread; -use std::time::Duration; +use crate::file; use bytes::Bytes; use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; -use crate::file; +use std::sync::mpsc; +use std::sync::Arc; +use std::thread; +use std::time::Duration; #[derive(Debug, Serialize, Deserialize)] struct Album { @@ -11,24 +13,26 @@ struct Album { images: Vec, } - fn get_response(url: &str) -> Result { - let client = reqwest::blocking::Client::builder() - .build()?; - + let client = reqwest::blocking::Client::builder().build()?; + let response = client .get(url) - .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + .header( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + ) .send()?; - + if response.status() == 200 { // 先获取头信息，再获取字节数据 - let content_encoding = response.headers() + let content_encoding = response + .headers() .get("content-encoding") .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); - + let bytes = response.bytes()?; let result = if content_encoding.contains("gzip") { @@ -49,7 +53,9 @@ fn get_response(url: &str) -> Result { }; result } else { - Err(reqwest::Error::from(response.error_for_status().unwrap_err())) + Err(reqwest::Error::from( + response.error_for_status().unwrap_err(), + )) } } @@ -78,7 +84,6 @@ fn get_html_title(body: String) -> String { } fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { - println!("{}", url); let mut flag = 1; let response = get_response(url); match response { @@ -99,17 +104,18 @@ fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { } pub(crate) fn download_images(uid: i32, title: &str) { + println!("Downloading images for {} {}...", uid, title); let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); let file_content = get_response(&base_url); let body = match file_content { Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"), Err(_e) => "".parse().unwrap(), }; - + let document = Html::parse_document(&body); let img_selector = Selector::parse("p[align='center'] img").unwrap(); let mut images = Vec::new(); - + // 获取尾页数 let mut total_pages = 1; let pages_selector = Selector::parse("#pages a").unwrap(); @@ -129,37 +135,73 @@ pub(crate) fn download_images(uid: i32, title: &str) { } } } - - // 收集所有图片 URL - for page in 1..=total_pages { - let page_url = if page == 1 { - base_url.clone() - } else { - format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page) - }; - - let page_content = get_response(&page_url); - match page_content { - Ok(value) => { - if value.is_empty() { - println!("Page {} not found, skipping", page); - continue; - } - let page_body = String::from_utf8(value.to_vec()).expect("Transforms error"); - let page_document = Html::parse_document(&page_body); - for img_element in page_document.select(&img_selector) { - if let Some(src) = img_element.value().attr("src") { - images.push(src.to_string()); + + // 多线程收集所有图片 URL + let (tx, rx) = mpsc::channel(); + let img_selector_arc = Arc::new(img_selector); + let mut page_handles = vec![]; + + // 创建页面解析线程池 + let max_page_threads = 8; // 限制并发页面解析数 + + for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) { + let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages); + + for page in chunk_start..=chunk_end { + let tx = tx.clone(); + let img_selector = Arc::clone(&img_selector_arc); + let base_url = base_url.clone(); + let uid = uid; + + let handle = thread::spawn(move || { + let page_url = if page == 1 { + base_url + } else { + format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page) + }; + + let mut page_images = Vec::new(); + + match get_response(&page_url) { + Ok(value) => { + if value.is_empty() { + println!("Page {} not found, skipping", page); + } else { + let page_body = + String::from_utf8(value.to_vec()).expect("Transforms error"); + let page_document = Html::parse_document(&page_body); + for img_element in page_document.select(&img_selector) { + if let Some(src) = img_element.value().attr("src") { + page_images.push(src.to_string()); + } + } + } + } + Err(e) => { + println!("Error fetching page {}: {}, skipping", page, e); } } - } - Err(e) => { - println!("Error fetching page {}: {}, skipping", page, e); - continue; - } + + tx.send((page, page_images)).unwrap(); + }); + page_handles.push(handle); + } + + // 等待当前批次完成 + for handle in page_handles.drain(..) { + handle.join().unwrap(); } } - + + // 收集所有页面的图片URL，按页码排序 + drop(tx); + let mut page_results: Vec<_> = rx.iter().collect(); + page_results.sort_by_key(|(page_num, _)| *page_num); + + for (_, page_images) in page_results { + images.extend(page_images); + } + // 生成 JSON 文件 let album = Album { title: title.to_string(), @@ -169,12 +211,64 @@ pub(crate) fn download_images(uid: i32, title: &str) { let json_path = format!("/mnt/share/data/albums/{}/index.json", uid); let json_bytes = Bytes::from(json); file::write_file(&json_path, json_bytes).expect("Failed to write JSON file"); - - // 执行下载 - let mut pic_num = 1; - for img_url in &album.images { - let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num); - download_image(img_url, &file_path, 3); - pic_num += 1; + + // 执行多线程下载 + let (tx, rx) = mpsc::channel(); + let images_arc = Arc::new(album.images.clone()); + let uid_arc = Arc::new(uid); + + // 创建线程池（限制并发数避免过多连接） + let max_threads = 8; // 限制并发线程数 + let mut handles = vec![]; + + for chunk_start in (0..album.images.len()).step_by(max_threads) { + let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len()); + + for i in chunk_start..chunk_end { + let tx = tx.clone(); + let images = Arc::clone(&images_arc); + let uid = Arc::clone(&uid_arc); + + let handle = thread::spawn(move || { + let img_url = &images[i]; + let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1); + let result = download_image(img_url, &file_path, 3); + + tx.send((i + 1, result)).unwrap(); + }); + handles.push(handle); + } + + // 等待当前批次完成 + for handle in handles.drain(..) { + handle.join().unwrap(); + } } -} \ No newline at end of file + + // 收集下载结果 + drop(tx); // 关闭发送端 + let mut successful_downloads = 0; + let mut failed_downloads = 0; + + for (pic_num, result) in rx { + match result { + 1 => { + successful_downloads += 1; + } + 0 => { + println!("图片 {} 下载失败（空内容）", pic_num); + failed_downloads += 1; + } + 2 => { + println!("图片 {} 下载失败（网络错误）", pic_num); + failed_downloads += 1; + } + _ => {} + } + } + + println!( + "下载完成：成功 {} 张，失败 {} 张", + successful_downloads, failed_downloads + ); +}