From 161837bd24795e730977be72d2a0972c72184ebe Mon Sep 17 00:00:00 2001 From: lychang <632945494@qq.com> Date: Tue, 17 Mar 2026 02:15:58 +0800 Subject: [PATCH] update get_response --- failed_index | 44 +++++++ src/main.rs | 211 ++++++++++++++++++++---------- src/web_parser.rs | 318 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 412 insertions(+), 161 deletions(-) create mode 100644 failed_index diff --git a/failed_index b/failed_index new file mode 100644 index 0000000..e551412 --- /dev/null +++ b/failed_index @@ -0,0 +1,44 @@ +6813 +6814 +6815 +6816 +6817 +6818 +6819 +6820 +6821 +6822 +6823 +6824 +6825 +6826 +6827 +6828 +6829 +6830 +6831 +6832 +6833 +6834 +6835 +6836 +6837 +6838 +6839 +6840 +6841 +6842 +6843 +6844 +6845 +6846 +6847 +6848 +6849 +6850 +6851 +6852 +6853 +6854 +6855 +6856 diff --git a/src/main.rs b/src/main.rs index 24e34e9..faf6bda 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,87 +1,168 @@ use std::str::FromStr; use std::thread; - use bytes::Bytes; use std::time::Duration; mod file; mod web_parser; +use web_parser::DownloadResult; -fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> bool { - if web_parser::download_images(idx, name) { - println!("相册 {} ({}) 下载成功", idx, name); - file::write_file("index", Bytes::from(idx.to_string())) - .unwrap_or_else(|_| panic!("Write index error: {}", idx)); - true - } else if times > 0 { - println!("相册 {} ({}) 下载失败，{}秒后重试", idx, name, duration); - thread::sleep(Duration::from_millis(duration * 1000)); - retry_download(idx, name, times - 1, duration) - } else { - println!("相册 {} ({}) 下载失败{}次,记录到失败列表", idx, name, times); - false +/// 重试下载相册 +/// +/// # 参数 +/// - `idx`: 相册索引 +/// - `name`: 相册名称 +/// - `times`: 重试次数 +/// - `duration`: 重试间隔（秒） +/// +/// # 返回值 +/// - `DownloadResult`: 下载结果 +fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> DownloadResult { + match web_parser::download_albums(idx, name) { + DownloadResult::Success => { + println!("✅ 相册 {} ({}) 下载成功", idx, name); + DownloadResult::Success + } + DownloadResult::NetworkError if times > 0 => { + println!("🔄 相册 {} ({}) 网络错误，{}秒后重试", idx, name, duration); + thread::sleep(Duration::from_millis(duration * 1000)); + retry_download(idx, name, times - 1, duration) + } + DownloadResult::Failed => { + println!("❌ 相册 {} ({}) 下载失败", idx, name); + DownloadResult::Failed + } + DownloadResult::NetworkError => { + println!("⚠️ 相册 {} ({}) 网络错误，记录到失败列表", idx, name); + DownloadResult::NetworkError + } + } +} + +/// 处理失败的下载记录 +/// +/// 从失败列表中读取记录，尝试重新下载 +fn process_failed_downloads() { + let failed_indexes = file::read_failed_indexes(); + if failed_indexes.is_empty() { + return; + } + + println!("🔄 发现 {} 个失败的下载记录，开始重试...", failed_indexes.len()); + + for failed_idx in failed_indexes { + match web_parser::get_title(failed_idx) { + Ok(name) => { + println!("🔄 重试下载索引 {} 相册 {}...", failed_idx, name); + match retry_download(failed_idx, &name, 1, 5) { + DownloadResult::Success => { + println!("✅ 相册 {} 重试成功", failed_idx); + file::remove_failed_index(failed_idx); + } + _ => println!("❌ 相册 {} 重试失败，保留在失败列表", failed_idx), + } + } + Err(DownloadResult::Failed) => { + println!("🗑️ 相册 {} 不存在，从失败列表中移除", failed_idx); + file::remove_failed_index(failed_idx); + } + Err(DownloadResult::NetworkError) => { + println!("⚠️ 相册 {} 获取标题时网络错误，保留在失败列表", failed_idx); + } + Err(DownloadResult::Success) => unreachable!("DownloadResult::Success should not appear in Err branch"), + } + thread::sleep(Duration::from_millis(100)); + } + println!("✅ 失败记录处理完成"); +} + +/// 更新索引文件 +/// +/// # 参数 +/// - `idx`: 当前索引值 +fn update_index(idx: i32) { + file::write_file("index", Bytes::from(idx.to_string())) + .unwrap_or_else(|_| panic!("❌ 写入索引文件错误: {}", idx)); +} + +/// 处理单个相册下载 +/// +/// # 参数 +/// - `idx`: 相册索引 +/// - `blank_count`: 空白计数引用 +/// +/// # 返回值 +/// - `bool`: 是否需要继续处理 +fn process_album(idx: i32, blank_count: &mut i32) -> bool { + match web_parser::get_title(idx) { + Ok(name) => { + println!("🚀 开始下载索引 {} 相册 {}...", idx, name); + match retry_download(idx, &name, 3, 15) { + DownloadResult::Success => { + // 下载成功：索引前进，重置空白计数 + *blank_count = 0; + true + } + DownloadResult::Failed => { + // 下载失败：索引前进，增加空白计数 + *blank_count += 1; + true + } + DownloadResult::NetworkError => { + // 网络错误：记录失败索引，索引前进，重置空白计数 + file::write_failed_index(idx); + *blank_count = 0; + true + } + } + } + Err(DownloadResult::Failed) => { + // 页面不存在：索引前进，增加空白计数 + println!("❌ 索引 {} 不存在内容", idx); + thread::sleep(Duration::from_millis(3000)); + *blank_count += 1; + true + } + Err(DownloadResult::NetworkError) => { + // 获取标题时网络错误：索引前进，增加空白计数 + println!("⚠️ 索引 {} 获取标题时网络错误，跳过", idx); + thread::sleep(Duration::from_millis(3000)); + *blank_count += 1; + true + } + Err(DownloadResult::Success) => unreachable!("DownloadResult::Success should not appear in Err branch"), } } fn main() { - let mut idx: i32; + // 先处理失败的下载记录 + process_failed_downloads(); + + // 初始化索引和空白计数 + let mut idx = file::read_file_string("index") + .ok() + .and_then(|s| i32::from_str(&s).ok()) + .unwrap_or(1); let mut blank_count = 0; - // 先处理失败的记录 - let mut failed_indexes = file::read_failed_indexes(); - if !failed_indexes.is_empty() { - println!( - "发现 {} 个失败的下载记录，开始重试...", - failed_indexes.len() - ); - - for failed_idx in failed_indexes.drain(..).collect::>() { - let name = web_parser::get_title(failed_idx); - if name != "" { - println!("重试下载索引 {} 相册 {}...", failed_idx, name); - let success = retry_download(failed_idx, &name, 2, 10); - if success { - println!("相册 {} 重试成功", failed_idx); - } else { - println!("相册 {} 重试失败，保留在失败列表", failed_idx); - } - } else { - println!("相册 {} 不存在，从失败列表中移除", failed_idx); - file::remove_failed_index(failed_idx); - } - thread::sleep(Duration::from_millis(100)); - } - println!("失败记录处理完成"); - } - - // 继续正常下载流程 - let index = file::read_file_string("index").expect("Read index error"); - if index.is_empty() { - idx = 1; - } else { - idx = i32::from_str(&index).expect("Index Corrupted"); - } + println!("🚀 下载程序启动，从索引 {} 开始...", idx); + // 主下载循环 loop { - let name = web_parser::get_title(idx); - if name == "" { - println!("索引 {} 不存在内容", idx); - thread::sleep(Duration::from_millis(3000)); + let should_continue = process_album(idx, &mut blank_count); + + if should_continue { idx += 1; - blank_count += 1; - } else { - println!("开始下载索引 {} 相册 {}...", idx, name); - let success = retry_download(idx, &name, 3, 15); - if success { - idx += 1; - } else { - file::write_failed_index(idx); - idx += 1; - } + update_index(idx); } + + // 检查是否达到空白计数上限 if blank_count >= 50 { - break; + println!("🔄 连续空白计数达到 {}，重新处理失败记录", blank_count); + process_failed_downloads(); + blank_count = 0; } + thread::sleep(Duration::from_millis(100)); } -} +} \ No newline at end of file diff --git a/src/web_parser.rs b/src/web_parser.rs index 72d0f31..5d360e8 100644 --- a/src/web_parser.rs +++ b/src/web_parser.rs @@ -7,43 +7,96 @@ use std::sync::Arc; use std::thread; use std::time::Duration; +/// 自定义错误类型，用于区分不同类型的网络错误 +#[derive(Debug)] +pub enum RequestError { + NotFound, // 404 错误 + NetworkError, // 网络错误 + OtherError(String), // 其他错误 +} + +impl From for RequestError { + fn from(err: reqwest::Error) -> Self { + if err.is_status() { + if let Some(status) = err.status() { + if status == reqwest::StatusCode::NOT_FOUND { + return RequestError::NotFound; + } + } + RequestError::OtherError(err.to_string()) + } else if err.is_timeout() || err.is_connect() { + RequestError::NetworkError + } else { + RequestError::OtherError(err.to_string()) + } + } +} + +/// 相册数据结构 #[derive(Debug, Serialize, Deserialize)] struct Album { title: String, images: Vec, } -fn get_response(url: &str) -> Result { +/// 下载结果枚举 +pub(crate) enum DownloadResult { + Success, + NetworkError, + Failed, +} + +/// 发送 HTTP 请求并获取响应 +/// +/// # 参数 +/// - `url`: 请求的 URL +/// +/// # 返回值 +/// - `Result`: 响应内容或错误 +fn get_response(url: &str) -> Result { let client = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(30)) - .build()?; + .build() + .map_err(|_| RequestError::NetworkError)?; let response = client .get(url) .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") - .send()?; + .send() + .map_err(RequestError::from)?; - if !response.status().is_success() { - return Err(response.error_for_status().unwrap_err().into()); + // 检查 HTTP 状态码 + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RequestError::NotFound); } - let content_encoding = response - .headers() + if !response.status().is_success() { + return Err(RequestError::OtherError(format!( + "HTTP {}: {}", + response.status(), + response.status().canonical_reason().unwrap_or("Unknown") + ))); + } + + // 先检查内容编码，再获取 bytes + let content_encoding = response.headers() .get("content-encoding") .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); - - let bytes = response.bytes()?; - + + let bytes = response.bytes().map_err(RequestError::from)?; + + // 处理 gzip 压缩 if content_encoding.contains("gzip") { + use std::io::Read; let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); let mut decompressed = Vec::new(); match decoder.read_to_end(&mut decompressed) { Ok(_) => Ok(Bytes::from(decompressed)), Err(e) => { - println!("Gzip decompression failed: {}, using raw data", e); + println!("⚠️ Gzip 解压失败: {}, 使用原始数据", e); Ok(bytes) } } @@ -52,64 +105,131 @@ fn get_response(url: &str) -> Result { } } -pub(crate) fn get_title(uid: i32) -> String { +/// 获取相册标题 +/// +/// # 参数 +/// - `uid`: 相册 ID +/// +/// # 返回值 +/// - `Result`: 相册标题或错误 +pub(crate) fn get_title(uid: i32) -> Result { let url = format!("https://www.meitu131.com/meinv/{}/", uid); - + match get_response(&url) { Ok(value) if !value.is_empty() => { let body = String::from_utf8_lossy(&value).to_string(); - get_html_title(body) + Ok(get_html_title(body)) } - _ => "".to_string(), + Ok(_) => Err(DownloadResult::NetworkError), + Err(RequestError::NotFound) => Err(DownloadResult::Failed), + Err(RequestError::NetworkError) => Err(DownloadResult::NetworkError), + Err(RequestError::OtherError(_)) => Err(DownloadResult::NetworkError), } } +/// 从 HTML 内容中提取标题 fn get_html_title(body: String) -> String { let document = Html::parse_document(&body); let title_selector = Selector::parse("h1").unwrap(); - let title_element = document.select(&title_selector).next().unwrap(); - title_element.text().next().unwrap().to_string() + document + .select(&title_selector) + .next() + .and_then(|element| element.text().next()) + .unwrap_or("未知标题") + .to_string() } -fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { +/// 下载单张图片 +/// +/// # 参数 +/// - `url`: 图片 URL +/// - `file_path`: 保存路径 +/// - `retry_times`: 重试次数 +/// +/// # 返回值 +/// - `DownloadResult`: 下载结果 +fn download_image(url: &str, file_path: &str, retry_times: i8) -> DownloadResult { match get_response(url) { Ok(value) if !value.is_empty() => { - file::write_file(file_path, value).expect("Write file error"); - 1 + file::write_file(file_path, value).expect("❌ 写入文件错误"); + DownloadResult::Success } - Ok(_) => 0, // 空内容 - Err(_) if retry_times > 1 => { + Ok(_) => DownloadResult::Failed, // 空内容 + Err(RequestError::NotFound) => DownloadResult::Failed, // 404 错误 + Err(RequestError::NetworkError) if retry_times > 1 => { thread::sleep(Duration::from_millis(1000)); download_image(url, file_path, retry_times - 1) } - Err(_) => 2, // 网络错误 + Err(RequestError::OtherError(_)) if retry_times > 1 => { + thread::sleep(Duration::from_millis(1000)); + download_image(url, file_path, retry_times - 1) + } + Err(RequestError::NetworkError) => DownloadResult::NetworkError, + Err(RequestError::OtherError(_)) => DownloadResult::NetworkError, } } -pub(crate) fn download_images(uid: i32, title: &str) -> bool { +/// 下载相册的所有图片 +/// +/// # 参数 +/// - `uid`: 相册 ID +/// - `title`: 相册标题 +/// +/// # 返回值 +/// - `DownloadResult`: 下载结果 +pub(crate) fn download_albums(uid: i32, title: &str) -> DownloadResult { let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); - let file_content = get_response(&base_url); - let body = match file_content { - Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"), - Err(_e) => "".parse().unwrap(), + + // 获取相册主页内容 + let body = match get_response(&base_url) { + Ok(value) => String::from_utf8(value.to_vec()).expect("❌ 转换错误"), + Err(RequestError::NotFound) => { + println!("❌ 相册 {} 不存在 (404)", uid); + return DownloadResult::Failed; + } + Err(RequestError::NetworkError) => { + println!("⚠️ 相册 {} 下载时网络波动 (500)", uid); + return DownloadResult::NetworkError; + } + Err(RequestError::OtherError(_)) => { + println!("⚠️ 相册 {} 下载时网络波动 (500)", uid); + return DownloadResult::NetworkError; + } }; let document = Html::parse_document(&body); let img_selector = Selector::parse("p[align='center'] img").unwrap(); - let mut images = Vec::new(); + + // 获取总页数 + let total_pages = get_total_pages(&document); + + // 多线程收集所有图片 URL + let images = collect_image_urls(uid, total_pages, &img_selector, &base_url); + + if images.len() < total_pages as usize { + println!("⚠️ 相册 {} 下载时网络波动, 仅下载 {} 页", uid, images.len()); + return DownloadResult::NetworkError; + } - // 获取尾页数 - let mut total_pages = 1; + // 生成 JSON 文件 + generate_json_file(uid, title, &images); + + // 多线程下载图片 + download_images_multithreaded(uid, &images) +} + +/// 获取相册总页数 +fn get_total_pages(document: &Html) -> i32 { let pages_selector = Selector::parse("#pages a").unwrap(); + for page_element in document.select(&pages_selector) { if let Some(text) = page_element.text().next() { if text == "尾页" { if let Some(href) = page_element.value().attr("href") { - // 从尾页链接中提取页数 if let Some(page_num_str) = href.split("_").last() { if let Some(page_num) = page_num_str.split(".").next() { if let Ok(num) = page_num.parse::() { - total_pages = num; + return num; } } } @@ -117,23 +237,25 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool { } } } + 1 // 默认只有一页 +} - // 多线程收集所有图片 URL +/// 多线程收集图片 URL +fn collect_image_urls(uid: i32, total_pages: i32, img_selector: &Selector, base_url: &str) -> Vec { let (tx, rx) = mpsc::channel(); - let img_selector_arc = Arc::new(img_selector); - let mut page_handles = vec![]; - + let img_selector_arc = Arc::new(img_selector.clone()); + // 创建页面解析线程池 - let max_page_threads = 8; // 限制并发页面解析数 - + let max_page_threads = 8; + for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) { let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages); + let mut page_handles = vec![]; for page in chunk_start..=chunk_end { let tx = tx.clone(); let img_selector = Arc::clone(&img_selector_arc); - let base_url = base_url.clone(); - let uid = uid; + let base_url = base_url.to_string(); let handle = thread::spawn(move || { let page_url = if page == 1 { @@ -145,22 +267,26 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool { let mut page_images = Vec::new(); match get_response(&page_url) { - Ok(value) => { - if value.is_empty() { - println!("Page {} not found, skipping", page); - } else { - let page_body = - String::from_utf8(value.to_vec()).expect("Transforms error"); - let page_document = Html::parse_document(&page_body); - for img_element in page_document.select(&img_selector) { - if let Some(src) = img_element.value().attr("src") { - page_images.push(src.to_string()); - } + Ok(value) if !value.is_empty() => { + let page_body = String::from_utf8(value.to_vec()).expect("❌ 转换错误"); + let page_document = Html::parse_document(&page_body); + for img_element in page_document.select(&img_selector) { + if let Some(src) = img_element.value().attr("src") { + page_images.push(src.to_string()); } } } - Err(e) => { - println!("Error fetching page {}: {}, skipping", page, e); + Ok(_) => { + println!("⚠️ 页面 {} 内容为空, 跳过", page); + } + Err(RequestError::NotFound) => { + println!("⚠️ 页面 {} 不存在 (404), 跳过", page); + } + Err(RequestError::NetworkError) => { + println!("⚠️ 获取页面 {} 时网络错误, 跳过", page); + } + Err(RequestError::OtherError(e)) => { + println!("⚠️ 获取页面 {} 时错误: {}, 跳过", page, e); } } @@ -170,90 +296,90 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool { } // 等待当前批次完成 - for handle in page_handles.drain(..) { + for handle in page_handles { handle.join().unwrap(); } } - // 收集所有页面的图片URL，按页码排序 + // 收集并排序所有页面的图片 URL drop(tx); let mut page_results: Vec<_> = rx.iter().collect(); page_results.sort_by_key(|(page_num, _)| *page_num); + + page_results.into_iter() + .flat_map(|(_, page_images)| page_images) + .collect() +} - for (_, page_images) in page_results { - images.extend(page_images); - } - - // 生成 JSON 文件 +/// 生成 JSON 文件 +fn generate_json_file(uid: i32, title: &str, images: &[String]) { let album = Album { title: title.to_string(), - images, + images: images.to_vec(), }; - let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album"); + let json = serde_json::to_string_pretty(&album).expect("❌ 序列化相册失败"); let json_path = format!("/mnt/share/data/albums/{}/index.json", uid); - let json_bytes = Bytes::from(json); - file::write_file(&json_path, json_bytes).expect("Failed to write JSON file"); + file::write_file(&json_path, Bytes::from(json)).expect("❌ 写入 JSON 文件失败"); +} - // 执行多线程下载 +/// 多线程下载图片 +fn download_images_multithreaded(uid: i32, images: &[String]) -> DownloadResult { let (tx, rx) = mpsc::channel(); - let images_arc = Arc::new(album.images.clone()); - let uid_arc = Arc::new(uid); + let images_arc = Arc::new(images.to_vec()); - // 创建线程池（限制并发数避免过多连接） - let max_threads = 8; // 限制并发线程数 - let mut handles = vec![]; - - for chunk_start in (0..album.images.len()).step_by(max_threads) { - let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len()); + // 创建线程池 + let max_threads = 8; + + for chunk_start in (0..images.len()).step_by(max_threads) { + let chunk_end = std::cmp::min(chunk_start + max_threads, images.len()); + let mut handles = vec![]; for i in chunk_start..chunk_end { let tx = tx.clone(); let images = Arc::clone(&images_arc); - let uid = Arc::clone(&uid_arc); let handle = thread::spawn(move || { let img_url = &images[i]; let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1); let result = download_image(img_url, &file_path, 3); - tx.send((i + 1, result)).unwrap(); }); handles.push(handle); } // 等待当前批次完成 - for handle in handles.drain(..) { + for handle in handles { handle.join().unwrap(); } } // 收集下载结果 - drop(tx); // 关闭发送端 - let mut successful_downloads = 0; - let mut failed_downloads = 0; + drop(tx); + let (mut successful, mut failed, mut network_errors) = (0, 0, 0); for (pic_num, result) in rx { match result { - 1 => { - successful_downloads += 1; + DownloadResult::Success => successful += 1, + DownloadResult::Failed => { + println!("❌ 图片 {} 下载失败（空内容或404）", pic_num); + failed += 1; } - 0 => { - println!("图片 {} 下载失败（空内容）", pic_num); - failed_downloads += 1; + DownloadResult::NetworkError => { + println!("⚠️ 图片 {} 下载失败（网络错误）", pic_num); + network_errors += 1; } - 2 => { - println!("图片 {} 下载失败（网络错误）", pic_num); - failed_downloads += 1; - } - _ => {} } } - println!( - "下载完成：成功 {} 张，失败 {} 张", - successful_downloads, failed_downloads - ); + println!("📊 下载完成：成功 {} 张，失败 {} 张，网络错误 {} 张", + successful, failed, network_errors); - // 返回执行结果：至少成功下载一张图片返回 true，否则返回 false - failed_downloads == 0 + // 返回最终结果 + if failed == 0 && network_errors == 0 { + DownloadResult::Success + } else if network_errors > 0 { + DownloadResult::NetworkError + } else { + DownloadResult::Failed + } } \ No newline at end of file