diff --git a/src/file.rs b/src/file.rs index 32770d0..983d8b3 100644 --- a/src/file.rs +++ b/src/file.rs @@ -1,4 +1,3 @@ -use bytes; use bytes::Bytes; use std::fs; use std::fs::File; @@ -18,14 +17,52 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro pub(crate) fn read_file_string(path: &str) -> Result { let path = Path::new(path); - let folder_path = path.parent(); - if !folder_path.unwrap().exists() { - fs::create_dir_all(folder_path.unwrap().as_os_str())?; + if let Some(parent) = path.parent() { + if !parent.exists() { + fs::create_dir_all(parent)?; + } } let mut file = File::open(path)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; - contents = contents.replace("\r\n", "\n"); - contents = contents.replace("\n", ""); - Ok(contents) + Ok(contents.trim().to_string()) } + +pub(crate) fn write_failed_index(idx: i32) { + let existing_content = read_file_string("failed_index").unwrap_or_default(); + + // 检查是否已存在该ID + if !existing_content.lines().any(|line| line.trim() == idx.to_string()) { + let new_content = if existing_content.is_empty() { + idx.to_string() + } else { + format!("{}\n{}", existing_content.trim(), idx) + }; + + write_file("failed_index", Bytes::from(new_content)) + .unwrap_or_else(|_| panic!("Write failed index error: {}", idx)); + } +} + +pub(crate) fn read_failed_indexes() -> Vec { + read_file_string("failed_index") + .unwrap_or_default() + .lines() + .filter_map(|line| line.trim().parse::().ok()) + .collect() +} +pub(crate) fn remove_failed_index(idx: i32) { + let new_failed_indexes: Vec = read_failed_indexes() + .into_iter() + .filter(|&failed_idx| failed_idx != idx) + .map(|i| i.to_string()) + .collect(); + + if new_failed_indexes.is_empty() { + let _ = std::fs::remove_file("failed_index"); + } else { + let content = new_failed_indexes.join("\n"); + write_file("failed_index", Bytes::from(content)) + .unwrap_or_else(|_| panic!("Update failed index error: {}", idx)); + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index e2a6323..24e34e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,38 +1,87 @@ use std::str::FromStr; use std::thread; -use std::time::Duration; use bytes::Bytes; - +use std::time::Duration; mod file; mod web_parser; +fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> bool { + if web_parser::download_images(idx, name) { + println!("相册 {} ({}) 下载成功", idx, name); + file::write_file("index", Bytes::from(idx.to_string())) + .unwrap_or_else(|_| panic!("Write index error: {}", idx)); + true + } else if times > 0 { + println!("相册 {} ({}) 下载失败,{}秒后重试", idx, name, duration); + thread::sleep(Duration::from_millis(duration * 1000)); + retry_download(idx, name, times - 1, duration) + } else { + println!("相册 {} ({}) 下载失败{}次,记录到失败列表", idx, name, times); + false + } +} fn main() { let mut idx: i32; let mut blank_count = 0; + + // 先处理失败的记录 + let mut failed_indexes = file::read_failed_indexes(); + if !failed_indexes.is_empty() { + println!( + "发现 {} 个失败的下载记录,开始重试...", + failed_indexes.len() + ); + + for failed_idx in failed_indexes.drain(..).collect::>() { + let name = web_parser::get_title(failed_idx); + if name != "" { + println!("重试下载索引 {} 相册 {}...", failed_idx, name); + let success = retry_download(failed_idx, &name, 2, 10); + if success { + println!("相册 {} 重试成功", failed_idx); + } else { + println!("相册 {} 重试失败,保留在失败列表", failed_idx); + } + } else { + println!("相册 {} 不存在,从失败列表中移除", failed_idx); + file::remove_failed_index(failed_idx); + } + thread::sleep(Duration::from_millis(100)); + } + println!("失败记录处理完成"); + } + + // 继续正常下载流程 let index = file::read_file_string("index").expect("Read index error"); if index.is_empty() { idx = 1; } else { idx = i32::from_str(&index).expect("Index Corrupted"); } + loop { let name = web_parser::get_title(idx); if name == "" { + println!("索引 {} 不存在内容", idx); thread::sleep(Duration::from_millis(3000)); idx += 1; blank_count += 1; } else { - web_parser::download_images(idx, &name); - idx += 1; - file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx)); - blank_count = 0; + println!("开始下载索引 {} 相册 {}...", idx, name); + let success = retry_download(idx, &name, 3, 15); + if success { + idx += 1; + } else { + file::write_failed_index(idx); + idx += 1; + } } - if blank_count >= 10 { + if blank_count >= 50 { break; } thread::sleep(Duration::from_millis(100)); } -} \ No newline at end of file +} diff --git a/src/web_parser.rs b/src/web_parser.rs index 7bbe67c..72d0f31 100644 --- a/src/web_parser.rs +++ b/src/web_parser.rs @@ -14,65 +14,53 @@ struct Album { } fn get_response(url: &str) -> Result { - let client = reqwest::blocking::Client::builder().build()?; + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(30)) + .build()?; let response = client .get(url) - .header( - "User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", - ) + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") .send()?; - if response.status() == 200 { - // 先获取头信息,再获取字节数据 - let content_encoding = response - .headers() - .get("content-encoding") - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); + if !response.status().is_success() { + return Err(response.error_for_status().unwrap_err().into()); + } - let bytes = response.bytes()?; + let content_encoding = response + .headers() + .get("content-encoding") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); - let result = if content_encoding.contains("gzip") { - // 手动解压缩 gzip 数据 - use std::io::Read; - let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); - let mut decompressed = Vec::new(); - match decoder.read_to_end(&mut decompressed) { - Ok(_) => Ok(Bytes::from(decompressed)), - Err(e) => { - // 如果解压缩失败,返回原始字节数据 - println!("Gzip decompression failed: {}, using raw data", e); - Ok(bytes) - } + let bytes = response.bytes()?; + + if content_encoding.contains("gzip") { + use std::io::Read; + let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); + let mut decompressed = Vec::new(); + match decoder.read_to_end(&mut decompressed) { + Ok(_) => Ok(Bytes::from(decompressed)), + Err(e) => { + println!("Gzip decompression failed: {}, using raw data", e); + Ok(bytes) } - } else { - Ok(bytes) - }; - result + } } else { - Err(reqwest::Error::from( - response.error_for_status().unwrap_err(), - )) + Ok(bytes) } } pub(crate) fn get_title(uid: i32) -> String { - let url = "https://www.meitu131.com/meinv/"; - let temp_url = format!("{}{}/", url, uid); - let file_content = get_response(&temp_url); - match file_content { - Ok(value) => { - if value.is_empty() { - return "".to_string(); - } - // 直接使用 lossy 转换避免 UTF-8 错误 + let url = format!("https://www.meitu131.com/meinv/{}/", uid); + + match get_response(&url) { + Ok(value) if !value.is_empty() => { let body = String::from_utf8_lossy(&value).to_string(); get_html_title(body) } - Err(_e) => "".to_string(), + _ => "".to_string(), } } @@ -84,27 +72,21 @@ fn get_html_title(body: String) -> String { } fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { - let mut flag = 1; - let response = get_response(url); - match response { - Ok(value) => { - if value.is_empty() { - flag = 0 - } else { - file::write_file(&file_path, value).expect("Write file error"); - } + match get_response(url) { + Ok(value) if !value.is_empty() => { + file::write_file(file_path, value).expect("Write file error"); + 1 } - Err(_e) => flag = 2, + Ok(_) => 0, // 空内容 + Err(_) if retry_times > 1 => { + thread::sleep(Duration::from_millis(1000)); + download_image(url, file_path, retry_times - 1) + } + Err(_) => 2, // 网络错误 } - if flag == 2 && retry_times > 1 { - thread::sleep(Duration::from_millis(1000)); - return download_image(url, file_path, retry_times - 1); - } - flag } -pub(crate) fn download_images(uid: i32, title: &str) { - println!("Downloading images for {} {}...", uid, title); +pub(crate) fn download_images(uid: i32, title: &str) -> bool { let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); let file_content = get_response(&base_url); let body = match file_content { @@ -271,4 +253,7 @@ pub(crate) fn download_images(uid: i32, title: &str) { "下载完成:成功 {} 张,失败 {} 张", successful_downloads, failed_downloads ); -} + + // 返回执行结果:至少成功下载一张图片返回 true,否则返回 false + failed_downloads == 0 +} \ No newline at end of file