update get_response

This commit is contained in:
lychang
2026-03-15 03:00:14 +08:00
parent f8134d566c
commit 6af97ee8ee
3 changed files with 147 additions and 76 deletions

View File

@@ -1,4 +1,3 @@
use bytes;
use bytes::Bytes; use bytes::Bytes;
use std::fs; use std::fs;
use std::fs::File; use std::fs::File;
@@ -18,14 +17,52 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> { pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
let path = Path::new(path); let path = Path::new(path);
let folder_path = path.parent(); if let Some(parent) = path.parent() {
if !folder_path.unwrap().exists() { if !parent.exists() {
fs::create_dir_all(folder_path.unwrap().as_os_str())?; fs::create_dir_all(parent)?;
}
} }
let mut file = File::open(path)?; let mut file = File::open(path)?;
let mut contents = String::new(); let mut contents = String::new();
file.read_to_string(&mut contents)?; file.read_to_string(&mut contents)?;
contents = contents.replace("\r\n", "\n"); Ok(contents.trim().to_string())
contents = contents.replace("\n", ""); }
Ok(contents)
pub(crate) fn write_failed_index(idx: i32) {
let existing_content = read_file_string("failed_index").unwrap_or_default();
// 检查是否已存在该ID
if !existing_content.lines().any(|line| line.trim() == idx.to_string()) {
let new_content = if existing_content.is_empty() {
idx.to_string()
} else {
format!("{}\n{}", existing_content.trim(), idx)
};
write_file("failed_index", Bytes::from(new_content))
.unwrap_or_else(|_| panic!("Write failed index error: {}", idx));
}
}
pub(crate) fn read_failed_indexes() -> Vec<i32> {
read_file_string("failed_index")
.unwrap_or_default()
.lines()
.filter_map(|line| line.trim().parse::<i32>().ok())
.collect()
}
pub(crate) fn remove_failed_index(idx: i32) {
let new_failed_indexes: Vec<String> = read_failed_indexes()
.into_iter()
.filter(|&failed_idx| failed_idx != idx)
.map(|i| i.to_string())
.collect();
if new_failed_indexes.is_empty() {
let _ = std::fs::remove_file("failed_index");
} else {
let content = new_failed_indexes.join("\n");
write_file("failed_index", Bytes::from(content))
.unwrap_or_else(|_| panic!("Update failed index error: {}", idx));
}
} }

View File

@@ -1,36 +1,85 @@
use std::str::FromStr; use std::str::FromStr;
use std::thread; use std::thread;
use std::time::Duration;
use bytes::Bytes; use bytes::Bytes;
use std::time::Duration;
mod file; mod file;
mod web_parser; mod web_parser;
fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> bool {
if web_parser::download_images(idx, name) {
println!("相册 {} ({}) 下载成功", idx, name);
file::write_file("index", Bytes::from(idx.to_string()))
.unwrap_or_else(|_| panic!("Write index error: {}", idx));
true
} else if times > 0 {
println!("相册 {} ({}) 下载失败,{}秒后重试", idx, name, duration);
thread::sleep(Duration::from_millis(duration * 1000));
retry_download(idx, name, times - 1, duration)
} else {
println!("相册 {} ({}) 下载失败{}次,记录到失败列表", idx, name, times);
false
}
}
fn main() { fn main() {
let mut idx: i32; let mut idx: i32;
let mut blank_count = 0; let mut blank_count = 0;
// 先处理失败的记录
let mut failed_indexes = file::read_failed_indexes();
if !failed_indexes.is_empty() {
println!(
"发现 {} 个失败的下载记录,开始重试...",
failed_indexes.len()
);
for failed_idx in failed_indexes.drain(..).collect::<Vec<_>>() {
let name = web_parser::get_title(failed_idx);
if name != "" {
println!("重试下载索引 {} 相册 {}...", failed_idx, name);
let success = retry_download(failed_idx, &name, 2, 10);
if success {
println!("相册 {} 重试成功", failed_idx);
} else {
println!("相册 {} 重试失败,保留在失败列表", failed_idx);
}
} else {
println!("相册 {} 不存在,从失败列表中移除", failed_idx);
file::remove_failed_index(failed_idx);
}
thread::sleep(Duration::from_millis(100));
}
println!("失败记录处理完成");
}
// 继续正常下载流程
let index = file::read_file_string("index").expect("Read index error"); let index = file::read_file_string("index").expect("Read index error");
if index.is_empty() { if index.is_empty() {
idx = 1; idx = 1;
} else { } else {
idx = i32::from_str(&index).expect("Index Corrupted"); idx = i32::from_str(&index).expect("Index Corrupted");
} }
loop { loop {
let name = web_parser::get_title(idx); let name = web_parser::get_title(idx);
if name == "" { if name == "" {
println!("索引 {} 不存在内容", idx);
thread::sleep(Duration::from_millis(3000)); thread::sleep(Duration::from_millis(3000));
idx += 1; idx += 1;
blank_count += 1; blank_count += 1;
} else { } else {
web_parser::download_images(idx, &name); println!("开始下载索引 {} 相册 {}...", idx, name);
idx += 1; let success = retry_download(idx, &name, 3, 15);
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx)); if success {
blank_count = 0; idx += 1;
} else {
file::write_failed_index(idx);
idx += 1;
}
} }
if blank_count >= 10 { if blank_count >= 50 {
break; break;
} }
thread::sleep(Duration::from_millis(100)); thread::sleep(Duration::from_millis(100));

View File

@@ -14,65 +14,53 @@ struct Album {
} }
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> { fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let client = reqwest::blocking::Client::builder().build()?; let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()?;
let response = client let response = client
.get(url) .get(url)
.header( .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
)
.send()?; .send()?;
if response.status() == 200 { if !response.status().is_success() {
// 先获取头信息,再获取字节数据 return Err(response.error_for_status().unwrap_err().into());
let content_encoding = response }
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = response.bytes()?; let content_encoding = response
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let result = if content_encoding.contains("gzip") { let bytes = response.bytes()?;
// 手动解压缩 gzip 数据
use std::io::Read; if content_encoding.contains("gzip") {
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); use std::io::Read;
let mut decompressed = Vec::new(); let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
match decoder.read_to_end(&mut decompressed) { let mut decompressed = Vec::new();
Ok(_) => Ok(Bytes::from(decompressed)), match decoder.read_to_end(&mut decompressed) {
Err(e) => { Ok(_) => Ok(Bytes::from(decompressed)),
// 如果解压缩失败,返回原始字节数据 Err(e) => {
println!("Gzip decompression failed: {}, using raw data", e); println!("Gzip decompression failed: {}, using raw data", e);
Ok(bytes) Ok(bytes)
}
} }
} else { }
Ok(bytes)
};
result
} else { } else {
Err(reqwest::Error::from( Ok(bytes)
response.error_for_status().unwrap_err(),
))
} }
} }
pub(crate) fn get_title(uid: i32) -> String { pub(crate) fn get_title(uid: i32) -> String {
let url = "https://www.meitu131.com/meinv/"; let url = format!("https://www.meitu131.com/meinv/{}/", uid);
let temp_url = format!("{}{}/", url, uid);
let file_content = get_response(&temp_url); match get_response(&url) {
match file_content { Ok(value) if !value.is_empty() => {
Ok(value) => {
if value.is_empty() {
return "".to_string();
}
// 直接使用 lossy 转换避免 UTF-8 错误
let body = String::from_utf8_lossy(&value).to_string(); let body = String::from_utf8_lossy(&value).to_string();
get_html_title(body) get_html_title(body)
} }
Err(_e) => "".to_string(), _ => "".to_string(),
} }
} }
@@ -84,27 +72,21 @@ fn get_html_title(body: String) -> String {
} }
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
let mut flag = 1; match get_response(url) {
let response = get_response(url); Ok(value) if !value.is_empty() => {
match response { file::write_file(file_path, value).expect("Write file error");
Ok(value) => { 1
if value.is_empty() {
flag = 0
} else {
file::write_file(&file_path, value).expect("Write file error");
}
} }
Err(_e) => flag = 2, Ok(_) => 0, // 空内容
Err(_) if retry_times > 1 => {
thread::sleep(Duration::from_millis(1000));
download_image(url, file_path, retry_times - 1)
}
Err(_) => 2, // 网络错误
} }
if flag == 2 && retry_times > 1 {
thread::sleep(Duration::from_millis(1000));
return download_image(url, file_path, retry_times - 1);
}
flag
} }
pub(crate) fn download_images(uid: i32, title: &str) { pub(crate) fn download_images(uid: i32, title: &str) -> bool {
println!("Downloading images for {} {}...", uid, title);
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let file_content = get_response(&base_url); let file_content = get_response(&base_url);
let body = match file_content { let body = match file_content {
@@ -271,4 +253,7 @@ pub(crate) fn download_images(uid: i32, title: &str) {
"下载完成:成功 {} 张,失败 {} 张", "下载完成:成功 {} 张,失败 {} 张",
successful_downloads, failed_downloads successful_downloads, failed_downloads
); );
// 返回执行结果:至少成功下载一张图片返回 true,否则返回 false
failed_downloads == 0
} }