update get_response
This commit is contained in:
51
src/file.rs
51
src/file.rs
@@ -1,4 +1,3 @@
|
|||||||
use bytes;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
@@ -18,14 +17,52 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
|
|||||||
|
|
||||||
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
||||||
let path = Path::new(path);
|
let path = Path::new(path);
|
||||||
let folder_path = path.parent();
|
if let Some(parent) = path.parent() {
|
||||||
if !folder_path.unwrap().exists() {
|
if !parent.exists() {
|
||||||
fs::create_dir_all(folder_path.unwrap().as_os_str())?;
|
fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let mut file = File::open(path)?;
|
let mut file = File::open(path)?;
|
||||||
let mut contents = String::new();
|
let mut contents = String::new();
|
||||||
file.read_to_string(&mut contents)?;
|
file.read_to_string(&mut contents)?;
|
||||||
contents = contents.replace("\r\n", "\n");
|
Ok(contents.trim().to_string())
|
||||||
contents = contents.replace("\n", "");
|
|
||||||
Ok(contents)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn write_failed_index(idx: i32) {
|
||||||
|
let existing_content = read_file_string("failed_index").unwrap_or_default();
|
||||||
|
|
||||||
|
// 检查是否已存在该ID
|
||||||
|
if !existing_content.lines().any(|line| line.trim() == idx.to_string()) {
|
||||||
|
let new_content = if existing_content.is_empty() {
|
||||||
|
idx.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{}\n{}", existing_content.trim(), idx)
|
||||||
|
};
|
||||||
|
|
||||||
|
write_file("failed_index", Bytes::from(new_content))
|
||||||
|
.unwrap_or_else(|_| panic!("Write failed index error: {}", idx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn read_failed_indexes() -> Vec<i32> {
|
||||||
|
read_file_string("failed_index")
|
||||||
|
.unwrap_or_default()
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| line.trim().parse::<i32>().ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
pub(crate) fn remove_failed_index(idx: i32) {
|
||||||
|
let new_failed_indexes: Vec<String> = read_failed_indexes()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|&failed_idx| failed_idx != idx)
|
||||||
|
.map(|i| i.to_string())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if new_failed_indexes.is_empty() {
|
||||||
|
let _ = std::fs::remove_file("failed_index");
|
||||||
|
} else {
|
||||||
|
let content = new_failed_indexes.join("\n");
|
||||||
|
write_file("failed_index", Bytes::from(content))
|
||||||
|
.unwrap_or_else(|_| panic!("Update failed index error: {}", idx));
|
||||||
|
}
|
||||||
|
}
|
||||||
65
src/main.rs
65
src/main.rs
@@ -1,38 +1,87 @@
|
|||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
mod file;
|
mod file;
|
||||||
mod web_parser;
|
mod web_parser;
|
||||||
|
|
||||||
|
fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> bool {
|
||||||
|
if web_parser::download_images(idx, name) {
|
||||||
|
println!("相册 {} ({}) 下载成功", idx, name);
|
||||||
|
file::write_file("index", Bytes::from(idx.to_string()))
|
||||||
|
.unwrap_or_else(|_| panic!("Write index error: {}", idx));
|
||||||
|
true
|
||||||
|
} else if times > 0 {
|
||||||
|
println!("相册 {} ({}) 下载失败,{}秒后重试", idx, name, duration);
|
||||||
|
thread::sleep(Duration::from_millis(duration * 1000));
|
||||||
|
retry_download(idx, name, times - 1, duration)
|
||||||
|
} else {
|
||||||
|
println!("相册 {} ({}) 下载失败{}次,记录到失败列表", idx, name, times);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let mut idx: i32;
|
let mut idx: i32;
|
||||||
let mut blank_count = 0;
|
let mut blank_count = 0;
|
||||||
|
|
||||||
|
// 先处理失败的记录
|
||||||
|
let mut failed_indexes = file::read_failed_indexes();
|
||||||
|
if !failed_indexes.is_empty() {
|
||||||
|
println!(
|
||||||
|
"发现 {} 个失败的下载记录,开始重试...",
|
||||||
|
failed_indexes.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
for failed_idx in failed_indexes.drain(..).collect::<Vec<_>>() {
|
||||||
|
let name = web_parser::get_title(failed_idx);
|
||||||
|
if name != "" {
|
||||||
|
println!("重试下载索引 {} 相册 {}...", failed_idx, name);
|
||||||
|
let success = retry_download(failed_idx, &name, 2, 10);
|
||||||
|
if success {
|
||||||
|
println!("相册 {} 重试成功", failed_idx);
|
||||||
|
} else {
|
||||||
|
println!("相册 {} 重试失败,保留在失败列表", failed_idx);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
println!("相册 {} 不存在,从失败列表中移除", failed_idx);
|
||||||
|
file::remove_failed_index(failed_idx);
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(100));
|
||||||
|
}
|
||||||
|
println!("失败记录处理完成");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 继续正常下载流程
|
||||||
let index = file::read_file_string("index").expect("Read index error");
|
let index = file::read_file_string("index").expect("Read index error");
|
||||||
if index.is_empty() {
|
if index.is_empty() {
|
||||||
idx = 1;
|
idx = 1;
|
||||||
} else {
|
} else {
|
||||||
idx = i32::from_str(&index).expect("Index Corrupted");
|
idx = i32::from_str(&index).expect("Index Corrupted");
|
||||||
}
|
}
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let name = web_parser::get_title(idx);
|
let name = web_parser::get_title(idx);
|
||||||
if name == "" {
|
if name == "" {
|
||||||
|
println!("索引 {} 不存在内容", idx);
|
||||||
thread::sleep(Duration::from_millis(3000));
|
thread::sleep(Duration::from_millis(3000));
|
||||||
idx += 1;
|
idx += 1;
|
||||||
blank_count += 1;
|
blank_count += 1;
|
||||||
} else {
|
} else {
|
||||||
web_parser::download_images(idx, &name);
|
println!("开始下载索引 {} 相册 {}...", idx, name);
|
||||||
idx += 1;
|
let success = retry_download(idx, &name, 3, 15);
|
||||||
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
|
if success {
|
||||||
blank_count = 0;
|
idx += 1;
|
||||||
|
} else {
|
||||||
|
file::write_failed_index(idx);
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if blank_count >= 10 {
|
if blank_count >= 50 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
thread::sleep(Duration::from_millis(100));
|
thread::sleep(Duration::from_millis(100));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,65 +14,53 @@ struct Album {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
||||||
let client = reqwest::blocking::Client::builder().build()?;
|
let client = reqwest::blocking::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()?;
|
||||||
|
|
||||||
let response = client
|
let response = client
|
||||||
.get(url)
|
.get(url)
|
||||||
.header(
|
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||||
"User-Agent",
|
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
||||||
)
|
|
||||||
.send()?;
|
.send()?;
|
||||||
|
|
||||||
if response.status() == 200 {
|
if !response.status().is_success() {
|
||||||
// 先获取头信息,再获取字节数据
|
return Err(response.error_for_status().unwrap_err().into());
|
||||||
let content_encoding = response
|
}
|
||||||
.headers()
|
|
||||||
.get("content-encoding")
|
|
||||||
.and_then(|v| v.to_str().ok())
|
|
||||||
.unwrap_or("")
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let bytes = response.bytes()?;
|
let content_encoding = response
|
||||||
|
.headers()
|
||||||
|
.get("content-encoding")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
let result = if content_encoding.contains("gzip") {
|
let bytes = response.bytes()?;
|
||||||
// 手动解压缩 gzip 数据
|
|
||||||
use std::io::Read;
|
if content_encoding.contains("gzip") {
|
||||||
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
use std::io::Read;
|
||||||
let mut decompressed = Vec::new();
|
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
||||||
match decoder.read_to_end(&mut decompressed) {
|
let mut decompressed = Vec::new();
|
||||||
Ok(_) => Ok(Bytes::from(decompressed)),
|
match decoder.read_to_end(&mut decompressed) {
|
||||||
Err(e) => {
|
Ok(_) => Ok(Bytes::from(decompressed)),
|
||||||
// 如果解压缩失败,返回原始字节数据
|
Err(e) => {
|
||||||
println!("Gzip decompression failed: {}, using raw data", e);
|
println!("Gzip decompression failed: {}, using raw data", e);
|
||||||
Ok(bytes)
|
Ok(bytes)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
}
|
||||||
Ok(bytes)
|
|
||||||
};
|
|
||||||
result
|
|
||||||
} else {
|
} else {
|
||||||
Err(reqwest::Error::from(
|
Ok(bytes)
|
||||||
response.error_for_status().unwrap_err(),
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn get_title(uid: i32) -> String {
|
pub(crate) fn get_title(uid: i32) -> String {
|
||||||
let url = "https://www.meitu131.com/meinv/";
|
let url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
let temp_url = format!("{}{}/", url, uid);
|
|
||||||
let file_content = get_response(&temp_url);
|
match get_response(&url) {
|
||||||
match file_content {
|
Ok(value) if !value.is_empty() => {
|
||||||
Ok(value) => {
|
|
||||||
if value.is_empty() {
|
|
||||||
return "".to_string();
|
|
||||||
}
|
|
||||||
// 直接使用 lossy 转换避免 UTF-8 错误
|
|
||||||
let body = String::from_utf8_lossy(&value).to_string();
|
let body = String::from_utf8_lossy(&value).to_string();
|
||||||
get_html_title(body)
|
get_html_title(body)
|
||||||
}
|
}
|
||||||
Err(_e) => "".to_string(),
|
_ => "".to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,27 +72,21 @@ fn get_html_title(body: String) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
||||||
let mut flag = 1;
|
match get_response(url) {
|
||||||
let response = get_response(url);
|
Ok(value) if !value.is_empty() => {
|
||||||
match response {
|
file::write_file(file_path, value).expect("Write file error");
|
||||||
Ok(value) => {
|
1
|
||||||
if value.is_empty() {
|
|
||||||
flag = 0
|
|
||||||
} else {
|
|
||||||
file::write_file(&file_path, value).expect("Write file error");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(_e) => flag = 2,
|
Ok(_) => 0, // 空内容
|
||||||
|
Err(_) if retry_times > 1 => {
|
||||||
|
thread::sleep(Duration::from_millis(1000));
|
||||||
|
download_image(url, file_path, retry_times - 1)
|
||||||
|
}
|
||||||
|
Err(_) => 2, // 网络错误
|
||||||
}
|
}
|
||||||
if flag == 2 && retry_times > 1 {
|
|
||||||
thread::sleep(Duration::from_millis(1000));
|
|
||||||
return download_image(url, file_path, retry_times - 1);
|
|
||||||
}
|
|
||||||
flag
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn download_images(uid: i32, title: &str) {
|
pub(crate) fn download_images(uid: i32, title: &str) -> bool {
|
||||||
println!("Downloading images for {} {}...", uid, title);
|
|
||||||
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
let file_content = get_response(&base_url);
|
let file_content = get_response(&base_url);
|
||||||
let body = match file_content {
|
let body = match file_content {
|
||||||
@@ -271,4 +253,7 @@ pub(crate) fn download_images(uid: i32, title: &str) {
|
|||||||
"下载完成:成功 {} 张,失败 {} 张",
|
"下载完成:成功 {} 张,失败 {} 张",
|
||||||
successful_downloads, failed_downloads
|
successful_downloads, failed_downloads
|
||||||
);
|
);
|
||||||
}
|
|
||||||
|
// 返回执行结果:至少成功下载一张图片返回 true,否则返回 false
|
||||||
|
failed_downloads == 0
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user