admin/rust-imspyder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update get_response

Browse Source
This commit is contained in:
lychang
2026-03-15 03:00:14 +08:00
parent f8134d566c
commit 6af97ee8ee
3 changed files with 147 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/file.rs
View File

@@ -1,4 +1,3 @@
use bytes;
use bytes::Bytes;
use std::fs;
use std::fs::File;
@@ -18,14 +17,52 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
let path = Path::new(path);
let folder_path = path.parent();
if !folder_path.unwrap().exists() {
fs::create_dir_all(folder_path.unwrap().as_os_str())?;
if let Some(parent) = path.parent() {
if !parent.exists() {
fs::create_dir_all(parent)?;
}
}
let mut file = File::open(path)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
contents = contents.replace("\r\n", "\n");
contents = contents.replace("\n", "");
Ok(contents)
Ok(contents.trim().to_string())
}
pub(crate) fn write_failed_index(idx: i32) {
let existing_content = read_file_string("failed_index").unwrap_or_default();
// 检查是否已存在该ID
if !existing_content.lines().any(|line| line.trim() == idx.to_string()) {
let new_content = if existing_content.is_empty() {
idx.to_string()
} else {
format!("{}\n{}", existing_content.trim(), idx)
};
write_file("failed_index", Bytes::from(new_content))
.unwrap_or_else(|_| panic!("Write failed index error: {}", idx));
}
}
pub(crate) fn read_failed_indexes() -> Vec<i32> {
read_file_string("failed_index")
.unwrap_or_default()
.lines()
.filter_map(|line| line.trim().parse::<i32>().ok())
.collect()
}
pub(crate) fn remove_failed_index(idx: i32) {
let new_failed_indexes: Vec<String> = read_failed_indexes()
.into_iter()
.filter(|&failed_idx| failed_idx != idx)
.map(|i| i.to_string())
.collect();
if new_failed_indexes.is_empty() {
let _ = std::fs::remove_file("failed_index");
} else {
let content = new_failed_indexes.join("\n");
write_file("failed_index", Bytes::from(content))
.unwrap_or_else(|_| panic!("Update failed index error: {}", idx));
}
}

65
src/main.rs
View File

@@ -1,38 +1,87 @@
use std::str::FromStr;
use std::thread;
use std::time::Duration;
use bytes::Bytes;
use std::time::Duration;
mod file;
mod web_parser;
fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> bool {
if web_parser::download_images(idx, name) {
println!("相册 {} ({}) 下载成功", idx, name);
file::write_file("index", Bytes::from(idx.to_string()))
.unwrap_or_else(|_| panic!("Write index error: {}", idx));
true
} else if times > 0 {
println!("相册 {} ({}) 下载失败，{}秒后重试", idx, name, duration);
thread::sleep(Duration::from_millis(duration * 1000));
retry_download(idx, name, times - 1, duration)
} else {
println!("相册 {} ({}) 下载失败{}次,记录到失败列表", idx, name, times);
false
}
}
fn main() {
let mut idx: i32;
let mut blank_count = 0;
// 先处理失败的记录
let mut failed_indexes = file::read_failed_indexes();
if !failed_indexes.is_empty() {
println!(
"发现 {} 个失败的下载记录，开始重试...",
failed_indexes.len()
);
for failed_idx in failed_indexes.drain(..).collect::<Vec<_>>() {
let name = web_parser::get_title(failed_idx);
if name != "" {
println!("重试下载索引 {} 相册 {}...", failed_idx, name);
let success = retry_download(failed_idx, &name, 2, 10);
if success {
println!("相册 {} 重试成功", failed_idx);
} else {
println!("相册 {} 重试失败，保留在失败列表", failed_idx);
}
} else {
println!("相册 {} 不存在，从失败列表中移除", failed_idx);
file::remove_failed_index(failed_idx);
}
thread::sleep(Duration::from_millis(100));
}
println!("失败记录处理完成");
}
// 继续正常下载流程
let index = file::read_file_string("index").expect("Read index error");
if index.is_empty() {
idx = 1;
} else {
idx = i32::from_str(&index).expect("Index Corrupted");
}
loop {
let name = web_parser::get_title(idx);
if name == "" {
println!("索引 {} 不存在内容", idx);
thread::sleep(Duration::from_millis(3000));
idx += 1;
blank_count += 1;
} else {
web_parser::download_images(idx, &name);
idx += 1;
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
blank_count = 0;
println!("开始下载索引 {} 相册 {}...", idx, name);
let success = retry_download(idx, &name, 3, 15);
if success {
idx += 1;
} else {
file::write_failed_index(idx);
idx += 1;
}
}
if blank_count >= 10 {
if blank_count >= 50 {
break;
}
thread::sleep(Duration::from_millis(100));
}
}
}

107
src/web_parser.rs
View File

@@ -14,65 +14,53 @@ struct Album {
}
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let client = reqwest::blocking::Client::builder().build()?;
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()?;
let response = client
.get(url)
.header(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.send()?;
if response.status() == 200 {
// 先获取头信息，再获取字节数据
let content_encoding = response
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
if !response.status().is_success() {
return Err(response.error_for_status().unwrap_err().into());
}
let bytes = response.bytes()?;
let content_encoding = response
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let result = if content_encoding.contains("gzip") {
// 手动解压缩 gzip 数据
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
let mut decompressed = Vec::new();
match decoder.read_to_end(&mut decompressed) {
Ok(_) => Ok(Bytes::from(decompressed)),
Err(e) => {
// 如果解压缩失败，返回原始字节数据
println!("Gzip decompression failed: {}, using raw data", e);
Ok(bytes)
}
let bytes = response.bytes()?;
if content_encoding.contains("gzip") {
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
let mut decompressed = Vec::new();
match decoder.read_to_end(&mut decompressed) {
Ok(_) => Ok(Bytes::from(decompressed)),
Err(e) => {
println!("Gzip decompression failed: {}, using raw data", e);
Ok(bytes)
}
} else {
Ok(bytes)
};
result
}
} else {
Err(reqwest::Error::from(
response.error_for_status().unwrap_err(),
))
Ok(bytes)
}
}
pub(crate) fn get_title(uid: i32) -> String {
let url = "https://www.meitu131.com/meinv/";
let temp_url = format!("{}{}/", url, uid);
let file_content = get_response(&temp_url);
match file_content {
Ok(value) => {
if value.is_empty() {
return "".to_string();
}
// 直接使用 lossy 转换避免 UTF-8 错误
let url = format!("https://www.meitu131.com/meinv/{}/", uid);
match get_response(&url) {
Ok(value) if !value.is_empty() => {
let body = String::from_utf8_lossy(&value).to_string();
get_html_title(body)
}
Err(_e) => "".to_string(),
_ => "".to_string(),
}
}
@@ -84,27 +72,21 @@ fn get_html_title(body: String) -> String {
}
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
let mut flag = 1;
let response = get_response(url);
match response {
Ok(value) => {
if value.is_empty() {
flag = 0
} else {
file::write_file(&file_path, value).expect("Write file error");
}
match get_response(url) {
Ok(value) if !value.is_empty() => {
file::write_file(file_path, value).expect("Write file error");
1
}
Err(_e) => flag = 2,
Ok(_) => 0, // 空内容
Err(_) if retry_times > 1 => {
thread::sleep(Duration::from_millis(1000));
download_image(url, file_path, retry_times - 1)
}
Err(_) => 2, // 网络错误
}
if flag == 2 && retry_times > 1 {
thread::sleep(Duration::from_millis(1000));
return download_image(url, file_path, retry_times - 1);
}
flag
}
pub(crate) fn download_images(uid: i32, title: &str) {
println!("Downloading images for {} {}...", uid, title);
pub(crate) fn download_images(uid: i32, title: &str) -> bool {
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let file_content = get_response(&base_url);
let body = match file_content {
@@ -271,4 +253,7 @@ pub(crate) fn download_images(uid: i32, title: &str) {
"下载完成：成功 {} 张，失败 {} 张",
successful_downloads, failed_downloads
);
}
// 返回执行结果：至少成功下载一张图片返回 true，否则返回 false
failed_downloads == 0
}