update get_response

This commit is contained in:
lychang
2026-03-17 02:15:58 +08:00
parent 6af97ee8ee
commit 161837bd24
3 changed files with 412 additions and 161 deletions

View File

@@ -7,43 +7,96 @@ use std::sync::Arc;
use std::thread;
use std::time::Duration;
/// 自定义错误类型,用于区分不同类型的网络错误
#[derive(Debug)]
pub enum RequestError {
NotFound, // 404 错误
NetworkError, // 网络错误
OtherError(String), // 其他错误
}
impl From<reqwest::Error> for RequestError {
fn from(err: reqwest::Error) -> Self {
if err.is_status() {
if let Some(status) = err.status() {
if status == reqwest::StatusCode::NOT_FOUND {
return RequestError::NotFound;
}
}
RequestError::OtherError(err.to_string())
} else if err.is_timeout() || err.is_connect() {
RequestError::NetworkError
} else {
RequestError::OtherError(err.to_string())
}
}
}
/// 相册数据结构
#[derive(Debug, Serialize, Deserialize)]
struct Album {
title: String,
images: Vec<String>,
}
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
/// 下载结果枚举
pub(crate) enum DownloadResult {
Success,
NetworkError,
Failed,
}
/// 发送 HTTP 请求并获取响应
///
/// # 参数
/// - `url`: 请求的 URL
///
/// # 返回值
/// - `Result<Bytes, RequestError>`: 响应内容或错误
fn get_response(url: &str) -> Result<Bytes, RequestError> {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()?;
.build()
.map_err(|_| RequestError::NetworkError)?;
let response = client
.get(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.send()?;
.send()
.map_err(RequestError::from)?;
if !response.status().is_success() {
return Err(response.error_for_status().unwrap_err().into());
// 检查 HTTP 状态码
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Err(RequestError::NotFound);
}
let content_encoding = response
.headers()
if !response.status().is_success() {
return Err(RequestError::OtherError(format!(
"HTTP {}: {}",
response.status(),
response.status().canonical_reason().unwrap_or("Unknown")
)));
}
// 先检查内容编码,再获取 bytes
let content_encoding = response.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = response.bytes()?;
let bytes = response.bytes().map_err(RequestError::from)?;
// 处理 gzip 压缩
if content_encoding.contains("gzip") {
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
let mut decompressed = Vec::new();
match decoder.read_to_end(&mut decompressed) {
Ok(_) => Ok(Bytes::from(decompressed)),
Err(e) => {
println!("Gzip decompression failed: {}, using raw data", e);
println!("⚠️ Gzip 解压失败: {}, 使用原始数据", e);
Ok(bytes)
}
}
@@ -52,64 +105,131 @@ fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
}
}
pub(crate) fn get_title(uid: i32) -> String {
/// 获取相册标题
///
/// # 参数
/// - `uid`: 相册 ID
///
/// # 返回值
/// - `Result<String, DownloadResult>`: 相册标题或错误
pub(crate) fn get_title(uid: i32) -> Result<String, DownloadResult> {
let url = format!("https://www.meitu131.com/meinv/{}/", uid);
match get_response(&url) {
Ok(value) if !value.is_empty() => {
let body = String::from_utf8_lossy(&value).to_string();
get_html_title(body)
Ok(get_html_title(body))
}
_ => "".to_string(),
Ok(_) => Err(DownloadResult::NetworkError),
Err(RequestError::NotFound) => Err(DownloadResult::Failed),
Err(RequestError::NetworkError) => Err(DownloadResult::NetworkError),
Err(RequestError::OtherError(_)) => Err(DownloadResult::NetworkError),
}
}
/// 从 HTML 内容中提取标题
fn get_html_title(body: String) -> String {
let document = Html::parse_document(&body);
let title_selector = Selector::parse("h1").unwrap();
let title_element = document.select(&title_selector).next().unwrap();
title_element.text().next().unwrap().to_string()
document
.select(&title_selector)
.next()
.and_then(|element| element.text().next())
.unwrap_or("未知标题")
.to_string()
}
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
/// 下载单张图片
///
/// # 参数
/// - `url`: 图片 URL
/// - `file_path`: 保存路径
/// - `retry_times`: 重试次数
///
/// # 返回值
/// - `DownloadResult`: 下载结果
fn download_image(url: &str, file_path: &str, retry_times: i8) -> DownloadResult {
match get_response(url) {
Ok(value) if !value.is_empty() => {
file::write_file(file_path, value).expect("Write file error");
1
file::write_file(file_path, value).expect("❌ 写入文件错误");
DownloadResult::Success
}
Ok(_) => 0, // 空内容
Err(_) if retry_times > 1 => {
Ok(_) => DownloadResult::Failed, // 空内容
Err(RequestError::NotFound) => DownloadResult::Failed, // 404 错误
Err(RequestError::NetworkError) if retry_times > 1 => {
thread::sleep(Duration::from_millis(1000));
download_image(url, file_path, retry_times - 1)
}
Err(_) => 2, // 网络错误
Err(RequestError::OtherError(_)) if retry_times > 1 => {
thread::sleep(Duration::from_millis(1000));
download_image(url, file_path, retry_times - 1)
}
Err(RequestError::NetworkError) => DownloadResult::NetworkError,
Err(RequestError::OtherError(_)) => DownloadResult::NetworkError,
}
}
pub(crate) fn download_images(uid: i32, title: &str) -> bool {
/// 下载相册的所有图片
///
/// # 参数
/// - `uid`: 相册 ID
/// - `title`: 相册标题
///
/// # 返回值
/// - `DownloadResult`: 下载结果
pub(crate) fn download_albums(uid: i32, title: &str) -> DownloadResult {
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let file_content = get_response(&base_url);
let body = match file_content {
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
Err(_e) => "".parse().unwrap(),
// 获取相册主页内容
let body = match get_response(&base_url) {
Ok(value) => String::from_utf8(value.to_vec()).expect("❌ 转换错误"),
Err(RequestError::NotFound) => {
println!("❌ 相册 {} 不存在 (404)", uid);
return DownloadResult::Failed;
}
Err(RequestError::NetworkError) => {
println!("⚠️ 相册 {} 下载时网络波动 (500)", uid);
return DownloadResult::NetworkError;
}
Err(RequestError::OtherError(_)) => {
println!("⚠️ 相册 {} 下载时网络波动 (500)", uid);
return DownloadResult::NetworkError;
}
};
let document = Html::parse_document(&body);
let img_selector = Selector::parse("p[align='center'] img").unwrap();
let mut images = Vec::new();
// 获取总页数
let total_pages = get_total_pages(&document);
// 多线程收集所有图片 URL
let images = collect_image_urls(uid, total_pages, &img_selector, &base_url);
if images.len() < total_pages as usize {
println!("⚠️ 相册 {} 下载时网络波动, 仅下载 {} 页", uid, images.len());
return DownloadResult::NetworkError;
}
// 获取尾页数
let mut total_pages = 1;
// 生成 JSON 文件
generate_json_file(uid, title, &images);
// 多线程下载图片
download_images_multithreaded(uid, &images)
}
/// 获取相册总页数
fn get_total_pages(document: &Html) -> i32 {
let pages_selector = Selector::parse("#pages a").unwrap();
for page_element in document.select(&pages_selector) {
if let Some(text) = page_element.text().next() {
if text == "尾页" {
if let Some(href) = page_element.value().attr("href") {
// 从尾页链接中提取页数
if let Some(page_num_str) = href.split("_").last() {
if let Some(page_num) = page_num_str.split(".").next() {
if let Ok(num) = page_num.parse::<i32>() {
total_pages = num;
return num;
}
}
}
@@ -117,23 +237,25 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool {
}
}
}
1 // 默认只有一页
}
// 多线程收集所有图片 URL
/// 多线程收集图片 URL
fn collect_image_urls(uid: i32, total_pages: i32, img_selector: &Selector, base_url: &str) -> Vec<String> {
let (tx, rx) = mpsc::channel();
let img_selector_arc = Arc::new(img_selector);
let mut page_handles = vec![];
let img_selector_arc = Arc::new(img_selector.clone());
// 创建页面解析线程池
let max_page_threads = 8; // 限制并发页面解析数
let max_page_threads = 8;
for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
let mut page_handles = vec![];
for page in chunk_start..=chunk_end {
let tx = tx.clone();
let img_selector = Arc::clone(&img_selector_arc);
let base_url = base_url.clone();
let uid = uid;
let base_url = base_url.to_string();
let handle = thread::spawn(move || {
let page_url = if page == 1 {
@@ -145,22 +267,26 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool {
let mut page_images = Vec::new();
match get_response(&page_url) {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
} else {
let page_body =
String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
page_images.push(src.to_string());
}
Ok(value) if !value.is_empty() => {
let page_body = String::from_utf8(value.to_vec()).expect("❌ 转换错误");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
page_images.push(src.to_string());
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
Ok(_) => {
println!("⚠️ 页面 {} 内容为空, 跳过", page);
}
Err(RequestError::NotFound) => {
println!("⚠️ 页面 {} 不存在 (404), 跳过", page);
}
Err(RequestError::NetworkError) => {
println!("⚠️ 获取页面 {} 时网络错误, 跳过", page);
}
Err(RequestError::OtherError(e)) => {
println!("⚠️ 获取页面 {} 时错误: {}, 跳过", page, e);
}
}
@@ -170,90 +296,90 @@ pub(crate) fn download_images(uid: i32, title: &str) -> bool {
}
// 等待当前批次完成
for handle in page_handles.drain(..) {
for handle in page_handles {
handle.join().unwrap();
}
}
// 收集所有页面的图片URL,按页码排序
// 收集并排序所有页面的图片 URL
drop(tx);
let mut page_results: Vec<_> = rx.iter().collect();
page_results.sort_by_key(|(page_num, _)| *page_num);
page_results.into_iter()
.flat_map(|(_, page_images)| page_images)
.collect()
}
for (_, page_images) in page_results {
images.extend(page_images);
}
// 生成 JSON 文件
/// 生成 JSON 文件
fn generate_json_file(uid: i32, title: &str, images: &[String]) {
let album = Album {
title: title.to_string(),
images,
images: images.to_vec(),
};
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
let json = serde_json::to_string_pretty(&album).expect("❌ 序列化相册失败");
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
let json_bytes = Bytes::from(json);
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
file::write_file(&json_path, Bytes::from(json)).expect("❌ 写入 JSON 文件失败");
}
// 执行多线程下载
/// 多线程下载图片
fn download_images_multithreaded(uid: i32, images: &[String]) -> DownloadResult {
let (tx, rx) = mpsc::channel();
let images_arc = Arc::new(album.images.clone());
let uid_arc = Arc::new(uid);
let images_arc = Arc::new(images.to_vec());
// 创建线程池(限制并发数避免过多连接)
let max_threads = 8; // 限制并发线程数
let mut handles = vec![];
for chunk_start in (0..album.images.len()).step_by(max_threads) {
let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len());
// 创建线程池
let max_threads = 8;
for chunk_start in (0..images.len()).step_by(max_threads) {
let chunk_end = std::cmp::min(chunk_start + max_threads, images.len());
let mut handles = vec![];
for i in chunk_start..chunk_end {
let tx = tx.clone();
let images = Arc::clone(&images_arc);
let uid = Arc::clone(&uid_arc);
let handle = thread::spawn(move || {
let img_url = &images[i];
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
let result = download_image(img_url, &file_path, 3);
tx.send((i + 1, result)).unwrap();
});
handles.push(handle);
}
// 等待当前批次完成
for handle in handles.drain(..) {
for handle in handles {
handle.join().unwrap();
}
}
// 收集下载结果
drop(tx); // 关闭发送端
let mut successful_downloads = 0;
let mut failed_downloads = 0;
drop(tx);
let (mut successful, mut failed, mut network_errors) = (0, 0, 0);
for (pic_num, result) in rx {
match result {
1 => {
successful_downloads += 1;
DownloadResult::Success => successful += 1,
DownloadResult::Failed => {
println!("❌ 图片 {} 下载失败(空内容或404)", pic_num);
failed += 1;
}
0 => {
println!("图片 {} 下载失败(空内容)", pic_num);
failed_downloads += 1;
DownloadResult::NetworkError => {
println!("⚠️ 图片 {} 下载失败(网络错误)", pic_num);
network_errors += 1;
}
2 => {
println!("图片 {} 下载失败(网络错误)", pic_num);
failed_downloads += 1;
}
_ => {}
}
}
println!(
"下载完成:成功 {} 张,失败 {} 张",
successful_downloads, failed_downloads
);
println!("📊 下载完成:成功 {} 张,失败 {} 张,网络错误 {} 张",
successful, failed, network_errors);
// 返回执行结果:至少成功下载一张图片返回 true,否则返回 false
failed_downloads == 0
// 返回最终结果
if failed == 0 && network_errors == 0 {
DownloadResult::Success
} else if network_errors > 0 {
DownloadResult::NetworkError
} else {
DownloadResult::Failed
}
}