update get_response

This commit is contained in:
lychang
2026-03-13 19:03:26 +08:00
parent 4d014143dd
commit f8134d566c
5 changed files with 152 additions and 56 deletions

1
.gitignore vendored
View File

@@ -1 +1,2 @@
/target /target
.idea

View File

@@ -12,4 +12,4 @@ bytes = "1.5.0"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
flate2 = "1.0" flate2 = "1.0"
openssl = { version = "0.10", features = ["vendored"] } # openssl = { version = "0.10", features = ["vendored"] }

View File

@@ -1,10 +1,9 @@
use bytes;
use bytes::Bytes;
use std::fs; use std::fs;
use std::fs::File; use std::fs::File;
use std::io::{Read, Write}; use std::io::{Read, Write};
use std::path::Path; use std::path::Path;
use bytes::Bytes;
use bytes;
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> { pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
let path = Path::new(path); let path = Path::new(path);
@@ -26,5 +25,7 @@ pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
let mut file = File::open(path)?; let mut file = File::open(path)?;
let mut contents = String::new(); let mut contents = String::new();
file.read_to_string(&mut contents)?; file.read_to_string(&mut contents)?;
contents = contents.replace("\r\n", "\n");
contents = contents.replace("\n", "");
Ok(contents) Ok(contents)
} }

View File

@@ -33,6 +33,6 @@ fn main() {
if blank_count >= 10 { if blank_count >= 10 {
break; break;
} }
thread::sleep(Duration::from_millis(50)); thread::sleep(Duration::from_millis(100));
} }
} }

View File

@@ -1,9 +1,11 @@
use std::thread; use crate::file;
use std::time::Duration;
use bytes::Bytes; use bytes::Bytes;
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::file; use std::sync::mpsc;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
struct Album { struct Album {
@@ -11,24 +13,26 @@ struct Album {
images: Vec<String>, images: Vec<String>,
} }
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> { fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder().build()?;
.build()?;
let response = client let response = client
.get(url) .get(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") .header(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
)
.send()?; .send()?;
if response.status() == 200 { if response.status() == 200 {
// 先获取头信息,再获取字节数据 // 先获取头信息,再获取字节数据
let content_encoding = response.headers() let content_encoding = response
.headers()
.get("content-encoding") .get("content-encoding")
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.unwrap_or("") .unwrap_or("")
.to_string(); .to_string();
let bytes = response.bytes()?; let bytes = response.bytes()?;
let result = if content_encoding.contains("gzip") { let result = if content_encoding.contains("gzip") {
@@ -49,7 +53,9 @@ fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
}; };
result result
} else { } else {
Err(reqwest::Error::from(response.error_for_status().unwrap_err())) Err(reqwest::Error::from(
response.error_for_status().unwrap_err(),
))
} }
} }
@@ -78,7 +84,6 @@ fn get_html_title(body: String) -> String {
} }
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
println!("{}", url);
let mut flag = 1; let mut flag = 1;
let response = get_response(url); let response = get_response(url);
match response { match response {
@@ -99,17 +104,18 @@ fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
} }
pub(crate) fn download_images(uid: i32, title: &str) { pub(crate) fn download_images(uid: i32, title: &str) {
println!("Downloading images for {} {}...", uid, title);
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let file_content = get_response(&base_url); let file_content = get_response(&base_url);
let body = match file_content { let body = match file_content {
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"), Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
Err(_e) => "".parse().unwrap(), Err(_e) => "".parse().unwrap(),
}; };
let document = Html::parse_document(&body); let document = Html::parse_document(&body);
let img_selector = Selector::parse("p[align='center'] img").unwrap(); let img_selector = Selector::parse("p[align='center'] img").unwrap();
let mut images = Vec::new(); let mut images = Vec::new();
// 获取尾页数 // 获取尾页数
let mut total_pages = 1; let mut total_pages = 1;
let pages_selector = Selector::parse("#pages a").unwrap(); let pages_selector = Selector::parse("#pages a").unwrap();
@@ -129,37 +135,73 @@ pub(crate) fn download_images(uid: i32, title: &str) {
} }
} }
} }
// 收集所有图片 URL // 多线程收集所有图片 URL
for page in 1..=total_pages { let (tx, rx) = mpsc::channel();
let page_url = if page == 1 { let img_selector_arc = Arc::new(img_selector);
base_url.clone() let mut page_handles = vec![];
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page) // 创建页面解析线程池
}; let max_page_threads = 8; // 限制并发页面解析数
let page_content = get_response(&page_url); for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
match page_content { let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
Ok(value) => {
if value.is_empty() { for page in chunk_start..=chunk_end {
println!("Page {} not found, skipping", page); let tx = tx.clone();
continue; let img_selector = Arc::clone(&img_selector_arc);
} let base_url = base_url.clone();
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error"); let uid = uid;
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) { let handle = thread::spawn(move || {
if let Some(src) = img_element.value().attr("src") { let page_url = if page == 1 {
images.push(src.to_string()); base_url
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
};
let mut page_images = Vec::new();
match get_response(&page_url) {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
} else {
let page_body =
String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
page_images.push(src.to_string());
}
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
} }
} }
}
Err(e) => { tx.send((page, page_images)).unwrap();
println!("Error fetching page {}: {}, skipping", page, e); });
continue; page_handles.push(handle);
} }
// 等待当前批次完成
for handle in page_handles.drain(..) {
handle.join().unwrap();
} }
} }
// 收集所有页面的图片URL,按页码排序
drop(tx);
let mut page_results: Vec<_> = rx.iter().collect();
page_results.sort_by_key(|(page_num, _)| *page_num);
for (_, page_images) in page_results {
images.extend(page_images);
}
// 生成 JSON 文件 // 生成 JSON 文件
let album = Album { let album = Album {
title: title.to_string(), title: title.to_string(),
@@ -169,12 +211,64 @@ pub(crate) fn download_images(uid: i32, title: &str) {
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid); let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
let json_bytes = Bytes::from(json); let json_bytes = Bytes::from(json);
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file"); file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
// 执行下载 // 执行多线程下载
let mut pic_num = 1; let (tx, rx) = mpsc::channel();
for img_url in &album.images { let images_arc = Arc::new(album.images.clone());
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num); let uid_arc = Arc::new(uid);
download_image(img_url, &file_path, 3);
pic_num += 1; // 创建线程池(限制并发数避免过多连接)
let max_threads = 8; // 限制并发线程数
let mut handles = vec![];
for chunk_start in (0..album.images.len()).step_by(max_threads) {
let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len());
for i in chunk_start..chunk_end {
let tx = tx.clone();
let images = Arc::clone(&images_arc);
let uid = Arc::clone(&uid_arc);
let handle = thread::spawn(move || {
let img_url = &images[i];
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
let result = download_image(img_url, &file_path, 3);
tx.send((i + 1, result)).unwrap();
});
handles.push(handle);
}
// 等待当前批次完成
for handle in handles.drain(..) {
handle.join().unwrap();
}
} }
}
// 收集下载结果
drop(tx); // 关闭发送端
let mut successful_downloads = 0;
let mut failed_downloads = 0;
for (pic_num, result) in rx {
match result {
1 => {
successful_downloads += 1;
}
0 => {
println!("图片 {} 下载失败(空内容)", pic_num);
failed_downloads += 1;
}
2 => {
println!("图片 {} 下载失败(网络错误)", pic_num);
failed_downloads += 1;
}
_ => {}
}
}
println!(
"下载完成:成功 {} 张,失败 {} 张",
successful_downloads, failed_downloads
);
}