259 lines
8.4 KiB
Rust
259 lines
8.4 KiB
Rust
use crate::file;
|
||
use bytes::Bytes;
|
||
use scraper::{Html, Selector};
|
||
use serde::{Deserialize, Serialize};
|
||
use std::sync::mpsc;
|
||
use std::sync::Arc;
|
||
use std::thread;
|
||
use std::time::Duration;
|
||
|
||
#[derive(Debug, Serialize, Deserialize)]
|
||
struct Album {
|
||
title: String,
|
||
images: Vec<String>,
|
||
}
|
||
|
||
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
||
let client = reqwest::blocking::Client::builder()
|
||
.timeout(Duration::from_secs(30))
|
||
.build()?;
|
||
|
||
let response = client
|
||
.get(url)
|
||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||
.send()?;
|
||
|
||
if !response.status().is_success() {
|
||
return Err(response.error_for_status().unwrap_err().into());
|
||
}
|
||
|
||
let content_encoding = response
|
||
.headers()
|
||
.get("content-encoding")
|
||
.and_then(|v| v.to_str().ok())
|
||
.unwrap_or("")
|
||
.to_string();
|
||
|
||
let bytes = response.bytes()?;
|
||
|
||
if content_encoding.contains("gzip") {
|
||
use std::io::Read;
|
||
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
||
let mut decompressed = Vec::new();
|
||
match decoder.read_to_end(&mut decompressed) {
|
||
Ok(_) => Ok(Bytes::from(decompressed)),
|
||
Err(e) => {
|
||
println!("Gzip decompression failed: {}, using raw data", e);
|
||
Ok(bytes)
|
||
}
|
||
}
|
||
} else {
|
||
Ok(bytes)
|
||
}
|
||
}
|
||
|
||
pub(crate) fn get_title(uid: i32) -> String {
|
||
let url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||
|
||
match get_response(&url) {
|
||
Ok(value) if !value.is_empty() => {
|
||
let body = String::from_utf8_lossy(&value).to_string();
|
||
get_html_title(body)
|
||
}
|
||
_ => "".to_string(),
|
||
}
|
||
}
|
||
|
||
fn get_html_title(body: String) -> String {
|
||
let document = Html::parse_document(&body);
|
||
let title_selector = Selector::parse("h1").unwrap();
|
||
let title_element = document.select(&title_selector).next().unwrap();
|
||
title_element.text().next().unwrap().to_string()
|
||
}
|
||
|
||
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
||
match get_response(url) {
|
||
Ok(value) if !value.is_empty() => {
|
||
file::write_file(file_path, value).expect("Write file error");
|
||
1
|
||
}
|
||
Ok(_) => 0, // 空内容
|
||
Err(_) if retry_times > 1 => {
|
||
thread::sleep(Duration::from_millis(1000));
|
||
download_image(url, file_path, retry_times - 1)
|
||
}
|
||
Err(_) => 2, // 网络错误
|
||
}
|
||
}
|
||
|
||
pub(crate) fn download_images(uid: i32, title: &str) -> bool {
|
||
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||
let file_content = get_response(&base_url);
|
||
let body = match file_content {
|
||
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
|
||
Err(_e) => "".parse().unwrap(),
|
||
};
|
||
|
||
let document = Html::parse_document(&body);
|
||
let img_selector = Selector::parse("p[align='center'] img").unwrap();
|
||
let mut images = Vec::new();
|
||
|
||
// 获取尾页数
|
||
let mut total_pages = 1;
|
||
let pages_selector = Selector::parse("#pages a").unwrap();
|
||
for page_element in document.select(&pages_selector) {
|
||
if let Some(text) = page_element.text().next() {
|
||
if text == "尾页" {
|
||
if let Some(href) = page_element.value().attr("href") {
|
||
// 从尾页链接中提取页数
|
||
if let Some(page_num_str) = href.split("_").last() {
|
||
if let Some(page_num) = page_num_str.split(".").next() {
|
||
if let Ok(num) = page_num.parse::<i32>() {
|
||
total_pages = num;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 多线程收集所有图片 URL
|
||
let (tx, rx) = mpsc::channel();
|
||
let img_selector_arc = Arc::new(img_selector);
|
||
let mut page_handles = vec![];
|
||
|
||
// 创建页面解析线程池
|
||
let max_page_threads = 8; // 限制并发页面解析数
|
||
|
||
for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
|
||
let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
|
||
|
||
for page in chunk_start..=chunk_end {
|
||
let tx = tx.clone();
|
||
let img_selector = Arc::clone(&img_selector_arc);
|
||
let base_url = base_url.clone();
|
||
let uid = uid;
|
||
|
||
let handle = thread::spawn(move || {
|
||
let page_url = if page == 1 {
|
||
base_url
|
||
} else {
|
||
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
||
};
|
||
|
||
let mut page_images = Vec::new();
|
||
|
||
match get_response(&page_url) {
|
||
Ok(value) => {
|
||
if value.is_empty() {
|
||
println!("Page {} not found, skipping", page);
|
||
} else {
|
||
let page_body =
|
||
String::from_utf8(value.to_vec()).expect("Transforms error");
|
||
let page_document = Html::parse_document(&page_body);
|
||
for img_element in page_document.select(&img_selector) {
|
||
if let Some(src) = img_element.value().attr("src") {
|
||
page_images.push(src.to_string());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("Error fetching page {}: {}, skipping", page, e);
|
||
}
|
||
}
|
||
|
||
tx.send((page, page_images)).unwrap();
|
||
});
|
||
page_handles.push(handle);
|
||
}
|
||
|
||
// 等待当前批次完成
|
||
for handle in page_handles.drain(..) {
|
||
handle.join().unwrap();
|
||
}
|
||
}
|
||
|
||
// 收集所有页面的图片URL，按页码排序
|
||
drop(tx);
|
||
let mut page_results: Vec<_> = rx.iter().collect();
|
||
page_results.sort_by_key(|(page_num, _)| *page_num);
|
||
|
||
for (_, page_images) in page_results {
|
||
images.extend(page_images);
|
||
}
|
||
|
||
// 生成 JSON 文件
|
||
let album = Album {
|
||
title: title.to_string(),
|
||
images,
|
||
};
|
||
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
|
||
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
|
||
let json_bytes = Bytes::from(json);
|
||
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
|
||
|
||
// 执行多线程下载
|
||
let (tx, rx) = mpsc::channel();
|
||
let images_arc = Arc::new(album.images.clone());
|
||
let uid_arc = Arc::new(uid);
|
||
|
||
// 创建线程池（限制并发数避免过多连接）
|
||
let max_threads = 8; // 限制并发线程数
|
||
let mut handles = vec![];
|
||
|
||
for chunk_start in (0..album.images.len()).step_by(max_threads) {
|
||
let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len());
|
||
|
||
for i in chunk_start..chunk_end {
|
||
let tx = tx.clone();
|
||
let images = Arc::clone(&images_arc);
|
||
let uid = Arc::clone(&uid_arc);
|
||
|
||
let handle = thread::spawn(move || {
|
||
let img_url = &images[i];
|
||
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
|
||
let result = download_image(img_url, &file_path, 3);
|
||
|
||
tx.send((i + 1, result)).unwrap();
|
||
});
|
||
handles.push(handle);
|
||
}
|
||
|
||
// 等待当前批次完成
|
||
for handle in handles.drain(..) {
|
||
handle.join().unwrap();
|
||
}
|
||
}
|
||
|
||
// 收集下载结果
|
||
drop(tx); // 关闭发送端
|
||
let mut successful_downloads = 0;
|
||
let mut failed_downloads = 0;
|
||
|
||
for (pic_num, result) in rx {
|
||
match result {
|
||
1 => {
|
||
successful_downloads += 1;
|
||
}
|
||
0 => {
|
||
println!("图片 {} 下载失败（空内容）", pic_num);
|
||
failed_downloads += 1;
|
||
}
|
||
2 => {
|
||
println!("图片 {} 下载失败（网络错误）", pic_num);
|
||
failed_downloads += 1;
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
println!(
|
||
"下载完成：成功 {} 张，失败 {} 张",
|
||
successful_downloads, failed_downloads
|
||
);
|
||
|
||
// 返回执行结果：至少成功下载一张图片返回 true，否则返回 false
|
||
failed_downloads == 0
|
||
} |