use crate::file; use bytes::Bytes; use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; use std::sync::mpsc; use std::sync::Arc; use std::thread; use std::time::Duration; #[derive(Debug, Serialize, Deserialize)] struct Album { title: String, images: Vec, } fn get_response(url: &str) -> Result { let client = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(30)) .build()?; let response = client .get(url) .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") .send()?; if !response.status().is_success() { return Err(response.error_for_status().unwrap_err().into()); } let content_encoding = response .headers() .get("content-encoding") .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); let bytes = response.bytes()?; if content_encoding.contains("gzip") { use std::io::Read; let mut decoder = flate2::read::GzDecoder::new(&bytes[..]); let mut decompressed = Vec::new(); match decoder.read_to_end(&mut decompressed) { Ok(_) => Ok(Bytes::from(decompressed)), Err(e) => { println!("Gzip decompression failed: {}, using raw data", e); Ok(bytes) } } } else { Ok(bytes) } } pub(crate) fn get_title(uid: i32) -> String { let url = format!("https://www.meitu131.com/meinv/{}/", uid); match get_response(&url) { Ok(value) if !value.is_empty() => { let body = String::from_utf8_lossy(&value).to_string(); get_html_title(body) } _ => "".to_string(), } } fn get_html_title(body: String) -> String { let document = Html::parse_document(&body); let title_selector = Selector::parse("h1").unwrap(); let title_element = document.select(&title_selector).next().unwrap(); title_element.text().next().unwrap().to_string() } fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 { match get_response(url) { Ok(value) if !value.is_empty() => { file::write_file(file_path, value).expect("Write file error"); 1 } Ok(_) => 0, // 空内容 Err(_) if retry_times > 1 => { thread::sleep(Duration::from_millis(1000)); download_image(url, file_path, retry_times - 1) } Err(_) => 2, // 网络错误 } } pub(crate) fn download_images(uid: i32, title: &str) -> bool { let base_url = format!("https://www.meitu131.com/meinv/{}/", uid); let file_content = get_response(&base_url); let body = match file_content { Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"), Err(_e) => "".parse().unwrap(), }; let document = Html::parse_document(&body); let img_selector = Selector::parse("p[align='center'] img").unwrap(); let mut images = Vec::new(); // 获取尾页数 let mut total_pages = 1; let pages_selector = Selector::parse("#pages a").unwrap(); for page_element in document.select(&pages_selector) { if let Some(text) = page_element.text().next() { if text == "尾页" { if let Some(href) = page_element.value().attr("href") { // 从尾页链接中提取页数 if let Some(page_num_str) = href.split("_").last() { if let Some(page_num) = page_num_str.split(".").next() { if let Ok(num) = page_num.parse::() { total_pages = num; } } } } } } } // 多线程收集所有图片 URL let (tx, rx) = mpsc::channel(); let img_selector_arc = Arc::new(img_selector); let mut page_handles = vec![]; // 创建页面解析线程池 let max_page_threads = 8; // 限制并发页面解析数 for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) { let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages); for page in chunk_start..=chunk_end { let tx = tx.clone(); let img_selector = Arc::clone(&img_selector_arc); let base_url = base_url.clone(); let uid = uid; let handle = thread::spawn(move || { let page_url = if page == 1 { base_url } else { format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page) }; let mut page_images = Vec::new(); match get_response(&page_url) { Ok(value) => { if value.is_empty() { println!("Page {} not found, skipping", page); } else { let page_body = String::from_utf8(value.to_vec()).expect("Transforms error"); let page_document = Html::parse_document(&page_body); for img_element in page_document.select(&img_selector) { if let Some(src) = img_element.value().attr("src") { page_images.push(src.to_string()); } } } } Err(e) => { println!("Error fetching page {}: {}, skipping", page, e); } } tx.send((page, page_images)).unwrap(); }); page_handles.push(handle); } // 等待当前批次完成 for handle in page_handles.drain(..) { handle.join().unwrap(); } } // 收集所有页面的图片URL,按页码排序 drop(tx); let mut page_results: Vec<_> = rx.iter().collect(); page_results.sort_by_key(|(page_num, _)| *page_num); for (_, page_images) in page_results { images.extend(page_images); } // 生成 JSON 文件 let album = Album { title: title.to_string(), images, }; let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album"); let json_path = format!("/mnt/share/data/albums/{}/index.json", uid); let json_bytes = Bytes::from(json); file::write_file(&json_path, json_bytes).expect("Failed to write JSON file"); // 执行多线程下载 let (tx, rx) = mpsc::channel(); let images_arc = Arc::new(album.images.clone()); let uid_arc = Arc::new(uid); // 创建线程池(限制并发数避免过多连接) let max_threads = 8; // 限制并发线程数 let mut handles = vec![]; for chunk_start in (0..album.images.len()).step_by(max_threads) { let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len()); for i in chunk_start..chunk_end { let tx = tx.clone(); let images = Arc::clone(&images_arc); let uid = Arc::clone(&uid_arc); let handle = thread::spawn(move || { let img_url = &images[i]; let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1); let result = download_image(img_url, &file_path, 3); tx.send((i + 1, result)).unwrap(); }); handles.push(handle); } // 等待当前批次完成 for handle in handles.drain(..) { handle.join().unwrap(); } } // 收集下载结果 drop(tx); // 关闭发送端 let mut successful_downloads = 0; let mut failed_downloads = 0; for (pic_num, result) in rx { match result { 1 => { successful_downloads += 1; } 0 => { println!("图片 {} 下载失败(空内容)", pic_num); failed_downloads += 1; } 2 => { println!("图片 {} 下载失败(网络错误)", pic_num); failed_downloads += 1; } _ => {} } } println!( "下载完成:成功 {} 张,失败 {} 张", successful_downloads, failed_downloads ); // 返回执行结果:至少成功下载一张图片返回 true,否则返回 false failed_downloads == 0 }