update get_response
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1 +1,2 @@
|
|||||||
/target
|
/target
|
||||||
|
.idea
|
||||||
|
|||||||
@@ -12,4 +12,4 @@ bytes = "1.5.0"
|
|||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
flate2 = "1.0"
|
flate2 = "1.0"
|
||||||
openssl = { version = "0.10", features = ["vendored"] }
|
# openssl = { version = "0.10", features = ["vendored"] }
|
||||||
@@ -1,10 +1,9 @@
|
|||||||
|
use bytes;
|
||||||
|
use bytes::Bytes;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use bytes::Bytes;
|
|
||||||
use bytes;
|
|
||||||
|
|
||||||
|
|
||||||
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
|
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
|
||||||
let path = Path::new(path);
|
let path = Path::new(path);
|
||||||
@@ -26,5 +25,7 @@ pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
|||||||
let mut file = File::open(path)?;
|
let mut file = File::open(path)?;
|
||||||
let mut contents = String::new();
|
let mut contents = String::new();
|
||||||
file.read_to_string(&mut contents)?;
|
file.read_to_string(&mut contents)?;
|
||||||
|
contents = contents.replace("\r\n", "\n");
|
||||||
|
contents = contents.replace("\n", "");
|
||||||
Ok(contents)
|
Ok(contents)
|
||||||
}
|
}
|
||||||
@@ -33,6 +33,6 @@ fn main() {
|
|||||||
if blank_count >= 10 {
|
if blank_count >= 10 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
thread::sleep(Duration::from_millis(50));
|
thread::sleep(Duration::from_millis(100));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,11 @@
|
|||||||
use std::thread;
|
use crate::file;
|
||||||
use std::time::Duration;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use scraper::{Html, Selector};
|
use scraper::{Html, Selector};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use crate::file;
|
use std::sync::mpsc;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct Album {
|
struct Album {
|
||||||
@@ -11,19 +13,21 @@ struct Album {
|
|||||||
images: Vec<String>,
|
images: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder().build()?;
|
||||||
.build()?;
|
|
||||||
|
|
||||||
let response = client
|
let response = client
|
||||||
.get(url)
|
.get(url)
|
||||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
.header(
|
||||||
|
"User-Agent",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
)
|
||||||
.send()?;
|
.send()?;
|
||||||
|
|
||||||
if response.status() == 200 {
|
if response.status() == 200 {
|
||||||
// 先获取头信息,再获取字节数据
|
// 先获取头信息,再获取字节数据
|
||||||
let content_encoding = response.headers()
|
let content_encoding = response
|
||||||
|
.headers()
|
||||||
.get("content-encoding")
|
.get("content-encoding")
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
@@ -49,7 +53,9 @@ fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
|||||||
};
|
};
|
||||||
result
|
result
|
||||||
} else {
|
} else {
|
||||||
Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
|
Err(reqwest::Error::from(
|
||||||
|
response.error_for_status().unwrap_err(),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,7 +84,6 @@ fn get_html_title(body: String) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
||||||
println!("{}", url);
|
|
||||||
let mut flag = 1;
|
let mut flag = 1;
|
||||||
let response = get_response(url);
|
let response = get_response(url);
|
||||||
match response {
|
match response {
|
||||||
@@ -99,6 +104,7 @@ fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn download_images(uid: i32, title: &str) {
|
pub(crate) fn download_images(uid: i32, title: &str) {
|
||||||
|
println!("Downloading images for {} {}...", uid, title);
|
||||||
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
let file_content = get_response(&base_url);
|
let file_content = get_response(&base_url);
|
||||||
let body = match file_content {
|
let body = match file_content {
|
||||||
@@ -130,34 +136,70 @@ pub(crate) fn download_images(uid: i32, title: &str) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 收集所有图片 URL
|
// 多线程收集所有图片 URL
|
||||||
for page in 1..=total_pages {
|
let (tx, rx) = mpsc::channel();
|
||||||
let page_url = if page == 1 {
|
let img_selector_arc = Arc::new(img_selector);
|
||||||
base_url.clone()
|
let mut page_handles = vec![];
|
||||||
} else {
|
|
||||||
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
|
||||||
};
|
|
||||||
|
|
||||||
let page_content = get_response(&page_url);
|
// 创建页面解析线程池
|
||||||
match page_content {
|
let max_page_threads = 8; // 限制并发页面解析数
|
||||||
Ok(value) => {
|
|
||||||
if value.is_empty() {
|
for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
|
||||||
println!("Page {} not found, skipping", page);
|
let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
|
||||||
continue;
|
|
||||||
}
|
for page in chunk_start..=chunk_end {
|
||||||
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
|
let tx = tx.clone();
|
||||||
let page_document = Html::parse_document(&page_body);
|
let img_selector = Arc::clone(&img_selector_arc);
|
||||||
for img_element in page_document.select(&img_selector) {
|
let base_url = base_url.clone();
|
||||||
if let Some(src) = img_element.value().attr("src") {
|
let uid = uid;
|
||||||
images.push(src.to_string());
|
|
||||||
|
let handle = thread::spawn(move || {
|
||||||
|
let page_url = if page == 1 {
|
||||||
|
base_url
|
||||||
|
} else {
|
||||||
|
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut page_images = Vec::new();
|
||||||
|
|
||||||
|
match get_response(&page_url) {
|
||||||
|
Ok(value) => {
|
||||||
|
if value.is_empty() {
|
||||||
|
println!("Page {} not found, skipping", page);
|
||||||
|
} else {
|
||||||
|
let page_body =
|
||||||
|
String::from_utf8(value.to_vec()).expect("Transforms error");
|
||||||
|
let page_document = Html::parse_document(&page_body);
|
||||||
|
for img_element in page_document.select(&img_selector) {
|
||||||
|
if let Some(src) = img_element.value().attr("src") {
|
||||||
|
page_images.push(src.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("Error fetching page {}: {}, skipping", page, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Err(e) => {
|
tx.send((page, page_images)).unwrap();
|
||||||
println!("Error fetching page {}: {}, skipping", page, e);
|
});
|
||||||
continue;
|
page_handles.push(handle);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 等待当前批次完成
|
||||||
|
for handle in page_handles.drain(..) {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 收集所有页面的图片URL,按页码排序
|
||||||
|
drop(tx);
|
||||||
|
let mut page_results: Vec<_> = rx.iter().collect();
|
||||||
|
page_results.sort_by_key(|(page_num, _)| *page_num);
|
||||||
|
|
||||||
|
for (_, page_images) in page_results {
|
||||||
|
images.extend(page_images);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 生成 JSON 文件
|
// 生成 JSON 文件
|
||||||
@@ -170,11 +212,63 @@ pub(crate) fn download_images(uid: i32, title: &str) {
|
|||||||
let json_bytes = Bytes::from(json);
|
let json_bytes = Bytes::from(json);
|
||||||
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
|
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
|
||||||
|
|
||||||
// 执行下载
|
// 执行多线程下载
|
||||||
let mut pic_num = 1;
|
let (tx, rx) = mpsc::channel();
|
||||||
for img_url in &album.images {
|
let images_arc = Arc::new(album.images.clone());
|
||||||
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
|
let uid_arc = Arc::new(uid);
|
||||||
download_image(img_url, &file_path, 3);
|
|
||||||
pic_num += 1;
|
// 创建线程池(限制并发数避免过多连接)
|
||||||
|
let max_threads = 8; // 限制并发线程数
|
||||||
|
let mut handles = vec![];
|
||||||
|
|
||||||
|
for chunk_start in (0..album.images.len()).step_by(max_threads) {
|
||||||
|
let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len());
|
||||||
|
|
||||||
|
for i in chunk_start..chunk_end {
|
||||||
|
let tx = tx.clone();
|
||||||
|
let images = Arc::clone(&images_arc);
|
||||||
|
let uid = Arc::clone(&uid_arc);
|
||||||
|
|
||||||
|
let handle = thread::spawn(move || {
|
||||||
|
let img_url = &images[i];
|
||||||
|
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
|
||||||
|
let result = download_image(img_url, &file_path, 3);
|
||||||
|
|
||||||
|
tx.send((i + 1, result)).unwrap();
|
||||||
|
});
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待当前批次完成
|
||||||
|
for handle in handles.drain(..) {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 收集下载结果
|
||||||
|
drop(tx); // 关闭发送端
|
||||||
|
let mut successful_downloads = 0;
|
||||||
|
let mut failed_downloads = 0;
|
||||||
|
|
||||||
|
for (pic_num, result) in rx {
|
||||||
|
match result {
|
||||||
|
1 => {
|
||||||
|
successful_downloads += 1;
|
||||||
|
}
|
||||||
|
0 => {
|
||||||
|
println!("图片 {} 下载失败(空内容)", pic_num);
|
||||||
|
failed_downloads += 1;
|
||||||
|
}
|
||||||
|
2 => {
|
||||||
|
println!("图片 {} 下载失败(网络错误)", pic_num);
|
||||||
|
failed_downloads += 1;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"下载完成:成功 {} 张,失败 {} 张",
|
||||||
|
successful_downloads, failed_downloads
|
||||||
|
);
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user