admin/rust-imspyder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update get_response

Browse Source
This commit is contained in:
lychang
2026-03-13 19:03:26 +08:00
parent 4d014143dd
commit f8134d566c
5 changed files with 152 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1 +1,2 @@
/target
.idea

2
Cargo.toml
View File

@@ -12,4 +12,4 @@ bytes = "1.5.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
flate2 = "1.0"
openssl = { version = "0.10", features = ["vendored"] }
# openssl = { version = "0.10", features = ["vendored"] }

9
src/file.rs
View File

@@ -1,10 +1,9 @@
use bytes;
use bytes::Bytes;
use std::fs;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use bytes::Bytes;
use bytes;
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
let path = Path::new(path);
@@ -26,5 +25,7 @@ pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
let mut file = File::open(path)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
contents = contents.replace("\r\n", "\n");
contents = contents.replace("\n", "");
Ok(contents)
}
}

2
src/main.rs
View File

@@ -33,6 +33,6 @@ fn main() {
if blank_count >= 10 {
break;
}
thread::sleep(Duration::from_millis(50));
thread::sleep(Duration::from_millis(100));
}
}

194
src/web_parser.rs
View File

@@ -1,9 +1,11 @@
use std::thread;
use std::time::Duration;
use crate::file;
use bytes::Bytes;
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use crate::file;
use std::sync::mpsc;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
#[derive(Debug, Serialize, Deserialize)]
struct Album {
@@ -11,24 +13,26 @@ struct Album {
images: Vec<String>,
}
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let client = reqwest::blocking::Client::builder()
.build()?;
let client = reqwest::blocking::Client::builder().build()?;
let response = client
.get(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.header(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
)
.send()?;
if response.status() == 200 {
// 先获取头信息，再获取字节数据
let content_encoding = response.headers()
let content_encoding = response
.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = response.bytes()?;
let result = if content_encoding.contains("gzip") {
@@ -49,7 +53,9 @@ fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
};
result
} else {
Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
Err(reqwest::Error::from(
response.error_for_status().unwrap_err(),
))
}
}
@@ -78,7 +84,6 @@ fn get_html_title(body: String) -> String {
}
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
println!("{}", url);
let mut flag = 1;
let response = get_response(url);
match response {
@@ -99,17 +104,18 @@ fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
}
pub(crate) fn download_images(uid: i32, title: &str) {
println!("Downloading images for {} {}...", uid, title);
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let file_content = get_response(&base_url);
let body = match file_content {
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
Err(_e) => "".parse().unwrap(),
};
let document = Html::parse_document(&body);
let img_selector = Selector::parse("p[align='center'] img").unwrap();
let mut images = Vec::new();
// 获取尾页数
let mut total_pages = 1;
let pages_selector = Selector::parse("#pages a").unwrap();
@@ -129,37 +135,73 @@ pub(crate) fn download_images(uid: i32, title: &str) {
}
}
}
// 收集所有图片 URL
for page in 1..=total_pages {
let page_url = if page == 1 {
base_url.clone()
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
};
let page_content = get_response(&page_url);
match page_content {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
continue;
}
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
images.push(src.to_string());
// 多线程收集所有图片 URL
let (tx, rx) = mpsc::channel();
let img_selector_arc = Arc::new(img_selector);
let mut page_handles = vec![];
// 创建页面解析线程池
let max_page_threads = 8; // 限制并发页面解析数
for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
for page in chunk_start..=chunk_end {
let tx = tx.clone();
let img_selector = Arc::clone(&img_selector_arc);
let base_url = base_url.clone();
let uid = uid;
let handle = thread::spawn(move || {
let page_url = if page == 1 {
base_url
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
};
let mut page_images = Vec::new();
match get_response(&page_url) {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
} else {
let page_body =
String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
page_images.push(src.to_string());
}
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
continue;
}
tx.send((page, page_images)).unwrap();
});
page_handles.push(handle);
}
// 等待当前批次完成
for handle in page_handles.drain(..) {
handle.join().unwrap();
}
}
// 收集所有页面的图片URL，按页码排序
drop(tx);
let mut page_results: Vec<_> = rx.iter().collect();
page_results.sort_by_key(|(page_num, _)| *page_num);
for (_, page_images) in page_results {
images.extend(page_images);
}
// 生成 JSON 文件
let album = Album {
title: title.to_string(),
@@ -169,12 +211,64 @@ pub(crate) fn download_images(uid: i32, title: &str) {
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
let json_bytes = Bytes::from(json);
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
// 执行下载
let mut pic_num = 1;
for img_url in &album.images {
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
download_image(img_url, &file_path, 3);
pic_num += 1;
// 执行多线程下载
let (tx, rx) = mpsc::channel();
let images_arc = Arc::new(album.images.clone());
let uid_arc = Arc::new(uid);
// 创建线程池（限制并发数避免过多连接）
let max_threads = 8; // 限制并发线程数
let mut handles = vec![];
for chunk_start in (0..album.images.len()).step_by(max_threads) {
let chunk_end = std::cmp::min(chunk_start + max_threads, album.images.len());
for i in chunk_start..chunk_end {
let tx = tx.clone();
let images = Arc::clone(&images_arc);
let uid = Arc::clone(&uid_arc);
let handle = thread::spawn(move || {
let img_url = &images[i];
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
let result = download_image(img_url, &file_path, 3);
tx.send((i + 1, result)).unwrap();
});
handles.push(handle);
}
// 等待当前批次完成
for handle in handles.drain(..) {
handle.join().unwrap();
}
}
}
// 收集下载结果
drop(tx); // 关闭发送端
let mut successful_downloads = 0;
let mut failed_downloads = 0;
for (pic_num, result) in rx {
match result {
1 => {
successful_downloads += 1;
}
0 => {
println!("图片 {} 下载失败（空内容）", pic_num);
failed_downloads += 1;
}
2 => {
println!("图片 {} 下载失败（网络错误）", pic_num);
failed_downloads += 1;
}
_ => {}
}
}
println!(
"下载完成：成功 {} 张，失败 {} 张",
successful_downloads, failed_downloads
);
}