Compare commits
5 Commits
a37772d8cd
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e45883130b | ||
|
|
161837bd24 | ||
|
|
6af97ee8ee | ||
|
|
f8134d566c | ||
|
|
4d014143dd |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1 +1,2 @@
|
|||||||
/target
|
/target
|
||||||
|
.idea
|
||||||
|
|||||||
@@ -1,14 +1,15 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "imspyder"
|
name = "imspyder"
|
||||||
version = "1.0.0"
|
version = "1.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = {version = "0.11", features = ["blocking"]}
|
reqwest = { version = "0.11", default-features = false, features = ["blocking","rustls-tls"] }
|
||||||
scraper = "0.18.1"
|
scraper = "0.18.1"
|
||||||
bytes = "1.5.0"
|
bytes = "1.5.0"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
flate2 = "1.0"
|
flate2 = "1.0"
|
||||||
|
# openssl = { version = "0.10", features = ["vendored"] }
|
||||||
44
failed_index
Normal file
44
failed_index
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
6813
|
||||||
|
6814
|
||||||
|
6815
|
||||||
|
6816
|
||||||
|
6817
|
||||||
|
6818
|
||||||
|
6819
|
||||||
|
6820
|
||||||
|
6821
|
||||||
|
6822
|
||||||
|
6823
|
||||||
|
6824
|
||||||
|
6825
|
||||||
|
6826
|
||||||
|
6827
|
||||||
|
6828
|
||||||
|
6829
|
||||||
|
6830
|
||||||
|
6831
|
||||||
|
6832
|
||||||
|
6833
|
||||||
|
6834
|
||||||
|
6835
|
||||||
|
6836
|
||||||
|
6837
|
||||||
|
6838
|
||||||
|
6839
|
||||||
|
6840
|
||||||
|
6841
|
||||||
|
6842
|
||||||
|
6843
|
||||||
|
6844
|
||||||
|
6845
|
||||||
|
6846
|
||||||
|
6847
|
||||||
|
6848
|
||||||
|
6849
|
||||||
|
6850
|
||||||
|
6851
|
||||||
|
6852
|
||||||
|
6853
|
||||||
|
6854
|
||||||
|
6855
|
||||||
|
6856
|
||||||
52
src/file.rs
52
src/file.rs
@@ -1,10 +1,8 @@
|
|||||||
|
use bytes::Bytes;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use bytes::Bytes;
|
|
||||||
use bytes;
|
|
||||||
|
|
||||||
|
|
||||||
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
|
pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Error> {
|
||||||
let path = Path::new(path);
|
let path = Path::new(path);
|
||||||
@@ -19,12 +17,52 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
|
|||||||
|
|
||||||
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
||||||
let path = Path::new(path);
|
let path = Path::new(path);
|
||||||
let folder_path = path.parent();
|
if let Some(parent) = path.parent() {
|
||||||
if !folder_path.unwrap().exists() {
|
if !parent.exists() {
|
||||||
fs::create_dir_all(folder_path.unwrap().as_os_str())?;
|
fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let mut file = File::open(path)?;
|
let mut file = File::open(path)?;
|
||||||
let mut contents = String::new();
|
let mut contents = String::new();
|
||||||
file.read_to_string(&mut contents)?;
|
file.read_to_string(&mut contents)?;
|
||||||
Ok(contents)
|
Ok(contents.trim().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn write_failed_index(idx: i32) {
|
||||||
|
let existing_content = read_file_string("failed_index").unwrap_or_default();
|
||||||
|
|
||||||
|
// 检查是否已存在该ID
|
||||||
|
if !existing_content.lines().any(|line| line.trim() == idx.to_string()) {
|
||||||
|
let new_content = if existing_content.is_empty() {
|
||||||
|
idx.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{}\n{}", existing_content.trim(), idx)
|
||||||
|
};
|
||||||
|
|
||||||
|
write_file("failed_index", Bytes::from(new_content))
|
||||||
|
.unwrap_or_else(|_| panic!("Write failed index error: {}", idx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn read_failed_indexes() -> Vec<i32> {
|
||||||
|
read_file_string("failed_index")
|
||||||
|
.unwrap_or_default()
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| line.trim().parse::<i32>().ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
pub(crate) fn remove_failed_index(idx: i32) {
|
||||||
|
let new_failed_indexes: Vec<String> = read_failed_indexes()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|&failed_idx| failed_idx != idx)
|
||||||
|
.map(|i| i.to_string())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if new_failed_indexes.is_empty() {
|
||||||
|
let _ = std::fs::remove_file("failed_index");
|
||||||
|
} else {
|
||||||
|
let content = new_failed_indexes.join("\n");
|
||||||
|
write_file("failed_index", Bytes::from(content))
|
||||||
|
.unwrap_or_else(|_| panic!("Update failed index error: {}", idx));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
175
src/main.rs
175
src/main.rs
@@ -1,38 +1,169 @@
|
|||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
mod file;
|
mod file;
|
||||||
mod web_parser;
|
mod web_parser;
|
||||||
|
use web_parser::DownloadResult;
|
||||||
|
|
||||||
|
/// 重试下载相册
|
||||||
|
///
|
||||||
|
/// # 参数
|
||||||
|
/// - `idx`: 相册索引
|
||||||
|
/// - `name`: 相册名称
|
||||||
|
/// - `times`: 重试次数
|
||||||
|
/// - `duration`: 重试间隔(秒)
|
||||||
|
///
|
||||||
|
/// # 返回值
|
||||||
|
/// - `DownloadResult`: 下载结果
|
||||||
|
fn retry_download(idx: i32, name: &str, times: i32, duration: u64) -> DownloadResult {
|
||||||
|
match web_parser::download_albums(idx, name) {
|
||||||
|
DownloadResult::Success => {
|
||||||
|
println!("✅ 相册 {} ({}) 下载成功", idx, name);
|
||||||
|
DownloadResult::Success
|
||||||
|
}
|
||||||
|
DownloadResult::NetworkError if times > 0 => {
|
||||||
|
println!("🔄 相册 {} ({}) 网络错误,{}秒后重试", idx, name, duration);
|
||||||
|
thread::sleep(Duration::from_millis(duration * 1000));
|
||||||
|
retry_download(idx, name, times - 1, duration)
|
||||||
|
}
|
||||||
|
DownloadResult::Failed => {
|
||||||
|
println!("❌ 相册 {} ({}) 下载失败", idx, name);
|
||||||
|
DownloadResult::Failed
|
||||||
|
}
|
||||||
|
DownloadResult::NetworkError => {
|
||||||
|
println!("⚠️ 相册 {} ({}) 网络错误,记录到失败列表", idx, name);
|
||||||
|
DownloadResult::NetworkError
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 处理失败的下载记录
|
||||||
|
///
|
||||||
|
/// 从失败列表中读取记录,尝试重新下载
|
||||||
|
fn process_failed_downloads() {
|
||||||
|
let failed_indexes = file::read_failed_indexes();
|
||||||
|
if failed_indexes.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("🔄 发现 {} 个失败的下载记录,开始重试...", failed_indexes.len());
|
||||||
|
|
||||||
|
for failed_idx in failed_indexes {
|
||||||
|
match web_parser::get_title(failed_idx) {
|
||||||
|
Ok(name) => {
|
||||||
|
println!("🔄 重试下载索引 {} 相册 {}...", failed_idx, name);
|
||||||
|
match retry_download(failed_idx, &name, 1, 5) {
|
||||||
|
DownloadResult::Success => {
|
||||||
|
println!("✅ 相册 {} 重试成功", failed_idx);
|
||||||
|
file::remove_failed_index(failed_idx);
|
||||||
|
}
|
||||||
|
_ => println!("❌ 相册 {} 重试失败,保留在失败列表", failed_idx),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(DownloadResult::Failed) => {
|
||||||
|
println!("🗑️ 相册 {} 不存在,从失败列表中移除", failed_idx);
|
||||||
|
file::remove_failed_index(failed_idx);
|
||||||
|
}
|
||||||
|
Err(DownloadResult::NetworkError) => {
|
||||||
|
println!("⚠️ 相册 {} 获取标题时网络错误,保留在失败列表", failed_idx);
|
||||||
|
}
|
||||||
|
Err(DownloadResult::Success) => unreachable!("DownloadResult::Success should not appear in Err branch"),
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(100));
|
||||||
|
}
|
||||||
|
println!("✅ 失败记录处理完成");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 更新索引文件
|
||||||
|
///
|
||||||
|
/// # 参数
|
||||||
|
/// - `idx`: 当前索引值
|
||||||
|
fn update_index(idx: i32) {
|
||||||
|
file::write_file("index", Bytes::from(idx.to_string()))
|
||||||
|
.unwrap_or_else(|_| panic!("❌ 写入索引文件错误: {}", idx));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 处理单个相册下载
|
||||||
|
///
|
||||||
|
/// # 参数
|
||||||
|
/// - `idx`: 相册索引
|
||||||
|
/// - `blank_count`: 空白计数引用
|
||||||
|
///
|
||||||
|
/// # 返回值
|
||||||
|
/// - `bool`: 是否需要继续处理
|
||||||
|
fn process_album(idx: i32, blank_count: &mut i32) -> bool {
|
||||||
|
match web_parser::get_title(idx) {
|
||||||
|
Ok(name) => {
|
||||||
|
println!("🚀 开始下载索引 {} 相册 {}...", idx, name);
|
||||||
|
match retry_download(idx, &name, 3, 15) {
|
||||||
|
DownloadResult::Success => {
|
||||||
|
// 下载成功:索引前进,重置空白计数
|
||||||
|
*blank_count = 0;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
DownloadResult::Failed => {
|
||||||
|
// 下载失败:索引前进,增加空白计数
|
||||||
|
*blank_count += 1;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
DownloadResult::NetworkError => {
|
||||||
|
// 网络错误:记录失败索引,索引前进,重置空白计数
|
||||||
|
file::write_failed_index(idx);
|
||||||
|
*blank_count = 0;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(DownloadResult::Failed) => {
|
||||||
|
// 页面不存在:索引前进,增加空白计数
|
||||||
|
println!("❌ 索引 {} 不存在内容", idx);
|
||||||
|
thread::sleep(Duration::from_millis(3000));
|
||||||
|
*blank_count += 1;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
Err(DownloadResult::NetworkError) => {
|
||||||
|
// 获取标题时网络错误:索引前进,增加空白计数
|
||||||
|
println!("⚠️ 索引 {} 获取标题时网络错误,跳过并记录到失败列表", idx);
|
||||||
|
file::write_failed_index(idx);
|
||||||
|
thread::sleep(Duration::from_millis(3000));
|
||||||
|
*blank_count += 1;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
Err(DownloadResult::Success) => unreachable!("DownloadResult::Success should not appear in Err branch"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let mut idx: i32;
|
// 先处理失败的下载记录
|
||||||
|
process_failed_downloads();
|
||||||
|
|
||||||
|
// 初始化索引和空白计数
|
||||||
|
let mut idx = file::read_file_string("index")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| i32::from_str(&s).ok())
|
||||||
|
.unwrap_or(1);
|
||||||
let mut blank_count = 0;
|
let mut blank_count = 0;
|
||||||
let index = file::read_file_string("index").expect("Read index error");
|
|
||||||
if index.is_empty() {
|
println!("🚀 下载程序启动,从索引 {} 开始...", idx);
|
||||||
idx = 1;
|
|
||||||
} else {
|
// 主下载循环
|
||||||
idx = i32::from_str(&index).expect("Index Corrupted");
|
|
||||||
}
|
|
||||||
loop {
|
loop {
|
||||||
let name = web_parser::get_title(idx);
|
let should_continue = process_album(idx, &mut blank_count);
|
||||||
if name == "" {
|
|
||||||
thread::sleep(Duration::from_millis(3000));
|
if should_continue {
|
||||||
idx += 1;
|
idx += 1;
|
||||||
blank_count += 1;
|
update_index(idx);
|
||||||
} else {
|
}
|
||||||
web_parser::download_images(idx, &name);
|
|
||||||
idx += 1;
|
// 检查是否达到空白计数上限
|
||||||
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
|
if blank_count >= 50 {
|
||||||
|
println!("🔄 连续空白计数达到 {},重新处理失败记录", blank_count);
|
||||||
|
process_failed_downloads();
|
||||||
blank_count = 0;
|
blank_count = 0;
|
||||||
}
|
}
|
||||||
if blank_count >= 10 {
|
|
||||||
break;
|
thread::sleep(Duration::from_millis(100));
|
||||||
}
|
|
||||||
thread::sleep(Duration::from_millis(50));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,127 +1,235 @@
|
|||||||
use std::thread;
|
use crate::file;
|
||||||
use std::time::Duration;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use scraper::{Html, Selector};
|
use scraper::{Html, Selector};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use crate::file;
|
use std::sync::mpsc;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
/// 自定义错误类型,用于区分不同类型的网络错误
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum RequestError {
|
||||||
|
NotFound, // 404 错误
|
||||||
|
NetworkError, // 网络错误
|
||||||
|
OtherError(String), // 其他错误
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<reqwest::Error> for RequestError {
|
||||||
|
fn from(err: reqwest::Error) -> Self {
|
||||||
|
if err.is_status() {
|
||||||
|
if let Some(status) = err.status() {
|
||||||
|
if status == reqwest::StatusCode::NOT_FOUND {
|
||||||
|
return RequestError::NotFound;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RequestError::OtherError(err.to_string())
|
||||||
|
} else if err.is_timeout() || err.is_connect() {
|
||||||
|
RequestError::NetworkError
|
||||||
|
} else {
|
||||||
|
RequestError::OtherError(err.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 相册数据结构
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct Album {
|
struct Album {
|
||||||
title: String,
|
title: String,
|
||||||
images: Vec<String>,
|
images: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 下载结果枚举
|
||||||
|
pub(crate) enum DownloadResult {
|
||||||
|
Success,
|
||||||
|
NetworkError,
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
|
||||||
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
/// 发送 HTTP 请求并获取响应
|
||||||
|
///
|
||||||
|
/// # 参数
|
||||||
|
/// - `url`: 请求的 URL
|
||||||
|
///
|
||||||
|
/// # 返回值
|
||||||
|
/// - `Result<Bytes, RequestError>`: 响应内容或错误
|
||||||
|
fn get_response(url: &str) -> Result<Bytes, RequestError> {
|
||||||
let client = reqwest::blocking::Client::builder()
|
let client = reqwest::blocking::Client::builder()
|
||||||
.build()?;
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.map_err(|_| RequestError::NetworkError)?;
|
||||||
|
|
||||||
let response = client
|
let response = client
|
||||||
.get(url)
|
.get(url)
|
||||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||||
.send()?;
|
.send()
|
||||||
|
.map_err(RequestError::from)?;
|
||||||
|
|
||||||
if response.status() == 200 {
|
// 检查 HTTP 状态码
|
||||||
// 先获取头信息,再获取字节数据
|
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||||||
|
return Err(RequestError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !response.status().is_success() {
|
||||||
|
return Err(RequestError::OtherError(format!(
|
||||||
|
"HTTP {}: {}",
|
||||||
|
response.status(),
|
||||||
|
response.status().canonical_reason().unwrap_or("Unknown")
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 先检查内容编码,再获取 bytes
|
||||||
let content_encoding = response.headers()
|
let content_encoding = response.headers()
|
||||||
.get("content-encoding")
|
.get("content-encoding")
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let bytes = response.bytes()?;
|
let bytes = response.bytes().map_err(RequestError::from)?;
|
||||||
|
|
||||||
|
// 处理 gzip 压缩
|
||||||
|
if content_encoding.contains("gzip") {
|
||||||
|
|
||||||
let result = if content_encoding.contains("gzip") {
|
|
||||||
// 手动解压缩 gzip 数据
|
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
||||||
let mut decompressed = Vec::new();
|
let mut decompressed = Vec::new();
|
||||||
match decoder.read_to_end(&mut decompressed) {
|
match decoder.read_to_end(&mut decompressed) {
|
||||||
Ok(_) => Ok(Bytes::from(decompressed)),
|
Ok(_) => Ok(Bytes::from(decompressed)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// 如果解压缩失败,返回原始字节数据
|
println!("⚠️ Gzip 解压失败: {}, 使用原始数据", e);
|
||||||
println!("Gzip decompression failed: {}, using raw data", e);
|
|
||||||
Ok(bytes)
|
Ok(bytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Ok(bytes)
|
Ok(bytes)
|
||||||
};
|
|
||||||
result
|
|
||||||
} else {
|
|
||||||
Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn get_title(uid: i32) -> String {
|
/// 获取相册标题
|
||||||
let url = "https://www.meitu131.com/meinv/";
|
///
|
||||||
let temp_url = format!("{}{}/", url, uid);
|
/// # 参数
|
||||||
let file_content = get_response(&temp_url);
|
/// - `uid`: 相册 ID
|
||||||
match file_content {
|
///
|
||||||
Ok(value) => {
|
/// # 返回值
|
||||||
if value.is_empty() {
|
/// - `Result<String, DownloadResult>`: 相册标题或错误
|
||||||
return "".to_string();
|
pub(crate) fn get_title(uid: i32) -> Result<String, DownloadResult> {
|
||||||
}
|
let url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
// 直接使用 lossy 转换避免 UTF-8 错误
|
|
||||||
|
match get_response(&url) {
|
||||||
|
Ok(value) if !value.is_empty() => {
|
||||||
let body = String::from_utf8_lossy(&value).to_string();
|
let body = String::from_utf8_lossy(&value).to_string();
|
||||||
get_html_title(body)
|
Ok(get_html_title(body))
|
||||||
}
|
}
|
||||||
Err(_e) => "".to_string(),
|
Ok(_) => Err(DownloadResult::NetworkError),
|
||||||
|
Err(RequestError::NotFound) => Err(DownloadResult::Failed),
|
||||||
|
Err(RequestError::NetworkError) => Err(DownloadResult::NetworkError),
|
||||||
|
Err(RequestError::OtherError(_)) => Err(DownloadResult::NetworkError),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 从 HTML 内容中提取标题
|
||||||
fn get_html_title(body: String) -> String {
|
fn get_html_title(body: String) -> String {
|
||||||
let document = Html::parse_document(&body);
|
let document = Html::parse_document(&body);
|
||||||
let title_selector = Selector::parse("h1").unwrap();
|
let title_selector = Selector::parse("h1").unwrap();
|
||||||
let title_element = document.select(&title_selector).next().unwrap();
|
document
|
||||||
title_element.text().next().unwrap().to_string()
|
.select(&title_selector)
|
||||||
|
.next()
|
||||||
|
.and_then(|element| element.text().next())
|
||||||
|
.unwrap_or("未知标题")
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
/// 下载单张图片
|
||||||
println!("{}", url);
|
///
|
||||||
let mut flag = 1;
|
/// # 参数
|
||||||
let response = get_response(url);
|
/// - `url`: 图片 URL
|
||||||
match response {
|
/// - `file_path`: 保存路径
|
||||||
Ok(value) => {
|
/// - `retry_times`: 重试次数
|
||||||
if value.is_empty() {
|
///
|
||||||
flag = 0
|
/// # 返回值
|
||||||
} else {
|
/// - `DownloadResult`: 下载结果
|
||||||
file::write_file(&file_path, value).expect("Write file error");
|
fn download_image(url: &str, file_path: &str, retry_times: i8) -> DownloadResult {
|
||||||
|
match get_response(url) {
|
||||||
|
Ok(value) if !value.is_empty() => {
|
||||||
|
file::write_file(file_path, value).expect("❌ 写入文件错误");
|
||||||
|
DownloadResult::Success
|
||||||
}
|
}
|
||||||
}
|
Ok(_) => DownloadResult::Failed, // 空内容
|
||||||
Err(_e) => flag = 2,
|
Err(RequestError::NotFound) => DownloadResult::Failed, // 404 错误
|
||||||
}
|
Err(RequestError::NetworkError) if retry_times > 1 => {
|
||||||
if flag == 2 && retry_times > 1 {
|
|
||||||
thread::sleep(Duration::from_millis(1000));
|
thread::sleep(Duration::from_millis(1000));
|
||||||
return download_image(url, file_path, retry_times - 1);
|
download_image(url, file_path, retry_times - 1)
|
||||||
|
}
|
||||||
|
Err(RequestError::OtherError(_)) if retry_times > 1 => {
|
||||||
|
thread::sleep(Duration::from_millis(1000));
|
||||||
|
download_image(url, file_path, retry_times - 1)
|
||||||
|
}
|
||||||
|
Err(RequestError::NetworkError) => DownloadResult::NetworkError,
|
||||||
|
Err(RequestError::OtherError(_)) => DownloadResult::NetworkError,
|
||||||
}
|
}
|
||||||
flag
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn download_images(uid: i32, title: &str) {
|
/// 下载相册的所有图片
|
||||||
|
///
|
||||||
|
/// # 参数
|
||||||
|
/// - `uid`: 相册 ID
|
||||||
|
/// - `title`: 相册标题
|
||||||
|
///
|
||||||
|
/// # 返回值
|
||||||
|
/// - `DownloadResult`: 下载结果
|
||||||
|
pub(crate) fn download_albums(uid: i32, title: &str) -> DownloadResult {
|
||||||
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
let file_content = get_response(&base_url);
|
|
||||||
let body = match file_content {
|
// 获取相册主页内容
|
||||||
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
|
let body = match get_response(&base_url) {
|
||||||
Err(_e) => "".parse().unwrap(),
|
Ok(value) => String::from_utf8(value.to_vec()).expect("❌ 转换错误"),
|
||||||
|
Err(RequestError::NotFound) => {
|
||||||
|
println!("❌ 相册 {} 不存在 (404)", uid);
|
||||||
|
return DownloadResult::Failed;
|
||||||
|
}
|
||||||
|
Err(RequestError::NetworkError) => {
|
||||||
|
println!("⚠️ 相册 {} 下载时网络波动 (500)", uid);
|
||||||
|
return DownloadResult::NetworkError;
|
||||||
|
}
|
||||||
|
Err(RequestError::OtherError(_)) => {
|
||||||
|
println!("⚠️ 相册 {} 下载时网络波动 (500)", uid);
|
||||||
|
return DownloadResult::NetworkError;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let document = Html::parse_document(&body);
|
let document = Html::parse_document(&body);
|
||||||
let img_selector = Selector::parse("p[align='center'] img").unwrap();
|
let img_selector = Selector::parse("p[align='center'] img").unwrap();
|
||||||
let mut images = Vec::new();
|
|
||||||
|
|
||||||
// 获取尾页数
|
// 获取总页数
|
||||||
let mut total_pages = 1;
|
let total_pages = get_total_pages(&document);
|
||||||
|
|
||||||
|
// 多线程收集所有图片 URL
|
||||||
|
let images = collect_image_urls(uid, total_pages, &img_selector, &base_url);
|
||||||
|
|
||||||
|
if images.len() < total_pages as usize {
|
||||||
|
println!("⚠️ 相册 {} 下载时网络波动, 仅下载 {} 页", uid, images.len());
|
||||||
|
return DownloadResult::NetworkError;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 生成 JSON 文件
|
||||||
|
generate_json_file(uid, title, &images);
|
||||||
|
|
||||||
|
// 多线程下载图片
|
||||||
|
download_images_multithreaded(uid, &images)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取相册总页数
|
||||||
|
fn get_total_pages(document: &Html) -> i32 {
|
||||||
let pages_selector = Selector::parse("#pages a").unwrap();
|
let pages_selector = Selector::parse("#pages a").unwrap();
|
||||||
|
|
||||||
for page_element in document.select(&pages_selector) {
|
for page_element in document.select(&pages_selector) {
|
||||||
if let Some(text) = page_element.text().next() {
|
if let Some(text) = page_element.text().next() {
|
||||||
if text == "尾页" {
|
if text == "尾页" {
|
||||||
if let Some(href) = page_element.value().attr("href") {
|
if let Some(href) = page_element.value().attr("href") {
|
||||||
// 从尾页链接中提取页数
|
|
||||||
if let Some(page_num_str) = href.split("_").last() {
|
if let Some(page_num_str) = href.split("_").last() {
|
||||||
if let Some(page_num) = page_num_str.split(".").next() {
|
if let Some(page_num) = page_num_str.split(".").next() {
|
||||||
if let Ok(num) = page_num.parse::<i32>() {
|
if let Ok(num) = page_num.parse::<i32>() {
|
||||||
total_pages = num;
|
return num;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -129,52 +237,149 @@ pub(crate) fn download_images(uid: i32, title: &str) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
1 // 默认只有一页
|
||||||
|
}
|
||||||
|
|
||||||
// 收集所有图片 URL
|
/// 多线程收集图片 URL
|
||||||
for page in 1..=total_pages {
|
fn collect_image_urls(uid: i32, total_pages: i32, img_selector: &Selector, base_url: &str) -> Vec<String> {
|
||||||
|
let (tx, rx) = mpsc::channel();
|
||||||
|
let img_selector_arc = Arc::new(img_selector.clone());
|
||||||
|
|
||||||
|
// 创建页面解析线程池
|
||||||
|
let max_page_threads = 8;
|
||||||
|
|
||||||
|
for chunk_start in (1..=total_pages).step_by(max_page_threads as usize) {
|
||||||
|
let chunk_end = std::cmp::min(chunk_start + max_page_threads as i32 - 1, total_pages);
|
||||||
|
let mut page_handles = vec![];
|
||||||
|
|
||||||
|
for page in chunk_start..=chunk_end {
|
||||||
|
let tx = tx.clone();
|
||||||
|
let img_selector = Arc::clone(&img_selector_arc);
|
||||||
|
let base_url = base_url.to_string();
|
||||||
|
|
||||||
|
let handle = thread::spawn(move || {
|
||||||
let page_url = if page == 1 {
|
let page_url = if page == 1 {
|
||||||
base_url.clone()
|
base_url
|
||||||
} else {
|
} else {
|
||||||
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
||||||
};
|
};
|
||||||
|
|
||||||
let page_content = get_response(&page_url);
|
let mut page_images = Vec::new();
|
||||||
match page_content {
|
|
||||||
Ok(value) => {
|
match get_response(&page_url) {
|
||||||
if value.is_empty() {
|
Ok(value) if !value.is_empty() => {
|
||||||
println!("Page {} not found, skipping", page);
|
let page_body = String::from_utf8(value.to_vec()).expect("❌ 转换错误");
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
|
|
||||||
let page_document = Html::parse_document(&page_body);
|
let page_document = Html::parse_document(&page_body);
|
||||||
for img_element in page_document.select(&img_selector) {
|
for img_element in page_document.select(&img_selector) {
|
||||||
if let Some(src) = img_element.value().attr("src") {
|
if let Some(src) = img_element.value().attr("src") {
|
||||||
images.push(src.to_string());
|
page_images.push(src.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Ok(_) => {
|
||||||
println!("Error fetching page {}: {}, skipping", page, e);
|
println!("⚠️ 页面 {} 内容为空, 跳过", page);
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
Err(RequestError::NotFound) => {
|
||||||
|
println!("⚠️ 页面 {} 不存在 (404), 跳过", page);
|
||||||
|
}
|
||||||
|
Err(RequestError::NetworkError) => {
|
||||||
|
println!("⚠️ 获取页面 {} 时网络错误, 跳过", page);
|
||||||
|
}
|
||||||
|
Err(RequestError::OtherError(e)) => {
|
||||||
|
println!("⚠️ 获取页面 {} 时错误: {}, 跳过", page, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 生成 JSON 文件
|
tx.send((page, page_images)).unwrap();
|
||||||
|
});
|
||||||
|
page_handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待当前批次完成
|
||||||
|
for handle in page_handles {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 收集并排序所有页面的图片 URL
|
||||||
|
drop(tx);
|
||||||
|
let mut page_results: Vec<_> = rx.iter().collect();
|
||||||
|
page_results.sort_by_key(|(page_num, _)| *page_num);
|
||||||
|
|
||||||
|
page_results.into_iter()
|
||||||
|
.flat_map(|(_, page_images)| page_images)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 生成 JSON 文件
|
||||||
|
fn generate_json_file(uid: i32, title: &str, images: &[String]) {
|
||||||
let album = Album {
|
let album = Album {
|
||||||
title: title.to_string(),
|
title: title.to_string(),
|
||||||
images,
|
images: images.to_vec(),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
|
let json = serde_json::to_string_pretty(&album).expect("❌ 序列化相册失败");
|
||||||
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
|
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
|
||||||
let json_bytes = Bytes::from(json);
|
file::write_file(&json_path, Bytes::from(json)).expect("❌ 写入 JSON 文件失败");
|
||||||
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
|
}
|
||||||
|
|
||||||
// 执行下载
|
/// 多线程下载图片
|
||||||
let mut pic_num = 1;
|
fn download_images_multithreaded(uid: i32, images: &[String]) -> DownloadResult {
|
||||||
for img_url in &album.images {
|
let (tx, rx) = mpsc::channel();
|
||||||
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
|
let images_arc = Arc::new(images.to_vec());
|
||||||
download_image(img_url, &file_path, 3);
|
|
||||||
pic_num += 1;
|
// 创建线程池
|
||||||
|
let max_threads = 8;
|
||||||
|
|
||||||
|
for chunk_start in (0..images.len()).step_by(max_threads) {
|
||||||
|
let chunk_end = std::cmp::min(chunk_start + max_threads, images.len());
|
||||||
|
let mut handles = vec![];
|
||||||
|
|
||||||
|
for i in chunk_start..chunk_end {
|
||||||
|
let tx = tx.clone();
|
||||||
|
let images = Arc::clone(&images_arc);
|
||||||
|
|
||||||
|
let handle = thread::spawn(move || {
|
||||||
|
let img_url = &images[i];
|
||||||
|
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, i + 1);
|
||||||
|
let result = download_image(img_url, &file_path, 3);
|
||||||
|
tx.send((i + 1, result)).unwrap();
|
||||||
|
});
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待当前批次完成
|
||||||
|
for handle in handles {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 收集下载结果
|
||||||
|
drop(tx);
|
||||||
|
let (mut successful, mut failed, mut network_errors) = (0, 0, 0);
|
||||||
|
|
||||||
|
for (pic_num, result) in rx {
|
||||||
|
match result {
|
||||||
|
DownloadResult::Success => successful += 1,
|
||||||
|
DownloadResult::Failed => {
|
||||||
|
println!("❌ 图片 {} 下载失败(空内容或404)", pic_num);
|
||||||
|
failed += 1;
|
||||||
|
}
|
||||||
|
DownloadResult::NetworkError => {
|
||||||
|
println!("⚠️ 图片 {} 下载失败(网络错误)", pic_num);
|
||||||
|
network_errors += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("📊 下载完成:成功 {} 张,失败 {} 张,网络错误 {} 张",
|
||||||
|
successful, failed, network_errors);
|
||||||
|
|
||||||
|
// 返回最终结果
|
||||||
|
if failed == 0 && network_errors == 0 {
|
||||||
|
DownloadResult::Success
|
||||||
|
} else if network_errors > 0 {
|
||||||
|
DownloadResult::NetworkError
|
||||||
|
} else {
|
||||||
|
DownloadResult::Failed
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user