update get_response

This commit is contained in:
lychang
2026-03-10 01:22:47 +08:00
parent abc601661e
commit a37772d8cd
4 changed files with 151 additions and 1607 deletions

1563
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -8,4 +8,7 @@ edition = "2021"
[dependencies] [dependencies]
reqwest = {version = "0.11", features = ["blocking"]} reqwest = {version = "0.11", features = ["blocking"]}
scraper = "0.18.1" scraper = "0.18.1"
bytes = "1.5.0" bytes = "1.5.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
flate2 = "1.0"

View File

@@ -10,23 +10,29 @@ mod web_parser;
fn main() { fn main() {
let mut continue_flag = true;
let mut idx: i32; let mut idx: i32;
let index = file::read_file_string("cache/index").expect("Read index error"); let mut blank_count = 0;
let index = file::read_file_string("index").expect("Read index error");
if index.is_empty() { if index.is_empty() {
idx = 1; idx = 1;
} else { } else {
idx = i32::from_str(&index).expect("Index Corrupted"); idx = i32::from_str(&index).expect("Index Corrupted");
} }
while continue_flag { loop {
let name = web_parser::get_title(idx); let name = web_parser::get_title(idx);
if name == "" { if name == "" {
continue_flag = false; thread::sleep(Duration::from_millis(3000));
idx += 1;
blank_count += 1;
} else { } else {
web_parser::download_images(idx, &name); web_parser::download_images(idx, &name);
idx += 1; idx += 1;
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
blank_count = 0;
} }
file::write_file("cache/index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx)); if blank_count >= 10 {
thread::sleep(Duration::from_millis(300)); break;
}
thread::sleep(Duration::from_millis(50));
} }
} }

View File

@@ -2,81 +2,179 @@ use std::thread;
use std::time::Duration; use std::time::Duration;
use bytes::Bytes; use bytes::Bytes;
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use crate::file; use crate::file;
#[derive(Debug, Serialize, Deserialize)]
struct Album {
title: String,
images: Vec<String>,
}
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> { fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let response = reqwest::blocking::get(url)?; let client = reqwest::blocking::Client::builder()
let mut body = Bytes::new(); .build()?;
let response = client
.get(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.send()?;
if response.status() == 200 { if response.status() == 200 {
body = response.bytes().unwrap() // 先获取头信息,再获取字节数据
let content_encoding = response.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = response.bytes()?;
let result = if content_encoding.contains("gzip") {
// 手动解压缩 gzip 数据
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
let mut decompressed = Vec::new();
match decoder.read_to_end(&mut decompressed) {
Ok(_) => Ok(Bytes::from(decompressed)),
Err(e) => {
// 如果解压缩失败,返回原始字节数据
println!("Gzip decompression failed: {}, using raw data", e);
Ok(bytes)
}
}
} else {
Ok(bytes)
};
result
} else {
Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
} }
Ok(body)
} }
pub(crate) fn get_title(uid: i32) -> String { pub(crate) fn get_title(uid: i32) -> String {
let body; let url = "https://www.meitu131.com/meinv/";
let url = "https://www.tuiimg.com/meinv/";
let temp_url = format!("{}{}/", url, uid); let temp_url = format!("{}{}/", url, uid);
let file_content = get_response(&temp_url); let file_content = get_response(&temp_url);
match file_content { match file_content {
Ok(value) => body = String::from_utf8(value.to_vec()).expect("Transforms error"), Ok(value) => {
Err(_e) => body = "".parse().unwrap(), if value.is_empty() {
return "".to_string();
}
// 直接使用 lossy 转换避免 UTF-8 错误
let body = String::from_utf8_lossy(&value).to_string();
get_html_title(body)
}
Err(_e) => "".to_string(),
} }
return get_html_title(body);
} }
fn get_html_title(body: String) -> String { fn get_html_title(body: String) -> String {
let document = Html::parse_document(&body); let document = Html::parse_document(&body);
let title_selector = Selector::parse("h1").unwrap(); let title_selector = Selector::parse("h1").unwrap();
let title_element = document.select(&title_selector).next().unwrap(); let title_element = document.select(&title_selector).next().unwrap();
return title_element.text().next().unwrap().to_string(); title_element.text().next().unwrap().to_string()
} }
fn download_image(url_path: &str, file_path: &str, retry_times: i8) -> i8 { fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
let url = format!("https://i.tuiimg.net/{}", url_path);
println!("{}", url); println!("{}", url);
let mut flag = 1; let mut flag = 1;
let response = get_response(&url); let response = get_response(url);
match response { match response {
Ok(value) => { Ok(value) => {
if value.is_empty() { if value.is_empty() {
flag = 0 flag = 0
} else { } else {
let file_path = format!("images/{}", file_path);
file::write_file(&file_path, value).expect("Write file error"); file::write_file(&file_path, value).expect("Write file error");
} }
} }
Err(_e) => flag = 2, Err(_e) => flag = 2,
} }
if flag == 2 && retry_times > 1 { if flag == 2 && retry_times > 1 {
thread::sleep(Duration::from_millis(5000)); thread::sleep(Duration::from_millis(1000));
return download_image(url_path, file_path, retry_times - 1); return download_image(url, file_path, retry_times - 1);
} }
return flag; flag
} }
pub(crate) fn download_images(uid: i32, title: &str) { pub(crate) fn download_images(uid: i32, title: &str) {
let page: i32 = uid / 500 + 1; let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let page_str; let file_content = get_response(&base_url);
if page < 10 { let body = match file_content {
page_str = format!("00{}", page); Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
} else if page < 100 { Err(_e) => "".parse().unwrap(),
page_str = format!("0{}", page); };
} else {
page_str = format!("{}", page); let document = Html::parse_document(&body);
} let img_selector = Selector::parse("p[align='center'] img").unwrap();
let mut continue_flag = true; let mut images = Vec::new();
let mut image_path;
let mut file_path; // 获取尾页数
let mut pic_num = 1; let mut total_pages = 1;
while continue_flag { let pages_selector = Selector::parse("#pages a").unwrap();
image_path = format!("{}/{}/{}.jpg", page_str, uid, pic_num); for page_element in document.select(&pages_selector) {
file_path = format!("{}/{}.jpg", title, pic_num); if let Some(text) = page_element.text().next() {
let status_code = download_image(&image_path, &file_path, 3); if text == "尾页" {
if status_code == 0 { if let Some(href) = page_element.value().attr("href") {
continue_flag = false; // 从尾页链接中提取页数
if let Some(page_num_str) = href.split("_").last() {
if let Some(page_num) = page_num_str.split(".").next() {
if let Ok(num) = page_num.parse::<i32>() {
total_pages = num;
}
}
}
}
}
} }
}
// 收集所有图片 URL
for page in 1..=total_pages {
let page_url = if page == 1 {
base_url.clone()
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
};
let page_content = get_response(&page_url);
match page_content {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
continue;
}
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
images.push(src.to_string());
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
continue;
}
}
}
// 生成 JSON 文件
let album = Album {
title: title.to_string(),
images,
};
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
let json_bytes = Bytes::from(json);
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
// 执行下载
let mut pic_num = 1;
for img_url in &album.images {
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
download_image(img_url, &file_path, 3);
pic_num += 1; pic_num += 1;
} }
} }