Compare commits

..

10 Commits

Author SHA1 Message Date
lychang
a37772d8cd update get_response 2026-03-10 01:22:47 +08:00
lychang
abc601661e update get_response 2024-01-12 22:28:13 +08:00
lychang
91c17d3a87 update get_response 2024-01-12 13:46:30 +08:00
lychang
195815564c update get_response 2024-01-12 13:44:58 +08:00
lychang
f3b265d330 update get_response 2024-01-12 12:25:38 +08:00
lychang
6c594bc461 update get_response 2024-01-12 12:20:44 +08:00
lychang
89237f3522 update code 2024-01-12 06:38:54 +08:00
lychang
260fe25aaf update code 2024-01-12 05:07:17 +08:00
lychang
71ebeb9c01 update code 2024-01-12 05:01:50 +08:00
lychang
6073607d24 update code 2024-01-12 04:54:22 +08:00
7 changed files with 212 additions and 1636 deletions

View File

@@ -4,13 +4,11 @@ stages:
- notify - notify
variables: variables:
BUILD_NAME: "rust/tuiimg" BUILD_NAME: "rust/imspyder"
SERVICE_NAME: "spyder" SERVICE_NAME: "imspyder"
IMAGE_TAG: "dev" IMAGE_TAG: "dev"
generate_image_dev: generate_image_dev:
variables:
IMAGE_TAG: dev
stage: generate_image stage: generate_image
image: docker:20.10.2 image: docker:20.10.2
script: script:
@@ -22,3 +20,22 @@ generate_image_dev:
except: except:
- triggers - triggers
deploy_dev:
variables:
CACHE_PATH: /mnt/data/cloud/cache
IMAGES_PATH: /mnt/data/cloud/images
stage: deploy
image: docker:20.10.2
script:
- CONTAINER_NAME=$(docker ps -aq --filter name=$SERVICE_NAME)
- echo $CONTAINER_NAME
- if [[ -n "$CONTAINER_NAME" ]]; then
docker rm -f $CONTAINER_NAME;
fi
- docker run -d --name $SERVICE_NAME -v $CACHE_PATH:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG
tags:
- '001'
only:
- dev
except:
- triggers

1563
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -8,4 +8,7 @@ edition = "2021"
[dependencies] [dependencies]
reqwest = {version = "0.11", features = ["blocking"]} reqwest = {version = "0.11", features = ["blocking"]}
scraper = "0.18.1" scraper = "0.18.1"
bytes = "1.5.0" bytes = "1.5.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
flate2 = "1.0"

View File

@@ -1,10 +1,9 @@
FROM rust:1.75 as builder FROM rust:1.75 as builder
WORKDIR /app WORKDIR /app
ARG APP=imspyder
RUN USER=root cargo new $APP RUN USER=root cargo new imspyder
WORKDIR /app/$APP WORKDIR /app/imspyder
RUN mkdir -p .cargo RUN mkdir -p .cargo
COPY config.toml .cargo/ COPY config.toml .cargo/
COPY Cargo.toml Cargo.lock ./ COPY Cargo.toml Cargo.lock ./
@@ -13,11 +12,12 @@ COPY ./src src
RUN cargo install --path . --color always RUN cargo install --path . --color always
# Copy the app to an base Docker image, here we use distroless # Copy the app to an base Docker image, here we use distroless
FROM alpine:latest FROM ubuntu:latest
ARG APP=imspyder
RUN apt-get update
RUN apt-get install -y ca-certificates
RUN update-ca-certificates
RUN mkdir -p /cache RUN mkdir -p /cache
COPY --from=builder /usr/local/cargo/bin/$APP /$APP COPY --from=builder /usr/local/cargo/bin/imspyder /imspyder
USER 1000 USER 1000
ENTRYPOINT ["/$APP"] ENTRYPOINT ["/imspyder"]

View File

@@ -18,6 +18,11 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
} }
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> { pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
let path = Path::new(path);
let folder_path = path.parent();
if !folder_path.unwrap().exists() {
fs::create_dir_all(folder_path.unwrap().as_os_str())?;
}
let mut file = File::open(path)?; let mut file = File::open(path)?;
let mut contents = String::new(); let mut contents = String::new();
file.read_to_string(&mut contents)?; file.read_to_string(&mut contents)?;

View File

@@ -10,23 +10,29 @@ mod web_parser;
fn main() { fn main() {
let mut continue_flag = true;
let mut idx: i32; let mut idx: i32;
let index = file::read_file_string("cache/index").expect("Read index error"); let mut blank_count = 0;
let index = file::read_file_string("index").expect("Read index error");
if index.is_empty() { if index.is_empty() {
idx = 1; idx = 1;
} else { } else {
idx = i32::from_str(&index).expect("Index Corrupted"); idx = i32::from_str(&index).expect("Index Corrupted");
} }
while continue_flag { loop {
let name = web_parser::get_title(idx); let name = web_parser::get_title(idx);
if name == "" { if name == "" {
continue_flag = false; thread::sleep(Duration::from_millis(3000));
idx += 1;
blank_count += 1;
} else { } else {
web_parser::download_images(idx, &name); web_parser::download_images(idx, &name);
idx += 1; idx += 1;
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
blank_count = 0;
} }
thread::sleep(Duration::from_millis(300)); if blank_count >= 10 {
break;
}
thread::sleep(Duration::from_millis(50));
} }
file::write_file("cache/index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
} }

View File

@@ -1,72 +1,180 @@
use std::thread;
use std::time::Duration;
use bytes::Bytes; use bytes::Bytes;
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use crate::file; use crate::file;
#[derive(Debug, Serialize, Deserialize)]
struct Album {
title: String,
images: Vec<String>,
}
fn get_response(url: &str) -> Bytes {
let response = reqwest::blocking::get(url).expect("Could not load url."); fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
let client = reqwest::blocking::Client::builder()
.build()?;
let response = client
.get(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.send()?;
if response.status() == 200 { if response.status() == 200 {
let body = response.bytes().unwrap(); // 先获取头信息,再获取字节数据
return body; let content_encoding = response.headers()
.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = response.bytes()?;
let result = if content_encoding.contains("gzip") {
// 手动解压缩 gzip 数据
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
let mut decompressed = Vec::new();
match decoder.read_to_end(&mut decompressed) {
Ok(_) => Ok(Bytes::from(decompressed)),
Err(e) => {
// 如果解压缩失败,返回原始字节数据
println!("Gzip decompression failed: {}, using raw data", e);
Ok(bytes)
}
}
} else {
Ok(bytes)
};
result
} else { } else {
return Bytes::new(); Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
} }
} }
pub(crate) fn get_html_title(body: String) -> String {
let document = Html::parse_document(&body);
let title_selector = Selector::parse("h1").unwrap();
let title_element = document.select(&title_selector).next().unwrap();
return title_element.text().next().unwrap().to_string();
}
pub(crate) fn download_image(url_path: &str, file_path: &str) -> bool {
let url = format!("https://i.tuiimg.net/{}", url_path);
println!("{}", url);
let response = get_response(&url);
if response.is_empty() {
return false;
}
let file_path = format!("images/{}", file_path);
file::write_file(&file_path, response).expect("Write file error");
return true;
}
pub(crate) fn get_title(uid: i32) -> String { pub(crate) fn get_title(uid: i32) -> String {
let body; let url = "https://www.meitu131.com/meinv/";
let url = "https://www.tuiimg.com/meinv/";
let temp_url = format!("{}{}/", url, uid); let temp_url = format!("{}{}/", url, uid);
let response = reqwest::blocking::get(temp_url).expect("Could not load url."); let file_content = get_response(&temp_url);
let status_code = response.status(); match file_content {
return if status_code != 200 { Ok(value) => {
"".parse().unwrap() if value.is_empty() {
} else { return "".to_string();
body = response.text().unwrap(); }
let name = get_html_title(body); // 直接使用 lossy 转换避免 UTF-8 错误
name let body = String::from_utf8_lossy(&value).to_string();
}; get_html_title(body)
}
Err(_e) => "".to_string(),
}
}
fn get_html_title(body: String) -> String {
let document = Html::parse_document(&body);
let title_selector = Selector::parse("h1").unwrap();
let title_element = document.select(&title_selector).next().unwrap();
title_element.text().next().unwrap().to_string()
}
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
println!("{}", url);
let mut flag = 1;
let response = get_response(url);
match response {
Ok(value) => {
if value.is_empty() {
flag = 0
} else {
file::write_file(&file_path, value).expect("Write file error");
}
}
Err(_e) => flag = 2,
}
if flag == 2 && retry_times > 1 {
thread::sleep(Duration::from_millis(1000));
return download_image(url, file_path, retry_times - 1);
}
flag
} }
pub(crate) fn download_images(uid: i32, title: &str) { pub(crate) fn download_images(uid: i32, title: &str) {
let page: i32 = uid / 500 + 1; let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
let page_str; let file_content = get_response(&base_url);
if page < 10 { let body = match file_content {
page_str = format!("00{}", page); Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
} else if page < 100 { Err(_e) => "".parse().unwrap(),
page_str = format!("0{}", page); };
} else {
page_str = format!("{}", page); let document = Html::parse_document(&body);
} let img_selector = Selector::parse("p[align='center'] img").unwrap();
let mut continue_flag = true; let mut images = Vec::new();
let mut image_path;
let mut file_path; // 获取尾页数
let mut pic_num = 1; let mut total_pages = 1;
while continue_flag { let pages_selector = Selector::parse("#pages a").unwrap();
image_path = format!("{}/{}/{}.jpg", page_str, uid, pic_num); for page_element in document.select(&pages_selector) {
file_path = format!("{}/{}.jpg", title, pic_num); if let Some(text) = page_element.text().next() {
if !download_image(&image_path, &file_path) { if text == "尾页" {
continue_flag = false; if let Some(href) = page_element.value().attr("href") {
// 从尾页链接中提取页数
if let Some(page_num_str) = href.split("_").last() {
if let Some(page_num) = page_num_str.split(".").next() {
if let Ok(num) = page_num.parse::<i32>() {
total_pages = num;
}
}
}
}
}
} }
}
// 收集所有图片 URL
for page in 1..=total_pages {
let page_url = if page == 1 {
base_url.clone()
} else {
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
};
let page_content = get_response(&page_url);
match page_content {
Ok(value) => {
if value.is_empty() {
println!("Page {} not found, skipping", page);
continue;
}
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
let page_document = Html::parse_document(&page_body);
for img_element in page_document.select(&img_selector) {
if let Some(src) = img_element.value().attr("src") {
images.push(src.to_string());
}
}
}
Err(e) => {
println!("Error fetching page {}: {}, skipping", page, e);
continue;
}
}
}
// 生成 JSON 文件
let album = Album {
title: title.to_string(),
images,
};
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
let json_bytes = Bytes::from(json);
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
// 执行下载
let mut pic_num = 1;
for img_url in &album.images {
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
download_image(img_url, &file_path, 3);
pic_num += 1; pic_num += 1;
} }
} }