Compare commits
10 Commits
9c0c61a96a
...
a37772d8cd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a37772d8cd | ||
|
|
abc601661e | ||
|
|
91c17d3a87 | ||
|
|
195815564c | ||
|
|
f3b265d330 | ||
|
|
6c594bc461 | ||
|
|
89237f3522 | ||
|
|
260fe25aaf | ||
|
|
71ebeb9c01 | ||
|
|
6073607d24 |
@@ -4,13 +4,11 @@ stages:
|
|||||||
- notify
|
- notify
|
||||||
|
|
||||||
variables:
|
variables:
|
||||||
BUILD_NAME: "rust/tuiimg"
|
BUILD_NAME: "rust/imspyder"
|
||||||
SERVICE_NAME: "spyder"
|
SERVICE_NAME: "imspyder"
|
||||||
IMAGE_TAG: "dev"
|
IMAGE_TAG: "dev"
|
||||||
|
|
||||||
generate_image_dev:
|
generate_image_dev:
|
||||||
variables:
|
|
||||||
IMAGE_TAG: dev
|
|
||||||
stage: generate_image
|
stage: generate_image
|
||||||
image: docker:20.10.2
|
image: docker:20.10.2
|
||||||
script:
|
script:
|
||||||
@@ -22,3 +20,22 @@ generate_image_dev:
|
|||||||
except:
|
except:
|
||||||
- triggers
|
- triggers
|
||||||
|
|
||||||
|
deploy_dev:
|
||||||
|
variables:
|
||||||
|
CACHE_PATH: /mnt/data/cloud/cache
|
||||||
|
IMAGES_PATH: /mnt/data/cloud/images
|
||||||
|
stage: deploy
|
||||||
|
image: docker:20.10.2
|
||||||
|
script:
|
||||||
|
- CONTAINER_NAME=$(docker ps -aq --filter name=$SERVICE_NAME)
|
||||||
|
- echo $CONTAINER_NAME
|
||||||
|
- if [[ -n "$CONTAINER_NAME" ]]; then
|
||||||
|
docker rm -f $CONTAINER_NAME;
|
||||||
|
fi
|
||||||
|
- docker run -d --name $SERVICE_NAME -v $CACHE_PATH:/cache -v $IMAGES_PATH:/images $BUILD_NAME:$IMAGE_TAG
|
||||||
|
tags:
|
||||||
|
- '001'
|
||||||
|
only:
|
||||||
|
- dev
|
||||||
|
except:
|
||||||
|
- triggers
|
||||||
1563
Cargo.lock
generated
1563
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -9,3 +9,6 @@ edition = "2021"
|
|||||||
reqwest = {version = "0.11", features = ["blocking"]}
|
reqwest = {version = "0.11", features = ["blocking"]}
|
||||||
scraper = "0.18.1"
|
scraper = "0.18.1"
|
||||||
bytes = "1.5.0"
|
bytes = "1.5.0"
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
|
flate2 = "1.0"
|
||||||
16
Dockerfile
16
Dockerfile
@@ -1,10 +1,9 @@
|
|||||||
FROM rust:1.75 as builder
|
FROM rust:1.75 as builder
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
ARG APP=imspyder
|
|
||||||
|
|
||||||
RUN USER=root cargo new $APP
|
RUN USER=root cargo new imspyder
|
||||||
WORKDIR /app/$APP
|
WORKDIR /app/imspyder
|
||||||
RUN mkdir -p .cargo
|
RUN mkdir -p .cargo
|
||||||
COPY config.toml .cargo/
|
COPY config.toml .cargo/
|
||||||
COPY Cargo.toml Cargo.lock ./
|
COPY Cargo.toml Cargo.lock ./
|
||||||
@@ -13,11 +12,12 @@ COPY ./src src
|
|||||||
RUN cargo install --path . --color always
|
RUN cargo install --path . --color always
|
||||||
|
|
||||||
# Copy the app to an base Docker image, here we use distroless
|
# Copy the app to an base Docker image, here we use distroless
|
||||||
FROM alpine:latest
|
FROM ubuntu:latest
|
||||||
|
|
||||||
ARG APP=imspyder
|
|
||||||
|
|
||||||
|
RUN apt-get update
|
||||||
|
RUN apt-get install -y ca-certificates
|
||||||
|
RUN update-ca-certificates
|
||||||
RUN mkdir -p /cache
|
RUN mkdir -p /cache
|
||||||
COPY --from=builder /usr/local/cargo/bin/$APP /$APP
|
COPY --from=builder /usr/local/cargo/bin/imspyder /imspyder
|
||||||
USER 1000
|
USER 1000
|
||||||
ENTRYPOINT ["/$APP"]
|
ENTRYPOINT ["/imspyder"]
|
||||||
@@ -18,6 +18,11 @@ pub(crate) fn write_file(path: &str, content: Bytes) -> Result<(), std::io::Erro
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
pub(crate) fn read_file_string(path: &str) -> Result<String, std::io::Error> {
|
||||||
|
let path = Path::new(path);
|
||||||
|
let folder_path = path.parent();
|
||||||
|
if !folder_path.unwrap().exists() {
|
||||||
|
fs::create_dir_all(folder_path.unwrap().as_os_str())?;
|
||||||
|
}
|
||||||
let mut file = File::open(path)?;
|
let mut file = File::open(path)?;
|
||||||
let mut contents = String::new();
|
let mut contents = String::new();
|
||||||
file.read_to_string(&mut contents)?;
|
file.read_to_string(&mut contents)?;
|
||||||
|
|||||||
18
src/main.rs
18
src/main.rs
@@ -10,23 +10,29 @@ mod web_parser;
|
|||||||
|
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let mut continue_flag = true;
|
|
||||||
let mut idx: i32;
|
let mut idx: i32;
|
||||||
let index = file::read_file_string("cache/index").expect("Read index error");
|
let mut blank_count = 0;
|
||||||
|
let index = file::read_file_string("index").expect("Read index error");
|
||||||
if index.is_empty() {
|
if index.is_empty() {
|
||||||
idx = 1;
|
idx = 1;
|
||||||
} else {
|
} else {
|
||||||
idx = i32::from_str(&index).expect("Index Corrupted");
|
idx = i32::from_str(&index).expect("Index Corrupted");
|
||||||
}
|
}
|
||||||
while continue_flag {
|
loop {
|
||||||
let name = web_parser::get_title(idx);
|
let name = web_parser::get_title(idx);
|
||||||
if name == "" {
|
if name == "" {
|
||||||
continue_flag = false;
|
thread::sleep(Duration::from_millis(3000));
|
||||||
|
idx += 1;
|
||||||
|
blank_count += 1;
|
||||||
} else {
|
} else {
|
||||||
web_parser::download_images(idx, &name);
|
web_parser::download_images(idx, &name);
|
||||||
idx += 1;
|
idx += 1;
|
||||||
|
file::write_file("index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
|
||||||
|
blank_count = 0;
|
||||||
}
|
}
|
||||||
thread::sleep(Duration::from_millis(300));
|
if blank_count >= 10 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(50));
|
||||||
}
|
}
|
||||||
file::write_file("cache/index", Bytes::from(idx.to_string())).expect(&format!("Write index error: {}", idx));
|
|
||||||
}
|
}
|
||||||
@@ -1,72 +1,180 @@
|
|||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use scraper::{Html, Selector};
|
use scraper::{Html, Selector};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use crate::file;
|
use crate::file;
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct Album {
|
||||||
|
title: String,
|
||||||
|
images: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
fn get_response(url: &str) -> Result<Bytes, reqwest::Error> {
|
||||||
|
let client = reqwest::blocking::Client::builder()
|
||||||
|
.build()?;
|
||||||
|
|
||||||
|
let response = client
|
||||||
|
.get(url)
|
||||||
|
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||||
|
.send()?;
|
||||||
|
|
||||||
fn get_response(url: &str) -> Bytes {
|
|
||||||
let response = reqwest::blocking::get(url).expect("Could not load url.");
|
|
||||||
if response.status() == 200 {
|
if response.status() == 200 {
|
||||||
let body = response.bytes().unwrap();
|
// 先获取头信息,再获取字节数据
|
||||||
return body;
|
let content_encoding = response.headers()
|
||||||
|
.get("content-encoding")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let bytes = response.bytes()?;
|
||||||
|
|
||||||
|
let result = if content_encoding.contains("gzip") {
|
||||||
|
// 手动解压缩 gzip 数据
|
||||||
|
use std::io::Read;
|
||||||
|
let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
|
||||||
|
let mut decompressed = Vec::new();
|
||||||
|
match decoder.read_to_end(&mut decompressed) {
|
||||||
|
Ok(_) => Ok(Bytes::from(decompressed)),
|
||||||
|
Err(e) => {
|
||||||
|
// 如果解压缩失败,返回原始字节数据
|
||||||
|
println!("Gzip decompression failed: {}, using raw data", e);
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Ok(bytes)
|
||||||
|
};
|
||||||
|
result
|
||||||
} else {
|
} else {
|
||||||
return Bytes::new();
|
Err(reqwest::Error::from(response.error_for_status().unwrap_err()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn get_html_title(body: String) -> String {
|
|
||||||
let document = Html::parse_document(&body);
|
|
||||||
let title_selector = Selector::parse("h1").unwrap();
|
|
||||||
let title_element = document.select(&title_selector).next().unwrap();
|
|
||||||
return title_element.text().next().unwrap().to_string();
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn download_image(url_path: &str, file_path: &str) -> bool {
|
|
||||||
let url = format!("https://i.tuiimg.net/{}", url_path);
|
|
||||||
println!("{}", url);
|
|
||||||
let response = get_response(&url);
|
|
||||||
if response.is_empty() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let file_path = format!("images/{}", file_path);
|
|
||||||
file::write_file(&file_path, response).expect("Write file error");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn get_title(uid: i32) -> String {
|
pub(crate) fn get_title(uid: i32) -> String {
|
||||||
let body;
|
let url = "https://www.meitu131.com/meinv/";
|
||||||
let url = "https://www.tuiimg.com/meinv/";
|
|
||||||
let temp_url = format!("{}{}/", url, uid);
|
let temp_url = format!("{}{}/", url, uid);
|
||||||
let response = reqwest::blocking::get(temp_url).expect("Could not load url.");
|
let file_content = get_response(&temp_url);
|
||||||
let status_code = response.status();
|
match file_content {
|
||||||
return if status_code != 200 {
|
Ok(value) => {
|
||||||
"".parse().unwrap()
|
if value.is_empty() {
|
||||||
} else {
|
return "".to_string();
|
||||||
body = response.text().unwrap();
|
}
|
||||||
let name = get_html_title(body);
|
// 直接使用 lossy 转换避免 UTF-8 错误
|
||||||
name
|
let body = String::from_utf8_lossy(&value).to_string();
|
||||||
};
|
get_html_title(body)
|
||||||
|
}
|
||||||
|
Err(_e) => "".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_html_title(body: String) -> String {
|
||||||
|
let document = Html::parse_document(&body);
|
||||||
|
let title_selector = Selector::parse("h1").unwrap();
|
||||||
|
let title_element = document.select(&title_selector).next().unwrap();
|
||||||
|
title_element.text().next().unwrap().to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn download_image(url: &str, file_path: &str, retry_times: i8) -> i8 {
|
||||||
|
println!("{}", url);
|
||||||
|
let mut flag = 1;
|
||||||
|
let response = get_response(url);
|
||||||
|
match response {
|
||||||
|
Ok(value) => {
|
||||||
|
if value.is_empty() {
|
||||||
|
flag = 0
|
||||||
|
} else {
|
||||||
|
file::write_file(&file_path, value).expect("Write file error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_e) => flag = 2,
|
||||||
|
}
|
||||||
|
if flag == 2 && retry_times > 1 {
|
||||||
|
thread::sleep(Duration::from_millis(1000));
|
||||||
|
return download_image(url, file_path, retry_times - 1);
|
||||||
|
}
|
||||||
|
flag
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn download_images(uid: i32, title: &str) {
|
pub(crate) fn download_images(uid: i32, title: &str) {
|
||||||
let page: i32 = uid / 500 + 1;
|
let base_url = format!("https://www.meitu131.com/meinv/{}/", uid);
|
||||||
let page_str;
|
let file_content = get_response(&base_url);
|
||||||
if page < 10 {
|
let body = match file_content {
|
||||||
page_str = format!("00{}", page);
|
Ok(value) => String::from_utf8(value.to_vec()).expect("Transforms error"),
|
||||||
} else if page < 100 {
|
Err(_e) => "".parse().unwrap(),
|
||||||
page_str = format!("0{}", page);
|
};
|
||||||
} else {
|
|
||||||
page_str = format!("{}", page);
|
let document = Html::parse_document(&body);
|
||||||
}
|
let img_selector = Selector::parse("p[align='center'] img").unwrap();
|
||||||
let mut continue_flag = true;
|
let mut images = Vec::new();
|
||||||
let mut image_path;
|
|
||||||
let mut file_path;
|
// 获取尾页数
|
||||||
let mut pic_num = 1;
|
let mut total_pages = 1;
|
||||||
while continue_flag {
|
let pages_selector = Selector::parse("#pages a").unwrap();
|
||||||
image_path = format!("{}/{}/{}.jpg", page_str, uid, pic_num);
|
for page_element in document.select(&pages_selector) {
|
||||||
file_path = format!("{}/{}.jpg", title, pic_num);
|
if let Some(text) = page_element.text().next() {
|
||||||
if !download_image(&image_path, &file_path) {
|
if text == "尾页" {
|
||||||
continue_flag = false;
|
if let Some(href) = page_element.value().attr("href") {
|
||||||
|
// 从尾页链接中提取页数
|
||||||
|
if let Some(page_num_str) = href.split("_").last() {
|
||||||
|
if let Some(page_num) = page_num_str.split(".").next() {
|
||||||
|
if let Ok(num) = page_num.parse::<i32>() {
|
||||||
|
total_pages = num;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 收集所有图片 URL
|
||||||
|
for page in 1..=total_pages {
|
||||||
|
let page_url = if page == 1 {
|
||||||
|
base_url.clone()
|
||||||
|
} else {
|
||||||
|
format!("https://www.meitu131.com/meinv/{}/index_{}.html", uid, page)
|
||||||
|
};
|
||||||
|
|
||||||
|
let page_content = get_response(&page_url);
|
||||||
|
match page_content {
|
||||||
|
Ok(value) => {
|
||||||
|
if value.is_empty() {
|
||||||
|
println!("Page {} not found, skipping", page);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let page_body = String::from_utf8(value.to_vec()).expect("Transforms error");
|
||||||
|
let page_document = Html::parse_document(&page_body);
|
||||||
|
for img_element in page_document.select(&img_selector) {
|
||||||
|
if let Some(src) = img_element.value().attr("src") {
|
||||||
|
images.push(src.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("Error fetching page {}: {}, skipping", page, e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 生成 JSON 文件
|
||||||
|
let album = Album {
|
||||||
|
title: title.to_string(),
|
||||||
|
images,
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string_pretty(&album).expect("Failed to serialize album");
|
||||||
|
let json_path = format!("/mnt/share/data/albums/{}/index.json", uid);
|
||||||
|
let json_bytes = Bytes::from(json);
|
||||||
|
file::write_file(&json_path, json_bytes).expect("Failed to write JSON file");
|
||||||
|
|
||||||
|
// 执行下载
|
||||||
|
let mut pic_num = 1;
|
||||||
|
for img_url in &album.images {
|
||||||
|
let file_path = format!("/mnt/share/data/albums/{}/{}.jpg", uid, pic_num);
|
||||||
|
download_image(img_url, &file_path, 3);
|
||||||
pic_num += 1;
|
pic_num += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user