diff --git a/Cargo.lock b/Cargo.lock
index 715c5c4..d4e9fdf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1231,6 +1231,7 @@ dependencies = [
"chrono",
"config",
"image",
+ "mimalloc",
"rand 0.8.5",
"reqwest",
"serde",
@@ -1343,6 +1344,16 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
+dependencies = [
+ "cc",
+ "libc",
+]
+
[[package]]
name = "libredox"
version = "0.1.14"
@@ -1432,6 +1443,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+[[package]]
+name = "mimalloc"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
+dependencies = [
+ "libmimalloc-sys",
+]
+
[[package]]
name = "mime"
version = "0.3.17"
diff --git a/Cargo.toml b/Cargo.toml
index 192ac88..8ac156f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,4 +18,5 @@ config = "0.13"
rand = "0.8"
tracing = "0.1"
tracing-subscriber = "0.3"
-thiserror = "1.0"
\ No newline at end of file
+thiserror = "1.0"
+mimalloc = "0.1.48"
diff --git a/assets/js/album.js b/assets/js/album.js
index 2732a85..6864aa3 100644
--- a/assets/js/album.js
+++ b/assets/js/album.js
@@ -118,7 +118,7 @@ function displayImages() {
let dataSrc = '';
if (image.id) {
- dataSrc = `/api/thumbnail/${image.id}`;
+ dataSrc = `/api/image/${image.id}`;
imageUrl = '/assets/placeholder.svg'; // 初始使用占位图
} else if (image.original_url) {
dataSrc = image.original_url;
diff --git a/assets/js/app.js b/assets/js/app.js
index 7d38bf6..2d52ba3 100644
--- a/assets/js/app.js
+++ b/assets/js/app.js
@@ -205,7 +205,7 @@ function appendAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
- dataSrc = `/api/thumbnail/${album.cover_image_id}`;
+ dataSrc = `/api/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -402,7 +402,7 @@ function displayAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
- dataSrc = `/api/thumbnail/${album.cover_image_id}`;
+ dataSrc = `/api/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -730,7 +730,7 @@ function displayFavorites(favoriteImages) {
` : `${favoriteImages.map((fav, index) => `
-
diff --git a/config.toml b/config.toml
index 2ca5044..a7bb2e7 100644
--- a/config.toml
+++ b/config.toml
@@ -10,7 +10,7 @@ images_dir = "./albums"
thumbnails_dir = "./thumbnails"
[queue]
-max_concurrent_tasks = 6
-queue_size_limit = 200
+max_concurrent_tasks = 2
+queue_size_limit = 500
high_request_rate_threshold = 200.0
idle_request_rate_threshold = 10.0
\ No newline at end of file
diff --git a/src/database.rs b/src/database.rs
index ea610f1..80dd2b4 100644
--- a/src/database.rs
+++ b/src/database.rs
@@ -5,6 +5,7 @@ use uuid::Uuid;
pub struct Database {
pub pool: SqlitePool,
+ config: Config,
}
impl Database {
@@ -53,7 +54,10 @@ impl Database {
sqlx::migrate!("./migrations").run(&pool).await?;
// 确保image_count字段存在并正确初始化
- let db = Database { pool };
+ let db = Database {
+ pool,
+ config: config.clone(), // 保存配置副本
+ };
db.ensure_image_count_field().await?;
Ok(db)
@@ -783,4 +787,60 @@ impl Database {
Ok(())
}
+
+ /// 获取配置
+ pub fn get_config(&self) -> &Config {
+ &self.config
+ }
+
+ /// 获取未处理的图片(即缩略图可能不存在的图片)
+ pub async fn get_unprocessed_images(&self, limit: i64) -> Result
, sqlx::Error> {
+ let rows = sqlx::query(
+ r#"
+ SELECT * FROM images
+ ORDER BY created_at DESC
+ LIMIT ?
+ "#
+ )
+ .bind(limit)
+ .fetch_all(&self.pool)
+ .await?;
+
+ let mut images = Vec::new();
+ for row in rows {
+ let image_id_str = row.get::("id");
+ let image_id = match Uuid::parse_str(&image_id_str) {
+ Ok(uuid) => uuid,
+ Err(e) => {
+ eprintln!("无效的图片UUID格式: {} - 错误: {}", image_id_str, e);
+ continue;
+ }
+ };
+
+ let album_id_str = row.get::("album_id");
+ let album_id = match Uuid::parse_str(&album_id_str) {
+ Ok(uuid) => uuid,
+ Err(e) => {
+ eprintln!("无效的相册UUID格式: {} - 错误: {}", album_id_str, e);
+ continue;
+ }
+ };
+
+ images.push(Image {
+ id: image_id,
+ album_id,
+ filename: row.get("filename"),
+ original_url: row.get("original_url"),
+ title: row.get("title"),
+ description: row.get("description"),
+ file_size: row.get("file_size"),
+ width: row.get("width"),
+ height: row.get("height"),
+ mime_type: row.get("mime_type"),
+ created_at: row.get::("created_at"),
+ });
+ }
+
+ Ok(images)
+ }
}
\ No newline at end of file
diff --git a/src/image_processor.rs b/src/image_processor.rs
index f528a38..be88d10 100644
--- a/src/image_processor.rs
+++ b/src/image_processor.rs
@@ -1,9 +1,9 @@
-use std::path::Path;
-use std::fs;
-use image::{GenericImageView, ImageFormat};
-use uuid::Uuid;
use crate::config::StorageConfig;
+use image::{GenericImageView, ImageFormat};
+use std::fs;
+use std::path::Path;
use thiserror::Error;
+use uuid::Uuid;
#[derive(Error, Debug)]
pub enum ImageProcessorError {
@@ -18,7 +18,10 @@ pub enum ImageProcessorError {
// 手动实现 From 以支持 ? 操作符
impl From for ImageProcessorError {
fn from(err: reqwest::Error) -> Self {
- ImageProcessorError::Io(std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))
+ ImageProcessorError::Io(std::io::Error::new(
+ std::io::ErrorKind::Other,
+ err.to_string(),
+ ))
}
}
@@ -41,7 +44,11 @@ impl ImageProcessor {
Self { config }
}
- pub async fn save_image(&self, image_data: &[u8], filename: &str) -> Result {
+ pub async fn save_image(
+ &self,
+ image_data: &[u8],
+ filename: &str,
+ ) -> Result {
let image_id = Uuid::new_v4();
let file_extension = Path::new(filename)
.extension()
@@ -60,7 +67,10 @@ impl ImageProcessor {
let file_size = image_data.len() as i64;
// 生成缩略图
- let thumbnail_path = self.config.thumbnails_dir.join(format!("thumb_{}", new_filename));
+ let thumbnail_path = self
+ .config
+ .thumbnails_dir
+ .join(format!("thumb_{}", new_filename));
self.generate_thumbnail(&img, &thumbnail_path, 0, 0)?;
Ok(ImageInfo {
@@ -83,12 +93,12 @@ impl ImageProcessor {
) -> Result<(), ImageProcessorError> {
// 使用统一的缩略图尺寸(400x300,保持宽高比)
let thumbnail = img.thumbnail(400, 300);
-
+
// 确保输出目录存在
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)?;
}
-
+
thumbnail.save_with_format(output_path, ImageFormat::Jpeg)?;
Ok(())
@@ -104,21 +114,6 @@ impl ImageProcessor {
}
}
- /// 快速生成缩略图(低质量但快速)
- pub async fn generate_fast_thumbnail(&self, image_data: &[u8], _filename: &str) -> Result, ImageProcessorError> {
- // 使用快速解码和缩放
- let img = image::load_from_memory(image_data)?;
-
- // 快速缩放到小尺寸(200x200)
- let thumbnail = img.thumbnail(200, 200);
-
- // 使用低质量JPEG压缩以加快编码速度
- let mut output = Vec::new();
- thumbnail.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(50))?;
-
- Ok(output)
- }
-
/// 快速调整图片尺寸(最大宽度)
pub async fn resize_image_to_max(&self, image_data: &[u8], max_width: u32) -> Vec {
match image::load_from_memory(image_data) {
@@ -127,12 +122,18 @@ impl ImageProcessor {
if width <= max_width {
return image_data.to_vec(); // 无需调整
}
-
+
let new_height = (height as f32 * max_width as f32 / width as f32) as u32;
let resized = img.thumbnail(max_width, new_height);
-
+
let mut output = Vec::new();
- if resized.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(75)).is_ok() {
+ if resized
+ .write_to(
+ &mut std::io::Cursor::new(&mut output),
+ image::ImageOutputFormat::Jpeg(75),
+ )
+ .is_ok()
+ {
output
} else {
image_data.to_vec() // 失败时返回原图
@@ -146,10 +147,6 @@ impl ImageProcessor {
format!("/images/{}", filename)
}
- pub fn get_thumbnail_url(&self, filename: &str) -> String {
- format!("/thumbnails/thumb_{}", filename)
- }
-
pub async fn download_and_save_image(
&self,
url: &str,
@@ -161,42 +158,23 @@ impl ImageProcessor {
self.save_image(&image_data, filename).await
}
- /// 从图片数据生成缩略图并保存到文件
- pub async fn generate_thumbnail_from_data(
- &self,
- image_data: &[u8],
- thumbnail_filename: &str,
- ) -> Result<(), ImageProcessorError> {
- // 使用统一的缩略图生成方法
- let thumbnail_data = self.generate_thumbnail_from_memory(image_data, thumbnail_filename).await?;
-
- // 确保缩略图目录存在
- fs::create_dir_all(&self.config.thumbnails_dir)?;
-
- // 保存缩略图到文件
- let thumbnail_path = self.config.thumbnails_dir.join(thumbnail_filename);
- fs::write(&thumbnail_path, thumbnail_data)?;
-
- Ok(())
- }
+ /// 检测图片格式
+ fn detect_image_format(&self, image_data: &[u8]) -> Option {
+ if image_data.len() < 12 {
+ return None;
+ }
- /// 从内存中的图片数据生成缩略图数据(不保存到文件)
- pub async fn generate_thumbnail_from_memory(
- &self,
- image_data: &[u8],
- _thumbnail_filename: &str,
- ) -> Result, ImageProcessorError> {
- // 加载图片
- let img = image::load_from_memory(image_data)?;
-
- // 生成缩略图(600x600,保持宽高比)
- let thumbnail = img.thumbnail(600, 600);
-
- // 将缩略图数据保存到内存缓冲区
- let mut buffer = std::io::Cursor::new(Vec::new());
- thumbnail.write_to(&mut buffer, ImageFormat::Jpeg)?;
-
- Ok(buffer.into_inner())
+ match &image_data[0..12] {
+ b"\xff\xd8\xff" => Some("JPEG".to_string()),
+ b"\x89PNG\r\n\x1a\n" => Some("PNG".to_string()),
+ b"GIF87a" => Some("GIF87a".to_string()),
+ b"GIF89a" => Some("GIF89a".to_string()),
+ b"BM" => Some("BMP".to_string()),
+ b"RIFF" if image_data.len() > 12 && &image_data[8..12] == b"WEBP" => {
+ Some("WebP".to_string())
+ }
+ _ => None,
+ }
}
}
@@ -208,4 +186,4 @@ pub struct ImageInfo {
pub height: i32,
pub file_size: i64,
pub mime_type: String,
-}
\ No newline at end of file
+}
diff --git a/src/import.rs b/src/import.rs
index 2151146..3f6e7f8 100644
--- a/src/import.rs
+++ b/src/import.rs
@@ -37,11 +37,7 @@ async fn batch_insert_images_and_set_cover(
// 获取第一张图片的信息
if let Ok(Some(first_image)) = db.get_image(*first_image_id).await {
- // 生成缩略图
- match generate_thumbnail_for_image(&first_image, album_path, folder_name).await {
- Ok(_) => println!(" 封面缩略图生成成功"),
- Err(e) => eprintln!(" 封面缩略图生成失败: {}", e),
- }
+ println!(" 封面图片已设置: {}", first_image.filename);
}
}
@@ -55,7 +51,7 @@ async fn batch_insert_images_and_set_cover(
}
-pub async fn import_existing_data(config: &Config, db: Arc, task_queue: Option>) -> Result<(), Box> {
+pub async fn import_existing_data(config: &Config, db: Arc, _task_queue: Option>) -> Result<(), Box> {
println!("开始扫描图片目录并导入新相册...");
let albums_dir = Path::new(&config.storage.images_dir);
@@ -143,7 +139,7 @@ pub async fn import_existing_data(config: &Config, db: Arc, task_queue
}
// 如果有队列系统,使用队列异步处理相册导入
- if let Some(queue) = task_queue {
+ if let Some(queue) = _task_queue {
println!("🚀 使用队列系统异步导入相册...");
for (folder_name, path) in album_folders {
@@ -475,40 +471,4 @@ async fn reimport_single_album(
batch_insert_images_and_set_cover(&db, images_to_insert, album_id, path, folder_name).await
}
-/// 为图片生成缩略图(使用ImageProcessor)
-async fn generate_thumbnail_for_image(
- image: &Image,
- album_path: &Path,
- _folder_name: &str,
-) -> Result<(), Box> {
- // 构建图片文件路径
- let image_filename = image.filename.split('/').last().unwrap_or(&image.filename);
- let image_path = album_path.join(image_filename);
-
- if !image_path.exists() {
- return Err(format!("图片文件不存在: {}", image_path.display()).into());
- }
-
- // 读取图片数据
- let image_data = fs::read(&image_path)?;
-
- // 创建临时的ImageProcessor实例
- // 注意:这里使用默认的缩略图目录路径
- let config = crate::config::StorageConfig {
- images_dir: Path::new("./albums").to_path_buf(),
- thumbnails_dir: Path::new("./thumbnails").to_path_buf(),
- };
-
- let image_processor = crate::image_processor::ImageProcessor::new(config);
-
- // 生成缩略图
- match image_processor.generate_thumbnail_from_data(&image_data, &image.id.to_string()).await {
- Ok(_) => {
- println!(" 缩略图生成成功 (ID: {})", image.id);
- Ok(())
- }
- Err(e) => {
- Err(format!("缩略图生成失败: {:?}", e).into())
- }
- }
-}
\ No newline at end of file
+// 缩略图生成功能已移除
\ No newline at end of file
diff --git a/src/main.rs b/src/main.rs
index 55f1df3..569a5fb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,22 +3,23 @@ mod database;
mod image_processor;
mod import;
mod models;
-mod routes;
mod queue;
+mod routes;
+use crate::models::AlbumsCache;
use actix_files as afs;
-use actix_web::{App, HttpResponse, HttpServer, web, middleware};
+use actix_web::{App, HttpResponse, HttpServer, middleware, web};
use std::fs;
use std::sync::Arc;
-use tokio::time::{interval, Duration};
-use tracing::{info, error};
-use crate::models::AlbumsCache;
-
+use tokio::time::{Duration, interval};
+use tracing::{error, info};
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[actix_web::main]
async fn main() -> std::io::Result<()> {
// 初始化结构化日志
tracing_subscriber::fmt::init();
-
+
// 加载配置
let config = match config::Config::load() {
Ok(cfg) => cfg,
@@ -29,7 +30,7 @@ async fn main() -> std::io::Result<()> {
};
info!("启动图片展示服务...");
- info!("服务地址: http://{}", config.server_address());
+ info!("服务地址: http://localhost:{} http://{}", config.server.port, config.server_address());
// 初始化数据库
let db = match database::Database::new(&config).await {
@@ -46,11 +47,12 @@ async fn main() -> std::io::Result<()> {
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// 初始化智能任务队列
- let task_queue = Arc::new(queue::TaskQueue::new(
- config.queue.max_concurrent_tasks, // 最大并发任务数
- config.queue.queue_size_limit, // 队列大小限制
- config.queue.high_request_rate_threshold, // 高请求频率阈值
- config.queue.idle_request_rate_threshold, // 空闲请求频率阈值
+ let task_queue = Arc::new(queue::TaskQueue::new_with_db(
+ config.queue.max_concurrent_tasks, // 最大并发任务数
+ config.queue.queue_size_limit, // 队列大小限制
+ config.queue.high_request_rate_threshold, // 高请求频率阈值
+ config.queue.idle_request_rate_threshold, // 空闲请求频率阈值
+ Arc::clone(&db), // 数据库实例
));
// 在后台线程中自动导入新相册(不阻塞HTTP服务启动)
@@ -65,15 +67,6 @@ async fn main() -> std::io::Result<()> {
}
});
- // 启动缩略图生成Worker
- let thumbnail_worker = queue::ThumbnailWorker::new(
- Arc::clone(&task_queue),
- Arc::clone(&image_processor),
- );
- tokio::spawn(async move {
- thumbnail_worker.start().await;
- });
-
// 启动相册导入Worker
let album_import_worker = queue::AlbumImportWorker::new(
Arc::clone(&task_queue),
@@ -125,13 +118,18 @@ async fn main() -> std::io::Result<()> {
}
});
+ // 启动定时清理过期任务的任务(每小时执行一次)
+ let task_queue_for_cleanup = Arc::clone(&task_queue);
+ tokio::spawn(async move {
+ task_queue_for_cleanup.start_periodic_cleanup(Duration::from_secs(60 * 60)).await; // 1小时
+ });
+
println!("API接口:");
println!(" - GET /api/albums - 获取所有相册");
println!(" - GET /api/search?q=关键词 - 搜索相册");
println!(" - GET /api/album/:id - 获取相册详情");
println!(" - POST /api/albums - 创建新相册");
- println!(" - GET /api/thumbnail/:id - 获取缩略图");
- println!(" - GET /api/original/:id - 获取原始图片");
+ println!(" - GET /api/image/:id - 获取图片(已移除缩略图功能)");
println!(" - GET /api/favorites - 获取收藏列表");
println!(" - GET /api/favorites/status/:id - 检查收藏状态");
println!(" - POST /api/favorites - 添加收藏");
@@ -141,7 +139,7 @@ async fn main() -> std::io::Result<()> {
// 创建数据目录
fs::create_dir_all(&config.storage.images_dir).unwrap_or(());
- fs::create_dir_all(&config.storage.thumbnails_dir).unwrap_or(());
+ // 不再创建缩略图目录,因为已移除缩略图功能
fs::create_dir_all(&config.database.url[7..]).unwrap_or(());
// 在闭包外提取需要的路径
@@ -156,29 +154,39 @@ async fn main() -> std::io::Result<()> {
.app_data(web::Data::new(Arc::clone(&db)))
.app_data(web::Data::new(Arc::clone(&image_processor)))
.app_data(web::Data::new(Arc::clone(&albums_cache)))
- .app_data(web::Data::new(Arc::clone(&task_queue)))
+ // 不再需要任务队列作为app_data,因为已移除缩略图功能
// API路由
.route("/api/albums", web::get().to(routes::get_albums))
.route("/api/search", web::get().to(routes::search_albums))
.route("/api/album/random", web::get().to(routes::get_random_album))
.route("/api/album/{id}", web::get().to(routes::get_album_detail))
.route("/api/albums", web::post().to(routes::create_album))
- .route("/api/thumbnail/{id}", web::get().to(routes::get_thumbnail))
- .route("/api/original/{id}", web::get().to(routes::get_original_image))
+ .route("/api/image/{id}", web::get().to(routes::get_image))
// 收藏功能API路由
.route("/api/favorites", web::get().to(routes::get_favorites))
- .route("/api/favorites/status/{id}", web::get().to(routes::check_favorite_status))
+ .route(
+ "/api/favorites/status/{id}",
+ web::get().to(routes::check_favorite_status),
+ )
.route("/api/favorites", web::post().to(routes::add_favorite))
- .route("/api/favorites/{id}", web::delete().to(routes::remove_favorite))
- .route("/api/favorites/image/{id}", web::delete().to(routes::remove_favorite_by_image))
+ .route(
+ "/api/favorites/{id}",
+ web::delete().to(routes::remove_favorite),
+ )
+ .route(
+ "/api/favorites/image/{id}",
+ web::delete().to(routes::remove_favorite_by_image),
+ )
// 队列状态API路由
.route("/api/queue/stats", web::get().to(routes::get_queue_stats))
// 静态文件服务 - 启用缓存
- .service(afs::Files::new("/assets", "./assets")
- .show_files_listing()
- .prefer_utf8(true)
- .use_last_modified(true)
- .use_etag(true))
+ .service(
+ afs::Files::new("/assets", "./assets")
+ .show_files_listing()
+ .prefer_utf8(true)
+ .use_last_modified(true)
+ .use_etag(true),
+ )
// HTML页面路由
.route(
"/",
diff --git a/src/models.rs b/src/models.rs
index 5c650ab..b69cbd6 100644
--- a/src/models.rs
+++ b/src/models.rs
@@ -123,7 +123,7 @@ impl AlbumsCache {
}
pub fn update(&mut self, albums: Vec) {
- self.albums = albums.clone();
+ self.albums = albums; // 不再克隆整个向量,而是移动所有权
self.last_updated = chrono::Utc::now();
// 构建搜索索引
@@ -138,15 +138,15 @@ impl AlbumsCache {
self.search_index
.entry(word.to_string())
.or_insert_with(Vec::new)
- .push(album.clone());
+ .push(album.clone()); // 这里仍需要克隆单个元素
}
}
// 为完整标题创建索引
self.search_index
- .entry(title_lower.clone())
+ .entry(title_lower)
.or_insert_with(Vec::new)
- .push(album.clone());
+ .push(album.clone()); // 这里仍需要克隆单个元素
}
info!(
diff --git a/src/queue.rs b/src/queue.rs
index 23e8079..6f7d3b2 100644
--- a/src/queue.rs
+++ b/src/queue.rs
@@ -9,6 +9,7 @@ use tracing::{info, warn, error, debug};
use uuid::Uuid;
use image::GenericImageView;
use crate::models::{AlbumIndex, CreateAlbum, CreateImage};
+use crate::database::Database;
/// 任务类型
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -74,7 +75,8 @@ pub struct TaskTypeStats {
/// 智能任务队列
pub struct TaskQueue {
- queue: Mutex>,
+ high_priority_queue: Mutex>,
+ normal_priority_queue: Mutex>,
active_tasks: AtomicUsize,
completed_tasks: AtomicUsize,
failed_tasks: AtomicUsize,
@@ -90,12 +92,15 @@ pub struct TaskQueue {
// 配置阈值
high_request_rate_threshold: f64,
idle_request_rate_threshold: f64,
+ // 数据库引用用于定时任务
+ db: Option>,
}
impl TaskQueue {
pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self {
Self {
- queue: Mutex::new(VecDeque::new()),
+ high_priority_queue: Mutex::new(VecDeque::new()),
+ normal_priority_queue: Mutex::new(VecDeque::new()),
active_tasks: AtomicUsize::new(0),
completed_tasks: AtomicUsize::new(0),
failed_tasks: AtomicUsize::new(0),
@@ -111,42 +116,95 @@ impl TaskQueue {
// 配置阈值
high_request_rate_threshold,
idle_request_rate_threshold,
+ db: None,
+ }
+ }
+
+ pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc) -> Self {
+ Self {
+ high_priority_queue: Mutex::new(VecDeque::new()),
+ normal_priority_queue: Mutex::new(VecDeque::new()),
+ active_tasks: AtomicUsize::new(0),
+ completed_tasks: AtomicUsize::new(0),
+ failed_tasks: AtomicUsize::new(0),
+ max_concurrent_tasks,
+ queue_size_limit,
+ is_paused: AtomicBool::new(false),
+ notify: Notify::new(),
+ last_activity: Mutex::new(Instant::now()),
+ // HTTP请求频率监控
+ request_count: AtomicU64::new(0),
+ last_request_time: Mutex::new(Instant::now()),
+ request_window: Duration::from_secs(10),
+ // 配置阈值
+ high_request_rate_threshold,
+ idle_request_rate_threshold,
+ db: Some(db),
}
}
/// 添加任务到队列
pub async fn add_task(&self, task: Task) -> Result<(), String> {
- let mut queue = self.queue.lock().await;
+ let (_high_queue_len, _normal_queue_len) = {
+ let high_queue = self.high_priority_queue.lock().await;
+ let normal_queue = self.normal_priority_queue.lock().await;
+
+ // 检查队列总容量是否已满
+ if high_queue.len() + normal_queue.len() >= self.queue_size_limit {
+ return Err("队列已满,无法添加新任务".to_string());
+ }
- // 检查队列是否已满
- if queue.len() >= self.queue_size_limit {
- return Err("队列已满,无法添加新任务".to_string());
+ // 检查是否已有相同ID的任务
+ let exists_in_high = high_queue.iter().any(|t| t.id == task.id);
+ let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id);
+
+ if exists_in_high || exists_in_normal {
+ return Err("该任务已在队列中".to_string());
+ }
+
+ (high_queue.len(), normal_queue.len())
+ };
+
+ // 根据优先级添加到相应队列
+ match task.priority {
+ TaskPriority::High => {
+ let mut high_queue = self.high_priority_queue.lock().await;
+
+ // 高优先级队列内部也按优先级排序
+ let insert_position = high_queue
+ .iter()
+ .position(|t| t.priority > task.priority)
+ .unwrap_or(high_queue.len());
+
+ high_queue.insert(insert_position, task);
+
+ debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len());
+ }
+ _ => { // Medium and Low go to normal priority queue
+ let mut normal_queue = self.normal_priority_queue.lock().await;
+
+ // 按优先级排序
+ let insert_position = normal_queue
+ .iter()
+ .position(|t| t.priority > task.priority)
+ .unwrap_or(normal_queue.len());
+
+ normal_queue.insert(insert_position, task);
+
+ debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len());
+ }
}
- // 检查是否已有相同ID的任务
- if queue.iter().any(|t| t.id == task.id) {
- return Err("该任务已在队列中".to_string());
- }
-
- // 根据优先级插入任务(高优先级在前)
- let insert_position = queue
- .iter()
- .position(|t| t.priority > task.priority)
- .unwrap_or(queue.len());
-
- queue.insert(insert_position, task);
-
// 更新最后活动时间
*self.last_activity.lock().await = Instant::now();
// 通知工作线程
self.notify.notify_one();
- debug!("任务已添加到队列,当前队列大小: {}", queue.len());
Ok(())
}
- /// 从队列获取下一个任务
+ /// 从队列获取下一个任务(优先处理高优先级队列)
pub async fn next_task(&self) -> Option {
// 检查是否暂停
if self.is_paused.load(Ordering::Acquire) {
@@ -158,8 +216,18 @@ impl TaskQueue {
return None;
}
- let mut queue = self.queue.lock().await;
- if let Some(task) = queue.pop_front() {
+ // 优先从高优先级队列获取任务
+ {
+ let mut high_queue = self.high_priority_queue.lock().await;
+ if let Some(task) = high_queue.pop_front() {
+ self.active_tasks.fetch_add(1, Ordering::Release);
+ return Some(task);
+ }
+ }
+
+ // 如果高优先级队列为空,则从普通优先级队列获取任务
+ let mut normal_queue = self.normal_priority_queue.lock().await;
+ if let Some(task) = normal_queue.pop_front() {
self.active_tasks.fetch_add(1, Ordering::Release);
Some(task)
} else {
@@ -231,16 +299,31 @@ impl TaskQueue {
/// 获取队列统计信息
pub async fn get_stats(&self) -> QueueStats {
- let queue = self.queue.lock().await;
-
- // 按任务类型统计
- let mut type_stats = std::collections::HashMap::new();
- for task in queue.iter() {
- let stats = type_stats
- .entry(task.task_type.clone())
- .or_insert((0, 0, 0));
- stats.0 += 1; // pending
- }
+ let (high_queue_len, normal_queue_len, type_stats) = {
+ let high_queue = self.high_priority_queue.lock().await;
+ let normal_queue = self.normal_priority_queue.lock().await;
+
+ // 按任务类型统计
+ let mut type_stats = std::collections::HashMap::new();
+
+ // 统计高优先级队列中的任务
+ for task in high_queue.iter() {
+ let stats = type_stats
+ .entry(task.task_type.clone())
+ .or_insert((0, 0, 0));
+ stats.0 += 1; // pending (high priority)
+ }
+
+ // 统计普通优先级队列中的任务
+ for task in normal_queue.iter() {
+ let stats = type_stats
+ .entry(task.task_type.clone())
+ .or_insert((0, 0, 0));
+ stats.0 += 1; // pending (normal priority)
+ }
+
+ (high_queue.len(), normal_queue.len(), type_stats)
+ };
let task_type_stats = type_stats
.into_iter()
@@ -253,7 +336,7 @@ impl TaskQueue {
.collect();
QueueStats {
- pending_tasks: queue.len(),
+ pending_tasks: high_queue_len + normal_queue_len,
active_tasks: self.active_tasks.load(Ordering::Acquire),
completed_tasks: self.completed_tasks.load(Ordering::Acquire),
failed_tasks: self.failed_tasks.load(Ordering::Acquire),
@@ -268,15 +351,28 @@ impl TaskQueue {
/// 清理过期的任务(超过一定时间的任务)
pub async fn cleanup_expired_tasks(&self, max_age: Duration) {
- let mut queue = self.queue.lock().await;
let now = Instant::now();
- let initial_len = queue.len();
-
- queue.retain(|task| now.duration_since(task.created_at) <= max_age);
-
- let removed_count = initial_len - queue.len();
- if removed_count > 0 {
- warn!("清理了 {} 个过期任务", removed_count);
+
+ // 清理高优先级队列中的过期任务
+ {
+ let mut high_queue = self.high_priority_queue.lock().await;
+ let initial_len = high_queue.len();
+ high_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
+ let removed_count = initial_len - high_queue.len();
+ if removed_count > 0 {
+ info!("清理了 {} 个过期的高优先级任务", removed_count);
+ }
+ }
+
+ // 清理普通优先级队列中的过期任务
+ {
+ let mut normal_queue = self.normal_priority_queue.lock().await;
+ let initial_len = normal_queue.len();
+ normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
+ let removed_count = initial_len - normal_queue.len();
+ if removed_count > 0 {
+ info!("清理了 {} 个过期的普通优先级任务", removed_count);
+ }
}
}
@@ -290,6 +386,22 @@ impl TaskQueue {
false
}
+
+ /// 启动定期清理过期任务
+ pub async fn start_periodic_cleanup(self: Arc, interval_duration: Duration) {
+ let queue = Arc::clone(&self);
+
+ tokio::spawn(async move {
+ let mut interval = tokio::time::interval(interval_duration);
+
+ loop {
+ interval.tick().await;
+
+ // 清理超过24小时的任务
+ queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await;
+ }
+ });
+ }
}
/// 缩略图生成Worker
@@ -340,8 +452,17 @@ impl ThumbnailWorker {
}
} else {
// 如果不是缩略图任务,放回队列
- let mut queue = self.queue.queue.lock().await;
- queue.push_front(task);
+ // 根据任务优先级放回相应队列
+ match task.priority {
+ TaskPriority::High => {
+ let mut high_queue = self.queue.high_priority_queue.lock().await;
+ high_queue.push_front(task);
+ }
+ _ => {
+ let mut normal_queue = self.queue.normal_priority_queue.lock().await;
+ normal_queue.push_front(task);
+ }
+ }
}
} else {
// 队列为空,等待新任务通知
@@ -353,44 +474,15 @@ impl ThumbnailWorker {
}
}
- /// 处理缩略图任务
+ // 缩略图任务处理功能已移除,直接返回成功
async fn process_thumbnail_task(&self, task: Task) -> bool {
if let TaskData::ThumbnailGeneration {
image_id,
- original_data,
+ original_data: _,
} = task.data
{
- // 使用具体的错误处理,避免 Send trait 问题
- let result = self
- .image_processor
- .generate_thumbnail_from_memory(&original_data, &image_id.to_string())
- .await;
-
- match result {
- Ok(thumbnail_data) => {
- // 保存缩略图到文件
- let thumbnail_path = self
- .image_processor
- .config
- .thumbnails_dir
- .join(image_id.to_string());
-
- match tokio::fs::write(&thumbnail_path, &thumbnail_data).await {
- Ok(_) => {
- info!("✅ 缩略图生成成功: {:?}", image_id);
- true
- }
- Err(e) => {
- warn!("保存缩略图失败: {:?} - {}", image_id, e);
- false
- }
- }
- }
- Err(e) => {
- warn!("缩略图生成失败: {:?} - {:?}", image_id, e);
- false
- }
- }
+ info!("缩略图任务已跳过(功能已移除): {:?}", image_id);
+ true // 直接返回成功,避免阻塞队列
} else {
warn!("任务类型不匹配,期望缩略图生成任务");
false
@@ -445,8 +537,17 @@ impl AlbumImportWorker {
}
} else {
// 如果不是相册导入任务,放回队列并减少active_tasks计数器
- let mut queue = self.queue.queue.lock().await;
- queue.push_front(task);
+ // 根据任务优先级放回相应队列
+ match task.priority {
+ TaskPriority::High => {
+ let mut high_queue = self.queue.high_priority_queue.lock().await;
+ high_queue.push_front(task);
+ }
+ _ => {
+ let mut normal_queue = self.queue.normal_priority_queue.lock().await;
+ normal_queue.push_front(task);
+ }
+ }
// 手动减少active_tasks计数器,因为任务没有被处理
self.queue.active_tasks.fetch_sub(1, Ordering::Release);
}
diff --git a/src/routes.rs b/src/routes.rs
index 621bb8b..4eb9648 100644
--- a/src/routes.rs
+++ b/src/routes.rs
@@ -66,73 +66,7 @@ pub async fn get_favorites(db: web::Data>) -> impl Responder {
HttpResponse::Ok().json(favorites)
}
-/// 检查缩略图质量并在需要时升级为高质量版本
-async fn check_and_upgrade_thumbnail_quality(
- config: &crate::config::StorageConfig,
- image: &Image,
- original_path: &std::path::Path,
- task_queue: Option>,
-) -> Result<(), Box> {
- let thumbnail_path = config.thumbnails_dir.join(image.id.to_string());
-
- // 检查原始图片是否存在
- if !original_path.exists() {
- return Ok(()); // 原始图片不存在,无法升级
- }
-
- // 读取缩略图文件信息
- let thumbnail_metadata = match fs::metadata(&thumbnail_path) {
- Ok(meta) => meta,
- Err(_) => return Ok(()), // 无法读取缩略图元数据
- };
-
- // 检查缩略图文件大小(简单质量检测)
- // 如果缩略图小于20KB,可能质量较低,需要升级
- if thumbnail_metadata.len() < 20 * 1024 {
- // 读取原始图片数据
- let original_data = match fs::read(original_path) {
- Ok(data) => data,
- Err(e) => {
- eprintln!("读取原始图片失败,无法升级缩略图: {}", e);
- return Ok(());
- }
- };
-
- // 如果有队列系统,使用队列处理质量升级
- if let Some(queue) = task_queue {
- let task = queue::TaskFactory::create_thumbnail_task(
- image.id,
- original_data,
- queue::TaskPriority::Low, // 质量升级,低优先级
- );
-
- if let Err(e) = queue.add_task(task).await {
- eprintln!("添加缩略图质量升级任务失败: {}", e);
- }
- } else {
- // 如果没有队列系统,直接生成高质量缩略图
- let image_processor = ImageProcessor::new(config.clone());
- match image_processor
- .generate_thumbnail_from_memory(&original_data, &image.id.to_string())
- .await
- {
- Ok(high_quality_data) => {
- // 保存高质量缩略图
- if let Err(e) = fs::write(&thumbnail_path, &high_quality_data) {
- eprintln!("保存高质量缩略图失败: {}", e);
- } else {
- println!("✅ 缩略图已升级为高质量版本 (ID: {})", image.id);
- }
- }
- Err(e) => {
- eprintln!("生成高质量缩略图失败: {:?}", e);
- }
- }
- }
- }
-
- Ok(())
-}
+// 图片质量检查功能已移除,直接返回原始图片
// 检查图片是否被收藏
pub async fn check_favorite_status(
@@ -450,10 +384,10 @@ pub async fn create_album(
}
}
-pub async fn get_thumbnail(
+/// 获取原始图片(已移除缩略图功能)
+pub async fn get_image(
db: web::Data>,
image_processor: web::Data>,
- task_queue: web::Data>,
path: web::Path,
) -> impl Responder {
let image_id = match Uuid::parse_str(&path.into_inner()) {
@@ -467,93 +401,17 @@ pub async fn get_thumbnail(
match db.get_image(image_id).await {
Ok(Some(image)) => {
- // 优先返回缩略图,如果不存在则生成
- let thumbnail_path = image_processor
- .config
- .thumbnails_dir
- .join(image.id.to_string());
let original_path = image_processor.config.images_dir.join(&image.filename);
- if thumbnail_path.exists() {
- // 返回现有缩略图,并异步检查是否需要替换为高质量版本
- match fs::read(&thumbnail_path) {
+ if original_path.exists() {
+ match fs::read(&original_path) {
Ok(data) => {
- let mime_type = image_processor.detect_mime_type("jpg"); // 缩略图总是JPEG格式
-
- // 异步检查缩略图质量并可能替换为高质量版本
- let config_clone = image_processor.config.clone();
- let image_clone = image.clone();
- let original_path_clone = original_path.clone();
- let queue_clone = task_queue.get_ref().clone();
- tokio::spawn(async move {
- if let Err(e) = check_and_upgrade_thumbnail_quality(
- &config_clone,
- &image_clone,
- &original_path_clone,
- Some(queue_clone),
- )
- .await
- {
- eprintln!("缩略图质量检查失败: {}", e);
- }
- });
-
HttpResponse::Ok()
- .content_type(mime_type)
+ .content_type(image.mime_type.clone())
.insert_header(("Cache-Control", "public, max-age=31536000")) // 1年缓存
.insert_header(("ETag", format!("\"{}\"", image_id)))
.body(data)
}
- Err(e) => {
- eprintln!("读取缩略图失败: {}", e);
- HttpResponse::InternalServerError().json(serde_json::json!({
- "error": "读取缩略图失败"
- }))
- }
- }
- } else if original_path.exists() {
- // 先返回原始图片的缩略版本(快速响应)
- match fs::read(&original_path) {
- Ok(data) => {
- // 快速生成小尺寸缩略图(不进行复杂处理)
- match image_processor
- .generate_fast_thumbnail(&data, &image.id.to_string())
- .await
- {
- Ok(thumbnail_data) => {
- // 将高质量缩略图生成任务添加到队列
- let task = queue::TaskFactory::create_thumbnail_task(
- image.id,
- data.clone(),
- queue::TaskPriority::High, // 前端请求,高优先级
- );
-
- // 异步添加任务到队列
- let queue_clone = task_queue.clone();
- tokio::spawn(async move {
- if let Err(e) = queue_clone.add_task(task).await {
- eprintln!("添加缩略图生成任务失败: {}", e);
- }
- });
-
- // 返回快速生成的缩略图数据
- HttpResponse::Ok()
- .content_type("image/jpeg")
- .insert_header(("Cache-Control", "public, max-age=30")) // 0.5分钟缓存
- .insert_header(("ETag", format!("\"{}\"", image_id)))
- .body(thumbnail_data)
- }
- Err(e) => {
- eprintln!("快速缩略图生成失败: {:?}", e);
- // 如果快速生成失败,返回原始图片的小尺寸版本
- let small_data =
- image_processor.resize_image_to_max(&data, 800).await;
- HttpResponse::Ok()
- .content_type(image.mime_type)
- .body(small_data)
- }
- }
- }
Err(e) => {
eprintln!("读取原始图片失败: {}", e);
HttpResponse::InternalServerError().json(serde_json::json!({