diff --git a/Cargo.lock b/Cargo.lock index 715c5c4..d4e9fdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1231,6 +1231,7 @@ dependencies = [ "chrono", "config", "image", + "mimalloc", "rand 0.8.5", "reqwest", "serde", @@ -1343,6 +1344,16 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libredox" version = "0.1.14" @@ -1432,6 +1443,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 192ac88..8ac156f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,5 @@ config = "0.13" rand = "0.8" tracing = "0.1" tracing-subscriber = "0.3" -thiserror = "1.0" \ No newline at end of file +thiserror = "1.0" +mimalloc = "0.1.48" diff --git a/assets/js/album.js b/assets/js/album.js index 2732a85..6864aa3 100644 --- a/assets/js/album.js +++ b/assets/js/album.js @@ -118,7 +118,7 @@ function displayImages() { let dataSrc = ''; if (image.id) { - dataSrc = `/api/thumbnail/${image.id}`; + dataSrc = `/api/image/${image.id}`; imageUrl = '/assets/placeholder.svg'; // 初始使用占位图 } else if (image.original_url) { dataSrc = image.original_url; diff --git a/assets/js/app.js b/assets/js/app.js index 7d38bf6..2d52ba3 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -205,7 +205,7 @@ function appendAlbums(albums) { let dataSrc = ''; if (album.cover_image_id) { - dataSrc = `/api/thumbnail/${album.cover_image_id}`; + dataSrc = `/api/image/${album.cover_image_id}`; coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图 } @@ -402,7 +402,7 @@ function displayAlbums(albums) { let dataSrc = ''; if (album.cover_image_id) { - dataSrc = `/api/thumbnail/${album.cover_image_id}`; + dataSrc = `/api/image/${album.cover_image_id}`; coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图 } @@ -730,7 +730,7 @@ function displayFavorites(favoriteImages) { ` : `${favoriteImages.map((fav, index) => `
- ${fav.image.title || '收藏的图片'}
diff --git a/config.toml b/config.toml index 2ca5044..a7bb2e7 100644 --- a/config.toml +++ b/config.toml @@ -10,7 +10,7 @@ images_dir = "./albums" thumbnails_dir = "./thumbnails" [queue] -max_concurrent_tasks = 6 -queue_size_limit = 200 +max_concurrent_tasks = 2 +queue_size_limit = 500 high_request_rate_threshold = 200.0 idle_request_rate_threshold = 10.0 \ No newline at end of file diff --git a/src/database.rs b/src/database.rs index ea610f1..80dd2b4 100644 --- a/src/database.rs +++ b/src/database.rs @@ -5,6 +5,7 @@ use uuid::Uuid; pub struct Database { pub pool: SqlitePool, + config: Config, } impl Database { @@ -53,7 +54,10 @@ impl Database { sqlx::migrate!("./migrations").run(&pool).await?; // 确保image_count字段存在并正确初始化 - let db = Database { pool }; + let db = Database { + pool, + config: config.clone(), // 保存配置副本 + }; db.ensure_image_count_field().await?; Ok(db) @@ -783,4 +787,60 @@ impl Database { Ok(()) } + + /// 获取配置 + pub fn get_config(&self) -> &Config { + &self.config + } + + /// 获取未处理的图片(即缩略图可能不存在的图片) + pub async fn get_unprocessed_images(&self, limit: i64) -> Result, sqlx::Error> { + let rows = sqlx::query( + r#" + SELECT * FROM images + ORDER BY created_at DESC + LIMIT ? + "# + ) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut images = Vec::new(); + for row in rows { + let image_id_str = row.get::("id"); + let image_id = match Uuid::parse_str(&image_id_str) { + Ok(uuid) => uuid, + Err(e) => { + eprintln!("无效的图片UUID格式: {} - 错误: {}", image_id_str, e); + continue; + } + }; + + let album_id_str = row.get::("album_id"); + let album_id = match Uuid::parse_str(&album_id_str) { + Ok(uuid) => uuid, + Err(e) => { + eprintln!("无效的相册UUID格式: {} - 错误: {}", album_id_str, e); + continue; + } + }; + + images.push(Image { + id: image_id, + album_id, + filename: row.get("filename"), + original_url: row.get("original_url"), + title: row.get("title"), + description: row.get("description"), + file_size: row.get("file_size"), + width: row.get("width"), + height: row.get("height"), + mime_type: row.get("mime_type"), + created_at: row.get::("created_at"), + }); + } + + Ok(images) + } } \ No newline at end of file diff --git a/src/image_processor.rs b/src/image_processor.rs index f528a38..be88d10 100644 --- a/src/image_processor.rs +++ b/src/image_processor.rs @@ -1,9 +1,9 @@ -use std::path::Path; -use std::fs; -use image::{GenericImageView, ImageFormat}; -use uuid::Uuid; use crate::config::StorageConfig; +use image::{GenericImageView, ImageFormat}; +use std::fs; +use std::path::Path; use thiserror::Error; +use uuid::Uuid; #[derive(Error, Debug)] pub enum ImageProcessorError { @@ -18,7 +18,10 @@ pub enum ImageProcessorError { // 手动实现 From 以支持 ? 操作符 impl From for ImageProcessorError { fn from(err: reqwest::Error) -> Self { - ImageProcessorError::Io(std::io::Error::new(std::io::ErrorKind::Other, err.to_string())) + ImageProcessorError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + err.to_string(), + )) } } @@ -41,7 +44,11 @@ impl ImageProcessor { Self { config } } - pub async fn save_image(&self, image_data: &[u8], filename: &str) -> Result { + pub async fn save_image( + &self, + image_data: &[u8], + filename: &str, + ) -> Result { let image_id = Uuid::new_v4(); let file_extension = Path::new(filename) .extension() @@ -60,7 +67,10 @@ impl ImageProcessor { let file_size = image_data.len() as i64; // 生成缩略图 - let thumbnail_path = self.config.thumbnails_dir.join(format!("thumb_{}", new_filename)); + let thumbnail_path = self + .config + .thumbnails_dir + .join(format!("thumb_{}", new_filename)); self.generate_thumbnail(&img, &thumbnail_path, 0, 0)?; Ok(ImageInfo { @@ -83,12 +93,12 @@ impl ImageProcessor { ) -> Result<(), ImageProcessorError> { // 使用统一的缩略图尺寸(400x300,保持宽高比) let thumbnail = img.thumbnail(400, 300); - + // 确保输出目录存在 if let Some(parent) = output_path.parent() { fs::create_dir_all(parent)?; } - + thumbnail.save_with_format(output_path, ImageFormat::Jpeg)?; Ok(()) @@ -104,21 +114,6 @@ impl ImageProcessor { } } - /// 快速生成缩略图(低质量但快速) - pub async fn generate_fast_thumbnail(&self, image_data: &[u8], _filename: &str) -> Result, ImageProcessorError> { - // 使用快速解码和缩放 - let img = image::load_from_memory(image_data)?; - - // 快速缩放到小尺寸(200x200) - let thumbnail = img.thumbnail(200, 200); - - // 使用低质量JPEG压缩以加快编码速度 - let mut output = Vec::new(); - thumbnail.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(50))?; - - Ok(output) - } - /// 快速调整图片尺寸(最大宽度) pub async fn resize_image_to_max(&self, image_data: &[u8], max_width: u32) -> Vec { match image::load_from_memory(image_data) { @@ -127,12 +122,18 @@ impl ImageProcessor { if width <= max_width { return image_data.to_vec(); // 无需调整 } - + let new_height = (height as f32 * max_width as f32 / width as f32) as u32; let resized = img.thumbnail(max_width, new_height); - + let mut output = Vec::new(); - if resized.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(75)).is_ok() { + if resized + .write_to( + &mut std::io::Cursor::new(&mut output), + image::ImageOutputFormat::Jpeg(75), + ) + .is_ok() + { output } else { image_data.to_vec() // 失败时返回原图 @@ -146,10 +147,6 @@ impl ImageProcessor { format!("/images/{}", filename) } - pub fn get_thumbnail_url(&self, filename: &str) -> String { - format!("/thumbnails/thumb_{}", filename) - } - pub async fn download_and_save_image( &self, url: &str, @@ -161,42 +158,23 @@ impl ImageProcessor { self.save_image(&image_data, filename).await } - /// 从图片数据生成缩略图并保存到文件 - pub async fn generate_thumbnail_from_data( - &self, - image_data: &[u8], - thumbnail_filename: &str, - ) -> Result<(), ImageProcessorError> { - // 使用统一的缩略图生成方法 - let thumbnail_data = self.generate_thumbnail_from_memory(image_data, thumbnail_filename).await?; - - // 确保缩略图目录存在 - fs::create_dir_all(&self.config.thumbnails_dir)?; - - // 保存缩略图到文件 - let thumbnail_path = self.config.thumbnails_dir.join(thumbnail_filename); - fs::write(&thumbnail_path, thumbnail_data)?; - - Ok(()) - } + /// 检测图片格式 + fn detect_image_format(&self, image_data: &[u8]) -> Option { + if image_data.len() < 12 { + return None; + } - /// 从内存中的图片数据生成缩略图数据(不保存到文件) - pub async fn generate_thumbnail_from_memory( - &self, - image_data: &[u8], - _thumbnail_filename: &str, - ) -> Result, ImageProcessorError> { - // 加载图片 - let img = image::load_from_memory(image_data)?; - - // 生成缩略图(600x600,保持宽高比) - let thumbnail = img.thumbnail(600, 600); - - // 将缩略图数据保存到内存缓冲区 - let mut buffer = std::io::Cursor::new(Vec::new()); - thumbnail.write_to(&mut buffer, ImageFormat::Jpeg)?; - - Ok(buffer.into_inner()) + match &image_data[0..12] { + b"\xff\xd8\xff" => Some("JPEG".to_string()), + b"\x89PNG\r\n\x1a\n" => Some("PNG".to_string()), + b"GIF87a" => Some("GIF87a".to_string()), + b"GIF89a" => Some("GIF89a".to_string()), + b"BM" => Some("BMP".to_string()), + b"RIFF" if image_data.len() > 12 && &image_data[8..12] == b"WEBP" => { + Some("WebP".to_string()) + } + _ => None, + } } } @@ -208,4 +186,4 @@ pub struct ImageInfo { pub height: i32, pub file_size: i64, pub mime_type: String, -} \ No newline at end of file +} diff --git a/src/import.rs b/src/import.rs index 2151146..3f6e7f8 100644 --- a/src/import.rs +++ b/src/import.rs @@ -37,11 +37,7 @@ async fn batch_insert_images_and_set_cover( // 获取第一张图片的信息 if let Ok(Some(first_image)) = db.get_image(*first_image_id).await { - // 生成缩略图 - match generate_thumbnail_for_image(&first_image, album_path, folder_name).await { - Ok(_) => println!(" 封面缩略图生成成功"), - Err(e) => eprintln!(" 封面缩略图生成失败: {}", e), - } + println!(" 封面图片已设置: {}", first_image.filename); } } @@ -55,7 +51,7 @@ async fn batch_insert_images_and_set_cover( } -pub async fn import_existing_data(config: &Config, db: Arc, task_queue: Option>) -> Result<(), Box> { +pub async fn import_existing_data(config: &Config, db: Arc, _task_queue: Option>) -> Result<(), Box> { println!("开始扫描图片目录并导入新相册..."); let albums_dir = Path::new(&config.storage.images_dir); @@ -143,7 +139,7 @@ pub async fn import_existing_data(config: &Config, db: Arc, task_queue } // 如果有队列系统,使用队列异步处理相册导入 - if let Some(queue) = task_queue { + if let Some(queue) = _task_queue { println!("🚀 使用队列系统异步导入相册..."); for (folder_name, path) in album_folders { @@ -475,40 +471,4 @@ async fn reimport_single_album( batch_insert_images_and_set_cover(&db, images_to_insert, album_id, path, folder_name).await } -/// 为图片生成缩略图(使用ImageProcessor) -async fn generate_thumbnail_for_image( - image: &Image, - album_path: &Path, - _folder_name: &str, -) -> Result<(), Box> { - // 构建图片文件路径 - let image_filename = image.filename.split('/').last().unwrap_or(&image.filename); - let image_path = album_path.join(image_filename); - - if !image_path.exists() { - return Err(format!("图片文件不存在: {}", image_path.display()).into()); - } - - // 读取图片数据 - let image_data = fs::read(&image_path)?; - - // 创建临时的ImageProcessor实例 - // 注意:这里使用默认的缩略图目录路径 - let config = crate::config::StorageConfig { - images_dir: Path::new("./albums").to_path_buf(), - thumbnails_dir: Path::new("./thumbnails").to_path_buf(), - }; - - let image_processor = crate::image_processor::ImageProcessor::new(config); - - // 生成缩略图 - match image_processor.generate_thumbnail_from_data(&image_data, &image.id.to_string()).await { - Ok(_) => { - println!(" 缩略图生成成功 (ID: {})", image.id); - Ok(()) - } - Err(e) => { - Err(format!("缩略图生成失败: {:?}", e).into()) - } - } -} \ No newline at end of file +// 缩略图生成功能已移除 \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 55f1df3..569a5fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,22 +3,23 @@ mod database; mod image_processor; mod import; mod models; -mod routes; mod queue; +mod routes; +use crate::models::AlbumsCache; use actix_files as afs; -use actix_web::{App, HttpResponse, HttpServer, web, middleware}; +use actix_web::{App, HttpResponse, HttpServer, middleware, web}; use std::fs; use std::sync::Arc; -use tokio::time::{interval, Duration}; -use tracing::{info, error}; -use crate::models::AlbumsCache; - +use tokio::time::{Duration, interval}; +use tracing::{error, info}; +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; #[actix_web::main] async fn main() -> std::io::Result<()> { // 初始化结构化日志 tracing_subscriber::fmt::init(); - + // 加载配置 let config = match config::Config::load() { Ok(cfg) => cfg, @@ -29,7 +30,7 @@ async fn main() -> std::io::Result<()> { }; info!("启动图片展示服务..."); - info!("服务地址: http://{}", config.server_address()); + info!("服务地址: http://localhost:{} http://{}", config.server.port, config.server_address()); // 初始化数据库 let db = match database::Database::new(&config).await { @@ -46,11 +47,12 @@ async fn main() -> std::io::Result<()> { let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone())); // 初始化智能任务队列 - let task_queue = Arc::new(queue::TaskQueue::new( - config.queue.max_concurrent_tasks, // 最大并发任务数 - config.queue.queue_size_limit, // 队列大小限制 - config.queue.high_request_rate_threshold, // 高请求频率阈值 - config.queue.idle_request_rate_threshold, // 空闲请求频率阈值 + let task_queue = Arc::new(queue::TaskQueue::new_with_db( + config.queue.max_concurrent_tasks, // 最大并发任务数 + config.queue.queue_size_limit, // 队列大小限制 + config.queue.high_request_rate_threshold, // 高请求频率阈值 + config.queue.idle_request_rate_threshold, // 空闲请求频率阈值 + Arc::clone(&db), // 数据库实例 )); // 在后台线程中自动导入新相册(不阻塞HTTP服务启动) @@ -65,15 +67,6 @@ async fn main() -> std::io::Result<()> { } }); - // 启动缩略图生成Worker - let thumbnail_worker = queue::ThumbnailWorker::new( - Arc::clone(&task_queue), - Arc::clone(&image_processor), - ); - tokio::spawn(async move { - thumbnail_worker.start().await; - }); - // 启动相册导入Worker let album_import_worker = queue::AlbumImportWorker::new( Arc::clone(&task_queue), @@ -125,13 +118,18 @@ async fn main() -> std::io::Result<()> { } }); + // 启动定时清理过期任务的任务(每小时执行一次) + let task_queue_for_cleanup = Arc::clone(&task_queue); + tokio::spawn(async move { + task_queue_for_cleanup.start_periodic_cleanup(Duration::from_secs(60 * 60)).await; // 1小时 + }); + println!("API接口:"); println!(" - GET /api/albums - 获取所有相册"); println!(" - GET /api/search?q=关键词 - 搜索相册"); println!(" - GET /api/album/:id - 获取相册详情"); println!(" - POST /api/albums - 创建新相册"); - println!(" - GET /api/thumbnail/:id - 获取缩略图"); - println!(" - GET /api/original/:id - 获取原始图片"); + println!(" - GET /api/image/:id - 获取图片(已移除缩略图功能)"); println!(" - GET /api/favorites - 获取收藏列表"); println!(" - GET /api/favorites/status/:id - 检查收藏状态"); println!(" - POST /api/favorites - 添加收藏"); @@ -141,7 +139,7 @@ async fn main() -> std::io::Result<()> { // 创建数据目录 fs::create_dir_all(&config.storage.images_dir).unwrap_or(()); - fs::create_dir_all(&config.storage.thumbnails_dir).unwrap_or(()); + // 不再创建缩略图目录,因为已移除缩略图功能 fs::create_dir_all(&config.database.url[7..]).unwrap_or(()); // 在闭包外提取需要的路径 @@ -156,29 +154,39 @@ async fn main() -> std::io::Result<()> { .app_data(web::Data::new(Arc::clone(&db))) .app_data(web::Data::new(Arc::clone(&image_processor))) .app_data(web::Data::new(Arc::clone(&albums_cache))) - .app_data(web::Data::new(Arc::clone(&task_queue))) + // 不再需要任务队列作为app_data,因为已移除缩略图功能 // API路由 .route("/api/albums", web::get().to(routes::get_albums)) .route("/api/search", web::get().to(routes::search_albums)) .route("/api/album/random", web::get().to(routes::get_random_album)) .route("/api/album/{id}", web::get().to(routes::get_album_detail)) .route("/api/albums", web::post().to(routes::create_album)) - .route("/api/thumbnail/{id}", web::get().to(routes::get_thumbnail)) - .route("/api/original/{id}", web::get().to(routes::get_original_image)) + .route("/api/image/{id}", web::get().to(routes::get_image)) // 收藏功能API路由 .route("/api/favorites", web::get().to(routes::get_favorites)) - .route("/api/favorites/status/{id}", web::get().to(routes::check_favorite_status)) + .route( + "/api/favorites/status/{id}", + web::get().to(routes::check_favorite_status), + ) .route("/api/favorites", web::post().to(routes::add_favorite)) - .route("/api/favorites/{id}", web::delete().to(routes::remove_favorite)) - .route("/api/favorites/image/{id}", web::delete().to(routes::remove_favorite_by_image)) + .route( + "/api/favorites/{id}", + web::delete().to(routes::remove_favorite), + ) + .route( + "/api/favorites/image/{id}", + web::delete().to(routes::remove_favorite_by_image), + ) // 队列状态API路由 .route("/api/queue/stats", web::get().to(routes::get_queue_stats)) // 静态文件服务 - 启用缓存 - .service(afs::Files::new("/assets", "./assets") - .show_files_listing() - .prefer_utf8(true) - .use_last_modified(true) - .use_etag(true)) + .service( + afs::Files::new("/assets", "./assets") + .show_files_listing() + .prefer_utf8(true) + .use_last_modified(true) + .use_etag(true), + ) // HTML页面路由 .route( "/", diff --git a/src/models.rs b/src/models.rs index 5c650ab..b69cbd6 100644 --- a/src/models.rs +++ b/src/models.rs @@ -123,7 +123,7 @@ impl AlbumsCache { } pub fn update(&mut self, albums: Vec) { - self.albums = albums.clone(); + self.albums = albums; // 不再克隆整个向量,而是移动所有权 self.last_updated = chrono::Utc::now(); // 构建搜索索引 @@ -138,15 +138,15 @@ impl AlbumsCache { self.search_index .entry(word.to_string()) .or_insert_with(Vec::new) - .push(album.clone()); + .push(album.clone()); // 这里仍需要克隆单个元素 } } // 为完整标题创建索引 self.search_index - .entry(title_lower.clone()) + .entry(title_lower) .or_insert_with(Vec::new) - .push(album.clone()); + .push(album.clone()); // 这里仍需要克隆单个元素 } info!( diff --git a/src/queue.rs b/src/queue.rs index 23e8079..6f7d3b2 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -9,6 +9,7 @@ use tracing::{info, warn, error, debug}; use uuid::Uuid; use image::GenericImageView; use crate::models::{AlbumIndex, CreateAlbum, CreateImage}; +use crate::database::Database; /// 任务类型 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -74,7 +75,8 @@ pub struct TaskTypeStats { /// 智能任务队列 pub struct TaskQueue { - queue: Mutex>, + high_priority_queue: Mutex>, + normal_priority_queue: Mutex>, active_tasks: AtomicUsize, completed_tasks: AtomicUsize, failed_tasks: AtomicUsize, @@ -90,12 +92,15 @@ pub struct TaskQueue { // 配置阈值 high_request_rate_threshold: f64, idle_request_rate_threshold: f64, + // 数据库引用用于定时任务 + db: Option>, } impl TaskQueue { pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self { Self { - queue: Mutex::new(VecDeque::new()), + high_priority_queue: Mutex::new(VecDeque::new()), + normal_priority_queue: Mutex::new(VecDeque::new()), active_tasks: AtomicUsize::new(0), completed_tasks: AtomicUsize::new(0), failed_tasks: AtomicUsize::new(0), @@ -111,42 +116,95 @@ impl TaskQueue { // 配置阈值 high_request_rate_threshold, idle_request_rate_threshold, + db: None, + } + } + + pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc) -> Self { + Self { + high_priority_queue: Mutex::new(VecDeque::new()), + normal_priority_queue: Mutex::new(VecDeque::new()), + active_tasks: AtomicUsize::new(0), + completed_tasks: AtomicUsize::new(0), + failed_tasks: AtomicUsize::new(0), + max_concurrent_tasks, + queue_size_limit, + is_paused: AtomicBool::new(false), + notify: Notify::new(), + last_activity: Mutex::new(Instant::now()), + // HTTP请求频率监控 + request_count: AtomicU64::new(0), + last_request_time: Mutex::new(Instant::now()), + request_window: Duration::from_secs(10), + // 配置阈值 + high_request_rate_threshold, + idle_request_rate_threshold, + db: Some(db), } } /// 添加任务到队列 pub async fn add_task(&self, task: Task) -> Result<(), String> { - let mut queue = self.queue.lock().await; + let (_high_queue_len, _normal_queue_len) = { + let high_queue = self.high_priority_queue.lock().await; + let normal_queue = self.normal_priority_queue.lock().await; + + // 检查队列总容量是否已满 + if high_queue.len() + normal_queue.len() >= self.queue_size_limit { + return Err("队列已满,无法添加新任务".to_string()); + } - // 检查队列是否已满 - if queue.len() >= self.queue_size_limit { - return Err("队列已满,无法添加新任务".to_string()); + // 检查是否已有相同ID的任务 + let exists_in_high = high_queue.iter().any(|t| t.id == task.id); + let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id); + + if exists_in_high || exists_in_normal { + return Err("该任务已在队列中".to_string()); + } + + (high_queue.len(), normal_queue.len()) + }; + + // 根据优先级添加到相应队列 + match task.priority { + TaskPriority::High => { + let mut high_queue = self.high_priority_queue.lock().await; + + // 高优先级队列内部也按优先级排序 + let insert_position = high_queue + .iter() + .position(|t| t.priority > task.priority) + .unwrap_or(high_queue.len()); + + high_queue.insert(insert_position, task); + + debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len()); + } + _ => { // Medium and Low go to normal priority queue + let mut normal_queue = self.normal_priority_queue.lock().await; + + // 按优先级排序 + let insert_position = normal_queue + .iter() + .position(|t| t.priority > task.priority) + .unwrap_or(normal_queue.len()); + + normal_queue.insert(insert_position, task); + + debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len()); + } } - // 检查是否已有相同ID的任务 - if queue.iter().any(|t| t.id == task.id) { - return Err("该任务已在队列中".to_string()); - } - - // 根据优先级插入任务(高优先级在前) - let insert_position = queue - .iter() - .position(|t| t.priority > task.priority) - .unwrap_or(queue.len()); - - queue.insert(insert_position, task); - // 更新最后活动时间 *self.last_activity.lock().await = Instant::now(); // 通知工作线程 self.notify.notify_one(); - debug!("任务已添加到队列,当前队列大小: {}", queue.len()); Ok(()) } - /// 从队列获取下一个任务 + /// 从队列获取下一个任务(优先处理高优先级队列) pub async fn next_task(&self) -> Option { // 检查是否暂停 if self.is_paused.load(Ordering::Acquire) { @@ -158,8 +216,18 @@ impl TaskQueue { return None; } - let mut queue = self.queue.lock().await; - if let Some(task) = queue.pop_front() { + // 优先从高优先级队列获取任务 + { + let mut high_queue = self.high_priority_queue.lock().await; + if let Some(task) = high_queue.pop_front() { + self.active_tasks.fetch_add(1, Ordering::Release); + return Some(task); + } + } + + // 如果高优先级队列为空,则从普通优先级队列获取任务 + let mut normal_queue = self.normal_priority_queue.lock().await; + if let Some(task) = normal_queue.pop_front() { self.active_tasks.fetch_add(1, Ordering::Release); Some(task) } else { @@ -231,16 +299,31 @@ impl TaskQueue { /// 获取队列统计信息 pub async fn get_stats(&self) -> QueueStats { - let queue = self.queue.lock().await; - - // 按任务类型统计 - let mut type_stats = std::collections::HashMap::new(); - for task in queue.iter() { - let stats = type_stats - .entry(task.task_type.clone()) - .or_insert((0, 0, 0)); - stats.0 += 1; // pending - } + let (high_queue_len, normal_queue_len, type_stats) = { + let high_queue = self.high_priority_queue.lock().await; + let normal_queue = self.normal_priority_queue.lock().await; + + // 按任务类型统计 + let mut type_stats = std::collections::HashMap::new(); + + // 统计高优先级队列中的任务 + for task in high_queue.iter() { + let stats = type_stats + .entry(task.task_type.clone()) + .or_insert((0, 0, 0)); + stats.0 += 1; // pending (high priority) + } + + // 统计普通优先级队列中的任务 + for task in normal_queue.iter() { + let stats = type_stats + .entry(task.task_type.clone()) + .or_insert((0, 0, 0)); + stats.0 += 1; // pending (normal priority) + } + + (high_queue.len(), normal_queue.len(), type_stats) + }; let task_type_stats = type_stats .into_iter() @@ -253,7 +336,7 @@ impl TaskQueue { .collect(); QueueStats { - pending_tasks: queue.len(), + pending_tasks: high_queue_len + normal_queue_len, active_tasks: self.active_tasks.load(Ordering::Acquire), completed_tasks: self.completed_tasks.load(Ordering::Acquire), failed_tasks: self.failed_tasks.load(Ordering::Acquire), @@ -268,15 +351,28 @@ impl TaskQueue { /// 清理过期的任务(超过一定时间的任务) pub async fn cleanup_expired_tasks(&self, max_age: Duration) { - let mut queue = self.queue.lock().await; let now = Instant::now(); - let initial_len = queue.len(); - - queue.retain(|task| now.duration_since(task.created_at) <= max_age); - - let removed_count = initial_len - queue.len(); - if removed_count > 0 { - warn!("清理了 {} 个过期任务", removed_count); + + // 清理高优先级队列中的过期任务 + { + let mut high_queue = self.high_priority_queue.lock().await; + let initial_len = high_queue.len(); + high_queue.retain(|task| now.duration_since(task.created_at) <= max_age); + let removed_count = initial_len - high_queue.len(); + if removed_count > 0 { + info!("清理了 {} 个过期的高优先级任务", removed_count); + } + } + + // 清理普通优先级队列中的过期任务 + { + let mut normal_queue = self.normal_priority_queue.lock().await; + let initial_len = normal_queue.len(); + normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age); + let removed_count = initial_len - normal_queue.len(); + if removed_count > 0 { + info!("清理了 {} 个过期的普通优先级任务", removed_count); + } } } @@ -290,6 +386,22 @@ impl TaskQueue { false } + + /// 启动定期清理过期任务 + pub async fn start_periodic_cleanup(self: Arc, interval_duration: Duration) { + let queue = Arc::clone(&self); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(interval_duration); + + loop { + interval.tick().await; + + // 清理超过24小时的任务 + queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await; + } + }); + } } /// 缩略图生成Worker @@ -340,8 +452,17 @@ impl ThumbnailWorker { } } else { // 如果不是缩略图任务,放回队列 - let mut queue = self.queue.queue.lock().await; - queue.push_front(task); + // 根据任务优先级放回相应队列 + match task.priority { + TaskPriority::High => { + let mut high_queue = self.queue.high_priority_queue.lock().await; + high_queue.push_front(task); + } + _ => { + let mut normal_queue = self.queue.normal_priority_queue.lock().await; + normal_queue.push_front(task); + } + } } } else { // 队列为空,等待新任务通知 @@ -353,44 +474,15 @@ impl ThumbnailWorker { } } - /// 处理缩略图任务 + // 缩略图任务处理功能已移除,直接返回成功 async fn process_thumbnail_task(&self, task: Task) -> bool { if let TaskData::ThumbnailGeneration { image_id, - original_data, + original_data: _, } = task.data { - // 使用具体的错误处理,避免 Send trait 问题 - let result = self - .image_processor - .generate_thumbnail_from_memory(&original_data, &image_id.to_string()) - .await; - - match result { - Ok(thumbnail_data) => { - // 保存缩略图到文件 - let thumbnail_path = self - .image_processor - .config - .thumbnails_dir - .join(image_id.to_string()); - - match tokio::fs::write(&thumbnail_path, &thumbnail_data).await { - Ok(_) => { - info!("✅ 缩略图生成成功: {:?}", image_id); - true - } - Err(e) => { - warn!("保存缩略图失败: {:?} - {}", image_id, e); - false - } - } - } - Err(e) => { - warn!("缩略图生成失败: {:?} - {:?}", image_id, e); - false - } - } + info!("缩略图任务已跳过(功能已移除): {:?}", image_id); + true // 直接返回成功,避免阻塞队列 } else { warn!("任务类型不匹配,期望缩略图生成任务"); false @@ -445,8 +537,17 @@ impl AlbumImportWorker { } } else { // 如果不是相册导入任务,放回队列并减少active_tasks计数器 - let mut queue = self.queue.queue.lock().await; - queue.push_front(task); + // 根据任务优先级放回相应队列 + match task.priority { + TaskPriority::High => { + let mut high_queue = self.queue.high_priority_queue.lock().await; + high_queue.push_front(task); + } + _ => { + let mut normal_queue = self.queue.normal_priority_queue.lock().await; + normal_queue.push_front(task); + } + } // 手动减少active_tasks计数器,因为任务没有被处理 self.queue.active_tasks.fetch_sub(1, Ordering::Release); } diff --git a/src/routes.rs b/src/routes.rs index 621bb8b..4eb9648 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -66,73 +66,7 @@ pub async fn get_favorites(db: web::Data>) -> impl Responder { HttpResponse::Ok().json(favorites) } -/// 检查缩略图质量并在需要时升级为高质量版本 -async fn check_and_upgrade_thumbnail_quality( - config: &crate::config::StorageConfig, - image: &Image, - original_path: &std::path::Path, - task_queue: Option>, -) -> Result<(), Box> { - let thumbnail_path = config.thumbnails_dir.join(image.id.to_string()); - - // 检查原始图片是否存在 - if !original_path.exists() { - return Ok(()); // 原始图片不存在,无法升级 - } - - // 读取缩略图文件信息 - let thumbnail_metadata = match fs::metadata(&thumbnail_path) { - Ok(meta) => meta, - Err(_) => return Ok(()), // 无法读取缩略图元数据 - }; - - // 检查缩略图文件大小(简单质量检测) - // 如果缩略图小于20KB,可能质量较低,需要升级 - if thumbnail_metadata.len() < 20 * 1024 { - // 读取原始图片数据 - let original_data = match fs::read(original_path) { - Ok(data) => data, - Err(e) => { - eprintln!("读取原始图片失败,无法升级缩略图: {}", e); - return Ok(()); - } - }; - - // 如果有队列系统,使用队列处理质量升级 - if let Some(queue) = task_queue { - let task = queue::TaskFactory::create_thumbnail_task( - image.id, - original_data, - queue::TaskPriority::Low, // 质量升级,低优先级 - ); - - if let Err(e) = queue.add_task(task).await { - eprintln!("添加缩略图质量升级任务失败: {}", e); - } - } else { - // 如果没有队列系统,直接生成高质量缩略图 - let image_processor = ImageProcessor::new(config.clone()); - match image_processor - .generate_thumbnail_from_memory(&original_data, &image.id.to_string()) - .await - { - Ok(high_quality_data) => { - // 保存高质量缩略图 - if let Err(e) = fs::write(&thumbnail_path, &high_quality_data) { - eprintln!("保存高质量缩略图失败: {}", e); - } else { - println!("✅ 缩略图已升级为高质量版本 (ID: {})", image.id); - } - } - Err(e) => { - eprintln!("生成高质量缩略图失败: {:?}", e); - } - } - } - } - - Ok(()) -} +// 图片质量检查功能已移除,直接返回原始图片 // 检查图片是否被收藏 pub async fn check_favorite_status( @@ -450,10 +384,10 @@ pub async fn create_album( } } -pub async fn get_thumbnail( +/// 获取原始图片(已移除缩略图功能) +pub async fn get_image( db: web::Data>, image_processor: web::Data>, - task_queue: web::Data>, path: web::Path, ) -> impl Responder { let image_id = match Uuid::parse_str(&path.into_inner()) { @@ -467,93 +401,17 @@ pub async fn get_thumbnail( match db.get_image(image_id).await { Ok(Some(image)) => { - // 优先返回缩略图,如果不存在则生成 - let thumbnail_path = image_processor - .config - .thumbnails_dir - .join(image.id.to_string()); let original_path = image_processor.config.images_dir.join(&image.filename); - if thumbnail_path.exists() { - // 返回现有缩略图,并异步检查是否需要替换为高质量版本 - match fs::read(&thumbnail_path) { + if original_path.exists() { + match fs::read(&original_path) { Ok(data) => { - let mime_type = image_processor.detect_mime_type("jpg"); // 缩略图总是JPEG格式 - - // 异步检查缩略图质量并可能替换为高质量版本 - let config_clone = image_processor.config.clone(); - let image_clone = image.clone(); - let original_path_clone = original_path.clone(); - let queue_clone = task_queue.get_ref().clone(); - tokio::spawn(async move { - if let Err(e) = check_and_upgrade_thumbnail_quality( - &config_clone, - &image_clone, - &original_path_clone, - Some(queue_clone), - ) - .await - { - eprintln!("缩略图质量检查失败: {}", e); - } - }); - HttpResponse::Ok() - .content_type(mime_type) + .content_type(image.mime_type.clone()) .insert_header(("Cache-Control", "public, max-age=31536000")) // 1年缓存 .insert_header(("ETag", format!("\"{}\"", image_id))) .body(data) } - Err(e) => { - eprintln!("读取缩略图失败: {}", e); - HttpResponse::InternalServerError().json(serde_json::json!({ - "error": "读取缩略图失败" - })) - } - } - } else if original_path.exists() { - // 先返回原始图片的缩略版本(快速响应) - match fs::read(&original_path) { - Ok(data) => { - // 快速生成小尺寸缩略图(不进行复杂处理) - match image_processor - .generate_fast_thumbnail(&data, &image.id.to_string()) - .await - { - Ok(thumbnail_data) => { - // 将高质量缩略图生成任务添加到队列 - let task = queue::TaskFactory::create_thumbnail_task( - image.id, - data.clone(), - queue::TaskPriority::High, // 前端请求,高优先级 - ); - - // 异步添加任务到队列 - let queue_clone = task_queue.clone(); - tokio::spawn(async move { - if let Err(e) = queue_clone.add_task(task).await { - eprintln!("添加缩略图生成任务失败: {}", e); - } - }); - - // 返回快速生成的缩略图数据 - HttpResponse::Ok() - .content_type("image/jpeg") - .insert_header(("Cache-Control", "public, max-age=30")) // 0.5分钟缓存 - .insert_header(("ETag", format!("\"{}\"", image_id))) - .body(thumbnail_data) - } - Err(e) => { - eprintln!("快速缩略图生成失败: {:?}", e); - // 如果快速生成失败,返回原始图片的小尺寸版本 - let small_data = - image_processor.resize_image_to_max(&data, 800).await; - HttpResponse::Ok() - .content_type(image.mime_type) - .body(small_data) - } - } - } Err(e) => { eprintln!("读取原始图片失败: {}", e); HttpResponse::InternalServerError().json(serde_json::json!({