update code level

This commit is contained in:
lychang
2026-03-28 12:36:26 +08:00
parent 7a68eb959a
commit 675a402848
12 changed files with 372 additions and 386 deletions

20
Cargo.lock generated
View File

@@ -1231,6 +1231,7 @@ dependencies = [
"chrono",
"config",
"image",
"mimalloc",
"rand 0.8.5",
"reqwest",
"serde",
@@ -1343,6 +1344,16 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "libredox"
version = "0.1.14"
@@ -1432,6 +1443,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "mimalloc"
version = "0.1.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"

View File

@@ -18,4 +18,5 @@ config = "0.13"
rand = "0.8"
tracing = "0.1"
tracing-subscriber = "0.3"
thiserror = "1.0"
thiserror = "1.0"
mimalloc = "0.1.48"

View File

@@ -118,7 +118,7 @@ function displayImages() {
let dataSrc = '';
if (image.id) {
dataSrc = `/api/thumbnail/${image.id}`;
dataSrc = `/api/image/${image.id}`;
imageUrl = '/assets/placeholder.svg'; // 初始使用占位图
} else if (image.original_url) {
dataSrc = image.original_url;

View File

@@ -205,7 +205,7 @@ function appendAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
dataSrc = `/api/thumbnail/${album.cover_image_id}`;
dataSrc = `/api/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -402,7 +402,7 @@ function displayAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
dataSrc = `/api/thumbnail/${album.cover_image_id}`;
dataSrc = `/api/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -730,7 +730,7 @@ function displayFavorites(favoriteImages) {
` : `${favoriteImages.map((fav, index) => `
<div class="favorite-card" onclick="openFavoriteImage('${fav.favorite.album_id}')">
<div class="card-image">
<img src="${API_BASE}/thumbnail/${fav.image.id}"
<img src="${API_BASE}/image/${fav.image.id}"
alt="${fav.image.title || '收藏的图片'}"
loading="lazy">
<div class="card-overlay">

View File

@@ -10,7 +10,7 @@ images_dir = "./albums"
thumbnails_dir = "./thumbnails"
[queue]
max_concurrent_tasks = 6
queue_size_limit = 200
max_concurrent_tasks = 2
queue_size_limit = 500
high_request_rate_threshold = 200.0
idle_request_rate_threshold = 10.0

View File

@@ -5,6 +5,7 @@ use uuid::Uuid;
pub struct Database {
pub pool: SqlitePool,
config: Config,
}
impl Database {
@@ -53,7 +54,10 @@ impl Database {
sqlx::migrate!("./migrations").run(&pool).await?;
// 确保image_count字段存在并正确初始化
let db = Database { pool };
let db = Database {
pool,
config: config.clone(), // 保存配置副本
};
db.ensure_image_count_field().await?;
Ok(db)
@@ -783,4 +787,60 @@ impl Database {
Ok(())
}
/// 获取配置
pub fn get_config(&self) -> &Config {
&self.config
}
/// 获取未处理的图片(即缩略图可能不存在的图片)
pub async fn get_unprocessed_images(&self, limit: i64) -> Result<Vec<Image>, sqlx::Error> {
let rows = sqlx::query(
r#"
SELECT * FROM images
ORDER BY created_at DESC
LIMIT ?
"#
)
.bind(limit)
.fetch_all(&self.pool)
.await?;
let mut images = Vec::new();
for row in rows {
let image_id_str = row.get::<String, _>("id");
let image_id = match Uuid::parse_str(&image_id_str) {
Ok(uuid) => uuid,
Err(e) => {
eprintln!("无效的图片UUID格式: {} - 错误: {}", image_id_str, e);
continue;
}
};
let album_id_str = row.get::<String, _>("album_id");
let album_id = match Uuid::parse_str(&album_id_str) {
Ok(uuid) => uuid,
Err(e) => {
eprintln!("无效的相册UUID格式: {} - 错误: {}", album_id_str, e);
continue;
}
};
images.push(Image {
id: image_id,
album_id,
filename: row.get("filename"),
original_url: row.get("original_url"),
title: row.get("title"),
description: row.get("description"),
file_size: row.get("file_size"),
width: row.get("width"),
height: row.get("height"),
mime_type: row.get("mime_type"),
created_at: row.get::<String, _>("created_at"),
});
}
Ok(images)
}
}

View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::fs;
use image::{GenericImageView, ImageFormat};
use uuid::Uuid;
use crate::config::StorageConfig;
use image::{GenericImageView, ImageFormat};
use std::fs;
use std::path::Path;
use thiserror::Error;
use uuid::Uuid;
#[derive(Error, Debug)]
pub enum ImageProcessorError {
@@ -18,7 +18,10 @@ pub enum ImageProcessorError {
// 手动实现 From<reqwest::Error> 以支持 ? 操作符
impl From<reqwest::Error> for ImageProcessorError {
fn from(err: reqwest::Error) -> Self {
ImageProcessorError::Io(std::io::Error::new(std::io::ErrorKind::Other, err.to_string()))
ImageProcessorError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
err.to_string(),
))
}
}
@@ -41,7 +44,11 @@ impl ImageProcessor {
Self { config }
}
pub async fn save_image(&self, image_data: &[u8], filename: &str) -> Result<ImageInfo, ImageProcessorError> {
pub async fn save_image(
&self,
image_data: &[u8],
filename: &str,
) -> Result<ImageInfo, ImageProcessorError> {
let image_id = Uuid::new_v4();
let file_extension = Path::new(filename)
.extension()
@@ -60,7 +67,10 @@ impl ImageProcessor {
let file_size = image_data.len() as i64;
// 生成缩略图
let thumbnail_path = self.config.thumbnails_dir.join(format!("thumb_{}", new_filename));
let thumbnail_path = self
.config
.thumbnails_dir
.join(format!("thumb_{}", new_filename));
self.generate_thumbnail(&img, &thumbnail_path, 0, 0)?;
Ok(ImageInfo {
@@ -83,12 +93,12 @@ impl ImageProcessor {
) -> Result<(), ImageProcessorError> {
// 使用统一的缩略图尺寸(400x300,保持宽高比)
let thumbnail = img.thumbnail(400, 300);
// 确保输出目录存在
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)?;
}
thumbnail.save_with_format(output_path, ImageFormat::Jpeg)?;
Ok(())
@@ -104,21 +114,6 @@ impl ImageProcessor {
}
}
/// 快速生成缩略图(低质量但快速)
pub async fn generate_fast_thumbnail(&self, image_data: &[u8], _filename: &str) -> Result<Vec<u8>, ImageProcessorError> {
// 使用快速解码和缩放
let img = image::load_from_memory(image_data)?;
// 快速缩放到小尺寸(200x200)
let thumbnail = img.thumbnail(200, 200);
// 使用低质量JPEG压缩以加快编码速度
let mut output = Vec::new();
thumbnail.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(50))?;
Ok(output)
}
/// 快速调整图片尺寸(最大宽度)
pub async fn resize_image_to_max(&self, image_data: &[u8], max_width: u32) -> Vec<u8> {
match image::load_from_memory(image_data) {
@@ -127,12 +122,18 @@ impl ImageProcessor {
if width <= max_width {
return image_data.to_vec(); // 无需调整
}
let new_height = (height as f32 * max_width as f32 / width as f32) as u32;
let resized = img.thumbnail(max_width, new_height);
let mut output = Vec::new();
if resized.write_to(&mut std::io::Cursor::new(&mut output), image::ImageOutputFormat::Jpeg(75)).is_ok() {
if resized
.write_to(
&mut std::io::Cursor::new(&mut output),
image::ImageOutputFormat::Jpeg(75),
)
.is_ok()
{
output
} else {
image_data.to_vec() // 失败时返回原图
@@ -146,10 +147,6 @@ impl ImageProcessor {
format!("/images/{}", filename)
}
pub fn get_thumbnail_url(&self, filename: &str) -> String {
format!("/thumbnails/thumb_{}", filename)
}
pub async fn download_and_save_image(
&self,
url: &str,
@@ -161,42 +158,23 @@ impl ImageProcessor {
self.save_image(&image_data, filename).await
}
/// 从图片数据生成缩略图并保存到文件
pub async fn generate_thumbnail_from_data(
&self,
image_data: &[u8],
thumbnail_filename: &str,
) -> Result<(), ImageProcessorError> {
// 使用统一的缩略图生成方法
let thumbnail_data = self.generate_thumbnail_from_memory(image_data, thumbnail_filename).await?;
// 确保缩略图目录存在
fs::create_dir_all(&self.config.thumbnails_dir)?;
// 保存缩略图到文件
let thumbnail_path = self.config.thumbnails_dir.join(thumbnail_filename);
fs::write(&thumbnail_path, thumbnail_data)?;
Ok(())
}
/// 检测图片格式
fn detect_image_format(&self, image_data: &[u8]) -> Option<String> {
if image_data.len() < 12 {
return None;
}
/// 从内存中的图片数据生成缩略图数据(不保存到文件)
pub async fn generate_thumbnail_from_memory(
&self,
image_data: &[u8],
_thumbnail_filename: &str,
) -> Result<Vec<u8>, ImageProcessorError> {
// 加载图片
let img = image::load_from_memory(image_data)?;
// 生成缩略图(600x600,保持宽高比)
let thumbnail = img.thumbnail(600, 600);
// 将缩略图数据保存到内存缓冲区
let mut buffer = std::io::Cursor::new(Vec::new());
thumbnail.write_to(&mut buffer, ImageFormat::Jpeg)?;
Ok(buffer.into_inner())
match &image_data[0..12] {
b"\xff\xd8\xff" => Some("JPEG".to_string()),
b"\x89PNG\r\n\x1a\n" => Some("PNG".to_string()),
b"GIF87a" => Some("GIF87a".to_string()),
b"GIF89a" => Some("GIF89a".to_string()),
b"BM" => Some("BMP".to_string()),
b"RIFF" if image_data.len() > 12 && &image_data[8..12] == b"WEBP" => {
Some("WebP".to_string())
}
_ => None,
}
}
}
@@ -208,4 +186,4 @@ pub struct ImageInfo {
pub height: i32,
pub file_size: i64,
pub mime_type: String,
}
}

View File

@@ -37,11 +37,7 @@ async fn batch_insert_images_and_set_cover(
// 获取第一张图片的信息
if let Ok(Some(first_image)) = db.get_image(*first_image_id).await {
// 生成缩略图
match generate_thumbnail_for_image(&first_image, album_path, folder_name).await {
Ok(_) => println!(" 封面缩略图生成成功"),
Err(e) => eprintln!(" 封面缩略图生成失败: {}", e),
}
println!(" 封面图片已设置: {}", first_image.filename);
}
}
@@ -55,7 +51,7 @@ async fn batch_insert_images_and_set_cover(
}
pub async fn import_existing_data(config: &Config, db: Arc<Database>, task_queue: Option<Arc<crate::queue::TaskQueue>>) -> Result<(), Box<dyn std::error::Error>> {
pub async fn import_existing_data(config: &Config, db: Arc<Database>, _task_queue: Option<Arc<crate::queue::TaskQueue>>) -> Result<(), Box<dyn std::error::Error>> {
println!("开始扫描图片目录并导入新相册...");
let albums_dir = Path::new(&config.storage.images_dir);
@@ -143,7 +139,7 @@ pub async fn import_existing_data(config: &Config, db: Arc<Database>, task_queue
}
// 如果有队列系统,使用队列异步处理相册导入
if let Some(queue) = task_queue {
if let Some(queue) = _task_queue {
println!("🚀 使用队列系统异步导入相册...");
for (folder_name, path) in album_folders {
@@ -475,40 +471,4 @@ async fn reimport_single_album(
batch_insert_images_and_set_cover(&db, images_to_insert, album_id, path, folder_name).await
}
/// 为图片生成缩略图(使用ImageProcessor)
async fn generate_thumbnail_for_image(
image: &Image,
album_path: &Path,
_folder_name: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// 构建图片文件路径
let image_filename = image.filename.split('/').last().unwrap_or(&image.filename);
let image_path = album_path.join(image_filename);
if !image_path.exists() {
return Err(format!("图片文件不存在: {}", image_path.display()).into());
}
// 读取图片数据
let image_data = fs::read(&image_path)?;
// 创建临时的ImageProcessor实例
// 注意:这里使用默认的缩略图目录路径
let config = crate::config::StorageConfig {
images_dir: Path::new("./albums").to_path_buf(),
thumbnails_dir: Path::new("./thumbnails").to_path_buf(),
};
let image_processor = crate::image_processor::ImageProcessor::new(config);
// 生成缩略图
match image_processor.generate_thumbnail_from_data(&image_data, &image.id.to_string()).await {
Ok(_) => {
println!(" 缩略图生成成功 (ID: {})", image.id);
Ok(())
}
Err(e) => {
Err(format!("缩略图生成失败: {:?}", e).into())
}
}
}
// 缩略图生成功能已移除

View File

@@ -3,22 +3,23 @@ mod database;
mod image_processor;
mod import;
mod models;
mod routes;
mod queue;
mod routes;
use crate::models::AlbumsCache;
use actix_files as afs;
use actix_web::{App, HttpResponse, HttpServer, web, middleware};
use actix_web::{App, HttpResponse, HttpServer, middleware, web};
use std::fs;
use std::sync::Arc;
use tokio::time::{interval, Duration};
use tracing::{info, error};
use crate::models::AlbumsCache;
use tokio::time::{Duration, interval};
use tracing::{error, info};
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[actix_web::main]
async fn main() -> std::io::Result<()> {
// 初始化结构化日志
tracing_subscriber::fmt::init();
// 加载配置
let config = match config::Config::load() {
Ok(cfg) => cfg,
@@ -29,7 +30,7 @@ async fn main() -> std::io::Result<()> {
};
info!("启动图片展示服务...");
info!("服务地址: http://{}", config.server_address());
info!("服务地址: http://localhost:{} http://{}", config.server.port, config.server_address());
// 初始化数据库
let db = match database::Database::new(&config).await {
@@ -46,11 +47,12 @@ async fn main() -> std::io::Result<()> {
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// 初始化智能任务队列
let task_queue = Arc::new(queue::TaskQueue::new(
config.queue.max_concurrent_tasks, // 最大并发任务数
config.queue.queue_size_limit, // 队列大小限制
config.queue.high_request_rate_threshold, // 高请求频率阈值
config.queue.idle_request_rate_threshold, // 空闲请求频率阈值
let task_queue = Arc::new(queue::TaskQueue::new_with_db(
config.queue.max_concurrent_tasks, // 最大并发任务数
config.queue.queue_size_limit, // 队列大小限制
config.queue.high_request_rate_threshold, // 高请求频率阈值
config.queue.idle_request_rate_threshold, // 空闲请求频率阈值
Arc::clone(&db), // 数据库实例
));
// 在后台线程中自动导入新相册(不阻塞HTTP服务启动)
@@ -65,15 +67,6 @@ async fn main() -> std::io::Result<()> {
}
});
// 启动缩略图生成Worker
let thumbnail_worker = queue::ThumbnailWorker::new(
Arc::clone(&task_queue),
Arc::clone(&image_processor),
);
tokio::spawn(async move {
thumbnail_worker.start().await;
});
// 启动相册导入Worker
let album_import_worker = queue::AlbumImportWorker::new(
Arc::clone(&task_queue),
@@ -125,13 +118,18 @@ async fn main() -> std::io::Result<()> {
}
});
// 启动定时清理过期任务的任务(每小时执行一次)
let task_queue_for_cleanup = Arc::clone(&task_queue);
tokio::spawn(async move {
task_queue_for_cleanup.start_periodic_cleanup(Duration::from_secs(60 * 60)).await; // 1小时
});
println!("API接口:");
println!(" - GET /api/albums - 获取所有相册");
println!(" - GET /api/search?q=关键词 - 搜索相册");
println!(" - GET /api/album/:id - 获取相册详情");
println!(" - POST /api/albums - 创建新相册");
println!(" - GET /api/thumbnail/:id - 获取缩略图");
println!(" - GET /api/original/:id - 获取原始图片");
println!(" - GET /api/image/:id - 获取图片(已移除缩略图功能)");
println!(" - GET /api/favorites - 获取收藏列表");
println!(" - GET /api/favorites/status/:id - 检查收藏状态");
println!(" - POST /api/favorites - 添加收藏");
@@ -141,7 +139,7 @@ async fn main() -> std::io::Result<()> {
// 创建数据目录
fs::create_dir_all(&config.storage.images_dir).unwrap_or(());
fs::create_dir_all(&config.storage.thumbnails_dir).unwrap_or(());
// 不再创建缩略图目录,因为已移除缩略图功能
fs::create_dir_all(&config.database.url[7..]).unwrap_or(());
// 在闭包外提取需要的路径
@@ -156,29 +154,39 @@ async fn main() -> std::io::Result<()> {
.app_data(web::Data::new(Arc::clone(&db)))
.app_data(web::Data::new(Arc::clone(&image_processor)))
.app_data(web::Data::new(Arc::clone(&albums_cache)))
.app_data(web::Data::new(Arc::clone(&task_queue)))
// 不再需要任务队列作为app_data,因为已移除缩略图功能
// API路由
.route("/api/albums", web::get().to(routes::get_albums))
.route("/api/search", web::get().to(routes::search_albums))
.route("/api/album/random", web::get().to(routes::get_random_album))
.route("/api/album/{id}", web::get().to(routes::get_album_detail))
.route("/api/albums", web::post().to(routes::create_album))
.route("/api/thumbnail/{id}", web::get().to(routes::get_thumbnail))
.route("/api/original/{id}", web::get().to(routes::get_original_image))
.route("/api/image/{id}", web::get().to(routes::get_image))
// 收藏功能API路由
.route("/api/favorites", web::get().to(routes::get_favorites))
.route("/api/favorites/status/{id}", web::get().to(routes::check_favorite_status))
.route(
"/api/favorites/status/{id}",
web::get().to(routes::check_favorite_status),
)
.route("/api/favorites", web::post().to(routes::add_favorite))
.route("/api/favorites/{id}", web::delete().to(routes::remove_favorite))
.route("/api/favorites/image/{id}", web::delete().to(routes::remove_favorite_by_image))
.route(
"/api/favorites/{id}",
web::delete().to(routes::remove_favorite),
)
.route(
"/api/favorites/image/{id}",
web::delete().to(routes::remove_favorite_by_image),
)
// 队列状态API路由
.route("/api/queue/stats", web::get().to(routes::get_queue_stats))
// 静态文件服务 - 启用缓存
.service(afs::Files::new("/assets", "./assets")
.show_files_listing()
.prefer_utf8(true)
.use_last_modified(true)
.use_etag(true))
.service(
afs::Files::new("/assets", "./assets")
.show_files_listing()
.prefer_utf8(true)
.use_last_modified(true)
.use_etag(true),
)
// HTML页面路由
.route(
"/",

View File

@@ -123,7 +123,7 @@ impl AlbumsCache {
}
pub fn update(&mut self, albums: Vec<AlbumInfo>) {
self.albums = albums.clone();
self.albums = albums; // 不再克隆整个向量,而是移动所有权
self.last_updated = chrono::Utc::now();
// 构建搜索索引
@@ -138,15 +138,15 @@ impl AlbumsCache {
self.search_index
.entry(word.to_string())
.or_insert_with(Vec::new)
.push(album.clone());
.push(album.clone()); // 这里仍需要克隆单个元素
}
}
// 为完整标题创建索引
self.search_index
.entry(title_lower.clone())
.entry(title_lower)
.or_insert_with(Vec::new)
.push(album.clone());
.push(album.clone()); // 这里仍需要克隆单个元素
}
info!(

View File

@@ -9,6 +9,7 @@ use tracing::{info, warn, error, debug};
use uuid::Uuid;
use image::GenericImageView;
use crate::models::{AlbumIndex, CreateAlbum, CreateImage};
use crate::database::Database;
/// 任务类型
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -74,7 +75,8 @@ pub struct TaskTypeStats {
/// 智能任务队列
pub struct TaskQueue {
queue: Mutex<VecDeque<Task>>,
high_priority_queue: Mutex<VecDeque<Task>>,
normal_priority_queue: Mutex<VecDeque<Task>>,
active_tasks: AtomicUsize,
completed_tasks: AtomicUsize,
failed_tasks: AtomicUsize,
@@ -90,12 +92,15 @@ pub struct TaskQueue {
// 配置阈值
high_request_rate_threshold: f64,
idle_request_rate_threshold: f64,
// 数据库引用用于定时任务
db: Option<Arc<Database>>,
}
impl TaskQueue {
pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self {
Self {
queue: Mutex::new(VecDeque::new()),
high_priority_queue: Mutex::new(VecDeque::new()),
normal_priority_queue: Mutex::new(VecDeque::new()),
active_tasks: AtomicUsize::new(0),
completed_tasks: AtomicUsize::new(0),
failed_tasks: AtomicUsize::new(0),
@@ -111,42 +116,95 @@ impl TaskQueue {
// 配置阈值
high_request_rate_threshold,
idle_request_rate_threshold,
db: None,
}
}
pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc<Database>) -> Self {
Self {
high_priority_queue: Mutex::new(VecDeque::new()),
normal_priority_queue: Mutex::new(VecDeque::new()),
active_tasks: AtomicUsize::new(0),
completed_tasks: AtomicUsize::new(0),
failed_tasks: AtomicUsize::new(0),
max_concurrent_tasks,
queue_size_limit,
is_paused: AtomicBool::new(false),
notify: Notify::new(),
last_activity: Mutex::new(Instant::now()),
// HTTP请求频率监控
request_count: AtomicU64::new(0),
last_request_time: Mutex::new(Instant::now()),
request_window: Duration::from_secs(10),
// 配置阈值
high_request_rate_threshold,
idle_request_rate_threshold,
db: Some(db),
}
}
/// 添加任务到队列
pub async fn add_task(&self, task: Task) -> Result<(), String> {
let mut queue = self.queue.lock().await;
let (_high_queue_len, _normal_queue_len) = {
let high_queue = self.high_priority_queue.lock().await;
let normal_queue = self.normal_priority_queue.lock().await;
// 检查队列总容量是否已满
if high_queue.len() + normal_queue.len() >= self.queue_size_limit {
return Err("队列已满,无法添加新任务".to_string());
}
// 检查队列是否已满
if queue.len() >= self.queue_size_limit {
return Err("队列已满,无法添加新任务".to_string());
// 检查是否已有相同ID的任务
let exists_in_high = high_queue.iter().any(|t| t.id == task.id);
let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id);
if exists_in_high || exists_in_normal {
return Err("该任务已在队列中".to_string());
}
(high_queue.len(), normal_queue.len())
};
// 根据优先级添加到相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.high_priority_queue.lock().await;
// 高优先级队列内部也按优先级排序
let insert_position = high_queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(high_queue.len());
high_queue.insert(insert_position, task);
debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len());
}
_ => { // Medium and Low go to normal priority queue
let mut normal_queue = self.normal_priority_queue.lock().await;
// 按优先级排序
let insert_position = normal_queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(normal_queue.len());
normal_queue.insert(insert_position, task);
debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len());
}
}
// 检查是否已有相同ID的任务
if queue.iter().any(|t| t.id == task.id) {
return Err("该任务已在队列中".to_string());
}
// 根据优先级插入任务(高优先级在前)
let insert_position = queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(queue.len());
queue.insert(insert_position, task);
// 更新最后活动时间
*self.last_activity.lock().await = Instant::now();
// 通知工作线程
self.notify.notify_one();
debug!("任务已添加到队列,当前队列大小: {}", queue.len());
Ok(())
}
/// 从队列获取下一个任务
/// 从队列获取下一个任务(优先处理高优先级队列)
pub async fn next_task(&self) -> Option<Task> {
// 检查是否暂停
if self.is_paused.load(Ordering::Acquire) {
@@ -158,8 +216,18 @@ impl TaskQueue {
return None;
}
let mut queue = self.queue.lock().await;
if let Some(task) = queue.pop_front() {
// 优先从高优先级队列获取任务
{
let mut high_queue = self.high_priority_queue.lock().await;
if let Some(task) = high_queue.pop_front() {
self.active_tasks.fetch_add(1, Ordering::Release);
return Some(task);
}
}
// 如果高优先级队列为空,则从普通优先级队列获取任务
let mut normal_queue = self.normal_priority_queue.lock().await;
if let Some(task) = normal_queue.pop_front() {
self.active_tasks.fetch_add(1, Ordering::Release);
Some(task)
} else {
@@ -231,16 +299,31 @@ impl TaskQueue {
/// 获取队列统计信息
pub async fn get_stats(&self) -> QueueStats {
let queue = self.queue.lock().await;
// 按任务类型统计
let mut type_stats = std::collections::HashMap::new();
for task in queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending
}
let (high_queue_len, normal_queue_len, type_stats) = {
let high_queue = self.high_priority_queue.lock().await;
let normal_queue = self.normal_priority_queue.lock().await;
// 按任务类型统计
let mut type_stats = std::collections::HashMap::new();
// 统计高优先级队列中的任务
for task in high_queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending (high priority)
}
// 统计普通优先级队列中的任务
for task in normal_queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending (normal priority)
}
(high_queue.len(), normal_queue.len(), type_stats)
};
let task_type_stats = type_stats
.into_iter()
@@ -253,7 +336,7 @@ impl TaskQueue {
.collect();
QueueStats {
pending_tasks: queue.len(),
pending_tasks: high_queue_len + normal_queue_len,
active_tasks: self.active_tasks.load(Ordering::Acquire),
completed_tasks: self.completed_tasks.load(Ordering::Acquire),
failed_tasks: self.failed_tasks.load(Ordering::Acquire),
@@ -268,15 +351,28 @@ impl TaskQueue {
/// 清理过期的任务(超过一定时间的任务)
pub async fn cleanup_expired_tasks(&self, max_age: Duration) {
let mut queue = self.queue.lock().await;
let now = Instant::now();
let initial_len = queue.len();
queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - queue.len();
if removed_count > 0 {
warn!("清理了 {} 个过期任务", removed_count);
// 清理高优先级队列中的过期任务
{
let mut high_queue = self.high_priority_queue.lock().await;
let initial_len = high_queue.len();
high_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - high_queue.len();
if removed_count > 0 {
info!("清理了 {} 个过期的高优先级任务", removed_count);
}
}
// 清理普通优先级队列中的过期任务
{
let mut normal_queue = self.normal_priority_queue.lock().await;
let initial_len = normal_queue.len();
normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - normal_queue.len();
if removed_count > 0 {
info!("清理了 {} 个过期的普通优先级任务", removed_count);
}
}
}
@@ -290,6 +386,22 @@ impl TaskQueue {
false
}
/// 启动定期清理过期任务
pub async fn start_periodic_cleanup(self: Arc<Self>, interval_duration: Duration) {
let queue = Arc::clone(&self);
tokio::spawn(async move {
let mut interval = tokio::time::interval(interval_duration);
loop {
interval.tick().await;
// 清理超过24小时的任务
queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await;
}
});
}
}
/// 缩略图生成Worker
@@ -340,8 +452,17 @@ impl ThumbnailWorker {
}
} else {
// 如果不是缩略图任务,放回队列
let mut queue = self.queue.queue.lock().await;
queue.push_front(task);
// 根据任务优先级放回相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.queue.high_priority_queue.lock().await;
high_queue.push_front(task);
}
_ => {
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
normal_queue.push_front(task);
}
}
}
} else {
// 队列为空,等待新任务通知
@@ -353,44 +474,15 @@ impl ThumbnailWorker {
}
}
/// 处理缩略图任务
// 缩略图任务处理功能已移除,直接返回成功
async fn process_thumbnail_task(&self, task: Task) -> bool {
if let TaskData::ThumbnailGeneration {
image_id,
original_data,
original_data: _,
} = task.data
{
// 使用具体的错误处理,避免 Send trait 问题
let result = self
.image_processor
.generate_thumbnail_from_memory(&original_data, &image_id.to_string())
.await;
match result {
Ok(thumbnail_data) => {
// 保存缩略图到文件
let thumbnail_path = self
.image_processor
.config
.thumbnails_dir
.join(image_id.to_string());
match tokio::fs::write(&thumbnail_path, &thumbnail_data).await {
Ok(_) => {
info!("✅ 缩略图生成成功: {:?}", image_id);
true
}
Err(e) => {
warn!("保存缩略图失败: {:?} - {}", image_id, e);
false
}
}
}
Err(e) => {
warn!("缩略图生成失败: {:?} - {:?}", image_id, e);
false
}
}
info!("缩略图任务已跳过(功能已移除): {:?}", image_id);
true // 直接返回成功,避免阻塞队列
} else {
warn!("任务类型不匹配,期望缩略图生成任务");
false
@@ -445,8 +537,17 @@ impl AlbumImportWorker {
}
} else {
// 如果不是相册导入任务,放回队列并减少active_tasks计数器
let mut queue = self.queue.queue.lock().await;
queue.push_front(task);
// 根据任务优先级放回相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.queue.high_priority_queue.lock().await;
high_queue.push_front(task);
}
_ => {
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
normal_queue.push_front(task);
}
}
// 手动减少active_tasks计数器,因为任务没有被处理
self.queue.active_tasks.fetch_sub(1, Ordering::Release);
}

View File

@@ -66,73 +66,7 @@ pub async fn get_favorites(db: web::Data<Arc<Database>>) -> impl Responder {
HttpResponse::Ok().json(favorites)
}
/// 检查缩略图质量并在需要时升级为高质量版本
async fn check_and_upgrade_thumbnail_quality(
config: &crate::config::StorageConfig,
image: &Image,
original_path: &std::path::Path,
task_queue: Option<Arc<queue::TaskQueue>>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let thumbnail_path = config.thumbnails_dir.join(image.id.to_string());
// 检查原始图片是否存在
if !original_path.exists() {
return Ok(()); // 原始图片不存在,无法升级
}
// 读取缩略图文件信息
let thumbnail_metadata = match fs::metadata(&thumbnail_path) {
Ok(meta) => meta,
Err(_) => return Ok(()), // 无法读取缩略图元数据
};
// 检查缩略图文件大小(简单质量检测)
// 如果缩略图小于20KB,可能质量较低,需要升级
if thumbnail_metadata.len() < 20 * 1024 {
// 读取原始图片数据
let original_data = match fs::read(original_path) {
Ok(data) => data,
Err(e) => {
eprintln!("读取原始图片失败,无法升级缩略图: {}", e);
return Ok(());
}
};
// 如果有队列系统,使用队列处理质量升级
if let Some(queue) = task_queue {
let task = queue::TaskFactory::create_thumbnail_task(
image.id,
original_data,
queue::TaskPriority::Low, // 质量升级,低优先级
);
if let Err(e) = queue.add_task(task).await {
eprintln!("添加缩略图质量升级任务失败: {}", e);
}
} else {
// 如果没有队列系统,直接生成高质量缩略图
let image_processor = ImageProcessor::new(config.clone());
match image_processor
.generate_thumbnail_from_memory(&original_data, &image.id.to_string())
.await
{
Ok(high_quality_data) => {
// 保存高质量缩略图
if let Err(e) = fs::write(&thumbnail_path, &high_quality_data) {
eprintln!("保存高质量缩略图失败: {}", e);
} else {
println!("✅ 缩略图已升级为高质量版本 (ID: {})", image.id);
}
}
Err(e) => {
eprintln!("生成高质量缩略图失败: {:?}", e);
}
}
}
}
Ok(())
}
// 图片质量检查功能已移除,直接返回原始图片
// 检查图片是否被收藏
pub async fn check_favorite_status(
@@ -450,10 +384,10 @@ pub async fn create_album(
}
}
pub async fn get_thumbnail(
/// 获取原始图片(已移除缩略图功能)
pub async fn get_image(
db: web::Data<Arc<Database>>,
image_processor: web::Data<Arc<ImageProcessor>>,
task_queue: web::Data<Arc<queue::TaskQueue>>,
path: web::Path<String>,
) -> impl Responder {
let image_id = match Uuid::parse_str(&path.into_inner()) {
@@ -467,93 +401,17 @@ pub async fn get_thumbnail(
match db.get_image(image_id).await {
Ok(Some(image)) => {
// 优先返回缩略图,如果不存在则生成
let thumbnail_path = image_processor
.config
.thumbnails_dir
.join(image.id.to_string());
let original_path = image_processor.config.images_dir.join(&image.filename);
if thumbnail_path.exists() {
// 返回现有缩略图,并异步检查是否需要替换为高质量版本
match fs::read(&thumbnail_path) {
if original_path.exists() {
match fs::read(&original_path) {
Ok(data) => {
let mime_type = image_processor.detect_mime_type("jpg"); // 缩略图总是JPEG格式
// 异步检查缩略图质量并可能替换为高质量版本
let config_clone = image_processor.config.clone();
let image_clone = image.clone();
let original_path_clone = original_path.clone();
let queue_clone = task_queue.get_ref().clone();
tokio::spawn(async move {
if let Err(e) = check_and_upgrade_thumbnail_quality(
&config_clone,
&image_clone,
&original_path_clone,
Some(queue_clone),
)
.await
{
eprintln!("缩略图质量检查失败: {}", e);
}
});
HttpResponse::Ok()
.content_type(mime_type)
.content_type(image.mime_type.clone())
.insert_header(("Cache-Control", "public, max-age=31536000")) // 1年缓存
.insert_header(("ETag", format!("\"{}\"", image_id)))
.body(data)
}
Err(e) => {
eprintln!("读取缩略图失败: {}", e);
HttpResponse::InternalServerError().json(serde_json::json!({
"error": "读取缩略图失败"
}))
}
}
} else if original_path.exists() {
// 先返回原始图片的缩略版本(快速响应)
match fs::read(&original_path) {
Ok(data) => {
// 快速生成小尺寸缩略图(不进行复杂处理)
match image_processor
.generate_fast_thumbnail(&data, &image.id.to_string())
.await
{
Ok(thumbnail_data) => {
// 将高质量缩略图生成任务添加到队列
let task = queue::TaskFactory::create_thumbnail_task(
image.id,
data.clone(),
queue::TaskPriority::High, // 前端请求,高优先级
);
// 异步添加任务到队列
let queue_clone = task_queue.clone();
tokio::spawn(async move {
if let Err(e) = queue_clone.add_task(task).await {
eprintln!("添加缩略图生成任务失败: {}", e);
}
});
// 返回快速生成的缩略图数据
HttpResponse::Ok()
.content_type("image/jpeg")
.insert_header(("Cache-Control", "public, max-age=30")) // 0.5分钟缓存
.insert_header(("ETag", format!("\"{}\"", image_id)))
.body(thumbnail_data)
}
Err(e) => {
eprintln!("快速缩略图生成失败: {:?}", e);
// 如果快速生成失败,返回原始图片的小尺寸版本
let small_data =
image_processor.resize_image_to_max(&data, 800).await;
HttpResponse::Ok()
.content_type(image.mime_type)
.body(small_data)
}
}
}
Err(e) => {
eprintln!("读取原始图片失败: {}", e);
HttpResponse::InternalServerError().json(serde_json::json!({