update code level

This commit is contained in:
lychang
2026-03-28 12:36:26 +08:00
parent 7a68eb959a
commit 675a402848
12 changed files with 372 additions and 386 deletions

View File

@@ -9,6 +9,7 @@ use tracing::{info, warn, error, debug};
use uuid::Uuid;
use image::GenericImageView;
use crate::models::{AlbumIndex, CreateAlbum, CreateImage};
use crate::database::Database;
/// 任务类型
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -74,7 +75,8 @@ pub struct TaskTypeStats {
/// 智能任务队列
pub struct TaskQueue {
queue: Mutex<VecDeque<Task>>,
high_priority_queue: Mutex<VecDeque<Task>>,
normal_priority_queue: Mutex<VecDeque<Task>>,
active_tasks: AtomicUsize,
completed_tasks: AtomicUsize,
failed_tasks: AtomicUsize,
@@ -90,12 +92,15 @@ pub struct TaskQueue {
// 配置阈值
high_request_rate_threshold: f64,
idle_request_rate_threshold: f64,
// 数据库引用用于定时任务
db: Option<Arc<Database>>,
}
impl TaskQueue {
pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self {
Self {
queue: Mutex::new(VecDeque::new()),
high_priority_queue: Mutex::new(VecDeque::new()),
normal_priority_queue: Mutex::new(VecDeque::new()),
active_tasks: AtomicUsize::new(0),
completed_tasks: AtomicUsize::new(0),
failed_tasks: AtomicUsize::new(0),
@@ -111,42 +116,95 @@ impl TaskQueue {
// 配置阈值
high_request_rate_threshold,
idle_request_rate_threshold,
db: None,
}
}
pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc<Database>) -> Self {
Self {
high_priority_queue: Mutex::new(VecDeque::new()),
normal_priority_queue: Mutex::new(VecDeque::new()),
active_tasks: AtomicUsize::new(0),
completed_tasks: AtomicUsize::new(0),
failed_tasks: AtomicUsize::new(0),
max_concurrent_tasks,
queue_size_limit,
is_paused: AtomicBool::new(false),
notify: Notify::new(),
last_activity: Mutex::new(Instant::now()),
// HTTP请求频率监控
request_count: AtomicU64::new(0),
last_request_time: Mutex::new(Instant::now()),
request_window: Duration::from_secs(10),
// 配置阈值
high_request_rate_threshold,
idle_request_rate_threshold,
db: Some(db),
}
}
/// 添加任务到队列
pub async fn add_task(&self, task: Task) -> Result<(), String> {
let mut queue = self.queue.lock().await;
let (_high_queue_len, _normal_queue_len) = {
let high_queue = self.high_priority_queue.lock().await;
let normal_queue = self.normal_priority_queue.lock().await;
// 检查队列总容量是否已满
if high_queue.len() + normal_queue.len() >= self.queue_size_limit {
return Err("队列已满,无法添加新任务".to_string());
}
// 检查队列是否已满
if queue.len() >= self.queue_size_limit {
return Err("队列已满,无法添加新任务".to_string());
// 检查是否已有相同ID的任务
let exists_in_high = high_queue.iter().any(|t| t.id == task.id);
let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id);
if exists_in_high || exists_in_normal {
return Err("该任务已在队列中".to_string());
}
(high_queue.len(), normal_queue.len())
};
// 根据优先级添加到相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.high_priority_queue.lock().await;
// 高优先级队列内部也按优先级排序
let insert_position = high_queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(high_queue.len());
high_queue.insert(insert_position, task);
debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len());
}
_ => { // Medium and Low go to normal priority queue
let mut normal_queue = self.normal_priority_queue.lock().await;
// 按优先级排序
let insert_position = normal_queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(normal_queue.len());
normal_queue.insert(insert_position, task);
debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len());
}
}
// 检查是否已有相同ID的任务
if queue.iter().any(|t| t.id == task.id) {
return Err("该任务已在队列中".to_string());
}
// 根据优先级插入任务(高优先级在前)
let insert_position = queue
.iter()
.position(|t| t.priority > task.priority)
.unwrap_or(queue.len());
queue.insert(insert_position, task);
// 更新最后活动时间
*self.last_activity.lock().await = Instant::now();
// 通知工作线程
self.notify.notify_one();
debug!("任务已添加到队列,当前队列大小: {}", queue.len());
Ok(())
}
/// 从队列获取下一个任务
/// 从队列获取下一个任务(优先处理高优先级队列)
pub async fn next_task(&self) -> Option<Task> {
// 检查是否暂停
if self.is_paused.load(Ordering::Acquire) {
@@ -158,8 +216,18 @@ impl TaskQueue {
return None;
}
let mut queue = self.queue.lock().await;
if let Some(task) = queue.pop_front() {
// 优先从高优先级队列获取任务
{
let mut high_queue = self.high_priority_queue.lock().await;
if let Some(task) = high_queue.pop_front() {
self.active_tasks.fetch_add(1, Ordering::Release);
return Some(task);
}
}
// 如果高优先级队列为空,则从普通优先级队列获取任务
let mut normal_queue = self.normal_priority_queue.lock().await;
if let Some(task) = normal_queue.pop_front() {
self.active_tasks.fetch_add(1, Ordering::Release);
Some(task)
} else {
@@ -231,16 +299,31 @@ impl TaskQueue {
/// 获取队列统计信息
pub async fn get_stats(&self) -> QueueStats {
let queue = self.queue.lock().await;
// 按任务类型统计
let mut type_stats = std::collections::HashMap::new();
for task in queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending
}
let (high_queue_len, normal_queue_len, type_stats) = {
let high_queue = self.high_priority_queue.lock().await;
let normal_queue = self.normal_priority_queue.lock().await;
// 按任务类型统计
let mut type_stats = std::collections::HashMap::new();
// 统计高优先级队列中的任务
for task in high_queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending (high priority)
}
// 统计普通优先级队列中的任务
for task in normal_queue.iter() {
let stats = type_stats
.entry(task.task_type.clone())
.or_insert((0, 0, 0));
stats.0 += 1; // pending (normal priority)
}
(high_queue.len(), normal_queue.len(), type_stats)
};
let task_type_stats = type_stats
.into_iter()
@@ -253,7 +336,7 @@ impl TaskQueue {
.collect();
QueueStats {
pending_tasks: queue.len(),
pending_tasks: high_queue_len + normal_queue_len,
active_tasks: self.active_tasks.load(Ordering::Acquire),
completed_tasks: self.completed_tasks.load(Ordering::Acquire),
failed_tasks: self.failed_tasks.load(Ordering::Acquire),
@@ -268,15 +351,28 @@ impl TaskQueue {
/// 清理过期的任务(超过一定时间的任务)
pub async fn cleanup_expired_tasks(&self, max_age: Duration) {
let mut queue = self.queue.lock().await;
let now = Instant::now();
let initial_len = queue.len();
queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - queue.len();
if removed_count > 0 {
warn!("清理了 {} 个过期任务", removed_count);
// 清理高优先级队列中的过期任务
{
let mut high_queue = self.high_priority_queue.lock().await;
let initial_len = high_queue.len();
high_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - high_queue.len();
if removed_count > 0 {
info!("清理了 {} 个过期的高优先级任务", removed_count);
}
}
// 清理普通优先级队列中的过期任务
{
let mut normal_queue = self.normal_priority_queue.lock().await;
let initial_len = normal_queue.len();
normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
let removed_count = initial_len - normal_queue.len();
if removed_count > 0 {
info!("清理了 {} 个过期的普通优先级任务", removed_count);
}
}
}
@@ -290,6 +386,22 @@ impl TaskQueue {
false
}
/// 启动定期清理过期任务
pub async fn start_periodic_cleanup(self: Arc<Self>, interval_duration: Duration) {
let queue = Arc::clone(&self);
tokio::spawn(async move {
let mut interval = tokio::time::interval(interval_duration);
loop {
interval.tick().await;
// 清理超过24小时的任务
queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await;
}
});
}
}
/// 缩略图生成Worker
@@ -340,8 +452,17 @@ impl ThumbnailWorker {
}
} else {
// 如果不是缩略图任务,放回队列
let mut queue = self.queue.queue.lock().await;
queue.push_front(task);
// 根据任务优先级放回相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.queue.high_priority_queue.lock().await;
high_queue.push_front(task);
}
_ => {
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
normal_queue.push_front(task);
}
}
}
} else {
// 队列为空,等待新任务通知
@@ -353,44 +474,15 @@ impl ThumbnailWorker {
}
}
/// 处理缩略图任务
// 缩略图任务处理功能已移除,直接返回成功
async fn process_thumbnail_task(&self, task: Task) -> bool {
if let TaskData::ThumbnailGeneration {
image_id,
original_data,
original_data: _,
} = task.data
{
// 使用具体的错误处理,避免 Send trait 问题
let result = self
.image_processor
.generate_thumbnail_from_memory(&original_data, &image_id.to_string())
.await;
match result {
Ok(thumbnail_data) => {
// 保存缩略图到文件
let thumbnail_path = self
.image_processor
.config
.thumbnails_dir
.join(image_id.to_string());
match tokio::fs::write(&thumbnail_path, &thumbnail_data).await {
Ok(_) => {
info!("✅ 缩略图生成成功: {:?}", image_id);
true
}
Err(e) => {
warn!("保存缩略图失败: {:?} - {}", image_id, e);
false
}
}
}
Err(e) => {
warn!("缩略图生成失败: {:?} - {:?}", image_id, e);
false
}
}
info!("缩略图任务已跳过(功能已移除): {:?}", image_id);
true // 直接返回成功,避免阻塞队列
} else {
warn!("任务类型不匹配,期望缩略图生成任务");
false
@@ -445,8 +537,17 @@ impl AlbumImportWorker {
}
} else {
// 如果不是相册导入任务,放回队列并减少active_tasks计数器
let mut queue = self.queue.queue.lock().await;
queue.push_front(task);
// 根据任务优先级放回相应队列
match task.priority {
TaskPriority::High => {
let mut high_queue = self.queue.high_priority_queue.lock().await;
high_queue.push_front(task);
}
_ => {
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
normal_queue.push_front(task);
}
}
// 手动减少active_tasks计数器,因为任务没有被处理
self.queue.active_tasks.fetch_sub(1, Ordering::Release);
}