update code level
This commit is contained in:
259
src/queue.rs
259
src/queue.rs
@@ -9,6 +9,7 @@ use tracing::{info, warn, error, debug};
|
||||
use uuid::Uuid;
|
||||
use image::GenericImageView;
|
||||
use crate::models::{AlbumIndex, CreateAlbum, CreateImage};
|
||||
use crate::database::Database;
|
||||
|
||||
/// 任务类型
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -74,7 +75,8 @@ pub struct TaskTypeStats {
|
||||
|
||||
/// 智能任务队列
|
||||
pub struct TaskQueue {
|
||||
queue: Mutex<VecDeque<Task>>,
|
||||
high_priority_queue: Mutex<VecDeque<Task>>,
|
||||
normal_priority_queue: Mutex<VecDeque<Task>>,
|
||||
active_tasks: AtomicUsize,
|
||||
completed_tasks: AtomicUsize,
|
||||
failed_tasks: AtomicUsize,
|
||||
@@ -90,12 +92,15 @@ pub struct TaskQueue {
|
||||
// 配置阈值
|
||||
high_request_rate_threshold: f64,
|
||||
idle_request_rate_threshold: f64,
|
||||
// 数据库引用用于定时任务
|
||||
db: Option<Arc<Database>>,
|
||||
}
|
||||
|
||||
impl TaskQueue {
|
||||
pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self {
|
||||
Self {
|
||||
queue: Mutex::new(VecDeque::new()),
|
||||
high_priority_queue: Mutex::new(VecDeque::new()),
|
||||
normal_priority_queue: Mutex::new(VecDeque::new()),
|
||||
active_tasks: AtomicUsize::new(0),
|
||||
completed_tasks: AtomicUsize::new(0),
|
||||
failed_tasks: AtomicUsize::new(0),
|
||||
@@ -111,42 +116,95 @@ impl TaskQueue {
|
||||
// 配置阈值
|
||||
high_request_rate_threshold,
|
||||
idle_request_rate_threshold,
|
||||
db: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc<Database>) -> Self {
|
||||
Self {
|
||||
high_priority_queue: Mutex::new(VecDeque::new()),
|
||||
normal_priority_queue: Mutex::new(VecDeque::new()),
|
||||
active_tasks: AtomicUsize::new(0),
|
||||
completed_tasks: AtomicUsize::new(0),
|
||||
failed_tasks: AtomicUsize::new(0),
|
||||
max_concurrent_tasks,
|
||||
queue_size_limit,
|
||||
is_paused: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
last_activity: Mutex::new(Instant::now()),
|
||||
// HTTP请求频率监控
|
||||
request_count: AtomicU64::new(0),
|
||||
last_request_time: Mutex::new(Instant::now()),
|
||||
request_window: Duration::from_secs(10),
|
||||
// 配置阈值
|
||||
high_request_rate_threshold,
|
||||
idle_request_rate_threshold,
|
||||
db: Some(db),
|
||||
}
|
||||
}
|
||||
|
||||
/// 添加任务到队列
|
||||
pub async fn add_task(&self, task: Task) -> Result<(), String> {
|
||||
let mut queue = self.queue.lock().await;
|
||||
let (_high_queue_len, _normal_queue_len) = {
|
||||
let high_queue = self.high_priority_queue.lock().await;
|
||||
let normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 检查队列总容量是否已满
|
||||
if high_queue.len() + normal_queue.len() >= self.queue_size_limit {
|
||||
return Err("队列已满,无法添加新任务".to_string());
|
||||
}
|
||||
|
||||
// 检查队列是否已满
|
||||
if queue.len() >= self.queue_size_limit {
|
||||
return Err("队列已满,无法添加新任务".to_string());
|
||||
// 检查是否已有相同ID的任务
|
||||
let exists_in_high = high_queue.iter().any(|t| t.id == task.id);
|
||||
let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id);
|
||||
|
||||
if exists_in_high || exists_in_normal {
|
||||
return Err("该任务已在队列中".to_string());
|
||||
}
|
||||
|
||||
(high_queue.len(), normal_queue.len())
|
||||
};
|
||||
|
||||
// 根据优先级添加到相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
|
||||
// 高优先级队列内部也按优先级排序
|
||||
let insert_position = high_queue
|
||||
.iter()
|
||||
.position(|t| t.priority > task.priority)
|
||||
.unwrap_or(high_queue.len());
|
||||
|
||||
high_queue.insert(insert_position, task);
|
||||
|
||||
debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len());
|
||||
}
|
||||
_ => { // Medium and Low go to normal priority queue
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 按优先级排序
|
||||
let insert_position = normal_queue
|
||||
.iter()
|
||||
.position(|t| t.priority > task.priority)
|
||||
.unwrap_or(normal_queue.len());
|
||||
|
||||
normal_queue.insert(insert_position, task);
|
||||
|
||||
debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len());
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否已有相同ID的任务
|
||||
if queue.iter().any(|t| t.id == task.id) {
|
||||
return Err("该任务已在队列中".to_string());
|
||||
}
|
||||
|
||||
// 根据优先级插入任务(高优先级在前)
|
||||
let insert_position = queue
|
||||
.iter()
|
||||
.position(|t| t.priority > task.priority)
|
||||
.unwrap_or(queue.len());
|
||||
|
||||
queue.insert(insert_position, task);
|
||||
|
||||
// 更新最后活动时间
|
||||
*self.last_activity.lock().await = Instant::now();
|
||||
|
||||
// 通知工作线程
|
||||
self.notify.notify_one();
|
||||
|
||||
debug!("任务已添加到队列,当前队列大小: {}", queue.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 从队列获取下一个任务
|
||||
/// 从队列获取下一个任务(优先处理高优先级队列)
|
||||
pub async fn next_task(&self) -> Option<Task> {
|
||||
// 检查是否暂停
|
||||
if self.is_paused.load(Ordering::Acquire) {
|
||||
@@ -158,8 +216,18 @@ impl TaskQueue {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut queue = self.queue.lock().await;
|
||||
if let Some(task) = queue.pop_front() {
|
||||
// 优先从高优先级队列获取任务
|
||||
{
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
if let Some(task) = high_queue.pop_front() {
|
||||
self.active_tasks.fetch_add(1, Ordering::Release);
|
||||
return Some(task);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果高优先级队列为空,则从普通优先级队列获取任务
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
if let Some(task) = normal_queue.pop_front() {
|
||||
self.active_tasks.fetch_add(1, Ordering::Release);
|
||||
Some(task)
|
||||
} else {
|
||||
@@ -231,16 +299,31 @@ impl TaskQueue {
|
||||
|
||||
/// 获取队列统计信息
|
||||
pub async fn get_stats(&self) -> QueueStats {
|
||||
let queue = self.queue.lock().await;
|
||||
|
||||
// 按任务类型统计
|
||||
let mut type_stats = std::collections::HashMap::new();
|
||||
for task in queue.iter() {
|
||||
let stats = type_stats
|
||||
.entry(task.task_type.clone())
|
||||
.or_insert((0, 0, 0));
|
||||
stats.0 += 1; // pending
|
||||
}
|
||||
let (high_queue_len, normal_queue_len, type_stats) = {
|
||||
let high_queue = self.high_priority_queue.lock().await;
|
||||
let normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 按任务类型统计
|
||||
let mut type_stats = std::collections::HashMap::new();
|
||||
|
||||
// 统计高优先级队列中的任务
|
||||
for task in high_queue.iter() {
|
||||
let stats = type_stats
|
||||
.entry(task.task_type.clone())
|
||||
.or_insert((0, 0, 0));
|
||||
stats.0 += 1; // pending (high priority)
|
||||
}
|
||||
|
||||
// 统计普通优先级队列中的任务
|
||||
for task in normal_queue.iter() {
|
||||
let stats = type_stats
|
||||
.entry(task.task_type.clone())
|
||||
.or_insert((0, 0, 0));
|
||||
stats.0 += 1; // pending (normal priority)
|
||||
}
|
||||
|
||||
(high_queue.len(), normal_queue.len(), type_stats)
|
||||
};
|
||||
|
||||
let task_type_stats = type_stats
|
||||
.into_iter()
|
||||
@@ -253,7 +336,7 @@ impl TaskQueue {
|
||||
.collect();
|
||||
|
||||
QueueStats {
|
||||
pending_tasks: queue.len(),
|
||||
pending_tasks: high_queue_len + normal_queue_len,
|
||||
active_tasks: self.active_tasks.load(Ordering::Acquire),
|
||||
completed_tasks: self.completed_tasks.load(Ordering::Acquire),
|
||||
failed_tasks: self.failed_tasks.load(Ordering::Acquire),
|
||||
@@ -268,15 +351,28 @@ impl TaskQueue {
|
||||
|
||||
/// 清理过期的任务(超过一定时间的任务)
|
||||
pub async fn cleanup_expired_tasks(&self, max_age: Duration) {
|
||||
let mut queue = self.queue.lock().await;
|
||||
let now = Instant::now();
|
||||
let initial_len = queue.len();
|
||||
|
||||
queue.retain(|task| now.duration_since(task.created_at) <= max_age);
|
||||
|
||||
let removed_count = initial_len - queue.len();
|
||||
if removed_count > 0 {
|
||||
warn!("清理了 {} 个过期任务", removed_count);
|
||||
|
||||
// 清理高优先级队列中的过期任务
|
||||
{
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
let initial_len = high_queue.len();
|
||||
high_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
|
||||
let removed_count = initial_len - high_queue.len();
|
||||
if removed_count > 0 {
|
||||
info!("清理了 {} 个过期的高优先级任务", removed_count);
|
||||
}
|
||||
}
|
||||
|
||||
// 清理普通优先级队列中的过期任务
|
||||
{
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
let initial_len = normal_queue.len();
|
||||
normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
|
||||
let removed_count = initial_len - normal_queue.len();
|
||||
if removed_count > 0 {
|
||||
info!("清理了 {} 个过期的普通优先级任务", removed_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,6 +386,22 @@ impl TaskQueue {
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// 启动定期清理过期任务
|
||||
pub async fn start_periodic_cleanup(self: Arc<Self>, interval_duration: Duration) {
|
||||
let queue = Arc::clone(&self);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(interval_duration);
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
// 清理超过24小时的任务
|
||||
queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// 缩略图生成Worker
|
||||
@@ -340,8 +452,17 @@ impl ThumbnailWorker {
|
||||
}
|
||||
} else {
|
||||
// 如果不是缩略图任务,放回队列
|
||||
let mut queue = self.queue.queue.lock().await;
|
||||
queue.push_front(task);
|
||||
// 根据任务优先级放回相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.queue.high_priority_queue.lock().await;
|
||||
high_queue.push_front(task);
|
||||
}
|
||||
_ => {
|
||||
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
|
||||
normal_queue.push_front(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 队列为空,等待新任务通知
|
||||
@@ -353,44 +474,15 @@ impl ThumbnailWorker {
|
||||
}
|
||||
}
|
||||
|
||||
/// 处理缩略图任务
|
||||
// 缩略图任务处理功能已移除,直接返回成功
|
||||
async fn process_thumbnail_task(&self, task: Task) -> bool {
|
||||
if let TaskData::ThumbnailGeneration {
|
||||
image_id,
|
||||
original_data,
|
||||
original_data: _,
|
||||
} = task.data
|
||||
{
|
||||
// 使用具体的错误处理,避免 Send trait 问题
|
||||
let result = self
|
||||
.image_processor
|
||||
.generate_thumbnail_from_memory(&original_data, &image_id.to_string())
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(thumbnail_data) => {
|
||||
// 保存缩略图到文件
|
||||
let thumbnail_path = self
|
||||
.image_processor
|
||||
.config
|
||||
.thumbnails_dir
|
||||
.join(image_id.to_string());
|
||||
|
||||
match tokio::fs::write(&thumbnail_path, &thumbnail_data).await {
|
||||
Ok(_) => {
|
||||
info!("✅ 缩略图生成成功: {:?}", image_id);
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("保存缩略图失败: {:?} - {}", image_id, e);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("缩略图生成失败: {:?} - {:?}", image_id, e);
|
||||
false
|
||||
}
|
||||
}
|
||||
info!("缩略图任务已跳过(功能已移除): {:?}", image_id);
|
||||
true // 直接返回成功,避免阻塞队列
|
||||
} else {
|
||||
warn!("任务类型不匹配,期望缩略图生成任务");
|
||||
false
|
||||
@@ -445,8 +537,17 @@ impl AlbumImportWorker {
|
||||
}
|
||||
} else {
|
||||
// 如果不是相册导入任务,放回队列并减少active_tasks计数器
|
||||
let mut queue = self.queue.queue.lock().await;
|
||||
queue.push_front(task);
|
||||
// 根据任务优先级放回相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.queue.high_priority_queue.lock().await;
|
||||
high_queue.push_front(task);
|
||||
}
|
||||
_ => {
|
||||
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
|
||||
normal_queue.push_front(task);
|
||||
}
|
||||
}
|
||||
// 手动减少active_tasks计数器,因为任务没有被处理
|
||||
self.queue.active_tasks.fetch_sub(1, Ordering::Release);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user