update code level
This commit is contained in:
772
src/queue.rs
772
src/queue.rs
@@ -1,281 +1,68 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, AtomicBool};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
use tokio::time::sleep;
|
||||
use tracing::{info, warn, error, debug};
|
||||
use uuid::Uuid;
|
||||
use image::GenericImageView;
|
||||
use crate::models::{AlbumIndex, CreateAlbum, CreateImage};
|
||||
use crate::database::Database;
|
||||
|
||||
/// 任务类型
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum TaskType {
|
||||
ThumbnailGeneration, // 缩略图生成
|
||||
AlbumImport, // 相册导入
|
||||
}
|
||||
|
||||
/// 任务优先级
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum TaskPriority {
|
||||
High = 0, // 前端请求的缩略图
|
||||
Medium = 1, // 相册导入
|
||||
Low = 2, // 后台质量升级
|
||||
}
|
||||
|
||||
/// 通用任务结构
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Task {
|
||||
pub id: Uuid,
|
||||
pub task_type: TaskType,
|
||||
pub priority: TaskPriority,
|
||||
pub created_at: Instant,
|
||||
pub data: TaskData,
|
||||
}
|
||||
|
||||
/// 任务数据
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TaskData {
|
||||
ThumbnailGeneration {
|
||||
image_id: Uuid,
|
||||
original_data: Vec<u8>,
|
||||
},
|
||||
AlbumImport {
|
||||
album_path: String,
|
||||
album_data: Option<Vec<u8>>,
|
||||
},
|
||||
}
|
||||
use tokio::sync::Notify;
|
||||
use tracing::{info, debug};
|
||||
|
||||
/// 队列统计信息
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueueStats {
|
||||
pub pending_tasks: usize,
|
||||
pub active_tasks: usize,
|
||||
pub completed_tasks: usize,
|
||||
pub failed_tasks: usize,
|
||||
pub queue_size_limit: usize,
|
||||
pub is_paused: bool,
|
||||
pub task_type_stats: Vec<TaskTypeStats>,
|
||||
// HTTP请求频率信息
|
||||
pub request_frequency: f64,
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// 任务类型统计
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TaskTypeStats {
|
||||
pub task_type: TaskType,
|
||||
pub pending: usize,
|
||||
pub completed: usize,
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
/// 智能任务队列
|
||||
/// 简化的任务队列(仅用于记录HTTP请求频率)
|
||||
pub struct TaskQueue {
|
||||
high_priority_queue: Mutex<VecDeque<Task>>,
|
||||
normal_priority_queue: Mutex<VecDeque<Task>>,
|
||||
active_tasks: AtomicUsize,
|
||||
completed_tasks: AtomicUsize,
|
||||
failed_tasks: AtomicUsize,
|
||||
max_concurrent_tasks: usize,
|
||||
queue_size_limit: usize,
|
||||
is_paused: AtomicBool,
|
||||
notify: Notify,
|
||||
last_activity: Mutex<Instant>,
|
||||
// HTTP请求频率监控
|
||||
request_count: AtomicU64,
|
||||
last_request_time: Mutex<Instant>,
|
||||
last_request_time: std::sync::Mutex<Instant>,
|
||||
request_window: Duration,
|
||||
// 配置阈值
|
||||
high_request_rate_threshold: f64,
|
||||
idle_request_rate_threshold: f64,
|
||||
// 数据库引用用于定时任务
|
||||
db: Option<Arc<Database>>,
|
||||
is_paused: AtomicBool,
|
||||
notify: Notify,
|
||||
}
|
||||
|
||||
impl TaskQueue {
|
||||
pub fn new(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64) -> Self {
|
||||
pub fn new(_max_concurrent_tasks: usize, _queue_size_limit: usize, _high_request_rate_threshold: f64, _idle_request_rate_threshold: f64) -> Self {
|
||||
Self {
|
||||
high_priority_queue: Mutex::new(VecDeque::new()),
|
||||
normal_priority_queue: Mutex::new(VecDeque::new()),
|
||||
active_tasks: AtomicUsize::new(0),
|
||||
completed_tasks: AtomicUsize::new(0),
|
||||
failed_tasks: AtomicUsize::new(0),
|
||||
max_concurrent_tasks,
|
||||
queue_size_limit,
|
||||
is_paused: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
last_activity: Mutex::new(Instant::now()),
|
||||
// HTTP请求频率监控
|
||||
request_count: AtomicU64::new(0),
|
||||
last_request_time: Mutex::new(Instant::now()),
|
||||
last_request_time: std::sync::Mutex::new(Instant::now()),
|
||||
request_window: Duration::from_secs(10),
|
||||
// 配置阈值
|
||||
high_request_rate_threshold,
|
||||
idle_request_rate_threshold,
|
||||
db: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_db(max_concurrent_tasks: usize, queue_size_limit: usize, high_request_rate_threshold: f64, idle_request_rate_threshold: f64, db: Arc<Database>) -> Self {
|
||||
Self {
|
||||
high_priority_queue: Mutex::new(VecDeque::new()),
|
||||
normal_priority_queue: Mutex::new(VecDeque::new()),
|
||||
active_tasks: AtomicUsize::new(0),
|
||||
completed_tasks: AtomicUsize::new(0),
|
||||
failed_tasks: AtomicUsize::new(0),
|
||||
max_concurrent_tasks,
|
||||
queue_size_limit,
|
||||
is_paused: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
last_activity: Mutex::new(Instant::now()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_db(_max_concurrent_tasks: usize, _queue_size_limit: usize, _high_request_rate_threshold: f64, _idle_request_rate_threshold: f64, _db: Arc<crate::database::Database>) -> Self {
|
||||
Self {
|
||||
// HTTP请求频率监控
|
||||
request_count: AtomicU64::new(0),
|
||||
last_request_time: Mutex::new(Instant::now()),
|
||||
last_request_time: std::sync::Mutex::new(Instant::now()),
|
||||
request_window: Duration::from_secs(10),
|
||||
// 配置阈值
|
||||
high_request_rate_threshold,
|
||||
idle_request_rate_threshold,
|
||||
db: Some(db),
|
||||
is_paused: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 添加任务到队列
|
||||
pub async fn add_task(&self, task: Task) -> Result<(), String> {
|
||||
let (_high_queue_len, _normal_queue_len) = {
|
||||
let high_queue = self.high_priority_queue.lock().await;
|
||||
let normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 检查队列总容量是否已满
|
||||
if high_queue.len() + normal_queue.len() >= self.queue_size_limit {
|
||||
return Err("队列已满,无法添加新任务".to_string());
|
||||
}
|
||||
|
||||
// 检查是否已有相同ID的任务
|
||||
let exists_in_high = high_queue.iter().any(|t| t.id == task.id);
|
||||
let exists_in_normal = normal_queue.iter().any(|t| t.id == task.id);
|
||||
|
||||
if exists_in_high || exists_in_normal {
|
||||
return Err("该任务已在队列中".to_string());
|
||||
}
|
||||
|
||||
(high_queue.len(), normal_queue.len())
|
||||
};
|
||||
|
||||
// 根据优先级添加到相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
|
||||
// 高优先级队列内部也按优先级排序
|
||||
let insert_position = high_queue
|
||||
.iter()
|
||||
.position(|t| t.priority > task.priority)
|
||||
.unwrap_or(high_queue.len());
|
||||
|
||||
high_queue.insert(insert_position, task);
|
||||
|
||||
debug!("高优先级任务已添加,当前高优先级队列大小: {}", high_queue.len());
|
||||
}
|
||||
_ => { // Medium and Low go to normal priority queue
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 按优先级排序
|
||||
let insert_position = normal_queue
|
||||
.iter()
|
||||
.position(|t| t.priority > task.priority)
|
||||
.unwrap_or(normal_queue.len());
|
||||
|
||||
normal_queue.insert(insert_position, task);
|
||||
|
||||
debug!("普通优先级任务已添加,当前普通优先级队列大小: {}", normal_queue.len());
|
||||
}
|
||||
}
|
||||
|
||||
// 更新最后活动时间
|
||||
*self.last_activity.lock().await = Instant::now();
|
||||
|
||||
// 通知工作线程
|
||||
self.notify.notify_one();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 从队列获取下一个任务(优先处理高优先级队列)
|
||||
pub async fn next_task(&self) -> Option<Task> {
|
||||
// 检查是否暂停
|
||||
if self.is_paused.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 检查并发任务数量
|
||||
if self.active_tasks.load(Ordering::Acquire) >= self.max_concurrent_tasks {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 优先从高优先级队列获取任务
|
||||
{
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
if let Some(task) = high_queue.pop_front() {
|
||||
self.active_tasks.fetch_add(1, Ordering::Release);
|
||||
return Some(task);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果高优先级队列为空,则从普通优先级队列获取任务
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
if let Some(task) = normal_queue.pop_front() {
|
||||
self.active_tasks.fetch_add(1, Ordering::Release);
|
||||
Some(task)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 标记任务完成
|
||||
pub fn task_completed(&self, success: bool) {
|
||||
self.active_tasks.fetch_sub(1, Ordering::Release);
|
||||
if success {
|
||||
self.completed_tasks.fetch_add(1, Ordering::Release);
|
||||
} else {
|
||||
self.failed_tasks.fetch_add(1, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
/// 暂停队列处理
|
||||
pub fn pause(&self) {
|
||||
self.is_paused.store(true, Ordering::Release);
|
||||
info!("任务队列已暂停");
|
||||
}
|
||||
|
||||
/// 恢复队列处理
|
||||
pub fn resume(&self) {
|
||||
self.is_paused.store(false, Ordering::Release);
|
||||
self.notify.notify_one();
|
||||
info!("任务队列已恢复");
|
||||
}
|
||||
|
||||
/// 记录HTTP请求(在每次HTTP请求处理时调用)
|
||||
pub async fn record_http_request(&self) {
|
||||
let now = Instant::now();
|
||||
let mut last_request_time = self.last_request_time.lock().await;
|
||||
let mut last_request_time = self.last_request_time.lock().unwrap();
|
||||
|
||||
// 如果距离上次请求时间超过了窗口时间,重置计数器
|
||||
if now.duration_since(*last_request_time) > self.request_window {
|
||||
self.request_count.store(0, Ordering::Release);
|
||||
self.request_count.store(0, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
|
||||
// 增加请求计数
|
||||
self.request_count.fetch_add(1, Ordering::Release);
|
||||
self.request_count.fetch_add(1, std::sync::atomic::Ordering::Release);
|
||||
*last_request_time = now;
|
||||
}
|
||||
|
||||
/// 获取当前请求频率(请求数/秒)
|
||||
pub fn get_request_frequency(&self) -> f64 {
|
||||
let request_count = self.request_count.load(Ordering::Acquire) as f64;
|
||||
let request_count = self.request_count.load(std::sync::atomic::Ordering::Acquire) as f64;
|
||||
let window_seconds = self.request_window.as_secs_f64();
|
||||
|
||||
if window_seconds > 0.0 {
|
||||
@@ -285,522 +72,25 @@ impl TaskQueue {
|
||||
}
|
||||
}
|
||||
|
||||
/// 检查服务器负载是否过高(基于HTTP请求频率)
|
||||
pub async fn is_server_busy(&self) -> bool {
|
||||
let request_frequency = self.get_request_frequency();
|
||||
|
||||
// 基于HTTP请求频率判断业务繁忙度
|
||||
// 高请求频率(超过配置阈值)时认为服务器繁忙
|
||||
let is_high_request_rate = request_frequency > self.high_request_rate_threshold;
|
||||
|
||||
// 如果请求频率高,或者并发任务/队列大小超过阈值,认为服务器繁忙
|
||||
is_high_request_rate
|
||||
}
|
||||
|
||||
/// 获取队列统计信息
|
||||
pub async fn get_stats(&self) -> QueueStats {
|
||||
let (high_queue_len, normal_queue_len, type_stats) = {
|
||||
let high_queue = self.high_priority_queue.lock().await;
|
||||
let normal_queue = self.normal_priority_queue.lock().await;
|
||||
|
||||
// 按任务类型统计
|
||||
let mut type_stats = std::collections::HashMap::new();
|
||||
|
||||
// 统计高优先级队列中的任务
|
||||
for task in high_queue.iter() {
|
||||
let stats = type_stats
|
||||
.entry(task.task_type.clone())
|
||||
.or_insert((0, 0, 0));
|
||||
stats.0 += 1; // pending (high priority)
|
||||
}
|
||||
|
||||
// 统计普通优先级队列中的任务
|
||||
for task in normal_queue.iter() {
|
||||
let stats = type_stats
|
||||
.entry(task.task_type.clone())
|
||||
.or_insert((0, 0, 0));
|
||||
stats.0 += 1; // pending (normal priority)
|
||||
}
|
||||
|
||||
(high_queue.len(), normal_queue.len(), type_stats)
|
||||
};
|
||||
|
||||
let task_type_stats = type_stats
|
||||
.into_iter()
|
||||
.map(|(task_type, (pending, completed, failed))| TaskTypeStats {
|
||||
task_type,
|
||||
pending,
|
||||
completed,
|
||||
failed,
|
||||
})
|
||||
.collect();
|
||||
|
||||
QueueStats {
|
||||
pending_tasks: high_queue_len + normal_queue_len,
|
||||
active_tasks: self.active_tasks.load(Ordering::Acquire),
|
||||
completed_tasks: self.completed_tasks.load(Ordering::Acquire),
|
||||
failed_tasks: self.failed_tasks.load(Ordering::Acquire),
|
||||
queue_size_limit: self.queue_size_limit,
|
||||
is_paused: self.is_paused.load(Ordering::Acquire),
|
||||
task_type_stats,
|
||||
// 添加请求频率信息
|
||||
request_frequency: self.get_request_frequency(),
|
||||
request_count: self.request_count.load(Ordering::Acquire),
|
||||
request_count: self.request_count.load(std::sync::atomic::Ordering::Acquire),
|
||||
}
|
||||
}
|
||||
|
||||
/// 清理过期的任务(超过一定时间的任务)
|
||||
pub async fn cleanup_expired_tasks(&self, max_age: Duration) {
|
||||
let now = Instant::now();
|
||||
|
||||
// 清理高优先级队列中的过期任务
|
||||
{
|
||||
let mut high_queue = self.high_priority_queue.lock().await;
|
||||
let initial_len = high_queue.len();
|
||||
high_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
|
||||
let removed_count = initial_len - high_queue.len();
|
||||
if removed_count > 0 {
|
||||
info!("清理了 {} 个过期的高优先级任务", removed_count);
|
||||
}
|
||||
}
|
||||
|
||||
// 清理普通优先级队列中的过期任务
|
||||
{
|
||||
let mut normal_queue = self.normal_priority_queue.lock().await;
|
||||
let initial_len = normal_queue.len();
|
||||
normal_queue.retain(|task| now.duration_since(task.created_at) <= max_age);
|
||||
let removed_count = initial_len - normal_queue.len();
|
||||
if removed_count > 0 {
|
||||
info!("清理了 {} 个过期的普通优先级任务", removed_count);
|
||||
}
|
||||
}
|
||||
/// 暂停队列处理
|
||||
pub fn pause(&self) {
|
||||
self.is_paused.store(true, std::sync::atomic::Ordering::Release);
|
||||
info!("任务队列已暂停");
|
||||
}
|
||||
|
||||
/// 等待队列变为空闲状态
|
||||
pub async fn wait_for_idle(&self, timeout: Duration) -> bool {
|
||||
let start = Instant::now();
|
||||
|
||||
while start.elapsed() < timeout {
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// 启动定期清理过期任务
|
||||
pub async fn start_periodic_cleanup(self: Arc<Self>, interval_duration: Duration) {
|
||||
let queue = Arc::clone(&self);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(interval_duration);
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
// 清理超过24小时的任务
|
||||
queue.cleanup_expired_tasks(Duration::from_secs(24 * 3600)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// 缩略图生成Worker
|
||||
pub struct ThumbnailWorker {
|
||||
queue: Arc<TaskQueue>,
|
||||
image_processor: Arc<crate::image_processor::ImageProcessor>,
|
||||
}
|
||||
|
||||
impl ThumbnailWorker {
|
||||
pub fn new(
|
||||
queue: Arc<TaskQueue>,
|
||||
image_processor: Arc<crate::image_processor::ImageProcessor>,
|
||||
) -> Self {
|
||||
Self {
|
||||
queue,
|
||||
image_processor,
|
||||
}
|
||||
}
|
||||
|
||||
/// 启动缩略图Worker
|
||||
pub async fn start(self) {
|
||||
info!("启动缩略图生成Worker");
|
||||
|
||||
loop {
|
||||
// 检查队列中是否有任务
|
||||
if let Some(task) = self.queue.next_task().await {
|
||||
if matches!(task.task_type, TaskType::ThumbnailGeneration) {
|
||||
debug!("缩略图Worker开始处理任务: {:?}", task.id);
|
||||
|
||||
let success = self.process_thumbnail_task(task).await;
|
||||
self.queue.task_completed(success);
|
||||
|
||||
// 在处理完每个任务后检查服务器负载
|
||||
if self.queue.is_server_busy().await {
|
||||
info!("服务器负载较高,暂停队列处理");
|
||||
self.queue.pause();
|
||||
|
||||
// 每30秒检查一次服务器状态,直到服务器不忙时恢复队列
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
if !self.queue.is_server_busy().await {
|
||||
info!("服务器负载恢复正常,恢复队列处理");
|
||||
self.queue.resume();
|
||||
break;
|
||||
}
|
||||
info!("服务器仍然繁忙,继续等待...");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 如果不是缩略图任务,放回队列
|
||||
// 根据任务优先级放回相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.queue.high_priority_queue.lock().await;
|
||||
high_queue.push_front(task);
|
||||
}
|
||||
_ => {
|
||||
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
|
||||
normal_queue.push_front(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 队列为空,等待新任务通知
|
||||
self.queue.notify.notified().await;
|
||||
}
|
||||
|
||||
// 短暂休眠避免CPU占用过高
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// 缩略图任务处理功能已移除,直接返回成功
|
||||
async fn process_thumbnail_task(&self, task: Task) -> bool {
|
||||
if let TaskData::ThumbnailGeneration {
|
||||
image_id,
|
||||
original_data: _,
|
||||
} = task.data
|
||||
{
|
||||
info!("缩略图任务已跳过(功能已移除): {:?}", image_id);
|
||||
true // 直接返回成功,避免阻塞队列
|
||||
} else {
|
||||
warn!("任务类型不匹配,期望缩略图生成任务");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 相册导入Worker
|
||||
pub struct AlbumImportWorker {
|
||||
queue: Arc<TaskQueue>,
|
||||
db: Arc<crate::database::Database>,
|
||||
config: crate::config::StorageConfig,
|
||||
}
|
||||
|
||||
impl AlbumImportWorker {
|
||||
pub fn new(
|
||||
queue: Arc<TaskQueue>,
|
||||
db: Arc<crate::database::Database>,
|
||||
config: crate::config::StorageConfig,
|
||||
) -> Self {
|
||||
Self { queue, db, config }
|
||||
}
|
||||
|
||||
/// 启动相册导入Worker
|
||||
pub async fn start(self) {
|
||||
info!("启动相册导入Worker");
|
||||
|
||||
loop {
|
||||
// 检查队列中是否有任务
|
||||
if let Some(task) = self.queue.next_task().await {
|
||||
if matches!(task.task_type, TaskType::AlbumImport) {
|
||||
debug!("相册导入Worker开始处理任务: {:?}", task.id);
|
||||
|
||||
let success = self.process_album_import_task(task).await;
|
||||
self.queue.task_completed(success);
|
||||
|
||||
// 在处理完每个任务后检查服务器负载
|
||||
if self.queue.is_server_busy().await {
|
||||
info!("服务器负载较高,暂停队列处理");
|
||||
self.queue.pause();
|
||||
|
||||
// 每30秒检查一次服务器状态,直到服务器不忙时恢复队列
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
if !self.queue.is_server_busy().await {
|
||||
info!("服务器负载恢复正常,恢复队列处理");
|
||||
self.queue.resume();
|
||||
break;
|
||||
}
|
||||
info!("服务器仍然繁忙,继续等待...");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 如果不是相册导入任务,放回队列并减少active_tasks计数器
|
||||
// 根据任务优先级放回相应队列
|
||||
match task.priority {
|
||||
TaskPriority::High => {
|
||||
let mut high_queue = self.queue.high_priority_queue.lock().await;
|
||||
high_queue.push_front(task);
|
||||
}
|
||||
_ => {
|
||||
let mut normal_queue = self.queue.normal_priority_queue.lock().await;
|
||||
normal_queue.push_front(task);
|
||||
}
|
||||
}
|
||||
// 手动减少active_tasks计数器,因为任务没有被处理
|
||||
self.queue.active_tasks.fetch_sub(1, Ordering::Release);
|
||||
}
|
||||
} else {
|
||||
// 队列为空,等待新任务通知
|
||||
self.queue.notify.notified().await;
|
||||
}
|
||||
|
||||
// 短暂休眠避免CPU占用过高
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// 处理相册导入任务
|
||||
async fn process_album_import_task(&self, task: Task) -> bool {
|
||||
if let TaskData::AlbumImport {
|
||||
album_path,
|
||||
album_data: _,
|
||||
} = task.data
|
||||
{
|
||||
info!("开始处理相册导入任务: {}", album_path);
|
||||
|
||||
// 将相对路径与images_dir组合成完整路径
|
||||
let full_path = std::path::Path::new(&self.config.images_dir).join(&album_path);
|
||||
|
||||
// 检查相册文件夹是否存在
|
||||
if !full_path.exists() {
|
||||
error!("相册文件夹不存在: {}", full_path.display());
|
||||
return false;
|
||||
}
|
||||
|
||||
// 导入单个相册
|
||||
match self.import_single_album(&album_path, &full_path).await {
|
||||
Ok(count) => {
|
||||
info!("相册导入成功: {} (导入 {} 张图片)", album_path, count);
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!("相册导入失败: {} - {}", album_path, e);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("任务类型不匹配,期望相册导入任务");
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// 导入单个相册(按照import.rs中的逻辑实现)
|
||||
async fn import_single_album(&self, folder_name: &str, path: &std::path::Path) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("开始导入相册: {}", folder_name);
|
||||
|
||||
// 读取index.json文件
|
||||
let index_file_path = path.join("index.json");
|
||||
let album_title = if index_file_path.exists() {
|
||||
match std::fs::read_to_string(&index_file_path) {
|
||||
Ok(content) => {
|
||||
match serde_json::from_str::<AlbumIndex>(&content) {
|
||||
Ok(index) => {
|
||||
info!("从index.json读取相册标题: {}", index.title);
|
||||
index.title
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("解析index.json失败: {}, 使用文件夹名作为标题", e);
|
||||
folder_name.to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("读取index.json失败: {}, 使用文件夹名作为标题", e);
|
||||
folder_name.to_string()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!("index.json不存在,使用文件夹名作为标题");
|
||||
folder_name.to_string()
|
||||
};
|
||||
|
||||
// 创建新相册
|
||||
let new_album = CreateAlbum {
|
||||
title: album_title.clone(),
|
||||
document: folder_name.to_string(),
|
||||
description: Some(format!("从文件夹 {} 自动导入的相册", folder_name)),
|
||||
};
|
||||
|
||||
let album_id = self.db.create_album(new_album).await?;
|
||||
info!("创建相册: {} (ID: {})", album_title, album_id);
|
||||
|
||||
// 读取图片URL列表(如果有index.json)
|
||||
let image_urls = if index_file_path.exists() {
|
||||
match std::fs::read_to_string(&index_file_path) {
|
||||
Ok(content) => {
|
||||
match serde_json::from_str::<AlbumIndex>(&content) {
|
||||
Ok(index) => {
|
||||
info!("从index.json读取到 {} 个图片URL", index.images.len());
|
||||
Some(index.images)
|
||||
}
|
||||
Err(_) => None
|
||||
}
|
||||
}
|
||||
Err(_) => None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// 快速扫描图片文件
|
||||
let image_files = self.scan_image_files(path, folder_name)?;
|
||||
|
||||
if image_files.is_empty() {
|
||||
info!("相册 {} 中没有找到图片文件", folder_name);
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
info!("发现 {} 张图片,开始处理...", image_files.len());
|
||||
|
||||
// 并行处理图片信息获取
|
||||
let mut image_tasks = Vec::new();
|
||||
|
||||
for (index, (filename, image_path)) in image_files.iter().enumerate() {
|
||||
let image_url = image_urls.as_ref().and_then(|urls| {
|
||||
if urls.len() > index {
|
||||
Some(urls[index].clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let filename_clone = filename.clone();
|
||||
let image_path_clone = image_path.clone();
|
||||
let album_id_clone = album_id;
|
||||
let folder_name_clone = folder_name.to_string();
|
||||
|
||||
let task = tokio::task::spawn_blocking(move || {
|
||||
// 在阻塞线程中处理图片信息
|
||||
let (file_size, width, height) = match AlbumImportWorker::get_image_info_static(&image_path_clone) {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
eprintln!("获取图片信息失败: {} - {}", image_path_clone.display(), e);
|
||||
(0, 0, 0)
|
||||
}
|
||||
};
|
||||
|
||||
CreateImage {
|
||||
album_id: album_id_clone,
|
||||
filename: format!("{}/{}", folder_name_clone, filename_clone),
|
||||
original_url: image_url,
|
||||
title: Some(format!("图片 {}", index + 1)),
|
||||
description: Some(format!("从文件夹 {} 导入的图片", folder_name_clone)),
|
||||
file_size,
|
||||
width,
|
||||
height,
|
||||
mime_type: match image_path_clone.extension().and_then(|s| s.to_str()).unwrap_or("jpg").to_lowercase().as_str() {
|
||||
"jpg" | "jpeg" => "image/jpeg".to_string(),
|
||||
"png" => "image/png".to_string(),
|
||||
"gif" => "image/gif".to_string(),
|
||||
"webp" => "image/webp".to_string(),
|
||||
"bmp" => "image/bmp".to_string(),
|
||||
_ => "image/jpeg".to_string(),
|
||||
},
|
||||
}
|
||||
});
|
||||
|
||||
image_tasks.push(task);
|
||||
}
|
||||
|
||||
// 收集所有图片信息
|
||||
let mut images_to_insert = Vec::new();
|
||||
for task in image_tasks {
|
||||
match task.await {
|
||||
Ok(image) => images_to_insert.push(image),
|
||||
Err(e) => error!("图片处理任务失败: {}", e),
|
||||
}
|
||||
}
|
||||
|
||||
// 批量插入图片
|
||||
if !images_to_insert.is_empty() {
|
||||
match self.db.add_images(images_to_insert).await {
|
||||
Ok(ids) => {
|
||||
info!("成功导入 {} 张图片到相册 {}", ids.len(), folder_name);
|
||||
|
||||
// 更新相册图片数量
|
||||
if let Err(e) = self.db.update_album_image_count(album_id).await {
|
||||
error!("更新相册图片数量失败: {}", e);
|
||||
}
|
||||
|
||||
Ok(ids.len())
|
||||
}
|
||||
Err(e) => {
|
||||
error!("批量插入图片失败: {}", e);
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!("没有图片需要插入到相册 {}", folder_name);
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// 扫描图片文件(从import.rs迁移)
|
||||
fn scan_image_files(&self, path: &std::path::Path, _folder_name: &str) -> Result<Vec<(String, std::path::PathBuf)>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut image_files = Vec::new();
|
||||
|
||||
for entry in std::fs::read_dir(path)? {
|
||||
let entry = entry?;
|
||||
let file_path = entry.path();
|
||||
|
||||
if file_path.is_file() {
|
||||
if let Some(extension) = file_path.extension().and_then(|s| s.to_str()) {
|
||||
let ext_lower = extension.to_lowercase();
|
||||
if ext_lower == "jpg" || ext_lower == "jpeg" || ext_lower == "png" ||
|
||||
ext_lower == "gif" || ext_lower == "webp" || ext_lower == "bmp" {
|
||||
|
||||
if let Some(filename) = file_path.file_name().and_then(|s| s.to_str()) {
|
||||
image_files.push((filename.to_string(), file_path));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(image_files)
|
||||
}
|
||||
|
||||
|
||||
/// 静态方法:获取图片信息(用于spawn_blocking)
|
||||
fn get_image_info_static(image_path: &std::path::Path) -> Result<(i64, i32, i32), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let metadata = std::fs::metadata(image_path)?;
|
||||
let file_size = metadata.len() as i64;
|
||||
|
||||
// 使用image库获取图片尺寸
|
||||
let img = image::open(image_path)?;
|
||||
let (width, height) = img.dimensions();
|
||||
|
||||
Ok((file_size, width as i32, height as i32))
|
||||
}
|
||||
}
|
||||
|
||||
/// 任务工厂 - 用于创建不同类型的任务
|
||||
pub struct TaskFactory;
|
||||
|
||||
impl TaskFactory {
|
||||
/// 创建相册导入任务
|
||||
pub fn create_album_import_task(album_path: String, album_data: Option<Vec<u8>>) -> Task {
|
||||
Task {
|
||||
id: Uuid::new_v4(),
|
||||
task_type: TaskType::AlbumImport,
|
||||
priority: TaskPriority::Medium,
|
||||
created_at: Instant::now(),
|
||||
data: TaskData::AlbumImport {
|
||||
album_path,
|
||||
album_data,
|
||||
},
|
||||
}
|
||||
/// 恢复队列处理
|
||||
pub fn resume(&self) {
|
||||
self.is_paused.store(false, std::sync::atomic::Ordering::Release);
|
||||
self.notify.notify_one();
|
||||
info!("任务队列已恢复");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user