update code level
This commit is contained in:
136
src/main.rs
136
src/main.rs
@@ -7,10 +7,13 @@ mod routes;
|
||||
mod services;
|
||||
mod image_processor;
|
||||
mod queue;
|
||||
mod search_db;
|
||||
|
||||
use crate::cache::AlbumsCache;
|
||||
use crate::search_db::SearchDb;
|
||||
use actix_files as afs;
|
||||
use actix_web::{App, HttpResponse, HttpServer, middleware, web};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::sync::Arc;
|
||||
use tokio::time::{Duration, interval};
|
||||
@@ -20,6 +23,14 @@ use tracing::{error, info};
|
||||
async fn main() -> std::io::Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let export_to_search = args.iter().any(|a| a == "--export-search");
|
||||
let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db="))
|
||||
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db"))
|
||||
.unwrap_or("search.db");
|
||||
let clear_search = args.iter().any(|a| a == "--clear-search");
|
||||
let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint");
|
||||
|
||||
// 加载配置
|
||||
let config = match config::Config::load() {
|
||||
Ok(cfg) => cfg,
|
||||
@@ -48,45 +59,103 @@ async fn main() -> std::io::Result<()> {
|
||||
}
|
||||
};
|
||||
|
||||
if export_to_search {
|
||||
info!("[搜索导出] 开始将元数据导出到 search.db...");
|
||||
match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await {
|
||||
Ok(result) => {
|
||||
info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}",
|
||||
result.albums_processed, result.tags_created, result.persons_created,
|
||||
result.aliases_created, result.album_tags_created + result.album_persons_created);
|
||||
std::process::exit(0);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("[搜索导出] 导出失败: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if wal_checkpoint {
|
||||
info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成");
|
||||
info!("[WAL合并] 正在合并 search.db WAL...");
|
||||
let search_db = SearchDb::new("data/search.db").await
|
||||
.expect("search.db 连接失败");
|
||||
search_db.wal_checkpoint().await;
|
||||
info!("[WAL合并] 两个数据库均已合并完成");
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
|
||||
|
||||
// ── 初始化 search.db(运行时搜索专用)──
|
||||
let search_db_path = "data/search.db";
|
||||
let search_db = Arc::new(
|
||||
SearchDb::new(search_db_path).await
|
||||
.expect("search.db 连接失败")
|
||||
);
|
||||
info!("search.db 连接成功");
|
||||
search_db.wal_checkpoint().await;
|
||||
|
||||
// ── 从 search.db 的 dict 表加载标签词典 ──
|
||||
match engine::tag_dict::reload_from_search_db(&search_db).await {
|
||||
msg => info!("{}", msg),
|
||||
}
|
||||
|
||||
// ── 内存缓存(必须在管线任务之前创建)──
|
||||
let albums_cache = Arc::new(tokio::sync::RwLock::new(AlbumsCache::new()));
|
||||
let persons_cache = Arc::new(tokio::sync::RwLock::new(cache::PersonCache::default()));
|
||||
let tags_cache = Arc::new(tokio::sync::RwLock::new(cache::TagCache::default()));
|
||||
|
||||
// ── 启动管线(单次):导入 → 解析 → 合并 → 缓存刷新 ──
|
||||
// ── 启动管线(单次):导入 → 增量解析 → 条件全量 sync → 缓存刷新 ──
|
||||
// 不做全量 reparse/merge,只处理新发现的相册
|
||||
let boot_config = config.clone();
|
||||
let boot_db = Arc::clone(&db);
|
||||
let boot_search_db = Arc::clone(&search_db);
|
||||
let boot_albums_cache = Arc::clone(&albums_cache);
|
||||
let boot_persons_cache = Arc::clone(&persons_cache);
|
||||
let boot_tags_cache = Arc::clone(&tags_cache);
|
||||
tokio::spawn(async move {
|
||||
// ① 扫描导入新相册
|
||||
info!("[启动管线] ① 扫描导入新相册...");
|
||||
info!("[启动] ① 扫描导入新相册...");
|
||||
match services::import::import_existing_data(&boot_config, Arc::clone(&boot_db), None).await {
|
||||
Ok(_) => info!("[启动管线] 导入完成"),
|
||||
Err(e) => error!("[启动管线] 导入失败: {}", e),
|
||||
Ok(_) => info!("[启动] 导入完成"),
|
||||
Err(e) => error!("[启动] 导入失败: {}", e),
|
||||
}
|
||||
|
||||
// ② 全量解析 + 人物合并
|
||||
info!("[启动管线] ② 全量解析 + 人物合并...");
|
||||
let result = engine::pipeline::run_pipeline(&boot_db).await;
|
||||
info!("[启动管线] 完成:清洗标点{},移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{}",
|
||||
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
|
||||
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
|
||||
result.persons_updated, result.tags_updated,
|
||||
result.persons_merged, result.aliases_moved, result.tags_removed_person);
|
||||
// ② 查找新相册 / 半成品
|
||||
info!("[启动] ② 查找新相册...");
|
||||
let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await;
|
||||
let new_count = dirty.len();
|
||||
let mut full_sync = false;
|
||||
|
||||
// ③ 刷新所有内存缓存
|
||||
info!("[启动管线] ③ 刷新所有内存缓存...");
|
||||
cache::refresh_all_caches(&boot_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
|
||||
info!("[启动管线] 启动管线执行完毕");
|
||||
if new_count > 0 {
|
||||
info!("[启动] 发现 {} 个新相册,增量同步到 search.db...", new_count);
|
||||
let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await;
|
||||
info!("[启动] 增量同步完成:{} 个相册(解析在定时管线中执行)", synced);
|
||||
} else {
|
||||
info!("[启动] 没有新相册");
|
||||
// 检查 search.db 是否为空(首次启动)
|
||||
let album_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums")
|
||||
.fetch_one(&boot_search_db.pool).await.unwrap_or(0);
|
||||
if album_count == 0 {
|
||||
info!("[启动] search.db 为空,执行全量同步...");
|
||||
let synced = engine::pipeline::sync_albums_to_search_db(&boot_db, &boot_search_db).await;
|
||||
info!("[启动] 全量同步完成:{} 个相册", synced);
|
||||
full_sync = true;
|
||||
}
|
||||
}
|
||||
|
||||
// ③ 刷新所有内存缓存(从 search.db 读取)
|
||||
info!("[启动] ③ 刷新所有内存缓存...");
|
||||
cache::refresh_all_caches(&boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
|
||||
info!("[启动] 启动管线执行完毕(新相册={}, 全量同步={})", new_count, full_sync);
|
||||
});
|
||||
|
||||
// ── 定时管线(每 30 分钟):解析 → 合并 → 缓存刷新 ──
|
||||
// ── 定时管线(每 30 分钟):导入 → 增量解析 → 缓存刷新 ──
|
||||
// 不做全量 reparse/merge
|
||||
let timer_cache = Arc::clone(&albums_cache);
|
||||
let timer_db = Arc::clone(&db);
|
||||
let timer_search_db = Arc::clone(&search_db);
|
||||
let timer_persons_cache = Arc::clone(&persons_cache);
|
||||
let timer_tags_cache = Arc::clone(&tags_cache);
|
||||
tokio::spawn(async move {
|
||||
@@ -94,18 +163,28 @@ async fn main() -> std::io::Result<()> {
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
// ① 全量解析 + 人物合并(不重复 import)
|
||||
info!("[定时管线] ① 全量解析 + 人物合并...");
|
||||
let result = engine::pipeline::run_pipeline(&timer_db).await;
|
||||
info!("[定时管线] 完成:清洗标点{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{}",
|
||||
result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected,
|
||||
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
|
||||
result.persons_updated, result.tags_updated,
|
||||
result.persons_merged, result.aliases_moved, result.tags_removed_person);
|
||||
// ① 扫描导入新相册
|
||||
info!("[定时] ① 扫描导入新相册...");
|
||||
match services::import::import_existing_data(&timer_db.config, Arc::clone(&timer_db), None).await {
|
||||
Ok(_) => {},
|
||||
Err(e) => error!("[定时] 导入失败: {}", e),
|
||||
}
|
||||
|
||||
// ② 刷新所有内存缓存
|
||||
info!("[定时管线] ② 刷新所有内存缓存...");
|
||||
cache::refresh_all_caches(&timer_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
|
||||
// ② 查找新相册
|
||||
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
|
||||
if new_albums.is_empty() {
|
||||
continue; // 无新增,跳过
|
||||
}
|
||||
|
||||
info!("[定时] 发现 {} 个新相册,增量解析...", new_albums.len());
|
||||
engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await;
|
||||
|
||||
info!("[定时] 增量同步到 search.db...");
|
||||
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
|
||||
|
||||
// ③ 刷新缓存
|
||||
info!("[定时] 刷新缓存...");
|
||||
cache::refresh_all_caches(&timer_search_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -158,6 +237,7 @@ async fn main() -> std::io::Result<()> {
|
||||
.wrap(middleware::Compress::default())
|
||||
.wrap(middleware::Logger::default())
|
||||
.app_data(web::Data::new(Arc::clone(&db)))
|
||||
.app_data(web::Data::new(Arc::clone(&search_db)))
|
||||
.app_data(web::Data::new(Arc::clone(&image_processor)))
|
||||
.app_data(web::Data::new(Arc::clone(&albums_cache)))
|
||||
.app_data(web::Data::new(Arc::clone(&persons_cache)))
|
||||
|
||||
Reference in New Issue
Block a user