update code level

This commit is contained in:
lychang
2026-07-20 23:12:20 +08:00
parent 923f9bd7cf
commit 84409207e2
19 changed files with 1471 additions and 428 deletions

View File

@@ -7,10 +7,13 @@ mod routes;
mod services;
mod image_processor;
mod queue;
mod search_db;
use crate::cache::AlbumsCache;
use crate::search_db::SearchDb;
use actix_files as afs;
use actix_web::{App, HttpResponse, HttpServer, middleware, web};
use std::env;
use std::fs;
use std::sync::Arc;
use tokio::time::{Duration, interval};
@@ -20,6 +23,14 @@ use tracing::{error, info};
async fn main() -> std::io::Result<()> {
tracing_subscriber::fmt::init();
let args: Vec<String> = env::args().collect();
let export_to_search = args.iter().any(|a| a == "--export-search");
let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db="))
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db"))
.unwrap_or("search.db");
let clear_search = args.iter().any(|a| a == "--clear-search");
let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint");
// 加载配置
let config = match config::Config::load() {
Ok(cfg) => cfg,
@@ -48,45 +59,103 @@ async fn main() -> std::io::Result<()> {
}
};
if export_to_search {
info!("[搜索导出] 开始将元数据导出到 search.db...");
match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await {
Ok(result) => {
info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}",
result.albums_processed, result.tags_created, result.persons_created,
result.aliases_created, result.album_tags_created + result.album_persons_created);
std::process::exit(0);
}
Err(e) => {
error!("[搜索导出] 导出失败: {}", e);
std::process::exit(1);
}
}
}
if wal_checkpoint {
info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成");
info!("[WAL合并] 正在合并 search.db WAL...");
let search_db = SearchDb::new("data/search.db").await
.expect("search.db 连接失败");
search_db.wal_checkpoint().await;
info!("[WAL合并] 两个数据库均已合并完成");
std::process::exit(0);
}
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// ── 初始化 search.db(运行时搜索专用)──
let search_db_path = "data/search.db";
let search_db = Arc::new(
SearchDb::new(search_db_path).await
.expect("search.db 连接失败")
);
info!("search.db 连接成功");
search_db.wal_checkpoint().await;
// ── 从 search.db 的 dict 表加载标签词典 ──
match engine::tag_dict::reload_from_search_db(&search_db).await {
msg => info!("{}", msg),
}
// ── 内存缓存(必须在管线任务之前创建)──
let albums_cache = Arc::new(tokio::sync::RwLock::new(AlbumsCache::new()));
let persons_cache = Arc::new(tokio::sync::RwLock::new(cache::PersonCache::default()));
let tags_cache = Arc::new(tokio::sync::RwLock::new(cache::TagCache::default()));
// ── 启动管线(单次):导入 → 解析 → 合并 → 缓存刷新 ──
// ── 启动管线(单次):导入 → 增量解析 → 条件全量 sync → 缓存刷新 ──
// 不做全量 reparse/merge,只处理新发现的相册
let boot_config = config.clone();
let boot_db = Arc::clone(&db);
let boot_search_db = Arc::clone(&search_db);
let boot_albums_cache = Arc::clone(&albums_cache);
let boot_persons_cache = Arc::clone(&persons_cache);
let boot_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
// ① 扫描导入新相册
info!("[启动管线] ① 扫描导入新相册...");
info!("[启动] ① 扫描导入新相册...");
match services::import::import_existing_data(&boot_config, Arc::clone(&boot_db), None).await {
Ok(_) => info!("[启动管线] 导入完成"),
Err(e) => error!("[启动管线] 导入失败: {}", e),
Ok(_) => info!("[启动] 导入完成"),
Err(e) => error!("[启动] 导入失败: {}", e),
}
// ② 全量解析 + 人物合并
info!("[启动管线] ② 全量解析 + 人物合并...");
let result = engine::pipeline::run_pipeline(&boot_db).await;
info!("[启动管线] 完成:清洗标点{},移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,
result.persons_merged, result.aliases_moved, result.tags_removed_person);
// ② 查找新相册 / 半成品
info!("[启动] ② 查找新相册...");
let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await;
let new_count = dirty.len();
let mut full_sync = false;
// ③ 刷新所有内存缓存
info!("[启动管线] ③ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动管线] 启动管线执行完毕");
if new_count > 0 {
info!("[启动] 发现 {} 个新相册,增量同步到 search.db...", new_count);
let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await;
info!("[启动] 增量同步完成:{} 个相册(解析在定时管线中执行)", synced);
} else {
info!("[启动] 没有新相册");
// 检查 search.db 是否为空(首次启动)
let album_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums")
.fetch_one(&boot_search_db.pool).await.unwrap_or(0);
if album_count == 0 {
info!("[启动] search.db 为空,执行全量同步...");
let synced = engine::pipeline::sync_albums_to_search_db(&boot_db, &boot_search_db).await;
info!("[启动] 全量同步完成:{} 个相册", synced);
full_sync = true;
}
}
// ③ 刷新所有内存缓存(从 search.db 读取)
info!("[启动] ③ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动] 启动管线执行完毕(新相册={}, 全量同步={})", new_count, full_sync);
});
// ── 定时管线(每 30 分钟):解析 → 合并 → 缓存刷新 ──
// ── 定时管线(每 30 分钟):导入 → 增量解析 → 缓存刷新 ──
// 不做全量 reparse/merge
let timer_cache = Arc::clone(&albums_cache);
let timer_db = Arc::clone(&db);
let timer_search_db = Arc::clone(&search_db);
let timer_persons_cache = Arc::clone(&persons_cache);
let timer_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
@@ -94,18 +163,28 @@ async fn main() -> std::io::Result<()> {
loop {
interval.tick().await;
// ① 全量解析 + 人物合并(不重复 import)
info!("[定时管线] ① 全量解析 + 人物合并...");
let result = engine::pipeline::run_pipeline(&timer_db).await;
info!("[定时管线] 完成:清洗标点{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{}",
result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,
result.persons_merged, result.aliases_moved, result.tags_removed_person);
// ① 扫描导入新相册
info!("[定时] ① 扫描导入新相册...");
match services::import::import_existing_data(&timer_db.config, Arc::clone(&timer_db), None).await {
Ok(_) => {},
Err(e) => error!("[定时] 导入失败: {}", e),
}
// ② 刷新所有内存缓存
info!("[定时管线] ② 刷新所有内存缓存...");
cache::refresh_all_caches(&timer_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
// ② 查找新相册
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
if new_albums.is_empty() {
continue; // 无新增,跳过
}
info!("[定时] 发现 {} 个新相册,增量解析...", new_albums.len());
engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await;
info!("[定时] 增量同步到 search.db...");
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
// ③ 刷新缓存
info!("[定时] 刷新缓存...");
cache::refresh_all_caches(&timer_search_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
}
});
@@ -158,6 +237,7 @@ async fn main() -> std::io::Result<()> {
.wrap(middleware::Compress::default())
.wrap(middleware::Logger::default())
.app_data(web::Data::new(Arc::clone(&db)))
.app_data(web::Data::new(Arc::clone(&search_db)))
.app_data(web::Data::new(Arc::clone(&image_processor)))
.app_data(web::Data::new(Arc::clone(&albums_cache)))
.app_data(web::Data::new(Arc::clone(&persons_cache)))