update code level

This commit is contained in:
lychang
2026-07-21 02:01:52 +08:00
parent c8778c0ed1
commit 9c3ce4ec0d
11 changed files with 60 additions and 100 deletions

View File

@@ -28,7 +28,7 @@ impl Database {
let person_name = result.person_name.as_deref();
if let Some(name) = person_name {
// ── 标记处理时间(person_name 列已迁移到 search.db.bak)──
// ── 标记处理时间(person_name 列已迁移到 search.db)──
sqlx::query(
"UPDATE albums SET processed_at = datetime('now') WHERE id = ?"
)
@@ -60,7 +60,7 @@ impl Database {
.execute(&mut **tx).await?;
}
// ── 写标签(tags/all_words 列已迁移到 search.db.bak,只维护 album_tags 表)──
// ── 写标签(tags/all_words 列已迁移到 search.db,只维护 album_tags 表)──
if !result.tags.is_empty() {
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;

View File

@@ -406,7 +406,7 @@ pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = ""; // person_name_original 已迁移到 search.db.bak
let current = ""; // person_name_original 已迁移到 search.db
if let Some((_text, person, source)) = match_title(title, patterns) {
*stats.by_source.entry(source).or_insert(0) += 1;
@@ -458,7 +458,7 @@ pub async fn analyze_deep_matches(db: &Database) -> DeepMatchStats {
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = ""; // person_name_original 已迁移到 search.db.bak
let current = ""; // person_name_original 已迁移到 search.db
if title.is_empty() {
continue;

View File

@@ -166,7 +166,7 @@ pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
// ════════════════════════════════════════════════
pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
println!("[管线·PhaseB] 后校验标签完成:已迁移到 search.db.bak,跳过");
println!("[管线·PhaseB] 后校验标签完成:已迁移到 search.db,跳过");
0
}
@@ -184,7 +184,7 @@ struct PreprocessResult {
dirty: DirtySet,
}
/// 合并预处理:只做收集新相册,不再做标签清洗(已迁移到 search.db.bak)
/// 合并预处理:只做收集新相册,不再做标签清洗(已迁移到 search.db)
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
let mut result = PreprocessResult::default();
let dirty = phase_collect_new_albums(db).await;
@@ -309,15 +309,12 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
}
// ════════════════════════════════════════════════
// Phase S:同步管线结果到 search.db.bak
// Phase S:同步管线结果到 search.db
// 只同步 tags/persons/aliases 到 search.db 的关联表,通过 album_id 关联
// 不同步 albums 表数据(albums 由 imager.db 管理)
// ════════════════════════════════════════════════
/// 将管线解析结果从主库同步到 search.db.bak
///
/// 读取主库中所有相册的解析结果(tags json, person_name_cleaned 等),
/// 写入 search.db.bak 的结构化表(albums, tags, persons, album_tags, album_persons)。
///
/// 在管线执行完成后调用,确保搜索数据与主库一致。
/// 全量同步:将所有相册的 tags/persons 解析结果写入 search.db 的关联表
pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> usize {
let albums = db.get_all_albums_full().await;
let mut synced = 0usize;
@@ -325,17 +322,7 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
let mut person_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
// ── 1. 写入 albums 表 ──
let cover_id = album.cover_image_id.as_deref();
let img_count = album.image_count.unwrap_or(0);
if let Err(e) = search_db.upsert_album(&album.id, album.title.as_deref().unwrap_or(""),
cover_id, img_count, None).await {
eprintln!("[Sync->search.db.bak] upsert 相册 {} 失败: {}", album.id, e);
continue;
}
// ── 2. 写入 tags → album_tags ──
// 从 title 解析 tags,不再依赖 albums 表的 all_words 字段
// ── 1. 写入 tags → album_tags ──
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
@@ -357,8 +344,7 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
}
}
// ── 3. 写入 persons → album_persons ──
// 从 title 解析人物,不再依赖 albums 表的 person_name_cleaned 字段
// ── 2. 写入 persons → album_persons ──
search_db.clear_album_persons(&album.id).await.ok();
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
@@ -393,7 +379,7 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
synced += 1;
}
// ── 4. 更新 person counts ──
// ── 3. 更新 person counts ──
for (person_name, person_id) in &person_cache {
let albums_count: i64 = match sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
@@ -404,18 +390,15 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
search_db.update_person_counts(*person_id, albums_count, 0).await.ok();
}
println!("[Sync->search.db.bak] 同步完成:{} 个相册,{} 个人物,{} 个标签",
println!("[Sync->search.db] 同步完成:{} 个相册,{} 个人物,{} 个标签",
synced, person_cache.len(), tag_cache.len());
synced
}
/// 增量同步脏集相册到 search.db.bak
///
/// 只处理脏集中的相册 ID,从主库按需查询后写入 search.db.bak。
/// 逻辑与全量同步一致,但只处理脏集里的相册,避免全表扫描。
/// 增量同步:只处理脏集中的相册 ID,按需查询后写入 search.db 的关联表
pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &DirtySet) -> usize {
if dirty.is_empty() {
println!("[Sync->search.db.bak·增量] 脏集为空,跳过");
println!("[Sync->search.db·增量] 脏集为空,跳过");
return 0;
}
@@ -427,17 +410,7 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt
let mut person_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let img_count = album.image_count.unwrap_or(0);
// ── 1. upsert albums 表 ──
if let Err(e) = search_db.upsert_album(&album.id, title, album.cover_image_id.as_deref(), img_count, None).await {
eprintln!("[Sync->search.db.bak·增量] upsert 相册 {} 失败: {}", album.id, e);
continue;
}
// ── 2. 写入 tags → album_tags ──
// 从 title 解析 tags,不再依赖 albums 表的 all_words 字段
// ── 1. 写入 tags → album_tags ──
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
@@ -459,8 +432,7 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt
}
}
// ── 3. 写入 persons → album_persons ──
// 从 title 解析人物,不再依赖 albums 表的 person_name_cleaned 字段
// ── 2. 写入 persons → album_persons ──
search_db.clear_album_persons(&album.id).await.ok();
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
@@ -495,7 +467,7 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt
synced += 1;
}
// ── 4. 更新 person counts ──
// ── 3. 更新 person counts ──
for (person_name, person_id) in &person_cache {
let albums_count: i64 = match sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
@@ -506,7 +478,7 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt
search_db.update_person_counts(*person_id, albums_count, 0).await.ok();
}
println!("[Sync->search.db.bak·增量] 同步完成:{} 个相册,{} 个人物,{} 个标签",
println!("[Sync->search.db·增量] 同步完成:{} 个相册,{} 个人物,{} 个标签",
synced, person_cache.len(), tag_cache.len());
synced
}

View File

@@ -136,7 +136,7 @@ impl RecommendEngine {
let total = albums.len() as f64;
for album in &albums {
let aid = album.id.to_string();
// 从 search.db.bak 查该相册的人物
// 从 search.db 查该相册的人物
let person_rows = sqlx::query(
r#"SELECT p.name FROM persons p
JOIN album_persons ap ON p.id = ap.person_id
@@ -207,7 +207,7 @@ impl RecommendEngine {
}
}).collect();
// 为热门人物获取 top tags(从 search.db.bak)
// 为热门人物获取 top tags(从 search.db)
for rp in &mut results {
if let Some(person) = search_db.get_person_by_name(&rp.name).await {
let top_tags = search_db.get_person_top_tags(person.id, 5).await;

View File

@@ -533,13 +533,13 @@ pub fn get_tag_dict_stats() -> String {
}
// ════════════════════════════════════════════════
// 从 search.db.bak 的 dict 表加载
// 从 search.db 的 dict 表加载
// ════════════════════════════════════════════════
use crate::search_db::DictEntry;
impl TagDict {
/// 从 search.db.bak 的 dict 表条目加载词典数据
/// 从 search.db 的 dict 表条目加载词典数据
///
/// dict 表条目类型(通过 type 字段区分):
/// - `tag` — word 是标签,category 是分类名 → 加入 keymap
@@ -603,7 +603,7 @@ impl TagDict {
});
}
println!("[词典] 从 search.db.bak dict 表加载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
println!("[词典] 从 search.db dict 表加载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len());
@@ -612,16 +612,16 @@ impl TagDict {
}
}
/// 从 search.db.bak 的 dict 表重新加载标签词典
/// 从 search.db 的 dict 表重新加载标签词典
pub async fn reload_from_search_db(search_db: &crate::search_db::SearchDb) -> String {
let entries = match search_db.load_dict_all().await {
Ok(e) => e,
Err(e) => return format!("从 search.db.bak 加载 dict 失败: {}", e),
Err(e) => return format!("从 search.db 加载 dict 失败: {}", e),
};
let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染");
*dict = TagDict::load_from_dict_entries(&entries);
format!(
"词典已从 search.db.bak 重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
"词典已从 search.db 重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),

View File

@@ -26,8 +26,8 @@ async fn main() -> std::io::Result<()> {
let args: Vec<String> = env::args().collect();
let export_to_search = args.iter().any(|a| a == "--export-search");
let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db="))
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db.bak"))
.unwrap_or("search.db.bak");
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db"))
.unwrap_or("search.db");
let clear_search = args.iter().any(|a| a == "--clear-search");
let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint");
@@ -60,7 +60,7 @@ async fn main() -> std::io::Result<()> {
};
if export_to_search {
info!("[搜索导出] 开始将元数据导出到 search.db.bak...");
info!("[搜索导出] 开始将元数据导出到 search.db...");
match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await {
Ok(result) => {
info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}",
@@ -77,9 +77,9 @@ async fn main() -> std::io::Result<()> {
if wal_checkpoint {
info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成");
info!("[WAL合并] 正在合并 search.db.bak WAL...");
let search_db = SearchDb::new("data/search.db.bak").await
.expect("search.db.bak 连接失败");
info!("[WAL合并] 正在合并 search.db WAL...");
let search_db = SearchDb::new("data/search.db").await
.expect("search.db 连接失败");
search_db.wal_checkpoint().await;
info!("[WAL合并] 两个数据库均已合并完成");
std::process::exit(0);
@@ -87,16 +87,16 @@ async fn main() -> std::io::Result<()> {
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// ── 初始化 search.db.bak(运行时搜索专用)──
let search_db_path = "data/search.db.bak";
// ── 初始化 search.db(运行时搜索专用)──
let search_db_path = "data/search.db";
let search_db = Arc::new(
SearchDb::new(search_db_path).await
.expect("search.db.bak 连接失败")
.expect("search.db 连接失败")
);
info!("search.db.bak 连接成功");
info!("search.db 连接成功");
search_db.wal_checkpoint().await;
// ── 从 search.db.bak 的 dict 表加载标签词典 ──
// ── 从 search.db 的 dict 表加载标签词典 ──
match engine::tag_dict::reload_from_search_db(&search_db).await {
msg => info!("{}", msg),
}
@@ -126,29 +126,18 @@ async fn main() -> std::io::Result<()> {
info!("[启动] ② 查找新相册...");
let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await;
let new_count = dirty.len();
let mut full_sync = false;
if new_count > 0 {
info!("[启动] 发现 {} 个新相册,增量同步到 search.db.bak...", new_count);
info!("[启动] 发现 {} 个新相册,同步到 search.db...", new_count);
let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await;
info!("[启动] 增量同步完成:{} 个相册(解析在定时管线中执行)", synced);
info!("[启动] 增量同步完成:{} 个相册(仅 tags/persons)", synced);
} else {
info!("[启动] 没有新相册");
// 检查 search.db.bak 是否为空(首次启动)
let album_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums")
.fetch_one(&boot_search_db.pool).await.unwrap_or(0);
if album_count == 0 {
info!("[启动] search.db.bak 为空,执行全量同步...");
let synced = engine::pipeline::sync_albums_to_search_db(&boot_db, &boot_search_db).await;
info!("[启动] 全量同步完成:{} 个相册", synced);
full_sync = true;
}
}
// ③ 刷新所有内存缓存(从 search.db.bak 读取)
// ③ 刷新所有内存缓存(从 search.db 读取)
info!("[启动] ③ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_db, &boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动] 启动管线执行完毕(新相册={}, 全量同步={})", new_count, full_sync);
info!("[启动] 启动管线执行完毕(新相册={})", new_count);
});
// ── 定时管线(每 30 分钟):导入 → 增量解析 → 缓存刷新 ──
@@ -171,15 +160,15 @@ async fn main() -> std::io::Result<()> {
}
// ② 查找新相册
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
if new_albums.is_empty() {
continue; // 无新增,跳过
}
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
if new_albums.is_empty() {
continue; // 无新增,跳过
}
info!("[定时] 发现 {} 个新相册,增量解析...", new_albums.len());
engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await;
info!("[定时] 增量同步到 search.db.bak...");
info!("[定时] 同步到 search.db...");
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
// ③ 刷新缓存

View File

@@ -35,7 +35,7 @@ pub async fn search_people(
})
}
/// GET /api/people/{name} — 从 search.db.bak 查找人物详情(缓存查人物信息,search.db.bak 查相册列表)
/// GET /api/people/{name} — 从 search.db 查找人物详情(缓存查人物信息,search.db 查相册列表)
pub async fn get_person_detail(
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<PersonCache>>>,
@@ -48,7 +48,7 @@ pub async fn get_person_detail(
let p = found.into_iter().find(|p| p.name.to_lowercase() == name.to_lowercase());
match p {
Some(p) => {
// 相册列表从 search.db.bak 获取
// 相册列表从 search.db 获取
let albums = search_db.get_person_albums(p.id).await;
HttpResponse::Ok().json(PersonWithAlbums {
album_count: albums.len(),
@@ -69,7 +69,7 @@ pub async fn get_person_detail(
}
}
/// GET /api/people/{id}/aliases — 从 search.db.bak 查别名
/// GET /api/people/{id}/aliases — 从 search.db 查别名
pub async fn get_person_aliases(search_db: web::Data<Arc<SearchDb>>, path: web::Path<i64>) -> impl Responder {
let aliases = search_db.get_person_aliases(path.into_inner()).await;
HttpResponse::Ok().json(aliases)

View File

@@ -9,7 +9,6 @@ pub async fn pipeline_run(
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
let result = engine::pipeline::run_pipeline(db.get_ref()).await;
// 同步到 search.db.bak
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
@@ -28,7 +27,7 @@ pub async fn pipeline_run(
"tags_removed_person": result.tags_removed_person,
"search_db_synced": synced,
"message": format!(
"管线执行完毕:清洗标点{}相册,移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{},同步search.db{}",
"管线执行完毕:清洗标点{}相册,移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{},同步标签/人物到search.db{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,

View File

@@ -6,7 +6,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
/// GET /api/metadata/stats — 从 search.db.bak 获取统计数据
/// GET /api/metadata/stats — 从 search.db 获取统计数据
pub async fn get_metadata_stats(search_db: web::Data<Arc<SearchDb>>) -> impl Responder {
HttpResponse::Ok().json(search_db.get_metadata_stats().await)
}

View File

@@ -50,13 +50,13 @@ impl SearchDb {
/// 执行 WAL checkpoint(TRUNCATE),将 WAL 完全合并回主文件并清理 -wal / -shm 文件
///
/// 在启动时调用,确保 search.db.bak 文件干净,避免下次启动时 WAL 恢复开销。
/// 在启动时调用,确保 search.db 文件干净,避免下次启动时 WAL 恢复开销。
pub async fn wal_checkpoint(&self) {
match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
.execute(&self.pool).await
{
Ok(_) => println!("[search.db.bak] WAL checkpoint(TRUNCATE) 完成,数据库文件已合并"),
Err(e) => eprintln!("[search.db.bak] WAL checkpoint 失败: {}", e),
Ok(_) => println!("[search.db] WAL checkpoint(TRUNCATE) 完成,数据库文件已合并"),
Err(e) => eprintln!("[search.db] WAL checkpoint 失败: {}", e),
}
}

View File

@@ -36,11 +36,11 @@ pub async fn export_to_search_db(
search_db_path: &str,
clear_existing: bool,
) -> Result<SearchExportResult, String> {
let search_db = SearchDb::new(search_db_path).await.map_err(|e| format!("创建 search.db.bak 失败: {}", e))?;
let search_db = SearchDb::new(search_db_path).await.map_err(|e| format!("创建 search.db 失败: {}", e))?;
if clear_existing {
search_db.clear_all().await.map_err(|e| format!("清空 search.db.bak 失败: {}", e))?;
println!("[搜索导出] 已清空 search.db.bak 现有数据");
search_db.clear_all().await.map_err(|e| format!("清空 search.db 失败: {}", e))?;
println!("[搜索导出] 已清空 search.db 现有数据");
}
// ── 加载 dict 表 ──
@@ -87,7 +87,7 @@ pub async fn export_to_search_db(
}
}
// ── 2. 写入 search.db.bak ──
// ── 2. 写入 search.db ──
for (tag_name, category) in &parse_result.tags {
// 检查是否是 dict 命中的 tag
let from_dict = dict_entries.iter().any(|e| {