From 84409207e27df9e4ce7b905c9a6c78b33df2df16 Mon Sep 17 00:00:00 2001 From: lychang <632945494@qq.com> Date: Mon, 20 Jul 2026 23:12:20 +0800 Subject: [PATCH] update code level --- .gitignore | 3 +- dict/tags.json | 6 +- migrations/004_person_albums_indexes.sql | 3 + src/cache/mod.rs | 16 +- src/db/album.rs | 30 +- src/engine/pipeline.rs | 517 +++++++++-------------- src/engine/recommend.rs | 61 ++- src/engine/tag_dict.rs | 96 +++++ src/engine/title_parser.rs | 121 ++++++ src/main.rs | 136 ++++-- src/routes/album.rs | 5 +- src/routes/dict.rs | 9 +- src/routes/person.rs | 16 +- src/routes/pipeline.rs | 25 +- src/routes/recommend.rs | 20 +- src/search_db/mod.rs | 310 ++++++++++++++ src/search_db/search.rs | 342 +++++++++++++++ src/services/mod.rs | 4 +- src/services/search_export.rs | 179 ++++++++ 19 files changed, 1471 insertions(+), 428 deletions(-) create mode 100644 migrations/004_person_albums_indexes.sql create mode 100644 src/search_db/mod.rs create mode 100644 src/search_db/search.rs create mode 100644 src/services/search_export.rs diff --git a/.gitignore b/.gitignore index 0505d14..c8a2ba2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target /albums /.idea -/data \ No newline at end of file +/data +/scripts \ No newline at end of file diff --git a/dict/tags.json b/dict/tags.json index 9ca71a7..ce950e7 100644 --- a/dict/tags.json +++ b/dict/tags.json @@ -634,11 +634,11 @@ "www", "https", "http", - "Impact", - "写真", - "系列" + "Impact" ], "person_prefixes": [ + "写真", + "系列", "美女模特", "性感女神", "清纯美女", diff --git a/migrations/004_person_albums_indexes.sql b/migrations/004_person_albums_indexes.sql new file mode 100644 index 0000000..cd853a6 --- /dev/null +++ b/migrations/004_person_albums_indexes.sql @@ -0,0 +1,3 @@ +-- 为 person_albums 添加 album_id 索引，优化 JOIN 查询和按相册查找的性能 +-- person_id 已有主键索引 (person_id, album_id) 覆盖，无需重复创建 +CREATE INDEX IF NOT EXISTS idx_pa_album ON person_albums(album_id); diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 987b0a1..aff639d 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1,6 +1,6 @@ //! 缓存层 — 内存缓存管理 -use crate::db::Database; +use crate::search_db::SearchDb; use crate::models::*; use std::collections::HashMap; use std::sync::Arc; @@ -206,25 +206,25 @@ impl PersonCache { /// 刷新所有缓存（相册 + 标签 + 人物） pub async fn refresh_all_caches( - db: &Database, + search_db: &SearchDb, albums_cache: &Arc>, persons_cache: &Arc>, tags_cache: &Arc>, ) { // 1. 刷新相册 - if let Ok(albums) = db.get_all_albums().await { + if let Ok(albums) = search_db.get_all_albums().await { let mut c = albums_cache.write().await; c.update(albums.clone()); // 2. 刷新标签 - let popular_tags = db.get_popular_tags(200).await; - let tag_albums = db.get_all_tag_albums().await; + let popular_tags = search_db.get_popular_tags(200).await; + let tag_albums = search_db.get_all_tag_albums().await; let mut tc = tags_cache.write().await; tc.update(popular_tags, tag_albums); } // 3. 刷新人物 - let persons = db.get_all_persons_with_aliases().await; + let persons = search_db.get_all_persons_with_aliases().await; let mut pc = persons_cache.write().await; pc.update(persons); @@ -232,8 +232,8 @@ pub async fn refresh_all_caches( } /// 仅刷新人物缓存（管线执行后调用） -pub async fn refresh_persons_cache(db: &Database, persons_cache: &Arc>) { - let persons = db.get_all_persons_with_aliases().await; +pub async fn refresh_persons_cache(search_db: &SearchDb, persons_cache: &Arc>) { + let persons = search_db.get_all_persons_with_aliases().await; let mut pc = persons_cache.write().await; pc.update(persons); } diff --git a/src/db/album.rs b/src/db/album.rs index b67cc32..e20a5f1 100644 --- a/src/db/album.rs +++ b/src/db/album.rs @@ -172,9 +172,10 @@ impl Database { ).fetch_one(&self.pool).await.unwrap_or(0); if has_col == 0 { return Vec::new(); } + // 去掉 ORDER BY：管线处理不需要排序，省去排序开销 let rows = sqlx::query( r#"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count - FROM albums ORDER BY created_at DESC"# + FROM albums"# ).fetch_all(&self.pool).await.unwrap_or_default(); rows.iter().map(|r| AlbumMeta { @@ -187,6 +188,33 @@ impl Database { }).collect() } + /// 按 ID 列表批量查询相册元数据（用于脏集增量解析，避免全表扫描） + pub async fn get_albums_by_ids(&self, ids: &[String]) -> Vec { + if ids.is_empty() { return Vec::new(); } + + let placeholders: Vec = (0..ids.len()).map(|_| "?".to_string()).collect(); + let sql = format!( + "SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count + FROM albums WHERE id IN ({})", + placeholders.join(", ") + ); + + let mut query = sqlx::query(&sql); + for id in ids { + query = query.bind(id); + } + + let rows = query.fetch_all(&self.pool).await.unwrap_or_default(); + rows.iter().map(|r| AlbumMeta { + id: r.get("id"), title: r.get("title"), + person_name_original: r.get("person_name_original"), + person_name_cleaned: r.get("person_name_cleaned"), + tags: r.get("tags"), all_words: r.get("all_words"), + raw_title: r.get("raw_title"), processed_at: r.get("processed_at"), + image_count: r.get("image_count"), + }).collect() + } + // ── 写入 ── pub async fn create_album(&self, album: CreateAlbum) -> Result { diff --git a/src/engine/pipeline.rs b/src/engine/pipeline.rs index b755f6a..f3ac805 100644 --- a/src/engine/pipeline.rs +++ b/src/engine/pipeline.rs @@ -17,8 +17,10 @@ use crate::engine::title_parser; use crate::engine::tag_extractor; use crate::engine::person_merger; use crate::engine::tag_dict; +use crate::search_db::SearchDb; use std::collections::HashSet; use rayon::prelude::*; +use sqlx; // ════════════════════════════════════════════════ // 脏集 — 追踪需要重新解析的相册 ID @@ -44,62 +46,6 @@ impl Default for DirtySet { fn default() -> Self { Self::new() } } -// ════════════════════════════════════════════════ -// Phase 0：清洗已有标签中的头尾标点 -// ════════════════════════════════════════════════ - -pub async fn phase_clean_tags(db: &Database) -> usize { - let albums = db.get_all_albums_full().await; - let mut cleaned_count = 0usize; - let mut removed_tags = HashSet::new(); - - for album in &albums { - let tags_json = match album.tags.as_deref() { - Some(t) if !t.is_empty() && t != "[]" => t, - _ => continue, - }; - let old_tags: Vec = match serde_json::from_str(tags_json) { - Ok(t) => t, Err(_) => continue, - }; - if old_tags.is_empty() { continue; } - - let mut changed = false; - let mut new_tags = Vec::new(); - for tag in old_tags { - let cleaned = match tag_extractor::clean_tag(&tag) { - Some(t) => t, - None => { removed_tags.insert(tag); changed = true; continue; } - }; - if cleaned != tag { removed_tags.insert(tag); changed = true; } - new_tags.push(cleaned); - } - if !changed { continue; } - cleaned_count += 1; - - let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); - let all_words_json = album.all_words.as_deref().unwrap_or("[]"); - let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) - .unwrap_or_default() - .into_iter() - .filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; Some([cleaned, pair[1].clone()]) }) - .collect(); - let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); - db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await; - } - - if cleaned_count > 0 { - println!("[管线·Phase0] 标签清洗完成：清洗 {} 个相册，移除标签: {:?}", - cleaned_count, removed_tags.iter().collect::>()); - } else { - println!("[管线·Phase0] 标签清洗完成：无需要清洗的标签"); - } - cleaned_count -} - -// ════════════════════════════════════════════════ -// Phase 0B：移除过短的标签（少于2个汉字/字母） -// ════════════════════════════════════════════════ - /// 判断一个标签是否"过短"——有效字符（汉字+字母）少于 2 个 fn is_short_tag(tag: &str) -> bool { let mut valid_count = 0usize; @@ -114,269 +60,6 @@ fn is_short_tag(tag: &str) -> bool { valid_count < 2 } -/// Phase 0B：扫描所有标签，移除过短的标签（有效字符 < 2）， -/// 受影响的相册加入脏集等待重新解析。 -pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) { - let albums = db.get_all_albums_full().await; - let mut dirty = DirtySet::new(); - let mut removed_total = 0usize; - let mut removed_samples: HashSet = HashSet::new(); - - for album in &albums { - let tags_json = match album.tags.as_deref() { - Some(t) if !t.is_empty() && t != "[]" => t, - _ => continue, - }; - let old_tags: Vec = match serde_json::from_str(tags_json) { - Ok(t) => t, Err(_) => continue, - }; - if old_tags.is_empty() { continue; } - - let mut changed = false; - let mut new_tags = Vec::new(); - for tag in old_tags { - if is_short_tag(&tag) { - removed_samples.insert(tag); - changed = true; - removed_total += 1; - } else { - new_tags.push(tag); - } - } - if !changed { continue; } - - dirty.add(&album.id); - - let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); - let all_words_json = album.all_words.as_deref().unwrap_or("[]"); - let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) - .unwrap_or_default() - .into_iter() - .filter(|pair| !is_short_tag(&pair[0])) - .collect(); - let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); - db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await; - } - - if removed_total > 0 { - let samples: Vec<_> = removed_samples.iter().take(10).collect(); - println!("[管线·Phase0B] 短标签清理完成：移除 {} 个短标签，影响 {} 个相册，示例: {:?}", - removed_total, dirty.len(), samples); - } else { - println!("[管线·Phase0B] 短标签清理完成：无短标签"); - } - (dirty, removed_total) -} - -// ════════════════════════════════════════════════ -// Phase A1：检测标签中的人名并直接提取到 person 字段 -// ════════════════════════════════════════════════ - -/// Phase A1：检测标签中可能的人名并直接提取到 person_name -/// -/// 核心变化：不再只标记脏集等待 Phase 1，而是直接从 tags 中提取人名写入 person 字段。 -/// 因为 tags 中的人名比从 title 重解析更直接、更准确。 -/// -/// 两层检测 + 最终写入： -/// 1. 逐个标签 — 标签是已知人物或像人名 → 收集为候选人名（中文加权） -/// 2. jieba 合并（空格分隔）— 发现跨标签拆分的人名（["杨晨","晨Yome"]） -/// 3. 最佳候选人名 → 写入 person_name，从标签中移除关联标签 -/// -/// 返回 (脏集, 从标签中移除的人名标签数) -pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) { - let albums = db.get_all_albums_full().await; - let mut dirty = DirtySet::new(); - let mut person_like_removed = 0usize; - - let known_persons: HashSet = db.get_all_person_names().await - .into_iter().map(|(n, _)| n.to_lowercase()).collect(); - - for album in &albums { - let tags_json = match album.tags.as_deref() { - Some(t) if !t.is_empty() && t != "[]" => t, - _ => continue, - }; - let old_tags: Vec = match serde_json::from_str(tags_json) { - Ok(t) => t, Err(_) => continue, - }; - - let has_person = album.person_name_original.as_deref() - .map(|s| !s.is_empty()).unwrap_or(false); - - // 快速判断是否需要处理 - let has_known_tag = old_tags.iter().any(|t| known_persons.contains(&t.to_lowercase())); - let has_like_tag = !has_person && old_tags.iter().any(|t| - title_parser::is_likely_person_name(t) && !title_parser::is_excluded_name(t)); - if has_person && !has_known_tag && !has_like_tag { continue; } - - // ── 第一层：逐个标签收集候选人名 ── - let mut known_person_found: Option = None; - let mut person_like_tags: Vec = Vec::new(); - - for tag in &old_tags { - let tag_lower = tag.to_lowercase(); - if known_persons.contains(&tag_lower) { - known_person_found = Some(tag.clone()); - continue; - } - if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) { - person_like_tags.push(tag.clone()); - continue; - } - } - - // 从 person_like_tags 中选最优（中文加权） - let best_layer1 = person_like_tags.into_iter().max_by_key(|t| { - if let Some(cn) = title_parser::extract_cn(t) { - if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() } - } else { t.len() } - }); - - // ── 第二层：空格连接所有标签，用 jieba 提取人名 ── - let combined = old_tags.join(" "); - let (jieba_name, _) = title_parser::extract_person_and_tags(&combined); - - // ── 确定最终人名 ── - let final_person = known_person_found.or_else(|| { - match (&jieba_name, &best_layer1) { - (Some(j), Some(b)) => { - // jieba 结果更完整 → 用 jieba - let j_len: usize = j.len(); - let b_len: usize = b.len(); - if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) } - } - (Some(j), None) => Some(j.clone()), - (None, Some(b)) => Some(b.clone()), - (None, None) => None, - } - }); - - if let Some(ref person_name) = final_person { - // ── 从标签中移除所有人名相关标签 ── - let mut changed = false; - let mut clean_tags: Vec = Vec::new(); - let person_lower = person_name.to_lowercase(); - let (primary_cleaned, _) = title_parser::split_person_name_with_aliases(person_name); - let primary_lower = primary_cleaned.to_lowercase(); - - for tag in &old_tags { - let tag_lower = tag.to_lowercase(); - let should_remove = - tag_lower == person_lower // 完全匹配 - || tag_lower == primary_lower // 中文核心匹配 - || (tag.len() >= 2 && tag.chars().all(|c| c.is_ascii_alphabetic()) - && !tag_lower.starts_with("cos") - && person_lower.contains(&tag_lower)) // 英文后缀 - || (tag_lower.len() >= 2 && person_lower.contains(&tag_lower)); // 人名包含 - - if should_remove { - person_like_removed += 1; - changed = true; - continue; - } - clean_tags.push(tag.clone()); - } - - if changed || !has_person { - // ── 只清理标签中的人名，不写入 person 字段 ── - // 人名提取统一由 Phase 1 从标题解析完成，保证一个相册只有一个 pid - let tags_json_new = serde_json::to_string(&clean_tags).unwrap_or_else(|_| "[]".to_string()); - let all_words_json = album.all_words.as_deref().unwrap_or("[]"); - let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) - .unwrap_or_default() - .into_iter() - .filter(|pair| !known_persons.contains(&pair[0].to_lowercase())) - .collect(); - let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); - db.update_album_tags(&album.id, &tags_json_new, &words_json_new).await; - - dirty.add(&album.id); - } - } - } - - if person_like_removed > 0 { - println!("[管线·PhaseA1] 从标签中检测并提取人名：移除 {} 个人名标签，写入 {} 个相册的 person 字段", - person_like_removed, dirty.len()); - } else { - println!("[管线·PhaseA1] 检测标签中的人名：无异常"); - } - (dirty, person_like_removed) -} - -// ════════════════════════════════════════════════ -// Phase A2：清理 persons 表中的非人物 -// ════════════════════════════════════════════════ - -pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) { - let persons = db.get_all_persons_with_counts().await; - let mut dirty = DirtySet::new(); - let mut deleted = 0usize; - - for (pid, name, _albums, _images) in &persons { - if title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name) { - let album_list = db.get_person_albums(*pid).await; - let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); - for aid in &album_ids { dirty.add(aid); } - println!("[管线·PhaseA2] 删除非人物：{} (id={})，释放 {} 个相册", name, pid, album_ids.len()); - db.update_album_person_by_person_id(*pid).await; - db.delete_person_completely(*pid).await; - deleted += 1; - } - } - - if deleted > 0 { - println!("[管线·PhaseA2] 清理完成：删除 {} 个非人物，标记 {} 个脏相册", deleted, dirty.len()); - } else { - println!("[管线·PhaseA2] 清理完成：无非人物需要删除"); - } - (dirty, deleted) -} - -// ════════════════════════════════════════════════ -// Phase A2B：清理 persons 表中与停用词重叠的人物 -// ════════════════════════════════════════════════ - -/// Phase A2B：检查 persons 表中的人名是否在停用词词典中。 -/// -/// 当用户在词典中添加了新的停用词，而这些词恰好已被作为人物创建时， -/// 需要删除该人物并释放对应的相册，让它们在后续 Phase 1 中重新解析。 -/// 这样可以防止停用词（如 "photo"、"model" 等通用词）被错误识别为人物。 -pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) { - let persons = db.get_all_persons_with_counts().await; - let mut dirty = DirtySet::new(); - let mut deleted = 0usize; - - for (pid, name, _albums, _images) in &persons { - // 使用 tag_dict 的 is_excluded_person 进行判断（含包含匹配） - let is_excluded = tag_dict::get_tag_dict().is_excluded_person(name); - - // 也检查中文核心（如 "秀人" 是排除词，"秀人网" 也会包含） - let cn_check = title_parser::extract_cn(name) - .map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn)) - .unwrap_or(false); - - if is_excluded || cn_check { - let album_list = db.get_person_albums(*pid).await; - let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); - for aid in &album_ids { dirty.add(aid); } - println!("[管线·PhaseA2B] 排除人名命中：{} (id={})，释放 {} 个相册", name, pid, album_ids.len()); - // 清空相册的 person 字段 - db.update_album_person_by_person_id(*pid).await; - // 彻底删除该人物 - db.delete_person_completely(*pid).await; - deleted += 1; - } - } - - if deleted > 0 { - println!("[管线·PhaseA2B] 清理完成：移除 {} 个被排除的人物，标记 {} 个脏相册", deleted, dirty.len()); - } else { - println!("[管线·PhaseA2B] 清理完成：无被排除的人物"); - } - (dirty, deleted) -} - // ════════════════════════════════════════════════ // Phase A3：收集新相册 + 半成品 // ════════════════════════════════════════════════ @@ -412,7 +95,8 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u return (0, 0); } - let albums = db.get_all_albums_full().await; + let dirty_ids: Vec = dirty_set.clone().into_inner().into_iter().collect(); + let albums = db.get_albums_by_ids(&dirty_ids).await; // ── Phase 1a：并行解析（CPU bound，rayon） ── let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter() @@ -854,3 +538,196 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult { aliases_moved, tags_removed_person, } } + +// ════════════════════════════════════════════════ +// Phase S：同步管线结果到 search.db +// ════════════════════════════════════════════════ + +/// 将管线解析结果从主库同步到 search.db +/// +/// 读取主库中所有相册的解析结果（tags json, person_name_cleaned 等）， +/// 写入 search.db 的结构化表（albums, tags, persons, album_tags, album_persons）。 +/// +/// 在管线执行完成后调用，确保搜索数据与主库一致。 +pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> usize { + let albums = db.get_all_albums_full().await; + let mut synced = 0usize; + let mut tag_cache: std::collections::HashMap = std::collections::HashMap::new(); + let mut person_cache: std::collections::HashMap = std::collections::HashMap::new(); + + for album in &albums { + // ── 1. 写入 albums 表 ── + let cover_id: Option<&str> = None; + // 从主库获取 cover_image_id：因 get_all_albums_full 不返回 cover_image_id， + // 这里用默认值，主库已有的 cover 在下一次增量同步时会补全 + let img_count = album.image_count.unwrap_or(0); + if let Err(e) = search_db.upsert_album(&album.id, album.title.as_deref().unwrap_or(""), + cover_id, img_count, None).await { + eprintln!("[Sync->search.db] upsert 相册 {} 失败: {}", album.id, e); + continue; + } + + // ── 2. 写入 tags → album_tags ── + let tags_json = album.tags.as_deref().unwrap_or("[]"); + if tags_json != "[]" { + if let Ok(tags) = serde_json::from_str::>(tags_json) { + search_db.clear_album_tags(&album.id).await.ok(); + for (tag_name, category) in &tags { + let tag_id = match tag_cache.get(tag_name) { + Some(&id) => id, + None => { + match search_db.insert_tag(tag_name, Some(category)).await { + Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } + Err(_) => continue, + } + } + }; + search_db.insert_album_tag(&album.id, tag_id).await.ok(); + } + } + } + + // ── 3. 写入 persons → album_persons ── + search_db.clear_album_persons(&album.id).await.ok(); + if let Some(ref person_name) = album.person_name_cleaned { + if !person_name.is_empty() { + let person_id = match person_cache.get(person_name) { + Some(&id) => id, + None => { + match search_db.insert_person(person_name, None).await { + Ok(id) => { person_cache.insert(person_name.clone(), id); id } + Err(_) => continue, + } + } + }; + search_db.insert_album_person(&album.id, person_id).await.ok(); + + // 同时同步别名 + let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); + if !aliases.is_empty() { + for (alias_type, alias_name) in &aliases { + if alias_name != person_name { + search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + } + } + if primary != person_name.as_str() && !primary.is_empty() { + search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); + } + } + } + } + + synced += 1; + } + + // ── 4. 更新 person counts ── + for (person_name, person_id) in &person_cache { + let albums_count: i64 = match sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM person_albums WHERE person_id = ?" + ).bind(*person_id).fetch_one(&db.pool).await { + Ok(c) => c, + Err(_) => continue, + }; + search_db.update_person_counts(*person_id, albums_count, 0).await.ok(); + } + + println!("[Sync->search.db] 同步完成：{} 个相册，{} 个人物，{} 个标签", + synced, person_cache.len(), tag_cache.len()); + synced +} + +/// 增量同步脏集相册到 search.db +/// +/// 只处理脏集中的相册 ID，从主库按需查询后写入 search.db。 +/// 逻辑与全量同步一致，但只处理脏集里的相册，避免全表扫描。 +pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &DirtySet) -> usize { + if dirty.is_empty() { + println!("[Sync->search.db·增量] 脏集为空，跳过"); + return 0; + } + + let dirty_ids: Vec = dirty.clone().into_inner().into_iter().collect(); + let albums = db.get_albums_by_ids(&dirty_ids).await; + + let mut synced = 0usize; + let mut tag_cache: std::collections::HashMap = std::collections::HashMap::new(); + let mut person_cache: std::collections::HashMap = std::collections::HashMap::new(); + + for album in &albums { + let title = album.title.as_deref().unwrap_or(""); + let img_count = album.image_count.unwrap_or(0); + + // ── 1. upsert albums 表 ── + if let Err(e) = search_db.upsert_album(&album.id, title, None, img_count, None).await { + eprintln!("[Sync->search.db·增量] upsert 相册 {} 失败: {}", album.id, e); + continue; + } + + // ── 2. 写入 tags → album_tags ── + let tags_json = album.tags.as_deref().unwrap_or("[]"); + search_db.clear_album_tags(&album.id).await.ok(); + if tags_json != "[]" { + if let Ok(tags) = serde_json::from_str::>(tags_json) { + for (tag_name, category) in &tags { + let tag_id = match tag_cache.get(tag_name) { + Some(&id) => id, + None => { + match search_db.insert_tag(tag_name, Some(category)).await { + Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } + Err(_) => continue, + } + } + }; + search_db.insert_album_tag(&album.id, tag_id).await.ok(); + } + } + } + + // ── 3. 写入 persons → album_persons ── + search_db.clear_album_persons(&album.id).await.ok(); + if let Some(ref person_name) = album.person_name_cleaned { + if !person_name.is_empty() { + let person_id = match person_cache.get(person_name) { + Some(&id) => id, + None => { + match search_db.insert_person(person_name, None).await { + Ok(id) => { person_cache.insert(person_name.clone(), id); id } + Err(_) => continue, + } + } + }; + search_db.insert_album_person(&album.id, person_id).await.ok(); + + // 同时同步别名 + let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); + if !aliases.is_empty() { + for (alias_type, alias_name) in &aliases { + if alias_name != person_name { + search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + } + } + if primary != person_name.as_str() && !primary.is_empty() { + search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); + } + } + } + } + + synced += 1; + } + + // ── 4. 更新 person counts ── + for (person_name, person_id) in &person_cache { + let albums_count: i64 = match sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM person_albums WHERE person_id = ?" + ).bind(*person_id).fetch_one(&db.pool).await { + Ok(c) => c, + Err(_) => continue, + }; + search_db.update_person_counts(*person_id, albums_count, 0).await.ok(); + } + + println!("[Sync->search.db·增量] 同步完成：{} 个相册，{} 个人物，{} 个标签", + synced, person_cache.len(), tag_cache.len()); + synced +} diff --git a/src/engine/recommend.rs b/src/engine/recommend.rs index 488f733..576b447 100644 --- a/src/engine/recommend.rs +++ b/src/engine/recommend.rs @@ -5,7 +5,7 @@ //! 2. 人物推荐：基于名称匹配 + 相册数排序 //! 3. 关联推荐：基于标签共现关系 + 人物共现关系 -use crate::db::Database; +use crate::search_db::SearchDb; use crate::models::*; use crate::engine::tag_dict; use sqlx::Row; @@ -16,14 +16,14 @@ pub struct RecommendEngine; impl RecommendEngine { /// 标签自动补全推荐 pub async fn recommend_tags( - db: &Database, + search_db: &SearchDb, query: &str, limit: usize, ) -> RecommendResponse { let limit = limit.min(20); let q = query.trim().to_lowercase(); - let tags = db.get_popular_tags(200).await; + let tags = search_db.get_popular_tags(200).await; let mut items: Vec = Vec::new(); for t in &tags { @@ -62,14 +62,14 @@ impl RecommendEngine { /// 人物推荐 pub async fn recommend_persons( - db: &Database, + search_db: &SearchDb, query: &str, limit: usize, ) -> RecommendResponse { let limit = limit.min(20); let q = query.trim().to_lowercase(); - let persons = db.get_all_persons_with_aliases().await; + let persons = search_db.get_all_persons_with_aliases().await; let mut items: Vec = Vec::new(); for p in &persons { @@ -112,14 +112,14 @@ impl RecommendEngine { /// 关联推荐（共现标签 + 共现人物） pub async fn recommend_related( - db: &Database, + search_db: &SearchDb, context: &str, // 标签或人名 limit: usize, ) -> TagRecommendation { let limit = limit.min(15); // 获取共现标签 - let co_tags = db.get_tag_co_occurrences(context, limit as i64).await; + let co_tags = search_db.get_tag_co_occurrences(context, limit as i64).await; let related_tags: Vec = co_tags.iter().map(|c| { RecommendItem { value: c.co_tag.clone(), @@ -131,19 +131,22 @@ impl RecommendEngine { }).collect(); // 获取相关人物（通过标签） - let albums = db.search_albums_by_tag(context).await; + let albums = search_db.search_albums_by_tag(context).await; let mut person_scores: std::collections::HashMap = std::collections::HashMap::new(); let total = albums.len() as f64; for album in &albums { - let meta = db.get_all_albums_full().await; - for m in &meta { - if m.id == album.id.to_string() { - if let Some(ref name) = m.person_name_cleaned { - let entry = person_scores.entry(name.clone()).or_insert((0, 0.0)); - entry.0 += 1; - entry.1 = entry.0 as f64 / total; - } - } + let aid = album.id.to_string(); + // 从 search.db 查该相册的人物 + let person_rows = sqlx::query( + r#"SELECT p.name FROM persons p + JOIN album_persons ap ON p.id = ap.person_id + WHERE ap.album_id = ?"# + ).bind(&aid).fetch_all(&search_db.pool).await.unwrap_or_default(); + for row in &person_rows { + let name: String = row.get("name"); + let entry = person_scores.entry(name).or_insert((0, 0.0)); + entry.0 += 1; + entry.1 = entry.0 as f64 / total; } } @@ -167,9 +170,9 @@ impl RecommendEngine { } /// 热门人物 - pub async fn popular_persons(db: &Database, limit: usize) -> Vec { + pub async fn popular_persons(search_db: &SearchDb, limit: usize) -> Vec { let dict = tag_dict::get_tag_dict(); - let persons = db.get_all_persons_with_aliases().await; + let persons = search_db.get_all_persons_with_aliases().await; // 收集所有标签分类词（非人名候选） let all_tag_words: std::collections::HashSet = dict.tag_to_category.keys() @@ -204,23 +207,11 @@ impl RecommendEngine { } }).collect(); - // 为热门人物获取 top tags - // （简化：从 person_albums 查标签） + // 为热门人物获取 top tags（从 search.db） for rp in &mut results { - if let Some(person) = db.get_person_by_name(&rp.name).await { - let albums = db.get_person_albums(person.id).await; - let mut tag_counts: std::collections::HashMap = std::collections::HashMap::new(); - for album in &albums { - let aid = album.id.to_string(); - let rows = sqlx::query("SELECT tag FROM album_tags WHERE album_id = ?") - .bind(&aid).fetch_all(&db.pool).await.unwrap_or_default(); - for row in &rows { - *tag_counts.entry(row.get::("tag")).or_insert(0) += 1; - } - } - let mut tags: Vec<(String, i64)> = tag_counts.into_iter().collect(); - tags.sort_by(|a, b| b.1.cmp(&a.1)); - rp.top_tags = tags.into_iter().take(5).map(|(t, _)| t).collect(); + if let Some(person) = search_db.get_person_by_name(&rp.name).await { + let top_tags = search_db.get_person_top_tags(person.id, 5).await; + rp.top_tags = top_tags.into_iter().map(|(t, _)| t).collect(); } } results diff --git a/src/engine/tag_dict.rs b/src/engine/tag_dict.rs index ba3e825..735867a 100644 --- a/src/engine/tag_dict.rs +++ b/src/engine/tag_dict.rs @@ -531,3 +531,99 @@ pub fn get_tag_dict_stats() -> String { dict.included_person_names.len(), ) } + +// ════════════════════════════════════════════════ +// 从 search.db 的 dict 表加载 +// ════════════════════════════════════════════════ + +use crate::search_db::DictEntry; + +impl TagDict { + /// 从 search.db 的 dict 表条目加载词典数据 + /// + /// dict 表条目类型（通过 type 字段区分）： + /// - `tag` — word 是标签，category 是分类名 → 加入 keymap + /// - `person` — word 是人名 → 加入 included_person_names + /// - `stop_word` — word 是停用词 + /// - `person_prefix` — word 是人物前缀 + /// - `excluded_person` — word 是排除人名 + /// + /// 派生规则： + /// - 所有 tag 类型的 word 自动加入 excluded_person_names + /// - categories 结构基于 metadata（内建）和 keymap（从 tag 条目构建） + pub fn load_from_dict_entries(entries: &[DictEntry]) -> Self { + let mut dict = TagDict::default(); + + for entry in entries { + match entry.type_.as_str() { + "tag" => { + let cat = entry.category.as_deref().unwrap_or("other"); + dict.keymap.insert(entry.word.clone(), cat.to_string()); + dict.tag_to_category.insert(entry.word.to_lowercase(), cat.to_string()); + dict.excluded_person_names.insert(entry.word.to_lowercase()); + } + "person" => { + dict.included_person_names.insert(entry.word.to_lowercase()); + } + "stop_word" => { + dict.stop_words.insert(entry.word.to_lowercase()); + } + "person_prefix" => { + dict.person_prefixes.push(entry.word.clone()); + } + "excluded_person" => { + dict.excluded_person_names.insert(entry.word.to_lowercase()); + } + _ => {} + } + } + + // ── 构建 categories（基于 metadata + keymap） ── + let mut words_by_cat: HashMap> = HashMap::new(); + for (word, cat_name) in &dict.keymap { + words_by_cat.entry(cat_name.clone()).or_default().push(word.clone()); + } + for (name, meta) in &dict.metadata { + let words = words_by_cat.remove(name).unwrap_or_default(); + dict.categories.insert(name.clone(), CategoryInfo { + name: name.clone(), + label: meta.label.clone(), + color_bg: meta.color_bg.clone(), + color_fg: meta.color_fg.clone(), + words, + }); + } + for (cat_name, words) in words_by_cat { + dict.categories.insert(cat_name.clone(), CategoryInfo { + name: cat_name.clone(), + label: cat_name.clone(), + color_bg: "#f1f5f9".to_string(), + color_fg: "#64748b".to_string(), + words, + }); + } + + println!("[词典] 从 search.db dict 表加载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名", + dict.categories.len(), dict.keymap.len(), + dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(), + dict.included_person_names.len()); + + dict + } +} + +/// 从 search.db 的 dict 表重新加载标签词典 +pub async fn reload_from_search_db(search_db: &crate::search_db::SearchDb) -> String { + let entries = match search_db.load_dict_all().await { + Ok(e) => e, + Err(e) => return format!("从 search.db 加载 dict 失败: {}", e), + }; + let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染"); + *dict = TagDict::load_from_dict_entries(&entries); + format!( + "词典已从 search.db 重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名", + dict.categories.len(), dict.keymap.len(), + dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(), + dict.included_person_names.len(), + ) +} diff --git a/src/engine/title_parser.rs b/src/engine/title_parser.rs index 42b95b9..348c95c 100644 --- a/src/engine/title_parser.rs +++ b/src/engine/title_parser.rs @@ -452,6 +452,127 @@ pub fn normalize_tag(tag: &str) -> String { tag.trim().to_lowercase() } +// ── dict 优先匹配 ── + +/// 使用 dict 表优先匹配 + title_parser 回退的混合解析 +/// +/// 流程： +/// 1. dict 最长匹配提取 person 和 tags（带 merged_to 解析） +/// 2. 从标题移除已匹配的 dict 词 +/// 3. 剩余文本走 title_parser 解析（tag_dict 回退） +/// 4. 合并：dict 结果优先，parser 补充 +pub fn parse_album_title_with_dict( + album_id: &str, + title: &str, + dict_entries: &[crate::search_db::DictEntry], +) -> crate::models::parse::ParseResult { + let title = title.trim(); + if title.is_empty() { + return crate::models::parse::ParseResult::new(album_id.to_string()); + } + + // Phase 1: dict 优先匹配 + let (dict_person, dict_tags, remaining) = extract_with_dict(title, dict_entries); + + // Phase 2: title_parser 回退（解析剩余文本） + let parse_result = if remaining.is_empty() { + crate::models::parse::ParseResult::new(album_id.to_string()) + } else { + parse_album_title(album_id, &remaining) + }; + + // Phase 3: 合并结果 — dict 优先，parser 补充 + let mut result = crate::models::parse::ParseResult::new(album_id.to_string()); + + // person: dict 命中 > parser 命中 + result.person_name = dict_person.or(parse_result.person_name); + + // tags: dict 标签优先，parser 标签补充去重 + let mut seen = HashSet::new(); + for (t, _) in &dict_tags { + seen.insert(t.to_lowercase()); + result.tags.push((t.clone(), "other".to_string())); + } + for (t, c) in &parse_result.tags { + if seen.insert(t.to_lowercase()) { + result.tags.push((t.clone(), c.clone())); + } + } + + result +} + +/// dict 条目按词长降序匹配标题，返回 (person, tags, 剩余文本) +fn extract_with_dict( + title: &str, + dict_entries: &[crate::search_db::DictEntry], +) -> (Option, Vec<(String, String)>, String) { + // 按词长降序排列 + let mut sorted: Vec<&crate::search_db::DictEntry> = dict_entries.iter().collect(); + sorted.sort_by(|a, b| b.word.len().cmp(&a.word.len())); + + let title_lower = title.to_lowercase(); + let mut matched_ranges: Vec<(usize, usize)> = Vec::new(); + let mut person: Option = None; + let mut tags: Vec<(String, String)> = Vec::new(); + let mut seen_tag_words = HashSet::new(); + + for entry in &sorted { + let word_lower = entry.word.to_lowercase(); + + // 在当前标题中查找该词（忽略大小写） + let mut search_start = 0; + while let Some(pos) = title_lower[search_start..].find(&word_lower) { + let abs_start = search_start + pos; + let abs_end = abs_start + entry.word.len(); + + // 检查是否已被更长的 dict 词覆盖 + if matched_ranges.iter().any(|&(s, e)| s <= abs_start && abs_end <= e) { + break; + } + + matched_ranges.push((abs_start, abs_end)); + + match entry.type_.as_str() { + "person" if person.is_none() => { + person = Some(entry.merged_to.as_ref().unwrap_or(&entry.word).clone()); + } + "tag" => { + if seen_tag_words.insert(entry.word.clone()) { + tags.push((entry.word.clone(), entry.category.clone().unwrap_or_default())); + } + } + _ => {} + } + break; // 每个词只匹配一次 + } + } + + // 构建剩余文本：去掉已匹配的部分 + if matched_ranges.is_empty() { + return (person, tags, title.to_string()); + } + + matched_ranges.sort_by_key(|&(s, _)| s); + let mut remaining = String::new(); + let mut last = 0; + for &(s, e) in &matched_ranges { + if s > last { + remaining.push_str(&title[last..s]); + } + remaining.push(' '); + last = e; + } + if last < title.len() { + remaining.push_str(&title[last..]); + } + + // 合并连续空格 + let remaining = remaining.split_whitespace().collect::>().join(" "); + + (person, tags, remaining) +} + // ── 兼容函数（供 pipeline / person_merger 调用） ── /// 从标题中提取人物名（兼容函数） diff --git a/src/main.rs b/src/main.rs index 75e524f..b1178a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,10 +7,13 @@ mod routes; mod services; mod image_processor; mod queue; +mod search_db; use crate::cache::AlbumsCache; +use crate::search_db::SearchDb; use actix_files as afs; use actix_web::{App, HttpResponse, HttpServer, middleware, web}; +use std::env; use std::fs; use std::sync::Arc; use tokio::time::{Duration, interval}; @@ -20,6 +23,14 @@ use tracing::{error, info}; async fn main() -> std::io::Result<()> { tracing_subscriber::fmt::init(); + let args: Vec = env::args().collect(); + let export_to_search = args.iter().any(|a| a == "--export-search"); + let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db=")) + .map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db")) + .unwrap_or("search.db"); + let clear_search = args.iter().any(|a| a == "--clear-search"); + let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint"); + // 加载配置 let config = match config::Config::load() { Ok(cfg) => cfg, @@ -48,45 +59,103 @@ async fn main() -> std::io::Result<()> { } }; + if export_to_search { + info!("[搜索导出] 开始将元数据导出到 search.db..."); + match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await { + Ok(result) => { + info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}", + result.albums_processed, result.tags_created, result.persons_created, + result.aliases_created, result.album_tags_created + result.album_persons_created); + std::process::exit(0); + } + Err(e) => { + error!("[搜索导出] 导出失败: {}", e); + std::process::exit(1); + } + } + } + + if wal_checkpoint { + info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成"); + info!("[WAL合并] 正在合并 search.db WAL..."); + let search_db = SearchDb::new("data/search.db").await + .expect("search.db 连接失败"); + search_db.wal_checkpoint().await; + info!("[WAL合并] 两个数据库均已合并完成"); + std::process::exit(0); + } + let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone())); + // ── 初始化 search.db（运行时搜索专用）── + let search_db_path = "data/search.db"; + let search_db = Arc::new( + SearchDb::new(search_db_path).await + .expect("search.db 连接失败") + ); + info!("search.db 连接成功"); + search_db.wal_checkpoint().await; + + // ── 从 search.db 的 dict 表加载标签词典 ── + match engine::tag_dict::reload_from_search_db(&search_db).await { + msg => info!("{}", msg), + } + // ── 内存缓存（必须在管线任务之前创建）── let albums_cache = Arc::new(tokio::sync::RwLock::new(AlbumsCache::new())); let persons_cache = Arc::new(tokio::sync::RwLock::new(cache::PersonCache::default())); let tags_cache = Arc::new(tokio::sync::RwLock::new(cache::TagCache::default())); - // ── 启动管线（单次）：导入 → 解析 → 合并 → 缓存刷新 ── + // ── 启动管线（单次）：导入 → 增量解析 → 条件全量 sync → 缓存刷新 ── + // 不做全量 reparse/merge，只处理新发现的相册 let boot_config = config.clone(); let boot_db = Arc::clone(&db); + let boot_search_db = Arc::clone(&search_db); let boot_albums_cache = Arc::clone(&albums_cache); let boot_persons_cache = Arc::clone(&persons_cache); let boot_tags_cache = Arc::clone(&tags_cache); tokio::spawn(async move { // ① 扫描导入新相册 - info!("[启动管线] ① 扫描导入新相册..."); + info!("[启动] ① 扫描导入新相册..."); match services::import::import_existing_data(&boot_config, Arc::clone(&boot_db), None).await { - Ok(_) => info!("[启动管线] 导入完成"), - Err(e) => error!("[启动管线] 导入失败: {}", e), + Ok(_) => info!("[启动] 导入完成"), + Err(e) => error!("[启动] 导入失败: {}", e), } - // ② 全量解析 + 人物合并 - info!("[启动管线] ② 全量解析 + 人物合并..."); - let result = engine::pipeline::run_pipeline(&boot_db).await; - info!("[启动管线] 完成：清洗标点{}，移除短标签{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", - result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected, - result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found, - result.persons_updated, result.tags_updated, - result.persons_merged, result.aliases_moved, result.tags_removed_person); + // ② 查找新相册 / 半成品 + info!("[启动] ② 查找新相册..."); + let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await; + let new_count = dirty.len(); + let mut full_sync = false; - // ③ 刷新所有内存缓存 - info!("[启动管线] ③ 刷新所有内存缓存..."); - cache::refresh_all_caches(&boot_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await; - info!("[启动管线] 启动管线执行完毕"); + if new_count > 0 { + info!("[启动] 发现 {} 个新相册，增量同步到 search.db...", new_count); + let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await; + info!("[启动] 增量同步完成：{} 个相册（解析在定时管线中执行）", synced); + } else { + info!("[启动] 没有新相册"); + // 检查 search.db 是否为空（首次启动） + let album_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums") + .fetch_one(&boot_search_db.pool).await.unwrap_or(0); + if album_count == 0 { + info!("[启动] search.db 为空，执行全量同步..."); + let synced = engine::pipeline::sync_albums_to_search_db(&boot_db, &boot_search_db).await; + info!("[启动] 全量同步完成：{} 个相册", synced); + full_sync = true; + } + } + + // ③ 刷新所有内存缓存（从 search.db 读取） + info!("[启动] ③ 刷新所有内存缓存..."); + cache::refresh_all_caches(&boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await; + info!("[启动] 启动管线执行完毕（新相册={}, 全量同步={}）", new_count, full_sync); }); - // ── 定时管线（每 30 分钟）：解析 → 合并 → 缓存刷新 ── + // ── 定时管线（每 30 分钟）：导入 → 增量解析 → 缓存刷新 ── + // 不做全量 reparse/merge let timer_cache = Arc::clone(&albums_cache); let timer_db = Arc::clone(&db); + let timer_search_db = Arc::clone(&search_db); let timer_persons_cache = Arc::clone(&persons_cache); let timer_tags_cache = Arc::clone(&tags_cache); tokio::spawn(async move { @@ -94,18 +163,28 @@ async fn main() -> std::io::Result<()> { loop { interval.tick().await; - // ① 全量解析 + 人物合并（不重复 import） - info!("[定时管线] ① 全量解析 + 人物合并..."); - let result = engine::pipeline::run_pipeline(&timer_db).await; - info!("[定时管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", - result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected, - result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found, - result.persons_updated, result.tags_updated, - result.persons_merged, result.aliases_moved, result.tags_removed_person); + // ① 扫描导入新相册 + info!("[定时] ① 扫描导入新相册..."); + match services::import::import_existing_data(&timer_db.config, Arc::clone(&timer_db), None).await { + Ok(_) => {}, + Err(e) => error!("[定时] 导入失败: {}", e), + } - // ② 刷新所有内存缓存 - info!("[定时管线] ② 刷新所有内存缓存..."); - cache::refresh_all_caches(&timer_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await; + // ② 查找新相册 + let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await; + if new_albums.is_empty() { + continue; // 无新增，跳过 + } + + info!("[定时] 发现 {} 个新相册，增量解析...", new_albums.len()); + engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await; + + info!("[定时] 增量同步到 search.db..."); + engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await; + + // ③ 刷新缓存 + info!("[定时] 刷新缓存..."); + cache::refresh_all_caches(&timer_search_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await; } }); @@ -158,6 +237,7 @@ async fn main() -> std::io::Result<()> { .wrap(middleware::Compress::default()) .wrap(middleware::Logger::default()) .app_data(web::Data::new(Arc::clone(&db))) + .app_data(web::Data::new(Arc::clone(&search_db))) .app_data(web::Data::new(Arc::clone(&image_processor))) .app_data(web::Data::new(Arc::clone(&albums_cache))) .app_data(web::Data::new(Arc::clone(&persons_cache))) diff --git a/src/routes/album.rs b/src/routes/album.rs index 747d8c4..6524c9e 100644 --- a/src/routes/album.rs +++ b/src/routes/album.rs @@ -1,5 +1,6 @@ use crate::cache::AlbumsCache; use crate::db::Database; +use crate::search_db::SearchDb; use crate::models::*; use actix_web::{HttpResponse, Responder, web}; use std::collections::HashMap; @@ -25,7 +26,7 @@ pub async fn get_albums(cache: web::Data>>) -> impl Resp /// 规则：有任意过滤条件时 → 走 DB 懒加载（只查符合条件的相册） /// 无过滤条件时 → 走缓存全量返回 pub async fn search_albums( - db: web::Data>, + search_db: web::Data>, cache: web::Data>>, query: web::Query>, ) -> impl Responder { @@ -50,7 +51,7 @@ pub async fn search_albums( let person_arg = if person.is_empty() { None } else { Some(person) }; let q_arg = if q.is_empty() { None } else { Some(q) }; - match db.search_albums_with_filters(tag_arg, person_arg, q_arg, limit, offset).await { + match search_db.search_albums(tag_arg, person_arg, q_arg, limit, offset).await { Ok(results) => HttpResponse::Ok().json(results), Err(e) => HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{}", e)})), } diff --git a/src/routes/dict.rs b/src/routes/dict.rs index 3ccfdfd..db1d637 100644 --- a/src/routes/dict.rs +++ b/src/routes/dict.rs @@ -1,8 +1,11 @@ //! 标签词典管理 API — 查看/热重载 -use actix_web::{HttpResponse, Responder}; +use std::sync::Arc; + +use actix_web::{HttpResponse, Responder, web}; use crate::engine::tag_dict; +use crate::search_db::SearchDb; /// GET /api/dict/stats — 查看词典状态 pub async fn dict_stats() -> impl Responder { @@ -30,8 +33,8 @@ pub async fn dict_stats() -> impl Responder { } /// POST /api/dict/reload — 热重载词典 -pub async fn dict_reload() -> impl Responder { - let msg = tag_dict::reload_tag_dict(); +pub async fn dict_reload(search_db: web::Data>) -> impl Responder { + let msg = tag_dict::reload_from_search_db(search_db.get_ref()).await; println!("[API] 词典热重载: {}", msg); HttpResponse::Ok().json(serde_json::json!({ "success": true, diff --git a/src/routes/person.rs b/src/routes/person.rs index d8a0e17..7dd9d05 100644 --- a/src/routes/person.rs +++ b/src/routes/person.rs @@ -1,5 +1,5 @@ use crate::cache::PersonCache; -use crate::db::Database; +use crate::search_db::SearchDb; use crate::models::*; use actix_web::{HttpResponse, Responder, web}; use std::collections::HashMap; @@ -36,9 +36,9 @@ pub async fn search_people( }) } -/// GET /api/people/{name} — 从缓存查找人物详情（需要 DB 获取相册列表） +/// GET /api/people/{name} — 从缓存查找人物详情（相册列表从 search.db） pub async fn get_person_detail( - db: web::Data>, + search_db: web::Data>, cache: web::Data>>, path: web::Path, ) -> impl Responder { @@ -49,8 +49,8 @@ pub async fn get_person_detail( let p = found.into_iter().find(|p| p.name.to_lowercase() == name.to_lowercase()); match p { Some(p) => { - // 相册列表仍需查 DB - let albums = db.get_person_albums(p.id).await; + // 相册列表从 search.db 获取 + let albums = search_db.get_person_albums(p.id).await; HttpResponse::Ok().json(PersonWithAlbums { album_count: albums.len(), person: Person { @@ -70,8 +70,8 @@ pub async fn get_person_detail( } } -/// GET /api/people/{id}/aliases — 查 DB（别名表记录数可能较大，不适合缓存） -pub async fn get_person_aliases(db: web::Data>, path: web::Path) -> impl Responder { - let aliases = db.get_person_aliases(path.into_inner()).await; +/// GET /api/people/{id}/aliases — 从 search.db 查别名 +pub async fn get_person_aliases(search_db: web::Data>, path: web::Path) -> impl Responder { + let aliases = search_db.get_person_aliases(path.into_inner()).await; HttpResponse::Ok().json(aliases) } diff --git a/src/routes/pipeline.rs b/src/routes/pipeline.rs index 2d1b0fa..c835d15 100644 --- a/src/routes/pipeline.rs +++ b/src/routes/pipeline.rs @@ -1,29 +1,38 @@ use crate::db::Database; -use crate::engine::pipeline; +use crate::engine; +use crate::search_db::SearchDb; use actix_web::{HttpResponse, Responder, web}; use std::sync::Arc; -/// POST /api/pipeline/run — 执行完整管线：清洗→检测→清理→解析→合并→校验 -pub async fn pipeline_run(db: web::Data>) -> impl Responder { - let result = pipeline::run_pipeline(db.get_ref()).await; +pub async fn pipeline_run( + db: web::Data>, + search_db: web::Data>, +) -> impl Responder { + let result = engine::pipeline::run_pipeline(db.get_ref()).await; + // 同步到 search.db + let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await; + HttpResponse::Ok().json(serde_json::json!({ "success": true, "tags_cleaned": result.tags_cleaned, + "short_tags_removed": result.short_tags_removed, "person_like_tags_removed": result.person_like_tags_removed, "dirty_albums_detected": result.dirty_albums_detected, "non_persons_deleted": result.non_persons_deleted, + "stop_word_persons_removed": result.stop_word_persons_removed, "new_albums_found": result.new_albums_found, "persons_updated": result.persons_updated, "tags_updated": result.tags_updated, "persons_merged": result.persons_merged, "aliases_moved": result.aliases_moved, "tags_removed_person": result.tags_removed_person, + "search_db_synced": synced, "message": format!( - "管线执行完毕：清洗标点{}相册，检测人名标签{}，脏集{}相册，删除非人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", - result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected, - result.non_persons_deleted, result.new_albums_found, + "管线执行完毕：清洗标点{}相册，移除短标签{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}，同步search.db{}", + result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected, + result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found, result.persons_updated, result.tags_updated, - result.persons_merged, result.aliases_moved, result.tags_removed_person + result.persons_merged, result.aliases_moved, result.tags_removed_person, synced ) })) } \ No newline at end of file diff --git a/src/routes/recommend.rs b/src/routes/recommend.rs index 0ec5c5c..ff9c667 100644 --- a/src/routes/recommend.rs +++ b/src/routes/recommend.rs @@ -1,36 +1,36 @@ -use crate::db::Database; +use crate::search_db::SearchDb; use crate::engine::RecommendEngine; -#[allow(unused)] use crate::models::*; +use crate::models::*; use actix_web::{HttpResponse, Responder, web}; use std::collections::HashMap; use std::sync::Arc; /// 标签推荐（自动补全） -pub async fn recommend_tags(db: web::Data>, query: web::Query>) -> impl Responder { +pub async fn recommend_tags(search_db: web::Data>, query: web::Query>) -> impl Responder { let q = query.get("q").map(|s| s.as_str()).unwrap_or(""); let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10); - HttpResponse::Ok().json(RecommendEngine::recommend_tags(&db, q, limit).await) + HttpResponse::Ok().json(RecommendEngine::recommend_tags(&search_db, q, limit).await) } /// 人物推荐（自动补全） -pub async fn recommend_persons(db: web::Data>, query: web::Query>) -> impl Responder { +pub async fn recommend_persons(search_db: web::Data>, query: web::Query>) -> impl Responder { let q = query.get("q").map(|s| s.as_str()).unwrap_or(""); let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10); - HttpResponse::Ok().json(RecommendEngine::recommend_persons(&db, q, limit).await) + HttpResponse::Ok().json(RecommendEngine::recommend_persons(&search_db, q, limit).await) } /// 关联推荐（标签共现 + 共现人物） -pub async fn recommend_related(db: web::Data>, query: web::Query>) -> impl Responder { +pub async fn recommend_related(search_db: web::Data>, query: web::Query>) -> impl Responder { let context = match query.get("context") { Some(c) if !c.is_empty() => c, _ => return HttpResponse::BadRequest().json(serde_json::json!({"error": "请提供context参数"})), }; let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(15); - HttpResponse::Ok().json(RecommendEngine::recommend_related(&db, context, limit).await) + HttpResponse::Ok().json(RecommendEngine::recommend_related(&search_db, context, limit).await) } /// 热门人物 -pub async fn popular_persons(db: web::Data>, query: web::Query>) -> impl Responder { +pub async fn popular_persons(search_db: web::Data>, query: web::Query>) -> impl Responder { let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(20).min(50); - HttpResponse::Ok().json(RecommendEngine::popular_persons(&db, limit).await) + HttpResponse::Ok().json(RecommendEngine::popular_persons(&search_db, limit).await) } \ No newline at end of file diff --git a/src/search_db/mod.rs b/src/search_db/mod.rs new file mode 100644 index 0000000..61434d0 --- /dev/null +++ b/src/search_db/mod.rs @@ -0,0 +1,310 @@ +mod search; + +pub use search::*; + +use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row}; + +#[derive(Debug, Clone)] +pub struct DictEntry { + pub word: String, + pub type_: String, + pub category: Option, + pub freq: Option, + pub merged_to: Option, +} + +pub struct SearchDb { + pub pool: SqlitePool, +} + +impl SearchDb { + pub async fn new(db_path: &str) -> Result { + if let Some(parent) = std::path::Path::new(db_path).parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + if !std::path::Path::new(db_path).exists() { + std::fs::File::create(db_path)?; + } + + let url = format!("sqlite:{}", db_path.replace('\\', "/")); + + let pool = SqlitePoolOptions::new() + .max_connections(2) + .connect(&url) + .await?; + + sqlx::query("PRAGMA journal_mode=WAL;").execute(&pool).await?; + sqlx::query("PRAGMA synchronous=NORMAL;").execute(&pool).await?; + sqlx::query("PRAGMA busy_timeout=5000;").execute(&pool).await?; + sqlx::query("PRAGMA wal_autocheckpoint=1000;").execute(&pool).await?; + sqlx::query("PRAGMA journal_size_limit=67108864;").execute(&pool).await?; + sqlx::query("PRAGMA cache_size=-64000;").execute(&pool).await?; + sqlx::query("PRAGMA temp_store=MEMORY;").execute(&pool).await?; + + Self::create_tables(&pool).await?; + + Ok(Self { pool }) + } + + /// 执行 WAL checkpoint(TRUNCATE)，将 WAL 完全合并回主文件并清理 -wal / -shm 文件 + /// + /// 在启动时调用，确保 search.db 文件干净，避免下次启动时 WAL 恢复开销。 + pub async fn wal_checkpoint(&self) { + match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)") + .execute(&self.pool).await + { + Ok(_) => println!("[search.db] WAL checkpoint(TRUNCATE) 完成，数据库文件已合并"), + Err(e) => eprintln!("[search.db] WAL checkpoint 失败: {}", e), + } + } + + async fn create_tables(pool: &SqlitePool) -> Result<(), sqlx::Error> { + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + category TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + "#).execute(pool).await?; + + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS persons ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + display_name TEXT, + total_albums INTEGER DEFAULT 0, + total_images INTEGER DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + "#).execute(pool).await?; + + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS person_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + person_id INTEGER NOT NULL, + alias_type TEXT NOT NULL, + alias_name TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE + ) + "#).execute(pool).await?; + + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS album_tags ( + album_id TEXT NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY (album_id, tag_id), + FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE + ) + "#).execute(pool).await?; + + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS album_persons ( + album_id TEXT NOT NULL, + person_id INTEGER NOT NULL, + PRIMARY KEY (album_id, person_id), + FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE + ) + "#).execute(pool).await?; + + // ── 新增 albums 表：搜索返回需要的相册基础信息 ── + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS albums ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + cover_image_id TEXT, + image_count INTEGER DEFAULT 0, + created_at TEXT + ) + "#).execute(pool).await?; + + sqlx::query(r#" + CREATE TABLE IF NOT EXISTS dict ( + word TEXT NOT NULL PRIMARY KEY, + type TEXT NOT NULL, + category TEXT, + freq INTEGER, + merged_to TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + "#).execute(pool).await?; + + sqlx::query("CREATE INDEX IF NOT EXISTS idx_albums_title ON albums(title)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_tags_name ON tags(name)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_persons_name ON persons(name)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_album ON album_tags(album_id)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_tag ON album_tags(tag_id)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_album ON album_persons(album_id)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_person ON album_persons(person_id)").execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_person_aliases_name ON person_aliases(alias_name)").execute(pool).await?; + + // 兼容旧表：尝试添加 merged_to 列（已有则忽略） + let _ = sqlx::query("ALTER TABLE dict ADD COLUMN merged_to TEXT").execute(pool).await; + + Ok(()) + } + + // ── upsert 写入方法 ── + + pub async fn upsert_album(&self, album_id: &str, title: &str, + cover_image_id: Option<&str>, image_count: i64, + created_at: Option<&str>) -> Result<(), sqlx::Error> { + sqlx::query(r#" + INSERT INTO albums (id, title, cover_image_id, image_count, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + title = excluded.title, + cover_image_id = excluded.cover_image_id, + image_count = excluded.image_count, + created_at = excluded.created_at + "#) + .bind(album_id).bind(title).bind(cover_image_id) + .bind(image_count).bind(created_at) + .execute(&self.pool).await?; + Ok(()) + } + + pub async fn delete_album(&self, album_id: &str) -> Result<(), sqlx::Error> { + let mut tx = self.pool.begin().await?; + sqlx::query("DELETE FROM album_tags WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?; + sqlx::query("DELETE FROM album_persons WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?; + sqlx::query("DELETE FROM albums WHERE id = ?").bind(album_id).execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + pub async fn insert_tag(&self, name: &str, category: Option<&str>) -> Result { + let result = sqlx::query(r#" + INSERT OR IGNORE INTO tags (name, category) VALUES (?, ?) + "#) + .bind(name) + .bind(category) + .execute(&self.pool) + .await?; + + let id = result.last_insert_rowid(); + if id == 0 { + let found_id: i64 = sqlx::query_scalar("SELECT id FROM tags WHERE name = ?") + .bind(name) + .fetch_one(&self.pool) + .await?; + Ok(found_id) + } else { + Ok(id) + } + } + + pub async fn insert_person(&self, name: &str, display_name: Option<&str>) -> Result { + let result = sqlx::query(r#" + INSERT OR IGNORE INTO persons (name, display_name) VALUES (?, ?) + "#) + .bind(name) + .bind(display_name) + .execute(&self.pool) + .await?; + + let id = result.last_insert_rowid(); + if id == 0 { + let found_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?") + .bind(name) + .fetch_one(&self.pool) + .await?; + Ok(found_id) + } else { + Ok(id) + } + } + + pub async fn update_person_counts(&self, person_id: i64, total_albums: i64, total_images: i64) -> Result<(), sqlx::Error> { + sqlx::query("UPDATE persons SET total_albums = ?, total_images = ?, updated_at = datetime('now') WHERE id = ?") + .bind(total_albums).bind(total_images).bind(person_id) + .execute(&self.pool).await?; + Ok(()) + } + + pub async fn insert_person_alias(&self, person_id: i64, alias_type: &str, alias_name: &str) -> Result<(), sqlx::Error> { + sqlx::query(r#" + INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?) + "#) + .bind(person_id) + .bind(alias_type) + .bind(alias_name) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn insert_album_tag(&self, album_id: &str, tag_id: i64) -> Result<(), sqlx::Error> { + sqlx::query(r#" + INSERT OR IGNORE INTO album_tags (album_id, tag_id) VALUES (?, ?) + "#) + .bind(album_id) + .bind(tag_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn insert_album_person(&self, album_id: &str, person_id: i64) -> Result<(), sqlx::Error> { + sqlx::query(r#" + INSERT OR IGNORE INTO album_persons (album_id, person_id) VALUES (?, ?) + "#) + .bind(album_id) + .bind(person_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// 清空某相册的所有关联标签 + pub async fn clear_album_tags(&self, album_id: &str) -> Result<(), sqlx::Error> { + sqlx::query("DELETE FROM album_tags WHERE album_id = ?") + .bind(album_id).execute(&self.pool).await?; + Ok(()) + } + + /// 清空某相册的所有关联人物 + pub async fn clear_album_persons(&self, album_id: &str) -> Result<(), sqlx::Error> { + sqlx::query("DELETE FROM album_persons WHERE album_id = ?") + .bind(album_id).execute(&self.pool).await?; + Ok(()) + } + + pub async fn load_dict(&self) -> Result, sqlx::Error> { + let rows = sqlx::query("SELECT word, type, category, freq, merged_to FROM dict") + .fetch_all(&self.pool) + .await?; + + let entries = rows.iter().map(|r| DictEntry { + word: r.get("word"), + type_: r.get("type"), + category: r.get("category"), + freq: r.get("freq"), + merged_to: r.get("merged_to"), + }).collect(); + + Ok(entries) + } + + /// 加载 dict 表全部条目（不限类型），用于初始化 TagDict + pub async fn load_dict_all(&self) -> Result, sqlx::Error> { + self.load_dict().await + } + + pub async fn clear_all(&self) -> Result<(), sqlx::Error> { + let mut tx = self.pool.begin().await?; + sqlx::query("DELETE FROM album_tags").execute(&mut *tx).await?; + sqlx::query("DELETE FROM album_persons").execute(&mut *tx).await?; + sqlx::query("DELETE FROM person_aliases").execute(&mut *tx).await?; + sqlx::query("DELETE FROM tags").execute(&mut *tx).await?; + sqlx::query("DELETE FROM persons").execute(&mut *tx).await?; + sqlx::query("DELETE FROM albums").execute(&mut *tx).await?; + sqlx::query("DELETE FROM dict").execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } +} diff --git a/src/search_db/search.rs b/src/search_db/search.rs new file mode 100644 index 0000000..c73d075 --- /dev/null +++ b/src/search_db/search.rs @@ -0,0 +1,342 @@ +//! 搜索读取方法 — SearchDb 的只读接口 +//! +//! 所有搜索相关的数据读取都从这里进行，完全不碰主库。 + +use sqlx::Row; +use std::collections::HashMap; +use uuid::Uuid; + +use crate::models::*; +use super::SearchDb; + +impl SearchDb { + // ═══════════════════════════════════════ + // 相册搜索 + // ═══════════════════════════════════════ + + /// 组合搜索：按 tag/person/标题关键词 任意组合过滤（AND） + pub async fn search_albums( + &self, + tag: Option<&str>, + person: Option<&str>, + title_q: Option<&str>, + limit: i64, + offset: i64, + ) -> Result, sqlx::Error> { + let has_tag = tag.map(|s| !s.is_empty()).unwrap_or(false); + let has_person = person.map(|s| !s.is_empty()).unwrap_or(false); + let has_q = title_q.map(|s| !s.is_empty()).unwrap_or(false); + + if !has_tag && !has_person && !has_q { + return self.get_all_albums().await; + } + + let mut sql = String::from( + "SELECT DISTINCT a.id, a.title, a.cover_image_id, a.image_count, a.created_at + FROM albums a" + ); + let mut conditions: Vec = Vec::new(); + + if has_tag { + sql.push_str(" JOIN album_tags at ON a.id = at.album_id JOIN tags t ON at.tag_id = t.id"); + conditions.push("t.name = ?".to_string()); + } + if has_person { + sql.push_str(" JOIN album_persons ap ON a.id = ap.album_id JOIN persons p ON ap.person_id = p.id"); + conditions.push("(p.name = ? OR LOWER(p.name) LIKE ?)".to_string()); + } + + if !conditions.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conditions.join(" AND ")); + } + if has_q { + if conditions.is_empty() { + sql.push_str(" WHERE "); + } else { + sql.push_str(" AND "); + } + sql.push_str("a.title LIKE ?"); + } + + sql.push_str(" ORDER BY a.created_at DESC LIMIT ? OFFSET ?"); + + let mut query = sqlx::query(&sql); + + if has_tag { query = query.bind(tag.unwrap()); } + if has_person { + let p = person.unwrap(); + query = query.bind(p).bind(format!("%{}%", p)); + } + if has_q { query = query.bind(format!("%{}%", title_q.unwrap())); } + + query = query.bind(limit).bind(offset); + + let rows = query.fetch_all(&self.pool).await?; + Ok(rows.iter().map(|r| { + let id_str: String = r.get("id"); + AlbumInfo { + id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()), + title: r.get("title"), + document: String::new(), + description: None, + cover_image_id: r.get("cover_image_id"), + image_count: r.get("image_count"), + created_at: r.get("created_at"), + } + }).collect()) + } + + /// 获取全量相册列表（缓存烘焙用） + pub async fn get_all_albums(&self) -> Result, sqlx::Error> { + let rows = sqlx::query( + "SELECT id, title, cover_image_id, image_count, created_at FROM albums ORDER BY created_at DESC" + ).fetch_all(&self.pool).await?; + + Ok(rows.iter().map(|r| { + let id_str: String = r.get("id"); + AlbumInfo { + id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()), + title: r.get("title"), + document: String::new(), + description: None, + cover_image_id: r.get("cover_image_id"), + image_count: r.get("image_count"), + created_at: r.get("created_at"), + } + }).collect()) + } + + // ═══════════════════════════════════════ + // 标签搜索 + // ═══════════════════════════════════════ + + /// 获取热门标签（按使用次数排序） + pub async fn get_popular_tags(&self, limit: usize) -> Vec { + let limit = limit.min(500); + let rows = sqlx::query( + r#"SELECT t.name, COUNT(at.album_id) as cnt + FROM tags t + JOIN album_tags at ON t.id = at.tag_id + GROUP BY t.id, t.name + ORDER BY cnt DESC LIMIT ?"# + ).bind(limit as i64).fetch_all(&self.pool).await.unwrap_or_default(); + + rows.iter().map(|r| TagStat { + tag: r.get("name"), + count: r.get("cnt"), + album_count: 0, + }).collect() + } + + /// 获取标签倒排索引：tag → albums（缓存烘焙用） + pub async fn get_all_tag_albums(&self) -> HashMap> { + let rows = sqlx::query( + r#"SELECT t.name as tag_name, + a.id, a.title, a.cover_image_id, a.image_count, a.created_at + FROM tags t + JOIN album_tags at ON t.id = at.tag_id + JOIN albums a ON a.id = at.album_id + ORDER BY a.created_at DESC"# + ).fetch_all(&self.pool).await.unwrap_or_default(); + + let mut map: HashMap> = HashMap::new(); + for r in &rows { + let tag_name: String = r.get("tag_name"); + let id_str: String = r.get("id"); + let album = AlbumInfo { + id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()), + title: r.get("title"), + document: String::new(), + description: None, + cover_image_id: r.get("cover_image_id"), + image_count: r.get("image_count"), + created_at: r.get("created_at"), + }; + map.entry(tag_name).or_default().push(album); + } + map + } + + /// 按标签搜索相册 + pub async fn search_albums_by_tag(&self, tag: &str) -> Vec { + let rows = sqlx::query( + r#"SELECT a.id, a.title, a.cover_image_id, a.image_count, a.created_at + FROM albums a + JOIN album_tags at ON a.id = at.album_id + JOIN tags t ON at.tag_id = t.id + WHERE t.name = ? + ORDER BY a.created_at DESC"# + ).bind(tag).fetch_all(&self.pool).await.unwrap_or_default(); + + rows.iter().map(|r| { + let id_str: String = r.get("id"); + AlbumInfo { + id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()), + title: r.get("title"), + document: String::new(), + description: None, + cover_image_id: r.get("cover_image_id"), + image_count: r.get("image_count"), + created_at: r.get("created_at"), + } + }).collect() + } + + /// 获取标签共现数据（关联推荐用） + pub async fn get_tag_co_occurrences(&self, tag: &str, limit: i64) -> Vec { + let rows = sqlx::query( + r#"SELECT t2.name as co_tag, COUNT(*) as cnt + FROM album_tags at1 + JOIN album_tags at2 ON at1.album_id = at2.album_id AND at1.tag_id != at2.tag_id + JOIN tags t1 ON at1.tag_id = t1.id + JOIN tags t2 ON at2.tag_id = t2.id + WHERE t1.name = ? + GROUP BY t2.name + ORDER BY cnt DESC LIMIT ?"# + ).bind(tag).bind(limit).fetch_all(&self.pool).await.unwrap_or_default(); + + let total: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM album_tags at JOIN tags t ON at.tag_id = t.id WHERE t.name = ?" + ).bind(tag).fetch_one(&self.pool).await.unwrap_or(1); + + rows.iter().map(|r| TagCoOccurrence { + tag: tag.to_string(), + co_tag: r.get("co_tag"), + count: r.get("cnt"), + score: r.get::("cnt") as f64 / total as f64, + }).collect() + } + + // ═══════════════════════════════════════ + // 人物搜索 + // ═══════════════════════════════════════ + + /// 获取人物别名列表 + pub async fn get_person_aliases(&self, person_id: i64) -> Vec { + let rows = sqlx::query( + "SELECT id, person_id, alias_type, alias_name, created_at FROM person_aliases WHERE person_id = ?" + ).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default(); + + rows.iter().map(|r| PersonAlias { + id: r.get("id"), + person_id: r.get("person_id"), + alias_type: r.get("alias_type"), + alias_name: r.get("alias_name"), + created_at: r.get("created_at"), + }).collect() + } + + /// 获取所有人物（含别名、排序） + pub async fn get_all_persons_with_aliases(&self) -> Vec { + let persons = sqlx::query( + "SELECT id, name, display_name, total_albums, total_images FROM persons ORDER BY total_albums DESC" + ).fetch_all(&self.pool).await.unwrap_or_default(); + + let mut results = Vec::new(); + for r in &persons { + let id: i64 = r.get("id"); + let name: String = r.get("name"); + let aliases: Vec = sqlx::query_scalar( + "SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type" + ).bind(id).fetch_all(&self.pool).await.unwrap_or_default(); + + results.push(PersonWithCounts { + id, + name, + display_name: r.get("display_name"), + total_albums: r.get("total_albums"), + total_images: r.get("total_images"), + aliases, + }); + } + results + } + + /// 按名称查找人物 + pub async fn get_person_by_name(&self, name: &str) -> Option { + let row = sqlx::query( + "SELECT id, name, display_name, total_albums, total_images FROM persons WHERE name = ?" + ).bind(name).fetch_optional(&self.pool).await.ok()??; + + let id: i64 = row.get("id"); + let aliases: Vec = sqlx::query_scalar( + "SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type" + ).bind(id).fetch_all(&self.pool).await.unwrap_or_default(); + + Some(PersonWithCounts { + id, + name: row.get("name"), + display_name: row.get("display_name"), + total_albums: row.get("total_albums"), + total_images: row.get("total_images"), + aliases, + }) + } + + /// 获取人物的所有相册 + pub async fn get_person_albums(&self, person_id: i64) -> Vec { + let rows = sqlx::query( + r#"SELECT a.id, a.title, a.cover_image_id, a.image_count, a.created_at + FROM albums a + JOIN album_persons ap ON a.id = ap.album_id + WHERE ap.person_id = ? + ORDER BY a.created_at DESC"# + ).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default(); + + rows.iter().map(|r| { + let id_str: String = r.get("id"); + AlbumInfo { + id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()), + title: r.get("title"), + document: String::new(), + description: None, + cover_image_id: r.get("cover_image_id"), + image_count: r.get("image_count"), + created_at: r.get("created_at"), + } + }).collect() + } + + /// 获取人物的 top 标签（用于详情页） + pub async fn get_person_top_tags(&self, person_id: i64, limit: usize) -> Vec<(String, i64)> { + let rows = sqlx::query( + r#"SELECT t.name, COUNT(*) as cnt + FROM tags t + JOIN album_tags at ON t.id = at.tag_id + JOIN album_persons ap ON at.album_id = ap.album_id + WHERE ap.person_id = ? + GROUP BY t.name + ORDER BY cnt DESC LIMIT ?"# + ).bind(person_id).bind(limit as i64) + .fetch_all(&self.pool).await.unwrap_or_default(); + + rows.iter().map(|r| (r.get("name"), r.get("cnt"))).collect() + } + + /// 搜索人物（按名称/别名模糊匹配，内存） + pub fn search_persons_inline(persons: &[PersonWithCounts], query: &str) -> Vec { + let q = query.to_lowercase().trim().to_string(); + if q.is_empty() || q.len() < 2 { return Vec::new(); } + + let mut seen = std::collections::HashSet::new(); + let mut results = Vec::new(); + + for p in persons { + let name_lower = p.name.to_lowercase(); + let alias_matches = p.aliases.iter().any(|a| a.to_lowercase().contains(&q)); + + let matched = name_lower == q + || name_lower.starts_with(&q) + || name_lower.contains(&q) + || alias_matches; + + if matched && seen.insert(p.id) { + results.push(p.clone()); + } + } + + results.sort_by(|a, b| b.total_albums.cmp(&a.total_albums)); + results + } +} diff --git a/src/services/mod.rs b/src/services/mod.rs index 0da4431..88d0a1b 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1,5 +1,7 @@ //! 服务层 — 导入、处理等业务逻辑 pub mod import; +pub mod search_export; -#[allow(unused)] pub use import::*; \ No newline at end of file +#[allow(unused)] pub use import::*; +#[allow(unused)] pub use search_export::*; \ No newline at end of file diff --git a/src/services/search_export.rs b/src/services/search_export.rs new file mode 100644 index 0000000..850402c --- /dev/null +++ b/src/services/search_export.rs @@ -0,0 +1,179 @@ +use crate::db::Database; +use crate::engine::title_parser; +use crate::search_db::SearchDb; +use std::collections::HashMap; + +pub struct SearchExportResult { + pub albums_processed: usize, + pub tags_created: usize, + pub persons_created: usize, + pub aliases_created: usize, + pub album_tags_created: usize, + pub album_persons_created: usize, + pub dict_matched_persons: usize, + pub dict_matched_tags: usize, + pub dict_resolved_aliases: usize, +} + +impl Default for SearchExportResult { + fn default() -> Self { + Self { + albums_processed: 0, + tags_created: 0, + persons_created: 0, + aliases_created: 0, + album_tags_created: 0, + album_persons_created: 0, + dict_matched_persons: 0, + dict_matched_tags: 0, + dict_resolved_aliases: 0, + } + } +} + +pub async fn export_to_search_db( + source_db: &Database, + search_db_path: &str, + clear_existing: bool, +) -> Result { + let search_db = SearchDb::new(search_db_path).await.map_err(|e| format!("创建 search.db 失败: {}", e))?; + + if clear_existing { + search_db.clear_all().await.map_err(|e| format!("清空 search.db 失败: {}", e))?; + println!("[搜索导出] 已清空 search.db 现有数据"); + } + + // ── 加载 dict 表 ── + let dict_entries = search_db + .load_dict() + .await + .map_err(|e| format!("加载 dict 表失败: {}", e))?; + + let person_count = dict_entries.iter().filter(|e| e.type_ == "person").count(); + let tag_count = dict_entries.iter().filter(|e| e.type_ == "tag").count(); + let alias_count = dict_entries.iter().filter(|e| e.merged_to.is_some()).count(); + + println!( + "[搜索导出] dict 表加载完成: {} 条 (person={}, tag={}, 别名映射={})", + dict_entries.len(), + person_count, + tag_count, + alias_count, + ); + + let albums = source_db.get_all_albums().await.map_err(|e| format!("读取相册列表失败: {}", e))?; + println!("[搜索导出] 共读取 {} 个相册（来自 imager.db）", albums.len()); + + let mut tag_cache: HashMap = HashMap::new(); + let mut person_cache: HashMap = HashMap::new(); + let mut result = SearchExportResult::default(); + + for album in &albums { + let album_id = album.id.to_string(); + let title = &album.title; + + // ── 1. dict 优先解析（dict 优先匹配 → tag_dict 回退 → 合并） ── + let parse_result = title_parser::parse_album_title_with_dict(&album_id, title, &dict_entries); + + // 统计 dict 命中 + if let Some(ref person_name) = parse_result.person_name { + let from_dict = dict_entries.iter().any(|e| { + e.type_ == "person" + && (e.word.to_lowercase() == person_name.to_lowercase() + || e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == person_name.to_lowercase())) + }); + if from_dict { + result.dict_matched_persons += 1; + } + } + + // ── 2. 写入 search.db ── + for (tag_name, category) in &parse_result.tags { + // 检查是否是 dict 命中的 tag + let from_dict = dict_entries.iter().any(|e| { + e.type_ == "tag" && e.word.to_lowercase() == tag_name.to_lowercase() + }); + if from_dict { + result.dict_matched_tags += 1; + } + + let tag_id = match tag_cache.get(tag_name) { + Some(&id) => id, + None => { + let id = search_db + .insert_tag(tag_name, Some(category)) + .await + .map_err(|e| format!("插入标签 {} 失败: {}", tag_name, e))?; + tag_cache.insert(tag_name.clone(), id); + result.tags_created += 1; + id + } + }; + + search_db + .insert_album_tag(&album_id, tag_id) + .await + .map_err(|e| format!("关联相册标签 {} → {} 失败: {}", album_id, tag_name, e))?; + result.album_tags_created += 1; + } + + if let Some(person_name) = &parse_result.person_name { + // 检查是否有 merged_to 别名解析 + let resolved_from = dict_entries.iter().find(|e| { + e.type_ == "person" + && e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == person_name.to_lowercase()) + }); + if resolved_from.is_some() { + result.dict_resolved_aliases += 1; + } + + let person_id = match person_cache.get(person_name) { + Some(&id) => id, + None => { + let id = search_db + .insert_person(person_name, None) + .await + .map_err(|e| format!("插入人物 {} 失败: {}", person_name, e))?; + person_cache.insert(person_name.clone(), id); + result.persons_created += 1; + id + } + }; + + search_db + .insert_album_person(&album_id, person_id) + .await + .map_err(|e| format!("关联相册人物 {} → {} 失败: {}", album_id, person_name, e))?; + result.album_persons_created += 1; + + if let Some(primary) = &parse_result.person_primary { + if primary != person_name { + search_db + .insert_person_alias(person_id, "primary", primary) + .await + .map_err(|e| format!("插入别名 {} → {} 失败: {}", person_name, primary, e))?; + result.aliases_created += 1; + } + } + } + + result.albums_processed += 1; + + if result.albums_processed % 100 == 0 { + println!("[搜索导出] 已处理 {} / {} 相册", result.albums_processed, albums.len()); + } + } + + println!("[搜索导出] 完成："); + println!(" - 相册: {}", result.albums_processed); + println!(" - 标签: {}", result.tags_created); + println!(" - 人物: {}", result.persons_created); + println!(" - 别名: {}", result.aliases_created); + println!(" - 相册标签关联: {}", result.album_tags_created); + println!(" - 相册人物关联: {}", result.album_persons_created); + println!(" - dict 命中 person: {}", result.dict_matched_persons); + println!(" - dict 命中 tag: {}", result.dict_matched_tags); + println!(" - dict 别名解析: {}", result.dict_resolved_aliases); + + Ok(result) +}