admin/rust-imager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

update code level

Browse Source
This commit is contained in:
lychang
2026-07-20 23:12:20 +08:00
parent 923f9bd7cf
commit 84409207e2
19 changed files with 1471 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,4 +1,5 @@
/target
/albums
/.idea
/data
/data
/scripts

6
dict/tags.json
View File

@@ -634,11 +634,11 @@
"www",
"https",
"http",
"Impact",
"写真",
"系列"
"Impact"
],
"person_prefixes": [
"写真",
"系列",
"美女模特",
"性感女神",
"清纯美女",

3
migrations/004_person_albums_indexes.sql Normal file
View File

@@ -0,0 +1,3 @@
-- 为 person_albums 添加 album_id 索引，优化 JOIN 查询和按相册查找的性能
-- person_id 已有主键索引 (person_id, album_id) 覆盖，无需重复创建
CREATE INDEX IF NOT EXISTS idx_pa_album ON person_albums(album_id);

16
src/cache/mod.rs vendored
View File

@@ -1,6 +1,6 @@
//! 缓存层 — 内存缓存管理
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use std::collections::HashMap;
use std::sync::Arc;
@@ -206,25 +206,25 @@ impl PersonCache {
/// 刷新所有缓存（相册 + 标签 + 人物）
pub async fn refresh_all_caches(
db: &Database,
search_db: &SearchDb,
albums_cache: &Arc<RwLock<AlbumsCache>>,
persons_cache: &Arc<RwLock<PersonCache>>,
tags_cache: &Arc<RwLock<TagCache>>,
) {
// 1. 刷新相册
if let Ok(albums) = db.get_all_albums().await {
if let Ok(albums) = search_db.get_all_albums().await {
let mut c = albums_cache.write().await;
c.update(albums.clone());
// 2. 刷新标签
let popular_tags = db.get_popular_tags(200).await;
let tag_albums = db.get_all_tag_albums().await;
let popular_tags = search_db.get_popular_tags(200).await;
let tag_albums = search_db.get_all_tag_albums().await;
let mut tc = tags_cache.write().await;
tc.update(popular_tags, tag_albums);
}
// 3. 刷新人物
let persons = db.get_all_persons_with_aliases().await;
let persons = search_db.get_all_persons_with_aliases().await;
let mut pc = persons_cache.write().await;
pc.update(persons);
@@ -232,8 +232,8 @@ pub async fn refresh_all_caches(
}
/// 仅刷新人物缓存（管线执行后调用）
pub async fn refresh_persons_cache(db: &Database, persons_cache: &Arc<RwLock<PersonCache>>) {
let persons = db.get_all_persons_with_aliases().await;
pub async fn refresh_persons_cache(search_db: &SearchDb, persons_cache: &Arc<RwLock<PersonCache>>) {
let persons = search_db.get_all_persons_with_aliases().await;
let mut pc = persons_cache.write().await;
pc.update(persons);
}

30
src/db/album.rs
View File

@@ -172,9 +172,10 @@ impl Database {
).fetch_one(&self.pool).await.unwrap_or(0);
if has_col == 0 { return Vec::new(); }
// 去掉 ORDER BY：管线处理不需要排序，省去排序开销
let rows = sqlx::query(
r#"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count
FROM albums ORDER BY created_at DESC"#
FROM albums"#
).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
@@ -187,6 +188,33 @@ impl Database {
}).collect()
}
/// 按 ID 列表批量查询相册元数据（用于脏集增量解析，避免全表扫描）
pub async fn get_albums_by_ids(&self, ids: &[String]) -> Vec<AlbumMeta> {
if ids.is_empty() { return Vec::new(); }
let placeholders: Vec<String> = (0..ids.len()).map(|_| "?".to_string()).collect();
let sql = format!(
"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count
FROM albums WHERE id IN ({})",
placeholders.join(", ")
);
let mut query = sqlx::query(&sql);
for id in ids {
query = query.bind(id);
}
let rows = query.fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
id: r.get("id"), title: r.get("title"),
person_name_original: r.get("person_name_original"),
person_name_cleaned: r.get("person_name_cleaned"),
tags: r.get("tags"), all_words: r.get("all_words"),
raw_title: r.get("raw_title"), processed_at: r.get("processed_at"),
image_count: r.get("image_count"),
}).collect()
}
// ── 写入 ──
pub async fn create_album(&self, album: CreateAlbum) -> Result<Uuid, sqlx::Error> {

517
src/engine/pipeline.rs
View File

@@ -17,8 +17,10 @@ use crate::engine::title_parser;
use crate::engine::tag_extractor;
use crate::engine::person_merger;
use crate::engine::tag_dict;
use crate::search_db::SearchDb;
use std::collections::HashSet;
use rayon::prelude::*;
use sqlx;
// ════════════════════════════════════════════════
// 脏集 — 追踪需要重新解析的相册 ID
@@ -44,62 +46,6 @@ impl Default for DirtySet {
fn default() -> Self { Self::new() }
}
// ════════════════════════════════════════════════
// Phase 0：清洗已有标签中的头尾标点
// ════════════════════════════════════════════════
pub async fn phase_clean_tags(db: &Database) -> usize {
let albums = db.get_all_albums_full().await;
let mut cleaned_count = 0usize;
let mut removed_tags = HashSet::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
let mut changed = false;
let mut new_tags = Vec::new();
for tag in old_tags {
let cleaned = match tag_extractor::clean_tag(&tag) {
Some(t) => t,
None => { removed_tags.insert(tag); changed = true; continue; }
};
if cleaned != tag { removed_tags.insert(tag); changed = true; }
new_tags.push(cleaned);
}
if !changed { continue; }
cleaned_count += 1;
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; Some([cleaned, pair[1].clone()]) })
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
if cleaned_count > 0 {
println!("[管线·Phase0] 标签清洗完成：清洗 {} 个相册，移除标签: {:?}",
cleaned_count, removed_tags.iter().collect::<Vec<_>>());
} else {
println!("[管线·Phase0] 标签清洗完成：无需要清洗的标签");
}
cleaned_count
}
// ════════════════════════════════════════════════
// Phase 0B：移除过短的标签（少于2个汉字/字母）
// ════════════════════════════════════════════════
/// 判断一个标签是否"过短"——有效字符（汉字+字母）少于 2 个
fn is_short_tag(tag: &str) -> bool {
let mut valid_count = 0usize;
@@ -114,269 +60,6 @@ fn is_short_tag(tag: &str) -> bool {
valid_count < 2
}
/// Phase 0B：扫描所有标签，移除过短的标签（有效字符 < 2），
/// 受影响的相册加入脏集等待重新解析。
pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut removed_total = 0usize;
let mut removed_samples: HashSet<String> = HashSet::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
let mut changed = false;
let mut new_tags = Vec::new();
for tag in old_tags {
if is_short_tag(&tag) {
removed_samples.insert(tag);
changed = true;
removed_total += 1;
} else {
new_tags.push(tag);
}
}
if !changed { continue; }
dirty.add(&album.id);
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter(|pair| !is_short_tag(&pair[0]))
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
if removed_total > 0 {
let samples: Vec<_> = removed_samples.iter().take(10).collect();
println!("[管线·Phase0B] 短标签清理完成：移除 {} 个短标签，影响 {} 个相册，示例: {:?}",
removed_total, dirty.len(), samples);
} else {
println!("[管线·Phase0B] 短标签清理完成：无短标签");
}
(dirty, removed_total)
}
// ════════════════════════════════════════════════
// Phase A1：检测标签中的人名并直接提取到 person 字段
// ════════════════════════════════════════════════
/// Phase A1：检测标签中可能的人名并直接提取到 person_name
///
/// 核心变化：不再只标记脏集等待 Phase 1，而是直接从 tags 中提取人名写入 person 字段。
/// 因为 tags 中的人名比从 title 重解析更直接、更准确。
///
/// 两层检测 + 最终写入：
/// 1. 逐个标签 — 标签是已知人物或像人名 → 收集为候选人名（中文加权）
/// 2. jieba 合并（空格分隔）— 发现跨标签拆分的人名（["杨晨","晨Yome"]）
/// 3. 最佳候选人名 → 写入 person_name，从标签中移除关联标签
///
/// 返回 (脏集, 从标签中移除的人名标签数)
pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut person_like_removed = 0usize;
let known_persons: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
// 快速判断是否需要处理
let has_known_tag = old_tags.iter().any(|t| known_persons.contains(&t.to_lowercase()));
let has_like_tag = !has_person && old_tags.iter().any(|t|
title_parser::is_likely_person_name(t) && !title_parser::is_excluded_name(t));
if has_person && !has_known_tag && !has_like_tag { continue; }
// ── 第一层：逐个标签收集候选人名 ──
let mut known_person_found: Option<String> = None;
let mut person_like_tags: Vec<String> = Vec::new();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
if known_persons.contains(&tag_lower) {
known_person_found = Some(tag.clone());
continue;
}
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
person_like_tags.push(tag.clone());
continue;
}
}
// 从 person_like_tags 中选最优（中文加权）
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
if let Some(cn) = title_parser::extract_cn(t) {
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
} else { t.len() }
});
// ── 第二层：空格连接所有标签，用 jieba 提取人名 ──
let combined = old_tags.join(" ");
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
// ── 确定最终人名 ──
let final_person = known_person_found.or_else(|| {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
// jieba 结果更完整 → 用 jieba
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),
(None, None) => None,
}
});
if let Some(ref person_name) = final_person {
// ── 从标签中移除所有人名相关标签 ──
let mut changed = false;
let mut clean_tags: Vec<String> = Vec::new();
let person_lower = person_name.to_lowercase();
let (primary_cleaned, _) = title_parser::split_person_name_with_aliases(person_name);
let primary_lower = primary_cleaned.to_lowercase();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
let should_remove =
tag_lower == person_lower // 完全匹配
|| tag_lower == primary_lower // 中文核心匹配
|| (tag.len() >= 2 && tag.chars().all(|c| c.is_ascii_alphabetic())
&& !tag_lower.starts_with("cos")
&& person_lower.contains(&tag_lower)) // 英文后缀
|| (tag_lower.len() >= 2 && person_lower.contains(&tag_lower)); // 人名包含
if should_remove {
person_like_removed += 1;
changed = true;
continue;
}
clean_tags.push(tag.clone());
}
if changed || !has_person {
// ── 只清理标签中的人名，不写入 person 字段 ──
// 人名提取统一由 Phase 1 从标题解析完成，保证一个相册只有一个 pid
let tags_json_new = serde_json::to_string(&clean_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter(|pair| !known_persons.contains(&pair[0].to_lowercase()))
.collect();
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &words_json_new).await;
dirty.add(&album.id);
}
}
}
if person_like_removed > 0 {
println!("[管线·PhaseA1] 从标签中检测并提取人名：移除 {} 个人名标签，写入 {} 个相册的 person 字段",
person_like_removed, dirty.len());
} else {
println!("[管线·PhaseA1] 检测标签中的人名：无异常");
}
(dirty, person_like_removed)
}
// ════════════════════════════════════════════════
// Phase A2：清理 persons 表中的非人物
// ════════════════════════════════════════════════
pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut deleted = 0usize;
for (pid, name, _albums, _images) in &persons {
if title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name) {
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
println!("[管线·PhaseA2] 删除非人物：{} (id={})，释放 {} 个相册", name, pid, album_ids.len());
db.update_album_person_by_person_id(*pid).await;
db.delete_person_completely(*pid).await;
deleted += 1;
}
}
if deleted > 0 {
println!("[管线·PhaseA2] 清理完成：删除 {} 个非人物，标记 {} 个脏相册", deleted, dirty.len());
} else {
println!("[管线·PhaseA2] 清理完成：无非人物需要删除");
}
(dirty, deleted)
}
// ════════════════════════════════════════════════
// Phase A2B：清理 persons 表中与停用词重叠的人物
// ════════════════════════════════════════════════
/// Phase A2B：检查 persons 表中的人名是否在停用词词典中。
///
/// 当用户在词典中添加了新的停用词，而这些词恰好已被作为人物创建时，
/// 需要删除该人物并释放对应的相册，让它们在后续 Phase 1 中重新解析。
/// 这样可以防止停用词（如 "photo"、"model" 等通用词）被错误识别为人物。
pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut deleted = 0usize;
for (pid, name, _albums, _images) in &persons {
// 使用 tag_dict 的 is_excluded_person 进行判断（含包含匹配）
let is_excluded = tag_dict::get_tag_dict().is_excluded_person(name);
// 也检查中文核心（如 "秀人" 是排除词，"秀人网" 也会包含）
let cn_check = title_parser::extract_cn(name)
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
.unwrap_or(false);
if is_excluded || cn_check {
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
println!("[管线·PhaseA2B] 排除人名命中：{} (id={})，释放 {} 个相册", name, pid, album_ids.len());
// 清空相册的 person 字段
db.update_album_person_by_person_id(*pid).await;
// 彻底删除该人物
db.delete_person_completely(*pid).await;
deleted += 1;
}
}
if deleted > 0 {
println!("[管线·PhaseA2B] 清理完成：移除 {} 个被排除的人物，标记 {} 个脏相册", deleted, dirty.len());
} else {
println!("[管线·PhaseA2B] 清理完成：无被排除的人物");
}
(dirty, deleted)
}
// ════════════════════════════════════════════════
// Phase A3：收集新相册 + 半成品
// ════════════════════════════════════════════════
@@ -412,7 +95,8 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
return (0, 0);
}
let albums = db.get_all_albums_full().await;
let dirty_ids: Vec<String> = dirty_set.clone().into_inner().into_iter().collect();
let albums = db.get_albums_by_ids(&dirty_ids).await;
// ── Phase 1a：并行解析（CPU bound，rayon） ──
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
@@ -854,3 +538,196 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
aliases_moved, tags_removed_person,
}
}
// ════════════════════════════════════════════════
// Phase S：同步管线结果到 search.db
// ════════════════════════════════════════════════
/// 将管线解析结果从主库同步到 search.db
///
/// 读取主库中所有相册的解析结果（tags json, person_name_cleaned 等），
/// 写入 search.db 的结构化表（albums, tags, persons, album_tags, album_persons）。
///
/// 在管线执行完成后调用，确保搜索数据与主库一致。
pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> usize {
let albums = db.get_all_albums_full().await;
let mut synced = 0usize;
let mut tag_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
let mut person_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
// ── 1. 写入 albums 表 ──
let cover_id: Option<&str> = None;
// 从主库获取 cover_image_id：因 get_all_albums_full 不返回 cover_image_id，
// 这里用默认值，主库已有的 cover 在下一次增量同步时会补全
let img_count = album.image_count.unwrap_or(0);
if let Err(e) = search_db.upsert_album(&album.id, album.title.as_deref().unwrap_or(""),
cover_id, img_count, None).await {
eprintln!("[Sync->search.db] upsert 相册 {} 失败: {}", album.id, e);
continue;
}
// ── 2. 写入 tags → album_tags ──
let tags_json = album.tags.as_deref().unwrap_or("[]");
if tags_json != "[]" {
if let Ok(tags) = serde_json::from_str::<Vec<(String, String)>>(tags_json) {
search_db.clear_album_tags(&album.id).await.ok();
for (tag_name, category) in &tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
}
}
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
}
}
}
// ── 3. 写入 persons → album_persons ──
search_db.clear_album_persons(&album.id).await.ok();
if let Some(ref person_name) = album.person_name_cleaned {
if !person_name.is_empty() {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
}
}
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同时同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
}
}
synced += 1;
}
// ── 4. 更新 person counts ──
for (person_name, person_id) in &person_cache {
let albums_count: i64 = match sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(*person_id).fetch_one(&db.pool).await {
Ok(c) => c,
Err(_) => continue,
};
search_db.update_person_counts(*person_id, albums_count, 0).await.ok();
}
println!("[Sync->search.db] 同步完成：{} 个相册，{} 个人物，{} 个标签",
synced, person_cache.len(), tag_cache.len());
synced
}
/// 增量同步脏集相册到 search.db
///
/// 只处理脏集中的相册 ID，从主库按需查询后写入 search.db。
/// 逻辑与全量同步一致，但只处理脏集里的相册，避免全表扫描。
pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &DirtySet) -> usize {
if dirty.is_empty() {
println!("[Sync->search.db·增量] 脏集为空，跳过");
return 0;
}
let dirty_ids: Vec<String> = dirty.clone().into_inner().into_iter().collect();
let albums = db.get_albums_by_ids(&dirty_ids).await;
let mut synced = 0usize;
let mut tag_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
let mut person_cache: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let img_count = album.image_count.unwrap_or(0);
// ── 1. upsert albums 表 ──
if let Err(e) = search_db.upsert_album(&album.id, title, None, img_count, None).await {
eprintln!("[Sync->search.db·增量] upsert 相册 {} 失败: {}", album.id, e);
continue;
}
// ── 2. 写入 tags → album_tags ──
let tags_json = album.tags.as_deref().unwrap_or("[]");
search_db.clear_album_tags(&album.id).await.ok();
if tags_json != "[]" {
if let Ok(tags) = serde_json::from_str::<Vec<(String, String)>>(tags_json) {
for (tag_name, category) in &tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
}
}
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
}
}
}
// ── 3. 写入 persons → album_persons ──
search_db.clear_album_persons(&album.id).await.ok();
if let Some(ref person_name) = album.person_name_cleaned {
if !person_name.is_empty() {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
}
}
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同时同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
}
}
synced += 1;
}
// ── 4. 更新 person counts ──
for (person_name, person_id) in &person_cache {
let albums_count: i64 = match sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(*person_id).fetch_one(&db.pool).await {
Ok(c) => c,
Err(_) => continue,
};
search_db.update_person_counts(*person_id, albums_count, 0).await.ok();
}
println!("[Sync->search.db·增量] 同步完成：{} 个相册，{} 个人物，{} 个标签",
synced, person_cache.len(), tag_cache.len());
synced
}

61
src/engine/recommend.rs
View File

@@ -5,7 +5,7 @@
//! 2. 人物推荐：基于名称匹配 + 相册数排序
//! 3. 关联推荐：基于标签共现关系 + 人物共现关系
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use crate::engine::tag_dict;
use sqlx::Row;
@@ -16,14 +16,14 @@ pub struct RecommendEngine;
impl RecommendEngine {
/// 标签自动补全推荐
pub async fn recommend_tags(
db: &Database,
search_db: &SearchDb,
query: &str,
limit: usize,
) -> RecommendResponse {
let limit = limit.min(20);
let q = query.trim().to_lowercase();
let tags = db.get_popular_tags(200).await;
let tags = search_db.get_popular_tags(200).await;
let mut items: Vec<RecommendItem> = Vec::new();
for t in &tags {
@@ -62,14 +62,14 @@ impl RecommendEngine {
/// 人物推荐
pub async fn recommend_persons(
db: &Database,
search_db: &SearchDb,
query: &str,
limit: usize,
) -> RecommendResponse {
let limit = limit.min(20);
let q = query.trim().to_lowercase();
let persons = db.get_all_persons_with_aliases().await;
let persons = search_db.get_all_persons_with_aliases().await;
let mut items: Vec<RecommendItem> = Vec::new();
for p in &persons {
@@ -112,14 +112,14 @@ impl RecommendEngine {
/// 关联推荐（共现标签 + 共现人物）
pub async fn recommend_related(
db: &Database,
search_db: &SearchDb,
context: &str, // 标签或人名
limit: usize,
) -> TagRecommendation {
let limit = limit.min(15);
// 获取共现标签
let co_tags = db.get_tag_co_occurrences(context, limit as i64).await;
let co_tags = search_db.get_tag_co_occurrences(context, limit as i64).await;
let related_tags: Vec<RecommendItem> = co_tags.iter().map(|c| {
RecommendItem {
value: c.co_tag.clone(),
@@ -131,19 +131,22 @@ impl RecommendEngine {
}).collect();
// 获取相关人物（通过标签）
let albums = db.search_albums_by_tag(context).await;
let albums = search_db.search_albums_by_tag(context).await;
let mut person_scores: std::collections::HashMap<String, (i64, f64)> = std::collections::HashMap::new();
let total = albums.len() as f64;
for album in &albums {
let meta = db.get_all_albums_full().await;
for m in &meta {
if m.id == album.id.to_string() {
if let Some(ref name) = m.person_name_cleaned {
let entry = person_scores.entry(name.clone()).or_insert((0, 0.0));
entry.0 += 1;
entry.1 = entry.0 as f64 / total;
}
}
let aid = album.id.to_string();
// 从 search.db 查该相册的人物
let person_rows = sqlx::query(
r#"SELECT p.name FROM persons p
JOIN album_persons ap ON p.id = ap.person_id
WHERE ap.album_id = ?"#
).bind(&aid).fetch_all(&search_db.pool).await.unwrap_or_default();
for row in &person_rows {
let name: String = row.get("name");
let entry = person_scores.entry(name).or_insert((0, 0.0));
entry.0 += 1;
entry.1 = entry.0 as f64 / total;
}
}
@@ -167,9 +170,9 @@ impl RecommendEngine {
}
/// 热门人物
pub async fn popular_persons(db: &Database, limit: usize) -> Vec<PopularPerson> {
pub async fn popular_persons(search_db: &SearchDb, limit: usize) -> Vec<PopularPerson> {
let dict = tag_dict::get_tag_dict();
let persons = db.get_all_persons_with_aliases().await;
let persons = search_db.get_all_persons_with_aliases().await;
// 收集所有标签分类词（非人名候选）
let all_tag_words: std::collections::HashSet<String> = dict.tag_to_category.keys()
@@ -204,23 +207,11 @@ impl RecommendEngine {
}
}).collect();
// 为热门人物获取 top tags
// （简化：从 person_albums 查标签）
// 为热门人物获取 top tags（从 search.db）
for rp in &mut results {
if let Some(person) = db.get_person_by_name(&rp.name).await {
let albums = db.get_person_albums(person.id).await;
let mut tag_counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
let aid = album.id.to_string();
let rows = sqlx::query("SELECT tag FROM album_tags WHERE album_id = ?")
.bind(&aid).fetch_all(&db.pool).await.unwrap_or_default();
for row in &rows {
*tag_counts.entry(row.get::<String, _>("tag")).or_insert(0) += 1;
}
}
let mut tags: Vec<(String, i64)> = tag_counts.into_iter().collect();
tags.sort_by(|a, b| b.1.cmp(&a.1));
rp.top_tags = tags.into_iter().take(5).map(|(t, _)| t).collect();
if let Some(person) = search_db.get_person_by_name(&rp.name).await {
let top_tags = search_db.get_person_top_tags(person.id, 5).await;
rp.top_tags = top_tags.into_iter().map(|(t, _)| t).collect();
}
}
results

96
src/engine/tag_dict.rs
View File

@@ -531,3 +531,99 @@ pub fn get_tag_dict_stats() -> String {
dict.included_person_names.len(),
)
}
// ════════════════════════════════════════════════
// 从 search.db 的 dict 表加载
// ════════════════════════════════════════════════
use crate::search_db::DictEntry;
impl TagDict {
/// 从 search.db 的 dict 表条目加载词典数据
///
/// dict 表条目类型（通过 type 字段区分）：
/// - `tag` — word 是标签，category 是分类名 → 加入 keymap
/// - `person` — word 是人名 → 加入 included_person_names
/// - `stop_word` — word 是停用词
/// - `person_prefix` — word 是人物前缀
/// - `excluded_person` — word 是排除人名
///
/// 派生规则：
/// - 所有 tag 类型的 word 自动加入 excluded_person_names
/// - categories 结构基于 metadata（内建）和 keymap（从 tag 条目构建）
pub fn load_from_dict_entries(entries: &[DictEntry]) -> Self {
let mut dict = TagDict::default();
for entry in entries {
match entry.type_.as_str() {
"tag" => {
let cat = entry.category.as_deref().unwrap_or("other");
dict.keymap.insert(entry.word.clone(), cat.to_string());
dict.tag_to_category.insert(entry.word.to_lowercase(), cat.to_string());
dict.excluded_person_names.insert(entry.word.to_lowercase());
}
"person" => {
dict.included_person_names.insert(entry.word.to_lowercase());
}
"stop_word" => {
dict.stop_words.insert(entry.word.to_lowercase());
}
"person_prefix" => {
dict.person_prefixes.push(entry.word.clone());
}
"excluded_person" => {
dict.excluded_person_names.insert(entry.word.to_lowercase());
}
_ => {}
}
}
// ── 构建 categories（基于 metadata + keymap） ──
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &dict.keymap {
words_by_cat.entry(cat_name.clone()).or_default().push(word.clone());
}
for (name, meta) in &dict.metadata {
let words = words_by_cat.remove(name).unwrap_or_default();
dict.categories.insert(name.clone(), CategoryInfo {
name: name.clone(),
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
for (cat_name, words) in words_by_cat {
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label: cat_name.clone(),
color_bg: "#f1f5f9".to_string(),
color_fg: "#64748b".to_string(),
words,
});
}
println!("[词典] 从 search.db dict 表加载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len());
dict
}
}
/// 从 search.db 的 dict 表重新加载标签词典
pub async fn reload_from_search_db(search_db: &crate::search_db::SearchDb) -> String {
let entries = match search_db.load_dict_all().await {
Ok(e) => e,
Err(e) => return format!("从 search.db 加载 dict 失败: {}", e),
};
let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染");
*dict = TagDict::load_from_dict_entries(&entries);
format!(
"词典已从 search.db 重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)
}

121
src/engine/title_parser.rs
View File

@@ -452,6 +452,127 @@ pub fn normalize_tag(tag: &str) -> String {
tag.trim().to_lowercase()
}
// ── dict 优先匹配 ──
/// 使用 dict 表优先匹配 + title_parser 回退的混合解析
///
/// 流程：
/// 1. dict 最长匹配提取 person 和 tags（带 merged_to 解析）
/// 2. 从标题移除已匹配的 dict 词
/// 3. 剩余文本走 title_parser 解析（tag_dict 回退）
/// 4. 合并：dict 结果优先，parser 补充
pub fn parse_album_title_with_dict(
album_id: &str,
title: &str,
dict_entries: &[crate::search_db::DictEntry],
) -> crate::models::parse::ParseResult {
let title = title.trim();
if title.is_empty() {
return crate::models::parse::ParseResult::new(album_id.to_string());
}
// Phase 1: dict 优先匹配
let (dict_person, dict_tags, remaining) = extract_with_dict(title, dict_entries);
// Phase 2: title_parser 回退（解析剩余文本）
let parse_result = if remaining.is_empty() {
crate::models::parse::ParseResult::new(album_id.to_string())
} else {
parse_album_title(album_id, &remaining)
};
// Phase 3: 合并结果 — dict 优先，parser 补充
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
// person: dict 命中 > parser 命中
result.person_name = dict_person.or(parse_result.person_name);
// tags: dict 标签优先，parser 标签补充去重
let mut seen = HashSet::new();
for (t, _) in &dict_tags {
seen.insert(t.to_lowercase());
result.tags.push((t.clone(), "other".to_string()));
}
for (t, c) in &parse_result.tags {
if seen.insert(t.to_lowercase()) {
result.tags.push((t.clone(), c.clone()));
}
}
result
}
/// dict 条目按词长降序匹配标题，返回 (person, tags, 剩余文本)
fn extract_with_dict(
title: &str,
dict_entries: &[crate::search_db::DictEntry],
) -> (Option<String>, Vec<(String, String)>, String) {
// 按词长降序排列
let mut sorted: Vec<&crate::search_db::DictEntry> = dict_entries.iter().collect();
sorted.sort_by(|a, b| b.word.len().cmp(&a.word.len()));
let title_lower = title.to_lowercase();
let mut matched_ranges: Vec<(usize, usize)> = Vec::new();
let mut person: Option<String> = None;
let mut tags: Vec<(String, String)> = Vec::new();
let mut seen_tag_words = HashSet::new();
for entry in &sorted {
let word_lower = entry.word.to_lowercase();
// 在当前标题中查找该词（忽略大小写）
let mut search_start = 0;
while let Some(pos) = title_lower[search_start..].find(&word_lower) {
let abs_start = search_start + pos;
let abs_end = abs_start + entry.word.len();
// 检查是否已被更长的 dict 词覆盖
if matched_ranges.iter().any(|&(s, e)| s <= abs_start && abs_end <= e) {
break;
}
matched_ranges.push((abs_start, abs_end));
match entry.type_.as_str() {
"person" if person.is_none() => {
person = Some(entry.merged_to.as_ref().unwrap_or(&entry.word).clone());
}
"tag" => {
if seen_tag_words.insert(entry.word.clone()) {
tags.push((entry.word.clone(), entry.category.clone().unwrap_or_default()));
}
}
_ => {}
}
break; // 每个词只匹配一次
}
}
// 构建剩余文本：去掉已匹配的部分
if matched_ranges.is_empty() {
return (person, tags, title.to_string());
}
matched_ranges.sort_by_key(|&(s, _)| s);
let mut remaining = String::new();
let mut last = 0;
for &(s, e) in &matched_ranges {
if s > last {
remaining.push_str(&title[last..s]);
}
remaining.push(' ');
last = e;
}
if last < title.len() {
remaining.push_str(&title[last..]);
}
// 合并连续空格
let remaining = remaining.split_whitespace().collect::<Vec<_>>().join(" ");
(person, tags, remaining)
}
// ── 兼容函数（供 pipeline / person_merger 调用） ──
/// 从标题中提取人物名（兼容函数）

136
src/main.rs
View File

@@ -7,10 +7,13 @@ mod routes;
mod services;
mod image_processor;
mod queue;
mod search_db;
use crate::cache::AlbumsCache;
use crate::search_db::SearchDb;
use actix_files as afs;
use actix_web::{App, HttpResponse, HttpServer, middleware, web};
use std::env;
use std::fs;
use std::sync::Arc;
use tokio::time::{Duration, interval};
@@ -20,6 +23,14 @@ use tracing::{error, info};
async fn main() -> std::io::Result<()> {
tracing_subscriber::fmt::init();
let args: Vec<String> = env::args().collect();
let export_to_search = args.iter().any(|a| a == "--export-search");
let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db="))
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db"))
.unwrap_or("search.db");
let clear_search = args.iter().any(|a| a == "--clear-search");
let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint");
// 加载配置
let config = match config::Config::load() {
Ok(cfg) => cfg,
@@ -48,45 +59,103 @@ async fn main() -> std::io::Result<()> {
}
};
if export_to_search {
info!("[搜索导出] 开始将元数据导出到 search.db...");
match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await {
Ok(result) => {
info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}",
result.albums_processed, result.tags_created, result.persons_created,
result.aliases_created, result.album_tags_created + result.album_persons_created);
std::process::exit(0);
}
Err(e) => {
error!("[搜索导出] 导出失败: {}", e);
std::process::exit(1);
}
}
}
if wal_checkpoint {
info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成");
info!("[WAL合并] 正在合并 search.db WAL...");
let search_db = SearchDb::new("data/search.db").await
.expect("search.db 连接失败");
search_db.wal_checkpoint().await;
info!("[WAL合并] 两个数据库均已合并完成");
std::process::exit(0);
}
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// ── 初始化 search.db（运行时搜索专用）──
let search_db_path = "data/search.db";
let search_db = Arc::new(
SearchDb::new(search_db_path).await
.expect("search.db 连接失败")
);
info!("search.db 连接成功");
search_db.wal_checkpoint().await;
// ── 从 search.db 的 dict 表加载标签词典 ──
match engine::tag_dict::reload_from_search_db(&search_db).await {
msg => info!("{}", msg),
}
// ── 内存缓存（必须在管线任务之前创建）──
let albums_cache = Arc::new(tokio::sync::RwLock::new(AlbumsCache::new()));
let persons_cache = Arc::new(tokio::sync::RwLock::new(cache::PersonCache::default()));
let tags_cache = Arc::new(tokio::sync::RwLock::new(cache::TagCache::default()));
// ── 启动管线（单次）：导入 → 解析 → 合并 → 缓存刷新 ──
// ── 启动管线（单次）：导入 → 增量解析 → 条件全量 sync → 缓存刷新 ──
// 不做全量 reparse/merge，只处理新发现的相册
let boot_config = config.clone();
let boot_db = Arc::clone(&db);
let boot_search_db = Arc::clone(&search_db);
let boot_albums_cache = Arc::clone(&albums_cache);
let boot_persons_cache = Arc::clone(&persons_cache);
let boot_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
// ① 扫描导入新相册
info!("[启动管线] ① 扫描导入新相册...");
info!("[启动] ① 扫描导入新相册...");
match services::import::import_existing_data(&boot_config, Arc::clone(&boot_db), None).await {
Ok(_) => info!("[启动管线] 导入完成"),
Err(e) => error!("[启动管线] 导入失败: {}", e),
Ok(_) => info!("[启动] 导入完成"),
Err(e) => error!("[启动] 导入失败: {}", e),
}
// ② 全量解析 + 人物合并
info!("[启动管线] ② 全量解析 + 人物合并...");
let result = engine::pipeline::run_pipeline(&boot_db).await;
info!("[启动管线] 完成：清洗标点{}，移除短标签{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,
result.persons_merged, result.aliases_moved, result.tags_removed_person);
// ② 查找新相册 / 半成品
info!("[启动] ② 查找新相册...");
let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await;
let new_count = dirty.len();
let mut full_sync = false;
// ③ 刷新所有内存缓存
info!("[启动管线] ③ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动管线] 启动管线执行完毕");
if new_count > 0 {
info!("[启动] 发现 {} 个新相册，增量同步到 search.db...", new_count);
let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await;
info!("[启动] 增量同步完成：{} 个相册（解析在定时管线中执行）", synced);
} else {
info!("[启动] 没有新相册");
// 检查 search.db 是否为空（首次启动）
let album_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums")
.fetch_one(&boot_search_db.pool).await.unwrap_or(0);
if album_count == 0 {
info!("[启动] search.db 为空，执行全量同步...");
let synced = engine::pipeline::sync_albums_to_search_db(&boot_db, &boot_search_db).await;
info!("[启动] 全量同步完成：{} 个相册", synced);
full_sync = true;
}
}
// ③ 刷新所有内存缓存（从 search.db 读取）
info!("[启动] ③ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动] 启动管线执行完毕（新相册={}, 全量同步={}）", new_count, full_sync);
});
// ── 定时管线（每 30 分钟）：解析 → 合并 → 缓存刷新 ──
// ── 定时管线（每 30 分钟）：导入 → 增量解析 → 缓存刷新 ──
// 不做全量 reparse/merge
let timer_cache = Arc::clone(&albums_cache);
let timer_db = Arc::clone(&db);
let timer_search_db = Arc::clone(&search_db);
let timer_persons_cache = Arc::clone(&persons_cache);
let timer_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
@@ -94,18 +163,28 @@ async fn main() -> std::io::Result<()> {
loop {
interval.tick().await;
// ① 全量解析 + 人物合并（不重复 import）
info!("[定时管线] ① 全量解析 + 人物合并...");
let result = engine::pipeline::run_pipeline(&timer_db).await;
info!("[定时管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}",
result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,
result.persons_merged, result.aliases_moved, result.tags_removed_person);
// ① 扫描导入新相册
info!("[定时] ① 扫描导入新相册...");
match services::import::import_existing_data(&timer_db.config, Arc::clone(&timer_db), None).await {
Ok(_) => {},
Err(e) => error!("[定时] 导入失败: {}", e),
}
// ② 刷新所有内存缓存
info!("[定时管线] ② 刷新所有内存缓存...");
cache::refresh_all_caches(&timer_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
// ② 查找新相册
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
if new_albums.is_empty() {
continue; // 无新增，跳过
}
info!("[定时] 发现 {} 个新相册，增量解析...", new_albums.len());
engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await;
info!("[定时] 增量同步到 search.db...");
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
// ③ 刷新缓存
info!("[定时] 刷新缓存...");
cache::refresh_all_caches(&timer_search_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
}
});
@@ -158,6 +237,7 @@ async fn main() -> std::io::Result<()> {
.wrap(middleware::Compress::default())
.wrap(middleware::Logger::default())
.app_data(web::Data::new(Arc::clone(&db)))
.app_data(web::Data::new(Arc::clone(&search_db)))
.app_data(web::Data::new(Arc::clone(&image_processor)))
.app_data(web::Data::new(Arc::clone(&albums_cache)))
.app_data(web::Data::new(Arc::clone(&persons_cache)))

5
src/routes/album.rs
View File

@@ -1,5 +1,6 @@
use crate::cache::AlbumsCache;
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
@@ -25,7 +26,7 @@ pub async fn get_albums(cache: web::Data<Arc<RwLock<AlbumsCache>>>) -> impl Resp
/// 规则：有任意过滤条件时 → 走 DB 懒加载（只查符合条件的相册）
/// 无过滤条件时 → 走缓存全量返回
pub async fn search_albums(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<AlbumsCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
@@ -50,7 +51,7 @@ pub async fn search_albums(
let person_arg = if person.is_empty() { None } else { Some(person) };
let q_arg = if q.is_empty() { None } else { Some(q) };
match db.search_albums_with_filters(tag_arg, person_arg, q_arg, limit, offset).await {
match search_db.search_albums(tag_arg, person_arg, q_arg, limit, offset).await {
Ok(results) => HttpResponse::Ok().json(results),
Err(e) => HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{}", e)})),
}

9
src/routes/dict.rs
View File

@@ -1,8 +1,11 @@
//! 标签词典管理 API — 查看/热重载
use actix_web::{HttpResponse, Responder};
use std::sync::Arc;
use actix_web::{HttpResponse, Responder, web};
use crate::engine::tag_dict;
use crate::search_db::SearchDb;
/// GET /api/dict/stats — 查看词典状态
pub async fn dict_stats() -> impl Responder {
@@ -30,8 +33,8 @@ pub async fn dict_stats() -> impl Responder {
}
/// POST /api/dict/reload — 热重载词典
pub async fn dict_reload() -> impl Responder {
let msg = tag_dict::reload_tag_dict();
pub async fn dict_reload(search_db: web::Data<Arc<SearchDb>>) -> impl Responder {
let msg = tag_dict::reload_from_search_db(search_db.get_ref()).await;
println!("[API] 词典热重载: {}", msg);
HttpResponse::Ok().json(serde_json::json!({
"success": true,

16
src/routes/person.rs
View File

@@ -1,5 +1,5 @@
use crate::cache::PersonCache;
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
@@ -36,9 +36,9 @@ pub async fn search_people(
})
}
/// GET /api/people/{name} — 从缓存查找人物详情（需要 DB 获取相册列表）
/// GET /api/people/{name} — 从缓存查找人物详情（相册列表从 search.db）
pub async fn get_person_detail(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<PersonCache>>>,
path: web::Path<String>,
) -> impl Responder {
@@ -49,8 +49,8 @@ pub async fn get_person_detail(
let p = found.into_iter().find(|p| p.name.to_lowercase() == name.to_lowercase());
match p {
Some(p) => {
// 相册列表仍需查 DB
let albums = db.get_person_albums(p.id).await;
// 相册列表从 search.db 获取
let albums = search_db.get_person_albums(p.id).await;
HttpResponse::Ok().json(PersonWithAlbums {
album_count: albums.len(),
person: Person {
@@ -70,8 +70,8 @@ pub async fn get_person_detail(
}
}
/// GET /api/people/{id}/aliases — 查 DB（别名表记录数可能较大，不适合缓存）
pub async fn get_person_aliases(db: web::Data<Arc<Database>>, path: web::Path<i64>) -> impl Responder {
let aliases = db.get_person_aliases(path.into_inner()).await;
/// GET /api/people/{id}/aliases — 从 search.db 查别名
pub async fn get_person_aliases(search_db: web::Data<Arc<SearchDb>>, path: web::Path<i64>) -> impl Responder {
let aliases = search_db.get_person_aliases(path.into_inner()).await;
HttpResponse::Ok().json(aliases)
}

25
src/routes/pipeline.rs
View File

@@ -1,29 +1,38 @@
use crate::db::Database;
use crate::engine::pipeline;
use crate::engine;
use crate::search_db::SearchDb;
use actix_web::{HttpResponse, Responder, web};
use std::sync::Arc;
/// POST /api/pipeline/run — 执行完整管线：清洗→检测→清理→解析→合并→校验
pub async fn pipeline_run(db: web::Data<Arc<Database>>) -> impl Responder {
let result = pipeline::run_pipeline(db.get_ref()).await;
pub async fn pipeline_run(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
let result = engine::pipeline::run_pipeline(db.get_ref()).await;
// 同步到 search.db
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"tags_cleaned": result.tags_cleaned,
"short_tags_removed": result.short_tags_removed,
"person_like_tags_removed": result.person_like_tags_removed,
"dirty_albums_detected": result.dirty_albums_detected,
"non_persons_deleted": result.non_persons_deleted,
"stop_word_persons_removed": result.stop_word_persons_removed,
"new_albums_found": result.new_albums_found,
"persons_updated": result.persons_updated,
"tags_updated": result.tags_updated,
"persons_merged": result.persons_merged,
"aliases_moved": result.aliases_moved,
"tags_removed_person": result.tags_removed_person,
"search_db_synced": synced,
"message": format!(
"管线执行完毕：清洗标点{}相册，检测人名标签{}，脏集{}相册，删除非人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}",
result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.new_albums_found,
"管线执行完毕：清洗标点{}相册，移除短标签{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}，同步search.db{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated,
result.persons_merged, result.aliases_moved, result.tags_removed_person
result.persons_merged, result.aliases_moved, result.tags_removed_person, synced
)
}))
}

20
src/routes/recommend.rs
View File

@@ -1,36 +1,36 @@
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::engine::RecommendEngine;
#[allow(unused)] use crate::models::*;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
use std::sync::Arc;
/// 标签推荐（自动补全）
pub async fn recommend_tags(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
pub async fn recommend_tags(search_db: web::Data<Arc<SearchDb>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10);
HttpResponse::Ok().json(RecommendEngine::recommend_tags(&db, q, limit).await)
HttpResponse::Ok().json(RecommendEngine::recommend_tags(&search_db, q, limit).await)
}
/// 人物推荐（自动补全）
pub async fn recommend_persons(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
pub async fn recommend_persons(search_db: web::Data<Arc<SearchDb>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10);
HttpResponse::Ok().json(RecommendEngine::recommend_persons(&db, q, limit).await)
HttpResponse::Ok().json(RecommendEngine::recommend_persons(&search_db, q, limit).await)
}
/// 关联推荐（标签共现 + 共现人物）
pub async fn recommend_related(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
pub async fn recommend_related(search_db: web::Data<Arc<SearchDb>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
let context = match query.get("context") {
Some(c) if !c.is_empty() => c,
_ => return HttpResponse::BadRequest().json(serde_json::json!({"error": "请提供context参数"})),
};
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(15);
HttpResponse::Ok().json(RecommendEngine::recommend_related(&db, context, limit).await)
HttpResponse::Ok().json(RecommendEngine::recommend_related(&search_db, context, limit).await)
}
/// 热门人物
pub async fn popular_persons(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
pub async fn popular_persons(search_db: web::Data<Arc<SearchDb>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(20).min(50);
HttpResponse::Ok().json(RecommendEngine::popular_persons(&db, limit).await)
HttpResponse::Ok().json(RecommendEngine::popular_persons(&search_db, limit).await)
}

310
src/search_db/mod.rs Normal file
View File

@@ -0,0 +1,310 @@
mod search;
pub use search::*;
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row};
#[derive(Debug, Clone)]
pub struct DictEntry {
pub word: String,
pub type_: String,
pub category: Option<String>,
pub freq: Option<i64>,
pub merged_to: Option<String>,
}
pub struct SearchDb {
pub pool: SqlitePool,
}
impl SearchDb {
pub async fn new(db_path: &str) -> Result<Self, sqlx::Error> {
if let Some(parent) = std::path::Path::new(db_path).parent() {
if !parent.exists() {
std::fs::create_dir_all(parent)?;
}
}
if !std::path::Path::new(db_path).exists() {
std::fs::File::create(db_path)?;
}
let url = format!("sqlite:{}", db_path.replace('\\', "/"));
let pool = SqlitePoolOptions::new()
.max_connections(2)
.connect(&url)
.await?;
sqlx::query("PRAGMA journal_mode=WAL;").execute(&pool).await?;
sqlx::query("PRAGMA synchronous=NORMAL;").execute(&pool).await?;
sqlx::query("PRAGMA busy_timeout=5000;").execute(&pool).await?;
sqlx::query("PRAGMA wal_autocheckpoint=1000;").execute(&pool).await?;
sqlx::query("PRAGMA journal_size_limit=67108864;").execute(&pool).await?;
sqlx::query("PRAGMA cache_size=-64000;").execute(&pool).await?;
sqlx::query("PRAGMA temp_store=MEMORY;").execute(&pool).await?;
Self::create_tables(&pool).await?;
Ok(Self { pool })
}
/// 执行 WAL checkpoint(TRUNCATE)，将 WAL 完全合并回主文件并清理 -wal / -shm 文件
///
/// 在启动时调用，确保 search.db 文件干净，避免下次启动时 WAL 恢复开销。
pub async fn wal_checkpoint(&self) {
match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
.execute(&self.pool).await
{
Ok(_) => println!("[search.db] WAL checkpoint(TRUNCATE) 完成，数据库文件已合并"),
Err(e) => eprintln!("[search.db] WAL checkpoint 失败: {}", e),
}
}
async fn create_tables(pool: &SqlitePool) -> Result<(), sqlx::Error> {
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
category TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS persons (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
display_name TEXT,
total_albums INTEGER DEFAULT 0,
total_images INTEGER DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS person_aliases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
person_id INTEGER NOT NULL,
alias_type TEXT NOT NULL,
alias_name TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS album_tags (
album_id TEXT NOT NULL,
tag_id INTEGER NOT NULL,
PRIMARY KEY (album_id, tag_id),
FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS album_persons (
album_id TEXT NOT NULL,
person_id INTEGER NOT NULL,
PRIMARY KEY (album_id, person_id),
FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
// ── 新增 albums 表：搜索返回需要的相册基础信息 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS albums (
id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT '',
cover_image_id TEXT,
image_count INTEGER DEFAULT 0,
created_at TEXT
)
"#).execute(pool).await?;
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS dict (
word TEXT NOT NULL PRIMARY KEY,
type TEXT NOT NULL,
category TEXT,
freq INTEGER,
merged_to TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_albums_title ON albums(title)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_tags_name ON tags(name)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_persons_name ON persons(name)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_album ON album_tags(album_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_tag ON album_tags(tag_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_album ON album_persons(album_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_person ON album_persons(person_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_person_aliases_name ON person_aliases(alias_name)").execute(pool).await?;
// 兼容旧表：尝试添加 merged_to 列（已有则忽略）
let _ = sqlx::query("ALTER TABLE dict ADD COLUMN merged_to TEXT").execute(pool).await;
Ok(())
}
// ── upsert 写入方法 ──
pub async fn upsert_album(&self, album_id: &str, title: &str,
cover_image_id: Option<&str>, image_count: i64,
created_at: Option<&str>) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT INTO albums (id, title, cover_image_id, image_count, created_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
title = excluded.title,
cover_image_id = excluded.cover_image_id,
image_count = excluded.image_count,
created_at = excluded.created_at
"#)
.bind(album_id).bind(title).bind(cover_image_id)
.bind(image_count).bind(created_at)
.execute(&self.pool).await?;
Ok(())
}
pub async fn delete_album(&self, album_id: &str) -> Result<(), sqlx::Error> {
let mut tx = self.pool.begin().await?;
sqlx::query("DELETE FROM album_tags WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?;
sqlx::query("DELETE FROM album_persons WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?;
sqlx::query("DELETE FROM albums WHERE id = ?").bind(album_id).execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
pub async fn insert_tag(&self, name: &str, category: Option<&str>) -> Result<i64, sqlx::Error> {
let result = sqlx::query(r#"
INSERT OR IGNORE INTO tags (name, category) VALUES (?, ?)
"#)
.bind(name)
.bind(category)
.execute(&self.pool)
.await?;
let id = result.last_insert_rowid();
if id == 0 {
let found_id: i64 = sqlx::query_scalar("SELECT id FROM tags WHERE name = ?")
.bind(name)
.fetch_one(&self.pool)
.await?;
Ok(found_id)
} else {
Ok(id)
}
}
pub async fn insert_person(&self, name: &str, display_name: Option<&str>) -> Result<i64, sqlx::Error> {
let result = sqlx::query(r#"
INSERT OR IGNORE INTO persons (name, display_name) VALUES (?, ?)
"#)
.bind(name)
.bind(display_name)
.execute(&self.pool)
.await?;
let id = result.last_insert_rowid();
if id == 0 {
let found_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?")
.bind(name)
.fetch_one(&self.pool)
.await?;
Ok(found_id)
} else {
Ok(id)
}
}
pub async fn update_person_counts(&self, person_id: i64, total_albums: i64, total_images: i64) -> Result<(), sqlx::Error> {
sqlx::query("UPDATE persons SET total_albums = ?, total_images = ?, updated_at = datetime('now') WHERE id = ?")
.bind(total_albums).bind(total_images).bind(person_id)
.execute(&self.pool).await?;
Ok(())
}
pub async fn insert_person_alias(&self, person_id: i64, alias_type: &str, alias_name: &str) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)
"#)
.bind(person_id)
.bind(alias_type)
.bind(alias_name)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn insert_album_tag(&self, album_id: &str, tag_id: i64) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO album_tags (album_id, tag_id) VALUES (?, ?)
"#)
.bind(album_id)
.bind(tag_id)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn insert_album_person(&self, album_id: &str, person_id: i64) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO album_persons (album_id, person_id) VALUES (?, ?)
"#)
.bind(album_id)
.bind(person_id)
.execute(&self.pool)
.await?;
Ok(())
}
/// 清空某相册的所有关联标签
pub async fn clear_album_tags(&self, album_id: &str) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id).execute(&self.pool).await?;
Ok(())
}
/// 清空某相册的所有关联人物
pub async fn clear_album_persons(&self, album_id: &str) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM album_persons WHERE album_id = ?")
.bind(album_id).execute(&self.pool).await?;
Ok(())
}
pub async fn load_dict(&self) -> Result<Vec<DictEntry>, sqlx::Error> {
let rows = sqlx::query("SELECT word, type, category, freq, merged_to FROM dict")
.fetch_all(&self.pool)
.await?;
let entries = rows.iter().map(|r| DictEntry {
word: r.get("word"),
type_: r.get("type"),
category: r.get("category"),
freq: r.get("freq"),
merged_to: r.get("merged_to"),
}).collect();
Ok(entries)
}
/// 加载 dict 表全部条目（不限类型），用于初始化 TagDict
pub async fn load_dict_all(&self) -> Result<Vec<DictEntry>, sqlx::Error> {
self.load_dict().await
}
pub async fn clear_all(&self) -> Result<(), sqlx::Error> {
let mut tx = self.pool.begin().await?;
sqlx::query("DELETE FROM album_tags").execute(&mut *tx).await?;
sqlx::query("DELETE FROM album_persons").execute(&mut *tx).await?;
sqlx::query("DELETE FROM person_aliases").execute(&mut *tx).await?;
sqlx::query("DELETE FROM tags").execute(&mut *tx).await?;
sqlx::query("DELETE FROM persons").execute(&mut *tx).await?;
sqlx::query("DELETE FROM albums").execute(&mut *tx).await?;
sqlx::query("DELETE FROM dict").execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
}

342
src/search_db/search.rs Normal file
View File

@@ -0,0 +1,342 @@
//! 搜索读取方法 — SearchDb 的只读接口
//!
//! 所有搜索相关的数据读取都从这里进行，完全不碰主库。
use sqlx::Row;
use std::collections::HashMap;
use uuid::Uuid;
use crate::models::*;
use super::SearchDb;
impl SearchDb {
// ═══════════════════════════════════════
// 相册搜索
// ═══════════════════════════════════════
/// 组合搜索：按 tag/person/标题关键词 任意组合过滤（AND）
pub async fn search_albums(
&self,
tag: Option<&str>,
person: Option<&str>,
title_q: Option<&str>,
limit: i64,
offset: i64,
) -> Result<Vec<AlbumInfo>, sqlx::Error> {
let has_tag = tag.map(|s| !s.is_empty()).unwrap_or(false);
let has_person = person.map(|s| !s.is_empty()).unwrap_or(false);
let has_q = title_q.map(|s| !s.is_empty()).unwrap_or(false);
if !has_tag && !has_person && !has_q {
return self.get_all_albums().await;
}
let mut sql = String::from(
"SELECT DISTINCT a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM albums a"
);
let mut conditions: Vec<String> = Vec::new();
if has_tag {
sql.push_str(" JOIN album_tags at ON a.id = at.album_id JOIN tags t ON at.tag_id = t.id");
conditions.push("t.name = ?".to_string());
}
if has_person {
sql.push_str(" JOIN album_persons ap ON a.id = ap.album_id JOIN persons p ON ap.person_id = p.id");
conditions.push("(p.name = ? OR LOWER(p.name) LIKE ?)".to_string());
}
if !conditions.is_empty() {
sql.push_str(" WHERE ");
sql.push_str(&conditions.join(" AND "));
}
if has_q {
if conditions.is_empty() {
sql.push_str(" WHERE ");
} else {
sql.push_str(" AND ");
}
sql.push_str("a.title LIKE ?");
}
sql.push_str(" ORDER BY a.created_at DESC LIMIT ? OFFSET ?");
let mut query = sqlx::query(&sql);
if has_tag { query = query.bind(tag.unwrap()); }
if has_person {
let p = person.unwrap();
query = query.bind(p).bind(format!("%{}%", p));
}
if has_q { query = query.bind(format!("%{}%", title_q.unwrap())); }
query = query.bind(limit).bind(offset);
let rows = query.fetch_all(&self.pool).await?;
Ok(rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
}
}).collect())
}
/// 获取全量相册列表（缓存烘焙用）
pub async fn get_all_albums(&self) -> Result<Vec<AlbumInfo>, sqlx::Error> {
let rows = sqlx::query(
"SELECT id, title, cover_image_id, image_count, created_at FROM albums ORDER BY created_at DESC"
).fetch_all(&self.pool).await?;
Ok(rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
}
}).collect())
}
// ═══════════════════════════════════════
// 标签搜索
// ═══════════════════════════════════════
/// 获取热门标签（按使用次数排序）
pub async fn get_popular_tags(&self, limit: usize) -> Vec<TagStat> {
let limit = limit.min(500);
let rows = sqlx::query(
r#"SELECT t.name, COUNT(at.album_id) as cnt
FROM tags t
JOIN album_tags at ON t.id = at.tag_id
GROUP BY t.id, t.name
ORDER BY cnt DESC LIMIT ?"#
).bind(limit as i64).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| TagStat {
tag: r.get("name"),
count: r.get("cnt"),
album_count: 0,
}).collect()
}
/// 获取标签倒排索引：tag → albums（缓存烘焙用）
pub async fn get_all_tag_albums(&self) -> HashMap<String, Vec<AlbumInfo>> {
let rows = sqlx::query(
r#"SELECT t.name as tag_name,
a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM tags t
JOIN album_tags at ON t.id = at.tag_id
JOIN albums a ON a.id = at.album_id
ORDER BY a.created_at DESC"#
).fetch_all(&self.pool).await.unwrap_or_default();
let mut map: HashMap<String, Vec<AlbumInfo>> = HashMap::new();
for r in &rows {
let tag_name: String = r.get("tag_name");
let id_str: String = r.get("id");
let album = AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
};
map.entry(tag_name).or_default().push(album);
}
map
}
/// 按标签搜索相册
pub async fn search_albums_by_tag(&self, tag: &str) -> Vec<AlbumInfo> {
let rows = sqlx::query(
r#"SELECT a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM albums a
JOIN album_tags at ON a.id = at.album_id
JOIN tags t ON at.tag_id = t.id
WHERE t.name = ?
ORDER BY a.created_at DESC"#
).bind(tag).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
}
}).collect()
}
/// 获取标签共现数据（关联推荐用）
pub async fn get_tag_co_occurrences(&self, tag: &str, limit: i64) -> Vec<TagCoOccurrence> {
let rows = sqlx::query(
r#"SELECT t2.name as co_tag, COUNT(*) as cnt
FROM album_tags at1
JOIN album_tags at2 ON at1.album_id = at2.album_id AND at1.tag_id != at2.tag_id
JOIN tags t1 ON at1.tag_id = t1.id
JOIN tags t2 ON at2.tag_id = t2.id
WHERE t1.name = ?
GROUP BY t2.name
ORDER BY cnt DESC LIMIT ?"#
).bind(tag).bind(limit).fetch_all(&self.pool).await.unwrap_or_default();
let total: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM album_tags at JOIN tags t ON at.tag_id = t.id WHERE t.name = ?"
).bind(tag).fetch_one(&self.pool).await.unwrap_or(1);
rows.iter().map(|r| TagCoOccurrence {
tag: tag.to_string(),
co_tag: r.get("co_tag"),
count: r.get("cnt"),
score: r.get::<i64, _>("cnt") as f64 / total as f64,
}).collect()
}
// ═══════════════════════════════════════
// 人物搜索
// ═══════════════════════════════════════
/// 获取人物别名列表
pub async fn get_person_aliases(&self, person_id: i64) -> Vec<PersonAlias> {
let rows = sqlx::query(
"SELECT id, person_id, alias_type, alias_name, created_at FROM person_aliases WHERE person_id = ?"
).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| PersonAlias {
id: r.get("id"),
person_id: r.get("person_id"),
alias_type: r.get("alias_type"),
alias_name: r.get("alias_name"),
created_at: r.get("created_at"),
}).collect()
}
/// 获取所有人物（含别名、排序）
pub async fn get_all_persons_with_aliases(&self) -> Vec<PersonWithCounts> {
let persons = sqlx::query(
"SELECT id, name, display_name, total_albums, total_images FROM persons ORDER BY total_albums DESC"
).fetch_all(&self.pool).await.unwrap_or_default();
let mut results = Vec::new();
for r in &persons {
let id: i64 = r.get("id");
let name: String = r.get("name");
let aliases: Vec<String> = sqlx::query_scalar(
"SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type"
).bind(id).fetch_all(&self.pool).await.unwrap_or_default();
results.push(PersonWithCounts {
id,
name,
display_name: r.get("display_name"),
total_albums: r.get("total_albums"),
total_images: r.get("total_images"),
aliases,
});
}
results
}
/// 按名称查找人物
pub async fn get_person_by_name(&self, name: &str) -> Option<PersonWithCounts> {
let row = sqlx::query(
"SELECT id, name, display_name, total_albums, total_images FROM persons WHERE name = ?"
).bind(name).fetch_optional(&self.pool).await.ok()??;
let id: i64 = row.get("id");
let aliases: Vec<String> = sqlx::query_scalar(
"SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type"
).bind(id).fetch_all(&self.pool).await.unwrap_or_default();
Some(PersonWithCounts {
id,
name: row.get("name"),
display_name: row.get("display_name"),
total_albums: row.get("total_albums"),
total_images: row.get("total_images"),
aliases,
})
}
/// 获取人物的所有相册
pub async fn get_person_albums(&self, person_id: i64) -> Vec<AlbumInfo> {
let rows = sqlx::query(
r#"SELECT a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM albums a
JOIN album_persons ap ON a.id = ap.album_id
WHERE ap.person_id = ?
ORDER BY a.created_at DESC"#
).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
}
}).collect()
}
/// 获取人物的 top 标签（用于详情页）
pub async fn get_person_top_tags(&self, person_id: i64, limit: usize) -> Vec<(String, i64)> {
let rows = sqlx::query(
r#"SELECT t.name, COUNT(*) as cnt
FROM tags t
JOIN album_tags at ON t.id = at.tag_id
JOIN album_persons ap ON at.album_id = ap.album_id
WHERE ap.person_id = ?
GROUP BY t.name
ORDER BY cnt DESC LIMIT ?"#
).bind(person_id).bind(limit as i64)
.fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| (r.get("name"), r.get("cnt"))).collect()
}
/// 搜索人物（按名称/别名模糊匹配，内存）
pub fn search_persons_inline(persons: &[PersonWithCounts], query: &str) -> Vec<PersonWithCounts> {
let q = query.to_lowercase().trim().to_string();
if q.is_empty() || q.len() < 2 { return Vec::new(); }
let mut seen = std::collections::HashSet::new();
let mut results = Vec::new();
for p in persons {
let name_lower = p.name.to_lowercase();
let alias_matches = p.aliases.iter().any(|a| a.to_lowercase().contains(&q));
let matched = name_lower == q
|| name_lower.starts_with(&q)
|| name_lower.contains(&q)
|| alias_matches;
if matched && seen.insert(p.id) {
results.push(p.clone());
}
}
results.sort_by(|a, b| b.total_albums.cmp(&a.total_albums));
results
}
}

4
src/services/mod.rs
View File

@@ -1,5 +1,7 @@
//! 服务层 — 导入、处理等业务逻辑
pub mod import;
pub mod search_export;
#[allow(unused)] pub use import::*;
#[allow(unused)] pub use import::*;
#[allow(unused)] pub use search_export::*;

179
src/services/search_export.rs Normal file
View File

@@ -0,0 +1,179 @@
use crate::db::Database;
use crate::engine::title_parser;
use crate::search_db::SearchDb;
use std::collections::HashMap;
pub struct SearchExportResult {
pub albums_processed: usize,
pub tags_created: usize,
pub persons_created: usize,
pub aliases_created: usize,
pub album_tags_created: usize,
pub album_persons_created: usize,
pub dict_matched_persons: usize,
pub dict_matched_tags: usize,
pub dict_resolved_aliases: usize,
}
impl Default for SearchExportResult {
fn default() -> Self {
Self {
albums_processed: 0,
tags_created: 0,
persons_created: 0,
aliases_created: 0,
album_tags_created: 0,
album_persons_created: 0,
dict_matched_persons: 0,
dict_matched_tags: 0,
dict_resolved_aliases: 0,
}
}
}
pub async fn export_to_search_db(
source_db: &Database,
search_db_path: &str,
clear_existing: bool,
) -> Result<SearchExportResult, String> {
let search_db = SearchDb::new(search_db_path).await.map_err(|e| format!("创建 search.db 失败: {}", e))?;
if clear_existing {
search_db.clear_all().await.map_err(|e| format!("清空 search.db 失败: {}", e))?;
println!("[搜索导出] 已清空 search.db 现有数据");
}
// ── 加载 dict 表 ──
let dict_entries = search_db
.load_dict()
.await
.map_err(|e| format!("加载 dict 表失败: {}", e))?;
let person_count = dict_entries.iter().filter(|e| e.type_ == "person").count();
let tag_count = dict_entries.iter().filter(|e| e.type_ == "tag").count();
let alias_count = dict_entries.iter().filter(|e| e.merged_to.is_some()).count();
println!(
"[搜索导出] dict 表加载完成: {} 条 (person={}, tag={}, 别名映射={})",
dict_entries.len(),
person_count,
tag_count,
alias_count,
);
let albums = source_db.get_all_albums().await.map_err(|e| format!("读取相册列表失败: {}", e))?;
println!("[搜索导出] 共读取 {} 个相册（来自 imager.db）", albums.len());
let mut tag_cache: HashMap<String, i64> = HashMap::new();
let mut person_cache: HashMap<String, i64> = HashMap::new();
let mut result = SearchExportResult::default();
for album in &albums {
let album_id = album.id.to_string();
let title = &album.title;
// ── 1. dict 优先解析（dict 优先匹配 → tag_dict 回退 → 合并） ──
let parse_result = title_parser::parse_album_title_with_dict(&album_id, title, &dict_entries);
// 统计 dict 命中
if let Some(ref person_name) = parse_result.person_name {
let from_dict = dict_entries.iter().any(|e| {
e.type_ == "person"
&& (e.word.to_lowercase() == person_name.to_lowercase()
|| e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == person_name.to_lowercase()))
});
if from_dict {
result.dict_matched_persons += 1;
}
}
// ── 2. 写入 search.db ──
for (tag_name, category) in &parse_result.tags {
// 检查是否是 dict 命中的 tag
let from_dict = dict_entries.iter().any(|e| {
e.type_ == "tag" && e.word.to_lowercase() == tag_name.to_lowercase()
});
if from_dict {
result.dict_matched_tags += 1;
}
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
let id = search_db
.insert_tag(tag_name, Some(category))
.await
.map_err(|e| format!("插入标签 {} 失败: {}", tag_name, e))?;
tag_cache.insert(tag_name.clone(), id);
result.tags_created += 1;
id
}
};
search_db
.insert_album_tag(&album_id, tag_id)
.await
.map_err(|e| format!("关联相册标签 {} → {} 失败: {}", album_id, tag_name, e))?;
result.album_tags_created += 1;
}
if let Some(person_name) = &parse_result.person_name {
// 检查是否有 merged_to 别名解析
let resolved_from = dict_entries.iter().find(|e| {
e.type_ == "person"
&& e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == person_name.to_lowercase())
});
if resolved_from.is_some() {
result.dict_resolved_aliases += 1;
}
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
let id = search_db
.insert_person(person_name, None)
.await
.map_err(|e| format!("插入人物 {} 失败: {}", person_name, e))?;
person_cache.insert(person_name.clone(), id);
result.persons_created += 1;
id
}
};
search_db
.insert_album_person(&album_id, person_id)
.await
.map_err(|e| format!("关联相册人物 {} → {} 失败: {}", album_id, person_name, e))?;
result.album_persons_created += 1;
if let Some(primary) = &parse_result.person_primary {
if primary != person_name {
search_db
.insert_person_alias(person_id, "primary", primary)
.await
.map_err(|e| format!("插入别名 {} → {} 失败: {}", person_name, primary, e))?;
result.aliases_created += 1;
}
}
}
result.albums_processed += 1;
if result.albums_processed % 100 == 0 {
println!("[搜索导出] 已处理 {} / {} 相册", result.albums_processed, albums.len());
}
}
println!("[搜索导出] 完成：");
println!(" - 相册: {}", result.albums_processed);
println!(" - 标签: {}", result.tags_created);
println!(" - 人物: {}", result.persons_created);
println!(" - 别名: {}", result.aliases_created);
println!(" - 相册标签关联: {}", result.album_tags_created);
println!(" - 相册人物关联: {}", result.album_persons_created);
println!(" - dict 命中 person: {}", result.dict_matched_persons);
println!(" - dict 命中 tag: {}", result.dict_matched_tags);
println!(" - dict 别名解析: {}", result.dict_resolved_aliases);
Ok(result)
}