diff --git a/assets/js/app.js b/assets/js/app.js index 7b82439..9805727 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -1473,7 +1473,8 @@ async function filterByPerson(personName) { showSkeleton(6); const response = await fetch(`${API_BASE}/people/search?q=${encodeURIComponent(personName)}`); if (!response.ok) throw new Error('搜索失败'); - const persons = await response.json(); + const personData = await response.json(); + const persons = personData.persons || personData; if (persons && persons.length > 0) { try { diff --git a/migrations/005_cleanup.sql b/migrations/005_cleanup.sql new file mode 100644 index 0000000..38dd64d --- /dev/null +++ b/migrations/005_cleanup.sql @@ -0,0 +1,8 @@ +-- 移除不再使用的扩展列（数据已迁移到 search.db） +-- SQLite 3.35.0+ 支持 ALTER TABLE DROP COLUMN + +ALTER TABLE albums DROP COLUMN tags; +ALTER TABLE albums DROP COLUMN all_words; +ALTER TABLE albums DROP COLUMN person_name_original; +ALTER TABLE albums DROP COLUMN person_name_cleaned; +ALTER TABLE albums DROP COLUMN raw_title; diff --git a/src/db/album.rs b/src/db/album.rs index e20a5f1..5d7372d 100644 --- a/src/db/album.rs +++ b/src/db/album.rs @@ -167,34 +167,25 @@ impl Database { } pub async fn get_all_albums_full(&self) -> Vec { - let has_col = sqlx::query_scalar::<_, i64>( - "SELECT COUNT(*) FROM pragma_table_info('albums') WHERE name = 'person_name_original'" - ).fetch_one(&self.pool).await.unwrap_or(0); - if has_col == 0 { return Vec::new(); } - - // 去掉 ORDER BY：管线处理不需要排序，省去排序开销 let rows = sqlx::query( - r#"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count + r#"SELECT id, title, cover_image_id, image_count FROM albums"# ).fetch_all(&self.pool).await.unwrap_or_default(); rows.iter().map(|r| AlbumMeta { id: r.get("id"), title: r.get("title"), - person_name_original: r.get("person_name_original"), - person_name_cleaned: r.get("person_name_cleaned"), - tags: r.get("tags"), all_words: r.get("all_words"), - raw_title: r.get("raw_title"), processed_at: r.get("processed_at"), + cover_image_id: r.get("cover_image_id"), image_count: r.get("image_count"), }).collect() } - /// 按 ID 列表批量查询相册元数据（用于脏集增量解析，避免全表扫描） + /// 按 ID 列表批量查询相册元数据 pub async fn get_albums_by_ids(&self, ids: &[String]) -> Vec { if ids.is_empty() { return Vec::new(); } let placeholders: Vec = (0..ids.len()).map(|_| "?".to_string()).collect(); let sql = format!( - "SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count + "SELECT id, title, cover_image_id, image_count FROM albums WHERE id IN ({})", placeholders.join(", ") ); @@ -207,14 +198,19 @@ impl Database { let rows = query.fetch_all(&self.pool).await.unwrap_or_default(); rows.iter().map(|r| AlbumMeta { id: r.get("id"), title: r.get("title"), - person_name_original: r.get("person_name_original"), - person_name_cleaned: r.get("person_name_cleaned"), - tags: r.get("tags"), all_words: r.get("all_words"), - raw_title: r.get("raw_title"), processed_at: r.get("processed_at"), + cover_image_id: r.get("cover_image_id"), image_count: r.get("image_count"), }).collect() } + /// 获取所有未处理的相册 ID（processed_at IS NULL），用于管线收集新相册 + pub async fn get_unprocessed_album_ids(&self) -> Vec { + let rows = sqlx::query_scalar::<_, String>( + "SELECT id FROM albums WHERE processed_at IS NULL" + ).fetch_all(&self.pool).await.unwrap_or_default(); + rows + } + // ── 写入 ── pub async fn create_album(&self, album: CreateAlbum) -> Result { diff --git a/src/engine/person_merger.rs b/src/engine/person_merger.rs index 1f2799c..18593e2 100644 --- a/src/engine/person_merger.rs +++ b/src/engine/person_merger.rs @@ -406,7 +406,7 @@ pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats for album in &albums { let title = album.title.as_deref().unwrap_or(""); - let current = album.person_name_original.as_deref().unwrap_or(""); + let current = ""; // person_name_original 已迁移到 search.db if let Some((_text, person, source)) = match_title(title, patterns) { *stats.by_source.entry(source).or_insert(0) += 1; @@ -458,7 +458,7 @@ pub async fn analyze_deep_matches(db: &Database) -> DeepMatchStats { for album in &albums { let title = album.title.as_deref().unwrap_or(""); - let current = album.person_name_original.as_deref().unwrap_or(""); + let current = ""; // person_name_original 已迁移到 search.db if title.is_empty() { continue; diff --git a/src/engine/pipeline.rs b/src/engine/pipeline.rs index f3ac805..53a2017 100644 --- a/src/engine/pipeline.rs +++ b/src/engine/pipeline.rs @@ -14,7 +14,6 @@ use crate::db::Database; use crate::engine::title_parser; -use crate::engine::tag_extractor; use crate::engine::person_merger; use crate::engine::tag_dict; use crate::search_db::SearchDb; @@ -65,22 +64,10 @@ fn is_short_tag(tag: &str) -> bool { // ════════════════════════════════════════════════ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet { - let albums = db.get_all_albums_full().await; + let ids = db.get_unprocessed_album_ids().await; let mut dirty = DirtySet::new(); - let mut new_count = 0usize; - let mut partial_count = 0usize; - - for album in &albums { - let tags_empty = album.tags.as_deref() - .map(|t| t.is_empty() || t == "[]" || t == "null").unwrap_or(true); - if tags_empty { dirty.add(&album.id); new_count += 1; continue; } - - let has_person = album.person_name_original.as_deref() - .map(|s| !s.is_empty()).unwrap_or(false); - if !has_person { dirty.add(&album.id); partial_count += 1; } - } - - println!("[管线·PhaseA3] 收集相册：{} 个全新，{} 个半成品，脏集合计 {} 个", new_count, partial_count, dirty.len()); + for id in &ids { dirty.add(id); } + println!("[管线·PhaseA3] 收集相册：{} 个未处理", dirty.len()); dirty } @@ -179,61 +166,8 @@ pub async fn phase_merge_persons(db: &Database) -> (usize, usize) { // ════════════════════════════════════════════════ pub async fn phase_clean_tags_after_parse(db: &Database) -> usize { - let albums = db.get_all_albums_full().await; - let all_person_names: HashSet = db.get_all_person_names().await - .into_iter().map(|(n, _)| n.to_lowercase()).collect(); - let all_aliases: HashSet = db.get_all_aliases().await - .into_iter().map(|(a, _)| a.to_lowercase()).collect(); - let person_or_alias: HashSet = all_person_names.into_iter().chain(all_aliases).collect(); - - if person_or_alias.is_empty() { - println!("[管线·PhaseB] 后校验标签完成：无人物数据，跳过"); - return 0; - } - - let mut tag_removed = 0usize; - let mut updates: Vec<(String, String, String)> = Vec::new(); - - for album in &albums { - let tags_json = match album.tags.as_deref() { - Some(t) if !t.is_empty() && t != "[]" => t, _ => continue, - }; - let old_tags: Vec = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, }; - - let mut album_removed = 0usize; - let good_tags: Vec = old_tags.into_iter().filter(|t| { - if person_or_alias.contains(&t.to_lowercase()) { album_removed += 1; false } else { true } - }).collect(); - - if album_removed == 0 { continue; } - tag_removed += album_removed; - - let all_words_json = album.all_words.as_deref().unwrap_or("[]"); - let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) - .unwrap_or_default().into_iter().filter(|pair| !person_or_alias.contains(&pair[0].to_lowercase())).collect(); - - let tags_json_new = serde_json::to_string(&good_tags).unwrap_or_else(|_| "[]".to_string()); - let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); - updates.push((album.id.clone(), tags_json_new, words_json_new)); - } - - // ── 批量写入：单个事务 ── - if !updates.is_empty() { - let _guard = db.write_lock.lock().await; - if let Ok(mut tx) = db.pool.begin().await { - for (album_id, tags_json, all_words_json) in &updates { - db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await; - } - tx.commit().await.ok(); - } - } - - if tag_removed > 0 { - println!("[管线·PhaseB] 后校验标签完成：移除 {} 个人名/别名", tag_removed); - } else { - println!("[管线·PhaseB] 后校验标签完成：无需要修复的标签"); - } - tag_removed + println!("[管线·PhaseB] 后校验标签完成：已迁移到 search.db，跳过"); + 0 } // ════════════════════════════════════════════════ @@ -250,179 +184,14 @@ struct PreprocessResult { dirty: DirtySet, } -/// 合并预处理：一次相册扫描完成 4 件事 -/// 1. 清洗标签标点 -/// 2. 移除短标签 -/// 3. 检测并移除标签中的人名 -/// 4. 收集新相册/半成品 -/// -/// 所有写库操作在单个事务中完成，避免逐条事务的 WAL fsync 开销。 +/// 合并预处理：只做收集新相册，不再做标签清洗（已迁移到 search.db） async fn phase_preprocess_combined(db: &Database) -> PreprocessResult { - let albums = db.get_all_albums_full().await; - - let known_persons: HashSet = db.get_all_person_names().await - .into_iter().map(|(n, _)| n.to_lowercase()).collect(); - let mut result = PreprocessResult::default(); - // 收集需要更新的相册变更：(album_id, tags_json, all_words_json) - let mut updates: Vec<(String, String, String)> = Vec::new(); - - for album in &albums { - let tags_json = album.tags.as_deref(); - let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false); - let has_person = album.person_name_original.as_deref() - .map(|s| !s.is_empty()).unwrap_or(false); - - // ── 新相册/半成品检测 ── - if !has_tags || !has_person { - result.dirty.add(&album.id); - if !has_tags { - result.new_albums_found += 1; - } - } - - // ── 没有标签就不需要清洗/检测 ── - if !has_tags { continue; } - - let old_tags: Vec = match serde_json::from_str(tags_json.unwrap()) { - Ok(t) => t, Err(_) => continue, - }; - if old_tags.is_empty() { continue; } - - // ── 三步标签处理合并到一次遍历 ── - let mut changed = false; - let mut new_tags: Vec = Vec::new(); - let mut person_like_removed_this = 0usize; - - // 先找到最优人名（用于后续从标签中移除人名相关） - // 第一层：逐个标签收集候选人名 - let mut known_person_found: Option = None; - let mut person_like_tags: Vec = Vec::new(); - - for tag in &old_tags { - let tag_lower = tag.to_lowercase(); - if known_persons.contains(&tag_lower) { - known_person_found = Some(tag.clone()); - continue; - } - if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) { - person_like_tags.push(tag.clone()); - continue; - } - } - - let best_layer1 = person_like_tags.into_iter().max_by_key(|t| { - if let Some(cn) = title_parser::extract_cn(t) { - if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() } - } else { t.len() } - }); - - let combined = old_tags.join(" "); - let (jieba_name, _) = title_parser::extract_person_and_tags(&combined); - - let final_person = known_person_found.or_else(|| { - match (&jieba_name, &best_layer1) { - (Some(j), Some(b)) => { - let j_len: usize = j.len(); - let b_len: usize = b.len(); - if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) } - } - (Some(j), None) => Some(j.clone()), - (None, Some(b)) => Some(b.clone()), - (None, None) => None, - } - }); - - let person_lower = final_person.as_ref().map(|n| n.to_lowercase()); - let primary_lower = final_person.as_ref().and_then(|n| { - let (p, _) = title_parser::split_person_name_with_aliases(n); - if p.is_empty() { None } else { Some(p.to_lowercase()) } - }); - - for tag in old_tags { - // Step 1: 清洗标点 - let cleaned = match tag_extractor::clean_tag(&tag) { - Some(t) => t, - None => { changed = true; continue; } - }; - if cleaned != tag { changed = true; } - - // Step 2: 短标签过滤 - if is_short_tag(&cleaned) { - changed = true; - result.short_tags_removed += 1; - continue; - } - - // Step 3: 人名标签过滤 - let cleaned_lower = cleaned.to_lowercase(); - let mut is_person_tag = false; - if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) { - is_person_tag = - cleaned_lower == *pl - || cleaned_lower == *pr - || (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic()) - && !cleaned_lower.starts_with("cos") - && pl.contains(&cleaned_lower)) - || (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower)); - } - - if is_person_tag { - person_like_removed_this += 1; - changed = true; - continue; - } - - new_tags.push(cleaned); - } - - result.person_like_tags_removed += person_like_removed_this; - - if !changed { continue; } - result.tags_cleaned += 1; - result.dirty.add(&album.id); - - // 收集变更，等下批量写 - let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); - let all_words_json = album.all_words.as_deref().unwrap_or("[]"); - let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) - .unwrap_or_default() - .into_iter() - .filter_map(|pair| { - let cleaned = tag_extractor::clean_tag(&pair[0])?; - if is_short_tag(&cleaned) { return None; } - if known_persons.contains(&pair[0].to_lowercase()) { return None; } - Some([cleaned, pair[1].clone()]) - }) - .collect(); - let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); - updates.push((album.id.clone(), tags_json_new, all_words_json_new)); - } - - // ── 批量写入：单个事务 ── - if !updates.is_empty() { - let _guard = db.write_lock.lock().await; - let mut tx = match db.pool.begin().await { - Ok(t) => t, - Err(e) => { - eprintln!("[管线·Phase①] 开启事务失败: {}", e); - return result; - } - }; - - for (album_id, tags_json, all_words_json) in &updates { - db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await; - } - - if let Err(e) = tx.commit().await { - eprintln!("[管线·Phase①] 提交事务失败: {}", e); - } - } - - println!("[管线·Phase①] 合并预处理完成：清洗{}相册，移除短标签{}，移除人名标签{}，新相册{}，脏集合计{}", - result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, + let dirty = phase_collect_new_albums(db).await; + result.dirty = dirty; + result.new_albums_found = result.dirty.len(); + println!("[管线·Phase①] 合并预处理完成（精简模式）：新相册{}，脏集合计{}", result.new_albums_found, result.dirty.len()); - result } @@ -557,9 +326,7 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us for album in &albums { // ── 1. 写入 albums 表 ── - let cover_id: Option<&str> = None; - // 从主库获取 cover_image_id：因 get_all_albums_full 不返回 cover_image_id， - // 这里用默认值，主库已有的 cover 在下一次增量同步时会补全 + let cover_id = album.cover_image_id.as_deref(); let img_count = album.image_count.unwrap_or(0); if let Err(e) = search_db.upsert_album(&album.id, album.title.as_deref().unwrap_or(""), cover_id, img_count, None).await { @@ -568,50 +335,56 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us } // ── 2. 写入 tags → album_tags ── - let tags_json = album.tags.as_deref().unwrap_or("[]"); - if tags_json != "[]" { - if let Ok(tags) = serde_json::from_str::>(tags_json) { - search_db.clear_album_tags(&album.id).await.ok(); - for (tag_name, category) in &tags { - let tag_id = match tag_cache.get(tag_name) { - Some(&id) => id, - None => { - match search_db.insert_tag(tag_name, Some(category)).await { - Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } - Err(_) => continue, + // 从 title 解析 tags，不再依赖 albums 表的 all_words 字段 + if let Some(title) = album.title.as_deref() { + if !title.is_empty() { + let parse_result = title_parser::parse_album_title(&album.id, title); + if !parse_result.tags.is_empty() { + search_db.clear_album_tags(&album.id).await.ok(); + for (tag_name, category) in &parse_result.tags { + let tag_id = match tag_cache.get(tag_name) { + Some(&id) => id, + None => { + match search_db.insert_tag(tag_name, Some(category)).await { + Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } + Err(_) => continue, + } } - } - }; - search_db.insert_album_tag(&album.id, tag_id).await.ok(); + }; + search_db.insert_album_tag(&album.id, tag_id).await.ok(); + } } } } // ── 3. 写入 persons → album_persons ── + // 从 title 解析人物，不再依赖 albums 表的 person_name_cleaned 字段 search_db.clear_album_persons(&album.id).await.ok(); - if let Some(ref person_name) = album.person_name_cleaned { - if !person_name.is_empty() { - let person_id = match person_cache.get(person_name) { - Some(&id) => id, - None => { - match search_db.insert_person(person_name, None).await { - Ok(id) => { person_cache.insert(person_name.clone(), id); id } - Err(_) => continue, + if let Some(title) = album.title.as_deref() { + if !title.is_empty() { + let parse_result = title_parser::parse_album_title(&album.id, title); + if let Some(ref person_name) = parse_result.person_name { + let person_id = match person_cache.get(person_name) { + Some(&id) => id, + None => { + match search_db.insert_person(person_name, None).await { + Ok(id) => { person_cache.insert(person_name.clone(), id); id } + Err(_) => continue, + } } - } - }; - search_db.insert_album_person(&album.id, person_id).await.ok(); - - // 同时同步别名 - let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); - if !aliases.is_empty() { - for (alias_type, alias_name) in &aliases { - if alias_name != person_name { - search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + }; + search_db.insert_album_person(&album.id, person_id).await.ok(); + // 同步别名 + let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); + if !aliases.is_empty() { + for (alias_type, alias_name) in &aliases { + if alias_name != person_name { + search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + } + } + if primary != person_name.as_str() && !primary.is_empty() { + search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); } - } - if primary != person_name.as_str() && !primary.is_empty() { - search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); } } } @@ -658,56 +431,62 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt let img_count = album.image_count.unwrap_or(0); // ── 1. upsert albums 表 ── - if let Err(e) = search_db.upsert_album(&album.id, title, None, img_count, None).await { + if let Err(e) = search_db.upsert_album(&album.id, title, album.cover_image_id.as_deref(), img_count, None).await { eprintln!("[Sync->search.db·增量] upsert 相册 {} 失败: {}", album.id, e); continue; } // ── 2. 写入 tags → album_tags ── - let tags_json = album.tags.as_deref().unwrap_or("[]"); - search_db.clear_album_tags(&album.id).await.ok(); - if tags_json != "[]" { - if let Ok(tags) = serde_json::from_str::>(tags_json) { - for (tag_name, category) in &tags { - let tag_id = match tag_cache.get(tag_name) { - Some(&id) => id, - None => { - match search_db.insert_tag(tag_name, Some(category)).await { - Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } - Err(_) => continue, + // 从 title 解析 tags，不再依赖 albums 表的 all_words 字段 + if let Some(title) = album.title.as_deref() { + if !title.is_empty() { + let parse_result = title_parser::parse_album_title(&album.id, title); + if !parse_result.tags.is_empty() { + search_db.clear_album_tags(&album.id).await.ok(); + for (tag_name, category) in &parse_result.tags { + let tag_id = match tag_cache.get(tag_name) { + Some(&id) => id, + None => { + match search_db.insert_tag(tag_name, Some(category)).await { + Ok(id) => { tag_cache.insert(tag_name.clone(), id); id } + Err(_) => continue, + } } - } - }; - search_db.insert_album_tag(&album.id, tag_id).await.ok(); + }; + search_db.insert_album_tag(&album.id, tag_id).await.ok(); + } } } } // ── 3. 写入 persons → album_persons ── + // 从 title 解析人物，不再依赖 albums 表的 person_name_cleaned 字段 search_db.clear_album_persons(&album.id).await.ok(); - if let Some(ref person_name) = album.person_name_cleaned { - if !person_name.is_empty() { - let person_id = match person_cache.get(person_name) { - Some(&id) => id, - None => { - match search_db.insert_person(person_name, None).await { - Ok(id) => { person_cache.insert(person_name.clone(), id); id } - Err(_) => continue, + if let Some(title) = album.title.as_deref() { + if !title.is_empty() { + let parse_result = title_parser::parse_album_title(&album.id, title); + if let Some(ref person_name) = parse_result.person_name { + let person_id = match person_cache.get(person_name) { + Some(&id) => id, + None => { + match search_db.insert_person(person_name, None).await { + Ok(id) => { person_cache.insert(person_name.clone(), id); id } + Err(_) => continue, + } } - } - }; - search_db.insert_album_person(&album.id, person_id).await.ok(); - - // 同时同步别名 - let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); - if !aliases.is_empty() { - for (alias_type, alias_name) in &aliases { - if alias_name != person_name { - search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + }; + search_db.insert_album_person(&album.id, person_id).await.ok(); + // 同步别名 + let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name); + if !aliases.is_empty() { + for (alias_type, alias_name) in &aliases { + if alias_name != person_name { + search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok(); + } + } + if primary != person_name.as_str() && !primary.is_empty() { + search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); } - } - if primary != person_name.as_str() && !primary.is_empty() { - search_db.insert_person_alias(person_id, "primary", &primary).await.ok(); } } } diff --git a/src/models/album.rs b/src/models/album.rs index 19d99e2..5722188 100644 --- a/src/models/album.rs +++ b/src/models/album.rs @@ -47,16 +47,11 @@ pub struct AlbumIndex { pub images: Vec, } -/// 相册完整元数据（含扩展字段） +/// 相册完整元数据（精简版：仅保留 albums 表原始字段） #[derive(Debug, Serialize, Deserialize, Clone)] pub struct AlbumMeta { pub id: String, pub title: Option, - pub person_name_original: Option, - pub person_name_cleaned: Option, - pub tags: Option, - pub all_words: Option, - pub raw_title: Option, - pub processed_at: Option, + pub cover_image_id: Option, pub image_count: Option, } \ No newline at end of file