//! 处理管线 — 批量处理相册元数据 //! //! 核心架构:脏集追踪 + 增量解析 //! //! 精简后流程(6 阶段,2 次全库扫描): //! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描 //! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描 //! Phase ③ 增量解析脏集相册 → 只扫脏集 //! Phase ④ 人物合并 + 重计 → 1次人物扫描 //! Phase ⑤ 后校验标签中的人名 → 可优化 //! //! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集, //! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。 use crate::db::Database; use crate::engine::title_parser; use crate::engine::tag_extractor; use crate::engine::person_merger; use crate::engine::tag_dict; use std::collections::HashSet; use rayon::prelude::*; // ════════════════════════════════════════════════ // 脏集 — 追踪需要重新解析的相册 ID // ════════════════════════════════════════════════ /// 脏集:记录哪些相册需要重新解析 #[derive(Debug, Clone)] pub struct DirtySet { ids: HashSet, } impl DirtySet { pub fn new() -> Self { Self { ids: HashSet::new() } } pub fn add(&mut self, id: &str) { self.ids.insert(id.to_string()); } pub fn extend(&mut self, ids: impl IntoIterator) { self.ids.extend(ids); } pub fn contains(&self, id: &str) -> bool { self.ids.contains(id) } pub fn len(&self) -> usize { self.ids.len() } pub fn is_empty(&self) -> bool { self.ids.is_empty() } pub fn into_inner(self) -> HashSet { self.ids } } impl Default for DirtySet { fn default() -> Self { Self::new() } } // ════════════════════════════════════════════════ // Phase 0:清洗已有标签中的头尾标点 // ════════════════════════════════════════════════ pub async fn phase_clean_tags(db: &Database) -> usize { let albums = db.get_all_albums_full().await; let mut cleaned_count = 0usize; let mut removed_tags = HashSet::new(); for album in &albums { let tags_json = match album.tags.as_deref() { Some(t) if !t.is_empty() && t != "[]" => t, _ => continue, }; let old_tags: Vec = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, }; if old_tags.is_empty() { continue; } let mut changed = false; let mut new_tags = Vec::new(); for tag in old_tags { let cleaned = match tag_extractor::clean_tag(&tag) { Some(t) => t, None => { removed_tags.insert(tag); changed = true; continue; } }; if cleaned != tag { removed_tags.insert(tag); changed = true; } new_tags.push(cleaned); } if !changed { continue; } cleaned_count += 1; let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); let all_words_json = album.all_words.as_deref().unwrap_or("[]"); let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) .unwrap_or_default() .into_iter() .filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; Some([cleaned, pair[1].clone()]) }) .collect(); let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await; } if cleaned_count > 0 { println!("[管线·Phase0] 标签清洗完成:清洗 {} 个相册,移除标签: {:?}", cleaned_count, removed_tags.iter().collect::>()); } else { println!("[管线·Phase0] 标签清洗完成:无需要清洗的标签"); } cleaned_count } // ════════════════════════════════════════════════ // Phase 0B:移除过短的标签(少于2个汉字/字母) // ════════════════════════════════════════════════ /// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个 fn is_short_tag(tag: &str) -> bool { let mut valid_count = 0usize; for ch in tag.chars() { if ch.is_ascii_alphabetic() { valid_count += 1; } else if ('\u{4e00}'..='\u{9fff}').contains(&ch) { valid_count += 1; } // 数字、符号等不算有效字符 } valid_count < 2 } /// Phase 0B:扫描所有标签,移除过短的标签(有效字符 < 2), /// 受影响的相册加入脏集等待重新解析。 pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) { let albums = db.get_all_albums_full().await; let mut dirty = DirtySet::new(); let mut removed_total = 0usize; let mut removed_samples: HashSet = HashSet::new(); for album in &albums { let tags_json = match album.tags.as_deref() { Some(t) if !t.is_empty() && t != "[]" => t, _ => continue, }; let old_tags: Vec = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, }; if old_tags.is_empty() { continue; } let mut changed = false; let mut new_tags = Vec::new(); for tag in old_tags { if is_short_tag(&tag) { removed_samples.insert(tag); changed = true; removed_total += 1; } else { new_tags.push(tag); } } if !changed { continue; } dirty.add(&album.id); let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); let all_words_json = album.all_words.as_deref().unwrap_or("[]"); let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) .unwrap_or_default() .into_iter() .filter(|pair| !is_short_tag(&pair[0])) .collect(); let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await; } if removed_total > 0 { let samples: Vec<_> = removed_samples.iter().take(10).collect(); println!("[管线·Phase0B] 短标签清理完成:移除 {} 个短标签,影响 {} 个相册,示例: {:?}", removed_total, dirty.len(), samples); } else { println!("[管线·Phase0B] 短标签清理完成:无短标签"); } (dirty, removed_total) } // ════════════════════════════════════════════════ // Phase A1:检测标签中的人名并直接提取到 person 字段 // ════════════════════════════════════════════════ /// Phase A1:检测标签中可能的人名并直接提取到 person_name /// /// 核心变化:不再只标记脏集等待 Phase 1,而是直接从 tags 中提取人名写入 person 字段。 /// 因为 tags 中的人名比从 title 重解析更直接、更准确。 /// /// 两层检测 + 最终写入: /// 1. 逐个标签 — 标签是已知人物或像人名 → 收集为候选人名(中文加权) /// 2. jieba 合并(空格分隔)— 发现跨标签拆分的人名(["杨晨","晨Yome"]) /// 3. 最佳候选人名 → 写入 person_name,从标签中移除关联标签 /// /// 返回 (脏集, 从标签中移除的人名标签数) pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) { let albums = db.get_all_albums_full().await; let mut dirty = DirtySet::new(); let mut person_like_removed = 0usize; let known_persons: HashSet = db.get_all_person_names().await .into_iter().map(|(n, _)| n.to_lowercase()).collect(); for album in &albums { let tags_json = match album.tags.as_deref() { Some(t) if !t.is_empty() && t != "[]" => t, _ => continue, }; let old_tags: Vec = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, }; let has_person = album.person_name_original.as_deref() .map(|s| !s.is_empty()).unwrap_or(false); // 快速判断是否需要处理 let has_known_tag = old_tags.iter().any(|t| known_persons.contains(&t.to_lowercase())); let has_like_tag = !has_person && old_tags.iter().any(|t| title_parser::is_likely_person_name(t) && !title_parser::is_excluded_name(t)); if has_person && !has_known_tag && !has_like_tag { continue; } // ── 第一层:逐个标签收集候选人名 ── let mut known_person_found: Option = None; let mut person_like_tags: Vec = Vec::new(); for tag in &old_tags { let tag_lower = tag.to_lowercase(); if known_persons.contains(&tag_lower) { known_person_found = Some(tag.clone()); continue; } if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) { person_like_tags.push(tag.clone()); continue; } } // 从 person_like_tags 中选最优(中文加权) let best_layer1 = person_like_tags.into_iter().max_by_key(|t| { if let Some(cn) = title_parser::extract_cn(t) { if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() } } else { t.len() } }); // ── 第二层:空格连接所有标签,用 jieba 提取人名 ── let combined = old_tags.join(" "); let (jieba_name, _) = title_parser::extract_person_and_tags(&combined); // ── 确定最终人名 ── let final_person = known_person_found.or_else(|| { match (&jieba_name, &best_layer1) { (Some(j), Some(b)) => { // jieba 结果更完整 → 用 jieba let j_len: usize = j.len(); let b_len: usize = b.len(); if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) } } (Some(j), None) => Some(j.clone()), (None, Some(b)) => Some(b.clone()), (None, None) => None, } }); if let Some(ref person_name) = final_person { // ── 从标签中移除所有人名相关标签 ── let mut changed = false; let mut clean_tags: Vec = Vec::new(); let person_lower = person_name.to_lowercase(); let (primary_cleaned, _) = title_parser::split_person_name_with_aliases(person_name); let primary_lower = primary_cleaned.to_lowercase(); for tag in &old_tags { let tag_lower = tag.to_lowercase(); let should_remove = tag_lower == person_lower // 完全匹配 || tag_lower == primary_lower // 中文核心匹配 || (tag.len() >= 2 && tag.chars().all(|c| c.is_ascii_alphabetic()) && !tag_lower.starts_with("cos") && person_lower.contains(&tag_lower)) // 英文后缀 || (tag_lower.len() >= 2 && person_lower.contains(&tag_lower)); // 人名包含 if should_remove { person_like_removed += 1; changed = true; continue; } clean_tags.push(tag.clone()); } if changed || !has_person { // ── 只清理标签中的人名,不写入 person 字段 ── // 人名提取统一由 Phase 1 从标题解析完成,保证一个相册只有一个 pid let tags_json_new = serde_json::to_string(&clean_tags).unwrap_or_else(|_| "[]".to_string()); let all_words_json = album.all_words.as_deref().unwrap_or("[]"); let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) .unwrap_or_default() .into_iter() .filter(|pair| !known_persons.contains(&pair[0].to_lowercase())) .collect(); let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); db.update_album_tags(&album.id, &tags_json_new, &words_json_new).await; dirty.add(&album.id); } } } if person_like_removed > 0 { println!("[管线·PhaseA1] 从标签中检测并提取人名:移除 {} 个人名标签,写入 {} 个相册的 person 字段", person_like_removed, dirty.len()); } else { println!("[管线·PhaseA1] 检测标签中的人名:无异常"); } (dirty, person_like_removed) } // ════════════════════════════════════════════════ // Phase A2:清理 persons 表中的非人物 // ════════════════════════════════════════════════ pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) { let persons = db.get_all_persons_with_counts().await; let mut dirty = DirtySet::new(); let mut deleted = 0usize; for (pid, name, _albums, _images) in &persons { if title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name) { let album_list = db.get_person_albums(*pid).await; let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); for aid in &album_ids { dirty.add(aid); } println!("[管线·PhaseA2] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len()); db.update_album_person_by_person_id(*pid).await; db.delete_person_completely(*pid).await; deleted += 1; } } if deleted > 0 { println!("[管线·PhaseA2] 清理完成:删除 {} 个非人物,标记 {} 个脏相册", deleted, dirty.len()); } else { println!("[管线·PhaseA2] 清理完成:无非人物需要删除"); } (dirty, deleted) } // ════════════════════════════════════════════════ // Phase A2B:清理 persons 表中与停用词重叠的人物 // ════════════════════════════════════════════════ /// Phase A2B:检查 persons 表中的人名是否在停用词词典中。 /// /// 当用户在词典中添加了新的停用词,而这些词恰好已被作为人物创建时, /// 需要删除该人物并释放对应的相册,让它们在后续 Phase 1 中重新解析。 /// 这样可以防止停用词(如 "photo"、"model" 等通用词)被错误识别为人物。 pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) { let persons = db.get_all_persons_with_counts().await; let mut dirty = DirtySet::new(); let mut deleted = 0usize; for (pid, name, _albums, _images) in &persons { // 使用 tag_dict 的 is_excluded_person 进行判断(含包含匹配) let is_excluded = tag_dict::get_tag_dict().is_excluded_person(name); // 也检查中文核心(如 "秀人" 是排除词,"秀人网" 也会包含) let cn_check = title_parser::extract_cn(name) .map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn)) .unwrap_or(false); if is_excluded || cn_check { let album_list = db.get_person_albums(*pid).await; let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); for aid in &album_ids { dirty.add(aid); } println!("[管线·PhaseA2B] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len()); // 清空相册的 person 字段 db.update_album_person_by_person_id(*pid).await; // 彻底删除该人物 db.delete_person_completely(*pid).await; deleted += 1; } } if deleted > 0 { println!("[管线·PhaseA2B] 清理完成:移除 {} 个被排除的人物,标记 {} 个脏相册", deleted, dirty.len()); } else { println!("[管线·PhaseA2B] 清理完成:无被排除的人物"); } (dirty, deleted) } // ════════════════════════════════════════════════ // Phase A3:收集新相册 + 半成品 // ════════════════════════════════════════════════ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet { let albums = db.get_all_albums_full().await; let mut dirty = DirtySet::new(); let mut new_count = 0usize; let mut partial_count = 0usize; for album in &albums { let tags_empty = album.tags.as_deref() .map(|t| t.is_empty() || t == "[]" || t == "null").unwrap_or(true); if tags_empty { dirty.add(&album.id); new_count += 1; continue; } let has_person = album.person_name_original.as_deref() .map(|s| !s.is_empty()).unwrap_or(false); if !has_person { dirty.add(&album.id); partial_count += 1; } } println!("[管线·PhaseA3] 收集相册:{} 个全新,{} 个半成品,脏集合计 {} 个", new_count, partial_count, dirty.len()); dirty } // ════════════════════════════════════════════════ // Phase 1:增量解析 — 只处理脏集中的相册 // ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter) // └─ Phase 1b:批量写库(IO 密集,一个事务) pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) { if dirty_set.is_empty() { println!("[管线·Phase1] 增量解析跳过:脏集为空"); return (0, 0); } let albums = db.get_all_albums_full().await; // ── Phase 1a:并行解析(CPU bound,rayon) ── let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter() .filter(|a| dirty_set.contains(&a.id)) .collect(); let skipped = dirty_albums.iter().filter(|a| { a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true) }).count(); let total = dirty_albums.len(); let parsed: Vec = dirty_albums .par_iter() .filter_map(|album| { let title = album.title.as_deref()?; if title.is_empty() { return None; } Some(title_parser::parse_album_title(&album.id, title)) }) .collect(); let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count(); let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count(); // ── Phase 1b:批量写入 ── let _guard = db.write_lock.lock().await; let mut tx = db.pool.begin().await.expect("begin tx"); db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| { eprintln!("[管线·Phase1b] 批量写入失败: {}", e); }); tx.commit().await.expect("commit tx"); println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个", total, parsed.len(), skipped, person_count, tag_count); (person_count, tag_count) } // ════════════════════════════════════════════════ // Phase C:全量重计 — 确保所有 person 的计数准确 // ════════════════════════════════════════════════ /// Phase C:全量重计所有人物相册和图片计数 /// /// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums, /// 用 JOIN albums 获取实际 image_count 更新 total_images。 /// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。 pub async fn phase_recount_all(db: &Database) -> usize { let persons = db.get_all_persons_with_counts().await; let mut fixed = 0usize; for (pid, name, old_albums, _) in &persons { let actual = sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM person_albums WHERE person_id = ?" ).bind(pid).fetch_one(&db.pool).await.unwrap_or(0); if actual != *old_albums { db.update_person_counts(*pid).await; fixed += 1; } } if fixed > 0 { println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed); } else { println!("[管线·PhaseC] 全量重计完成:所有计数正确"); } fixed } // ════════════════════════════════════════════════ pub async fn phase_merge_persons(db: &Database) -> (usize, usize) { let result = person_merger::execute_merge(db).await; if result.persons_merged > 0 { println!("[管线·Phase2] 人物合并完成:分析 {} 组,合并 {} 人,迁移 {} 个别名", result.groups_analyzed, result.persons_merged, result.aliases_moved); } else { println!("[管线·Phase2] 人物合并完成:无需要合并的人物"); } (result.persons_merged, result.aliases_moved) } // ════════════════════════════════════════════════ // Phase B:后校验标签 — 用合并后的人名检查 // ════════════════════════════════════════════════ pub async fn phase_clean_tags_after_parse(db: &Database) -> usize { let albums = db.get_all_albums_full().await; let all_person_names: HashSet = db.get_all_person_names().await .into_iter().map(|(n, _)| n.to_lowercase()).collect(); let all_aliases: HashSet = db.get_all_aliases().await .into_iter().map(|(a, _)| a.to_lowercase()).collect(); let person_or_alias: HashSet = all_person_names.into_iter().chain(all_aliases).collect(); if person_or_alias.is_empty() { println!("[管线·PhaseB] 后校验标签完成:无人物数据,跳过"); return 0; } let mut tag_removed = 0usize; let mut updates: Vec<(String, String, String)> = Vec::new(); for album in &albums { let tags_json = match album.tags.as_deref() { Some(t) if !t.is_empty() && t != "[]" => t, _ => continue, }; let old_tags: Vec = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, }; let mut album_removed = 0usize; let good_tags: Vec = old_tags.into_iter().filter(|t| { if person_or_alias.contains(&t.to_lowercase()) { album_removed += 1; false } else { true } }).collect(); if album_removed == 0 { continue; } tag_removed += album_removed; let all_words_json = album.all_words.as_deref().unwrap_or("[]"); let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) .unwrap_or_default().into_iter().filter(|pair| !person_or_alias.contains(&pair[0].to_lowercase())).collect(); let tags_json_new = serde_json::to_string(&good_tags).unwrap_or_else(|_| "[]".to_string()); let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); updates.push((album.id.clone(), tags_json_new, words_json_new)); } // ── 批量写入:单个事务 ── if !updates.is_empty() { let _guard = db.write_lock.lock().await; if let Ok(mut tx) = db.pool.begin().await { for (album_id, tags_json, all_words_json) in &updates { db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await; } tx.commit().await.ok(); } } if tag_removed > 0 { println!("[管线·PhaseB] 后校验标签完成:移除 {} 个人名/别名", tag_removed); } else { println!("[管线·PhaseB] 后校验标签完成:无需要修复的标签"); } tag_removed } // ════════════════════════════════════════════════ // 合并阶段 — 减少全库扫描次数 // ════════════════════════════════════════════════ /// 合并预处理结果 #[derive(Debug, Clone, Default)] struct PreprocessResult { tags_cleaned: usize, short_tags_removed: usize, person_like_tags_removed: usize, new_albums_found: usize, dirty: DirtySet, } /// 合并预处理:一次相册扫描完成 4 件事 /// 1. 清洗标签标点 /// 2. 移除短标签 /// 3. 检测并移除标签中的人名 /// 4. 收集新相册/半成品 /// /// 所有写库操作在单个事务中完成,避免逐条事务的 WAL fsync 开销。 async fn phase_preprocess_combined(db: &Database) -> PreprocessResult { let albums = db.get_all_albums_full().await; let known_persons: HashSet = db.get_all_person_names().await .into_iter().map(|(n, _)| n.to_lowercase()).collect(); let mut result = PreprocessResult::default(); // 收集需要更新的相册变更:(album_id, tags_json, all_words_json) let mut updates: Vec<(String, String, String)> = Vec::new(); for album in &albums { let tags_json = album.tags.as_deref(); let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false); let has_person = album.person_name_original.as_deref() .map(|s| !s.is_empty()).unwrap_or(false); // ── 新相册/半成品检测 ── if !has_tags || !has_person { result.dirty.add(&album.id); if !has_tags { result.new_albums_found += 1; } } // ── 没有标签就不需要清洗/检测 ── if !has_tags { continue; } let old_tags: Vec = match serde_json::from_str(tags_json.unwrap()) { Ok(t) => t, Err(_) => continue, }; if old_tags.is_empty() { continue; } // ── 三步标签处理合并到一次遍历 ── let mut changed = false; let mut new_tags: Vec = Vec::new(); let mut person_like_removed_this = 0usize; // 先找到最优人名(用于后续从标签中移除人名相关) // 第一层:逐个标签收集候选人名 let mut known_person_found: Option = None; let mut person_like_tags: Vec = Vec::new(); for tag in &old_tags { let tag_lower = tag.to_lowercase(); if known_persons.contains(&tag_lower) { known_person_found = Some(tag.clone()); continue; } if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) { person_like_tags.push(tag.clone()); continue; } } let best_layer1 = person_like_tags.into_iter().max_by_key(|t| { if let Some(cn) = title_parser::extract_cn(t) { if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() } } else { t.len() } }); let combined = old_tags.join(" "); let (jieba_name, _) = title_parser::extract_person_and_tags(&combined); let final_person = known_person_found.or_else(|| { match (&jieba_name, &best_layer1) { (Some(j), Some(b)) => { let j_len: usize = j.len(); let b_len: usize = b.len(); if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) } } (Some(j), None) => Some(j.clone()), (None, Some(b)) => Some(b.clone()), (None, None) => None, } }); let person_lower = final_person.as_ref().map(|n| n.to_lowercase()); let primary_lower = final_person.as_ref().and_then(|n| { let (p, _) = title_parser::split_person_name_with_aliases(n); if p.is_empty() { None } else { Some(p.to_lowercase()) } }); for tag in old_tags { // Step 1: 清洗标点 let cleaned = match tag_extractor::clean_tag(&tag) { Some(t) => t, None => { changed = true; continue; } }; if cleaned != tag { changed = true; } // Step 2: 短标签过滤 if is_short_tag(&cleaned) { changed = true; result.short_tags_removed += 1; continue; } // Step 3: 人名标签过滤 let cleaned_lower = cleaned.to_lowercase(); let mut is_person_tag = false; if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) { is_person_tag = cleaned_lower == *pl || cleaned_lower == *pr || (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic()) && !cleaned_lower.starts_with("cos") && pl.contains(&cleaned_lower)) || (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower)); } if is_person_tag { person_like_removed_this += 1; changed = true; continue; } new_tags.push(cleaned); } result.person_like_tags_removed += person_like_removed_this; if !changed { continue; } result.tags_cleaned += 1; result.dirty.add(&album.id); // 收集变更,等下批量写 let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string()); let all_words_json = album.all_words.as_deref().unwrap_or("[]"); let all_words_new: Vec<[String; 2]> = serde_json::from_str::>(all_words_json) .unwrap_or_default() .into_iter() .filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; if is_short_tag(&cleaned) { return None; } if known_persons.contains(&pair[0].to_lowercase()) { return None; } Some([cleaned, pair[1].clone()]) }) .collect(); let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string()); updates.push((album.id.clone(), tags_json_new, all_words_json_new)); } // ── 批量写入:单个事务 ── if !updates.is_empty() { let _guard = db.write_lock.lock().await; let mut tx = match db.pool.begin().await { Ok(t) => t, Err(e) => { eprintln!("[管线·Phase①] 开启事务失败: {}", e); return result; } }; for (album_id, tags_json, all_words_json) in &updates { db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await; } if let Err(e) = tx.commit().await { eprintln!("[管线·Phase①] 提交事务失败: {}", e); } } println!("[管线·Phase①] 合并预处理完成:清洗{}相册,移除短标签{},移除人名标签{},新相册{},脏集合计{}", result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.new_albums_found, result.dirty.len()); result } /// 合并人物清理:一次人物遍历完成 2 件事 /// 1. 清理非人物 /// 2. 清理停用词重叠人物 async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) { let persons = db.get_all_persons_with_counts().await; let mut dirty = DirtySet::new(); let mut non_persons_deleted = 0usize; let mut stop_word_persons_removed = 0usize; for (pid, name, _albums, _images) in &persons { // 判断是否应该删除 let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name); let is_stop_word = { let excluded = tag_dict::get_tag_dict().is_excluded_person(name); let cn_check = title_parser::extract_cn(name) .map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn)) .unwrap_or(false); excluded || cn_check }; if !is_non_person && !is_stop_word { continue; } let album_list = db.get_person_albums(*pid).await; let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); for aid in &album_ids { dirty.add(aid); } if is_stop_word { println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len()); stop_word_persons_removed += 1; } else { println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len()); non_persons_deleted += 1; } db.update_album_person_by_person_id(*pid).await; db.delete_person_completely(*pid).await; } if non_persons_deleted > 0 || stop_word_persons_removed > 0 { println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}", non_persons_deleted, stop_word_persons_removed, dirty.len()); } else { println!("[管线·Phase②] 人物清理完成:无异常"); } (dirty, non_persons_deleted, stop_word_persons_removed) } // ════════════════════════════════════════════════ // Pipeline 一键执行 // ════════════════════════════════════════════════ #[derive(Debug, Clone, Default)] pub struct PipelineResult { pub tags_cleaned: usize, // Phase ① pub short_tags_removed: usize, // Phase ① pub person_like_tags_removed: usize, // Phase ① pub dirty_albums_detected: usize, // 总脏集 pub non_persons_deleted: usize, // Phase ② pub stop_word_persons_removed: usize, // Phase ② pub new_albums_found: usize, // Phase ① pub persons_updated: usize, // Phase ③ pub tags_updated: usize, // Phase ③ pub persons_merged: usize, // Phase ④ pub aliases_moved: usize, // Phase ④ pub tags_removed_person: usize, // Phase ⑤ } /// 一键执行精简管线(5 阶段,2-3 次全库扫描) /// /// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描 /// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描 /// Phase ③ 增量解析脏集 → 只扫脏集 /// Phase ④ 人物合并 + 重计 → 1次人物扫描 /// Phase ⑤ 后校验标签中的人名 pub async fn run_pipeline(db: &Database) -> PipelineResult { // Phase ①:合并预处理(1 次相册扫描) let pre = phase_preprocess_combined(db).await; let tags_cleaned = pre.tags_cleaned; let short_tags_removed = pre.short_tags_removed; let person_like_tags_removed = pre.person_like_tags_removed; let new_albums_found = pre.new_albums_found; // Phase ②:合并人物清理(1 次人物扫描) let (dirty_persons, non_persons_deleted, stop_word_persons_removed) = phase_clean_persons_combined(db).await; // 合并脏集 let mut master_dirty = pre.dirty; master_dirty.extend(dirty_persons.into_inner()); let dirty_albums_detected = master_dirty.len(); // Phase ③:增量解析 let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await; // Phase ④:人物合并(内部会做包含关系 + 核心合并) let (persons_merged, aliases_moved) = phase_merge_persons(db).await; // Phase C:全量重计(合并后修正计数) let _recounted = phase_recount_all(db).await; // Phase ⑤:后校验 let tags_removed_person = phase_clean_tags_after_parse(db).await; PipelineResult { tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected, non_persons_deleted, stop_word_persons_removed, new_albums_found, persons_updated, tags_updated, persons_merged, aliases_moved, tags_removed_person, } }