Files
rust-imager/src/engine/pipeline.rs
2026-07-20 07:43:29 +08:00

857 lines
38 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! 处理管线 — 批量处理相册元数据
//!
//! 核心架构:脏集追踪 + 增量解析
//!
//! 精简后流程(6 阶段,2 次全库扫描):
//! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描
//! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描
//! Phase ③ 增量解析脏集相册 → 只扫脏集
//! Phase ④ 人物合并 + 重计 → 1次人物扫描
//! Phase ⑤ 后校验标签中的人名 → 可优化
//!
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
use crate::db::Database;
use crate::engine::title_parser;
use crate::engine::tag_extractor;
use crate::engine::person_merger;
use crate::engine::tag_dict;
use std::collections::HashSet;
use rayon::prelude::*;
// ════════════════════════════════════════════════
// 脏集 — 追踪需要重新解析的相册 ID
// ════════════════════════════════════════════════
/// 脏集:记录哪些相册需要重新解析
#[derive(Debug, Clone)]
pub struct DirtySet {
ids: HashSet<String>,
}
impl DirtySet {
pub fn new() -> Self { Self { ids: HashSet::new() } }
pub fn add(&mut self, id: &str) { self.ids.insert(id.to_string()); }
pub fn extend(&mut self, ids: impl IntoIterator<Item = String>) { self.ids.extend(ids); }
pub fn contains(&self, id: &str) -> bool { self.ids.contains(id) }
pub fn len(&self) -> usize { self.ids.len() }
pub fn is_empty(&self) -> bool { self.ids.is_empty() }
pub fn into_inner(self) -> HashSet<String> { self.ids }
}
impl Default for DirtySet {
fn default() -> Self { Self::new() }
}
// ════════════════════════════════════════════════
// Phase 0:清洗已有标签中的头尾标点
// ════════════════════════════════════════════════
pub async fn phase_clean_tags(db: &Database) -> usize {
let albums = db.get_all_albums_full().await;
let mut cleaned_count = 0usize;
let mut removed_tags = HashSet::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
let mut changed = false;
let mut new_tags = Vec::new();
for tag in old_tags {
let cleaned = match tag_extractor::clean_tag(&tag) {
Some(t) => t,
None => { removed_tags.insert(tag); changed = true; continue; }
};
if cleaned != tag { removed_tags.insert(tag); changed = true; }
new_tags.push(cleaned);
}
if !changed { continue; }
cleaned_count += 1;
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; Some([cleaned, pair[1].clone()]) })
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
if cleaned_count > 0 {
println!("[管线·Phase0] 标签清洗完成:清洗 {} 个相册,移除标签: {:?}",
cleaned_count, removed_tags.iter().collect::<Vec<_>>());
} else {
println!("[管线·Phase0] 标签清洗完成:无需要清洗的标签");
}
cleaned_count
}
// ════════════════════════════════════════════════
// Phase 0B:移除过短的标签(少于2个汉字/字母)
// ════════════════════════════════════════════════
/// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个
fn is_short_tag(tag: &str) -> bool {
let mut valid_count = 0usize;
for ch in tag.chars() {
if ch.is_ascii_alphabetic() {
valid_count += 1;
} else if ('\u{4e00}'..='\u{9fff}').contains(&ch) {
valid_count += 1;
}
// 数字、符号等不算有效字符
}
valid_count < 2
}
/// Phase 0B:扫描所有标签,移除过短的标签(有效字符 < 2),
/// 受影响的相册加入脏集等待重新解析。
pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut removed_total = 0usize;
let mut removed_samples: HashSet<String> = HashSet::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
let mut changed = false;
let mut new_tags = Vec::new();
for tag in old_tags {
if is_short_tag(&tag) {
removed_samples.insert(tag);
changed = true;
removed_total += 1;
} else {
new_tags.push(tag);
}
}
if !changed { continue; }
dirty.add(&album.id);
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter(|pair| !is_short_tag(&pair[0]))
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
if removed_total > 0 {
let samples: Vec<_> = removed_samples.iter().take(10).collect();
println!("[管线·Phase0B] 短标签清理完成:移除 {} 个短标签,影响 {} 个相册,示例: {:?}",
removed_total, dirty.len(), samples);
} else {
println!("[管线·Phase0B] 短标签清理完成:无短标签");
}
(dirty, removed_total)
}
// ════════════════════════════════════════════════
// Phase A1:检测标签中的人名并直接提取到 person 字段
// ════════════════════════════════════════════════
/// Phase A1:检测标签中可能的人名并直接提取到 person_name
///
/// 核心变化:不再只标记脏集等待 Phase 1,而是直接从 tags 中提取人名写入 person 字段。
/// 因为 tags 中的人名比从 title 重解析更直接、更准确。
///
/// 两层检测 + 最终写入:
/// 1. 逐个标签 — 标签是已知人物或像人名 → 收集为候选人名(中文加权)
/// 2. jieba 合并(空格分隔)— 发现跨标签拆分的人名(["杨晨","晨Yome"])
/// 3. 最佳候选人名 → 写入 person_name,从标签中移除关联标签
///
/// 返回 (脏集, 从标签中移除的人名标签数)
pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut person_like_removed = 0usize;
let known_persons: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
// 快速判断是否需要处理
let has_known_tag = old_tags.iter().any(|t| known_persons.contains(&t.to_lowercase()));
let has_like_tag = !has_person && old_tags.iter().any(|t|
title_parser::is_likely_person_name(t) && !title_parser::is_excluded_name(t));
if has_person && !has_known_tag && !has_like_tag { continue; }
// ── 第一层:逐个标签收集候选人名 ──
let mut known_person_found: Option<String> = None;
let mut person_like_tags: Vec<String> = Vec::new();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
if known_persons.contains(&tag_lower) {
known_person_found = Some(tag.clone());
continue;
}
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
person_like_tags.push(tag.clone());
continue;
}
}
// 从 person_like_tags 中选最优(中文加权)
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
if let Some(cn) = title_parser::extract_cn(t) {
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
} else { t.len() }
});
// ── 第二层:空格连接所有标签,用 jieba 提取人名 ──
let combined = old_tags.join(" ");
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
// ── 确定最终人名 ──
let final_person = known_person_found.or_else(|| {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
// jieba 结果更完整 → 用 jieba
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),
(None, None) => None,
}
});
if let Some(ref person_name) = final_person {
// ── 从标签中移除所有人名相关标签 ──
let mut changed = false;
let mut clean_tags: Vec<String> = Vec::new();
let person_lower = person_name.to_lowercase();
let (primary_cleaned, _) = title_parser::split_person_name_with_aliases(person_name);
let primary_lower = primary_cleaned.to_lowercase();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
let should_remove =
tag_lower == person_lower // 完全匹配
|| tag_lower == primary_lower // 中文核心匹配
|| (tag.len() >= 2 && tag.chars().all(|c| c.is_ascii_alphabetic())
&& !tag_lower.starts_with("cos")
&& person_lower.contains(&tag_lower)) // 英文后缀
|| (tag_lower.len() >= 2 && person_lower.contains(&tag_lower)); // 人名包含
if should_remove {
person_like_removed += 1;
changed = true;
continue;
}
clean_tags.push(tag.clone());
}
if changed || !has_person {
// ── 只清理标签中的人名,不写入 person 字段 ──
// 人名提取统一由 Phase 1 从标题解析完成,保证一个相册只有一个 pid
let tags_json_new = serde_json::to_string(&clean_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter(|pair| !known_persons.contains(&pair[0].to_lowercase()))
.collect();
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &words_json_new).await;
dirty.add(&album.id);
}
}
}
if person_like_removed > 0 {
println!("[管线·PhaseA1] 从标签中检测并提取人名:移除 {} 个人名标签,写入 {} 个相册的 person 字段",
person_like_removed, dirty.len());
} else {
println!("[管线·PhaseA1] 检测标签中的人名:无异常");
}
(dirty, person_like_removed)
}
// ════════════════════════════════════════════════
// Phase A2:清理 persons 表中的非人物
// ════════════════════════════════════════════════
pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut deleted = 0usize;
for (pid, name, _albums, _images) in &persons {
if title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name) {
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
println!("[管线·PhaseA2] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
db.update_album_person_by_person_id(*pid).await;
db.delete_person_completely(*pid).await;
deleted += 1;
}
}
if deleted > 0 {
println!("[管线·PhaseA2] 清理完成:删除 {} 个非人物,标记 {} 个脏相册", deleted, dirty.len());
} else {
println!("[管线·PhaseA2] 清理完成:无非人物需要删除");
}
(dirty, deleted)
}
// ════════════════════════════════════════════════
// Phase A2B:清理 persons 表中与停用词重叠的人物
// ════════════════════════════════════════════════
/// Phase A2B:检查 persons 表中的人名是否在停用词词典中。
///
/// 当用户在词典中添加了新的停用词,而这些词恰好已被作为人物创建时,
/// 需要删除该人物并释放对应的相册,让它们在后续 Phase 1 中重新解析。
/// 这样可以防止停用词(如 "photo"、"model" 等通用词)被错误识别为人物。
pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut deleted = 0usize;
for (pid, name, _albums, _images) in &persons {
// 使用 tag_dict 的 is_excluded_person 进行判断(含包含匹配)
let is_excluded = tag_dict::get_tag_dict().is_excluded_person(name);
// 也检查中文核心(如 "秀人" 是排除词,"秀人网" 也会包含)
let cn_check = title_parser::extract_cn(name)
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
.unwrap_or(false);
if is_excluded || cn_check {
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
println!("[管线·PhaseA2B] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
// 清空相册的 person 字段
db.update_album_person_by_person_id(*pid).await;
// 彻底删除该人物
db.delete_person_completely(*pid).await;
deleted += 1;
}
}
if deleted > 0 {
println!("[管线·PhaseA2B] 清理完成:移除 {} 个被排除的人物,标记 {} 个脏相册", deleted, dirty.len());
} else {
println!("[管线·PhaseA2B] 清理完成:无被排除的人物");
}
(dirty, deleted)
}
// ════════════════════════════════════════════════
// Phase A3:收集新相册 + 半成品
// ════════════════════════════════════════════════
pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut new_count = 0usize;
let mut partial_count = 0usize;
for album in &albums {
let tags_empty = album.tags.as_deref()
.map(|t| t.is_empty() || t == "[]" || t == "null").unwrap_or(true);
if tags_empty { dirty.add(&album.id); new_count += 1; continue; }
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
if !has_person { dirty.add(&album.id); partial_count += 1; }
}
println!("[管线·PhaseA3] 收集相册:{} 个全新,{} 个半成品,脏集合计 {} 个", new_count, partial_count, dirty.len());
dirty
}
// ════════════════════════════════════════════════
// Phase 1:增量解析 — 只处理脏集中的相册
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
// └─ Phase 1b:批量写库(IO 密集,一个事务)
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
if dirty_set.is_empty() {
println!("[管线·Phase1] 增量解析跳过:脏集为空");
return (0, 0);
}
let albums = db.get_all_albums_full().await;
// ── Phase 1a:并行解析(CPU bound,rayon) ──
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
.filter(|a| dirty_set.contains(&a.id))
.collect();
let skipped = dirty_albums.iter().filter(|a| {
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
}).count();
let total = dirty_albums.len();
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
Some(title_parser::parse_album_title(&album.id, title))
})
.collect();
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
// ── Phase 1b:批量写入 ──
let _guard = db.write_lock.lock().await;
let mut tx = db.pool.begin().await.expect("begin tx");
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
});
tx.commit().await.expect("commit tx");
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
total, parsed.len(), skipped, person_count, tag_count);
(person_count, tag_count)
}
// ════════════════════════════════════════════════
// Phase C:全量重计 — 确保所有 person 的计数准确
// ════════════════════════════════════════════════
/// Phase C:全量重计所有人物相册和图片计数
///
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
pub async fn phase_recount_all(db: &Database) -> usize {
let persons = db.get_all_persons_with_counts().await;
let mut fixed = 0usize;
for (pid, name, old_albums, _) in &persons {
let actual = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
if actual != *old_albums {
db.update_person_counts(*pid).await;
fixed += 1;
}
}
if fixed > 0 {
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
} else {
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
}
fixed
}
// ════════════════════════════════════════════════
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
let result = person_merger::execute_merge(db).await;
if result.persons_merged > 0 {
println!("[管线·Phase2] 人物合并完成:分析 {} 组,合并 {} 人,迁移 {} 个别名",
result.groups_analyzed, result.persons_merged, result.aliases_moved);
} else {
println!("[管线·Phase2] 人物合并完成:无需要合并的人物");
}
(result.persons_merged, result.aliases_moved)
}
// ════════════════════════════════════════════════
// Phase B:后校验标签 — 用合并后的人名检查
// ════════════════════════════════════════════════
pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
let albums = db.get_all_albums_full().await;
let all_person_names: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
let all_aliases: HashSet<String> = db.get_all_aliases().await
.into_iter().map(|(a, _)| a.to_lowercase()).collect();
let person_or_alias: HashSet<String> = all_person_names.into_iter().chain(all_aliases).collect();
if person_or_alias.is_empty() {
println!("[管线·PhaseB] 后校验标签完成:无人物数据,跳过");
return 0;
}
let mut tag_removed = 0usize;
let mut updates: Vec<(String, String, String)> = Vec::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t, _ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, };
let mut album_removed = 0usize;
let good_tags: Vec<String> = old_tags.into_iter().filter(|t| {
if person_or_alias.contains(&t.to_lowercase()) { album_removed += 1; false } else { true }
}).collect();
if album_removed == 0 { continue; }
tag_removed += album_removed;
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default().into_iter().filter(|pair| !person_or_alias.contains(&pair[0].to_lowercase())).collect();
let tags_json_new = serde_json::to_string(&good_tags).unwrap_or_else(|_| "[]".to_string());
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
updates.push((album.id.clone(), tags_json_new, words_json_new));
}
// ── 批量写入:单个事务 ──
if !updates.is_empty() {
let _guard = db.write_lock.lock().await;
if let Ok(mut tx) = db.pool.begin().await {
for (album_id, tags_json, all_words_json) in &updates {
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
}
tx.commit().await.ok();
}
}
if tag_removed > 0 {
println!("[管线·PhaseB] 后校验标签完成:移除 {} 个人名/别名", tag_removed);
} else {
println!("[管线·PhaseB] 后校验标签完成:无需要修复的标签");
}
tag_removed
}
// ════════════════════════════════════════════════
// 合并阶段 — 减少全库扫描次数
// ════════════════════════════════════════════════
/// 合并预处理结果
#[derive(Debug, Clone, Default)]
struct PreprocessResult {
tags_cleaned: usize,
short_tags_removed: usize,
person_like_tags_removed: usize,
new_albums_found: usize,
dirty: DirtySet,
}
/// 合并预处理:一次相册扫描完成 4 件事
/// 1. 清洗标签标点
/// 2. 移除短标签
/// 3. 检测并移除标签中的人名
/// 4. 收集新相册/半成品
///
/// 所有写库操作在单个事务中完成,避免逐条事务的 WAL fsync 开销。
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
let albums = db.get_all_albums_full().await;
let known_persons: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
let mut result = PreprocessResult::default();
// 收集需要更新的相册变更:(album_id, tags_json, all_words_json)
let mut updates: Vec<(String, String, String)> = Vec::new();
for album in &albums {
let tags_json = album.tags.as_deref();
let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false);
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
// ── 新相册/半成品检测 ──
if !has_tags || !has_person {
result.dirty.add(&album.id);
if !has_tags {
result.new_albums_found += 1;
}
}
// ── 没有标签就不需要清洗/检测 ──
if !has_tags { continue; }
let old_tags: Vec<String> = match serde_json::from_str(tags_json.unwrap()) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
// ── 三步标签处理合并到一次遍历 ──
let mut changed = false;
let mut new_tags: Vec<String> = Vec::new();
let mut person_like_removed_this = 0usize;
// 先找到最优人名(用于后续从标签中移除人名相关)
// 第一层:逐个标签收集候选人名
let mut known_person_found: Option<String> = None;
let mut person_like_tags: Vec<String> = Vec::new();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
if known_persons.contains(&tag_lower) {
known_person_found = Some(tag.clone());
continue;
}
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
person_like_tags.push(tag.clone());
continue;
}
}
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
if let Some(cn) = title_parser::extract_cn(t) {
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
} else { t.len() }
});
let combined = old_tags.join(" ");
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
let final_person = known_person_found.or_else(|| {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),
(None, None) => None,
}
});
let person_lower = final_person.as_ref().map(|n| n.to_lowercase());
let primary_lower = final_person.as_ref().and_then(|n| {
let (p, _) = title_parser::split_person_name_with_aliases(n);
if p.is_empty() { None } else { Some(p.to_lowercase()) }
});
for tag in old_tags {
// Step 1: 清洗标点
let cleaned = match tag_extractor::clean_tag(&tag) {
Some(t) => t,
None => { changed = true; continue; }
};
if cleaned != tag { changed = true; }
// Step 2: 短标签过滤
if is_short_tag(&cleaned) {
changed = true;
result.short_tags_removed += 1;
continue;
}
// Step 3: 人名标签过滤
let cleaned_lower = cleaned.to_lowercase();
let mut is_person_tag = false;
if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) {
is_person_tag =
cleaned_lower == *pl
|| cleaned_lower == *pr
|| (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic())
&& !cleaned_lower.starts_with("cos")
&& pl.contains(&cleaned_lower))
|| (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower));
}
if is_person_tag {
person_like_removed_this += 1;
changed = true;
continue;
}
new_tags.push(cleaned);
}
result.person_like_tags_removed += person_like_removed_this;
if !changed { continue; }
result.tags_cleaned += 1;
result.dirty.add(&album.id);
// 收集变更,等下批量写
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter_map(|pair| {
let cleaned = tag_extractor::clean_tag(&pair[0])?;
if is_short_tag(&cleaned) { return None; }
if known_persons.contains(&pair[0].to_lowercase()) { return None; }
Some([cleaned, pair[1].clone()])
})
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
updates.push((album.id.clone(), tags_json_new, all_words_json_new));
}
// ── 批量写入:单个事务 ──
if !updates.is_empty() {
let _guard = db.write_lock.lock().await;
let mut tx = match db.pool.begin().await {
Ok(t) => t,
Err(e) => {
eprintln!("[管线·Phase①] 开启事务失败: {}", e);
return result;
}
};
for (album_id, tags_json, all_words_json) in &updates {
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
}
if let Err(e) = tx.commit().await {
eprintln!("[管线·Phase①] 提交事务失败: {}", e);
}
}
println!("[管线·Phase①] 合并预处理完成:清洗{}相册,移除短标签{},移除人名标签{},新相册{},脏集合计{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed,
result.new_albums_found, result.dirty.len());
result
}
/// 合并人物清理:一次人物遍历完成 2 件事
/// 1. 清理非人物
/// 2. 清理停用词重叠人物
async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut non_persons_deleted = 0usize;
let mut stop_word_persons_removed = 0usize;
for (pid, name, _albums, _images) in &persons {
// 判断是否应该删除
let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name);
let is_stop_word = {
let excluded = tag_dict::get_tag_dict().is_excluded_person(name);
let cn_check = title_parser::extract_cn(name)
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
.unwrap_or(false);
excluded || cn_check
};
if !is_non_person && !is_stop_word { continue; }
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
if is_stop_word {
println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
stop_word_persons_removed += 1;
} else {
println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
non_persons_deleted += 1;
}
db.update_album_person_by_person_id(*pid).await;
db.delete_person_completely(*pid).await;
}
if non_persons_deleted > 0 || stop_word_persons_removed > 0 {
println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}",
non_persons_deleted, stop_word_persons_removed, dirty.len());
} else {
println!("[管线·Phase②] 人物清理完成:无异常");
}
(dirty, non_persons_deleted, stop_word_persons_removed)
}
// ════════════════════════════════════════════════
// Pipeline 一键执行
// ════════════════════════════════════════════════
#[derive(Debug, Clone, Default)]
pub struct PipelineResult {
pub tags_cleaned: usize, // Phase ①
pub short_tags_removed: usize, // Phase ①
pub person_like_tags_removed: usize, // Phase ①
pub dirty_albums_detected: usize, // 总脏集
pub non_persons_deleted: usize, // Phase ②
pub stop_word_persons_removed: usize, // Phase ②
pub new_albums_found: usize, // Phase ①
pub persons_updated: usize, // Phase ③
pub tags_updated: usize, // Phase ③
pub persons_merged: usize, // Phase ④
pub aliases_moved: usize, // Phase ④
pub tags_removed_person: usize, // Phase ⑤
}
/// 一键执行精简管线(5 阶段,2-3 次全库扫描)
///
/// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
/// Phase ③ 增量解析脏集 → 只扫脏集
/// Phase ④ 人物合并 + 重计 → 1次人物扫描
/// Phase ⑤ 后校验标签中的人名
pub async fn run_pipeline(db: &Database) -> PipelineResult {
// Phase ①:合并预处理(1 次相册扫描)
let pre = phase_preprocess_combined(db).await;
let tags_cleaned = pre.tags_cleaned;
let short_tags_removed = pre.short_tags_removed;
let person_like_tags_removed = pre.person_like_tags_removed;
let new_albums_found = pre.new_albums_found;
// Phase ②:合并人物清理(1 次人物扫描)
let (dirty_persons, non_persons_deleted, stop_word_persons_removed) =
phase_clean_persons_combined(db).await;
// 合并脏集
let mut master_dirty = pre.dirty;
master_dirty.extend(dirty_persons.into_inner());
let dirty_albums_detected = master_dirty.len();
// Phase ③:增量解析
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
// Phase ④:人物合并(内部会做包含关系 + 核心合并)
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
// Phase C:全量重计(合并后修正计数)
let _recounted = phase_recount_all(db).await;
// Phase ⑤:后校验
let tags_removed_person = phase_clean_tags_after_parse(db).await;
PipelineResult {
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
non_persons_deleted, stop_word_persons_removed, new_albums_found,
persons_updated, tags_updated, persons_merged,
aliases_moved, tags_removed_person,
}
}