857 lines
38 KiB
Rust
857 lines
38 KiB
Rust
//! 处理管线 — 批量处理相册元数据
|
||
//!
|
||
//! 核心架构:脏集追踪 + 增量解析
|
||
//!
|
||
//! 精简后流程(6 阶段,2 次全库扫描):
|
||
//! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描
|
||
//! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描
|
||
//! Phase ③ 增量解析脏集相册 → 只扫脏集
|
||
//! Phase ④ 人物合并 + 重计 → 1次人物扫描
|
||
//! Phase ⑤ 后校验标签中的人名 → 可优化
|
||
//!
|
||
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
|
||
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
|
||
|
||
use crate::db::Database;
|
||
use crate::engine::title_parser;
|
||
use crate::engine::tag_extractor;
|
||
use crate::engine::person_merger;
|
||
use crate::engine::tag_dict;
|
||
use std::collections::HashSet;
|
||
use rayon::prelude::*;
|
||
|
||
// ════════════════════════════════════════════════
|
||
// 脏集 — 追踪需要重新解析的相册 ID
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// 脏集:记录哪些相册需要重新解析
|
||
#[derive(Debug, Clone)]
|
||
pub struct DirtySet {
|
||
ids: HashSet<String>,
|
||
}
|
||
|
||
impl DirtySet {
|
||
pub fn new() -> Self { Self { ids: HashSet::new() } }
|
||
pub fn add(&mut self, id: &str) { self.ids.insert(id.to_string()); }
|
||
pub fn extend(&mut self, ids: impl IntoIterator<Item = String>) { self.ids.extend(ids); }
|
||
pub fn contains(&self, id: &str) -> bool { self.ids.contains(id) }
|
||
pub fn len(&self) -> usize { self.ids.len() }
|
||
pub fn is_empty(&self) -> bool { self.ids.is_empty() }
|
||
pub fn into_inner(self) -> HashSet<String> { self.ids }
|
||
}
|
||
|
||
impl Default for DirtySet {
|
||
fn default() -> Self { Self::new() }
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase 0:清洗已有标签中的头尾标点
|
||
// ════════════════════════════════════════════════
|
||
|
||
pub async fn phase_clean_tags(db: &Database) -> usize {
|
||
let albums = db.get_all_albums_full().await;
|
||
let mut cleaned_count = 0usize;
|
||
let mut removed_tags = HashSet::new();
|
||
|
||
for album in &albums {
|
||
let tags_json = match album.tags.as_deref() {
|
||
Some(t) if !t.is_empty() && t != "[]" => t,
|
||
_ => continue,
|
||
};
|
||
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
|
||
Ok(t) => t, Err(_) => continue,
|
||
};
|
||
if old_tags.is_empty() { continue; }
|
||
|
||
let mut changed = false;
|
||
let mut new_tags = Vec::new();
|
||
for tag in old_tags {
|
||
let cleaned = match tag_extractor::clean_tag(&tag) {
|
||
Some(t) => t,
|
||
None => { removed_tags.insert(tag); changed = true; continue; }
|
||
};
|
||
if cleaned != tag { removed_tags.insert(tag); changed = true; }
|
||
new_tags.push(cleaned);
|
||
}
|
||
if !changed { continue; }
|
||
cleaned_count += 1;
|
||
|
||
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
|
||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.filter_map(|pair| { let cleaned = tag_extractor::clean_tag(&pair[0])?; Some([cleaned, pair[1].clone()]) })
|
||
.collect();
|
||
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
|
||
}
|
||
|
||
if cleaned_count > 0 {
|
||
println!("[管线·Phase0] 标签清洗完成:清洗 {} 个相册,移除标签: {:?}",
|
||
cleaned_count, removed_tags.iter().collect::<Vec<_>>());
|
||
} else {
|
||
println!("[管线·Phase0] 标签清洗完成:无需要清洗的标签");
|
||
}
|
||
cleaned_count
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase 0B:移除过短的标签(少于2个汉字/字母)
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个
|
||
fn is_short_tag(tag: &str) -> bool {
|
||
let mut valid_count = 0usize;
|
||
for ch in tag.chars() {
|
||
if ch.is_ascii_alphabetic() {
|
||
valid_count += 1;
|
||
} else if ('\u{4e00}'..='\u{9fff}').contains(&ch) {
|
||
valid_count += 1;
|
||
}
|
||
// 数字、符号等不算有效字符
|
||
}
|
||
valid_count < 2
|
||
}
|
||
|
||
/// Phase 0B:扫描所有标签,移除过短的标签(有效字符 < 2),
|
||
/// 受影响的相册加入脏集等待重新解析。
|
||
pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) {
|
||
let albums = db.get_all_albums_full().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut removed_total = 0usize;
|
||
let mut removed_samples: HashSet<String> = HashSet::new();
|
||
|
||
for album in &albums {
|
||
let tags_json = match album.tags.as_deref() {
|
||
Some(t) if !t.is_empty() && t != "[]" => t,
|
||
_ => continue,
|
||
};
|
||
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
|
||
Ok(t) => t, Err(_) => continue,
|
||
};
|
||
if old_tags.is_empty() { continue; }
|
||
|
||
let mut changed = false;
|
||
let mut new_tags = Vec::new();
|
||
for tag in old_tags {
|
||
if is_short_tag(&tag) {
|
||
removed_samples.insert(tag);
|
||
changed = true;
|
||
removed_total += 1;
|
||
} else {
|
||
new_tags.push(tag);
|
||
}
|
||
}
|
||
if !changed { continue; }
|
||
|
||
dirty.add(&album.id);
|
||
|
||
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
|
||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.filter(|pair| !is_short_tag(&pair[0]))
|
||
.collect();
|
||
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
|
||
}
|
||
|
||
if removed_total > 0 {
|
||
let samples: Vec<_> = removed_samples.iter().take(10).collect();
|
||
println!("[管线·Phase0B] 短标签清理完成:移除 {} 个短标签,影响 {} 个相册,示例: {:?}",
|
||
removed_total, dirty.len(), samples);
|
||
} else {
|
||
println!("[管线·Phase0B] 短标签清理完成:无短标签");
|
||
}
|
||
(dirty, removed_total)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase A1:检测标签中的人名并直接提取到 person 字段
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// Phase A1:检测标签中可能的人名并直接提取到 person_name
|
||
///
|
||
/// 核心变化:不再只标记脏集等待 Phase 1,而是直接从 tags 中提取人名写入 person 字段。
|
||
/// 因为 tags 中的人名比从 title 重解析更直接、更准确。
|
||
///
|
||
/// 两层检测 + 最终写入:
|
||
/// 1. 逐个标签 — 标签是已知人物或像人名 → 收集为候选人名(中文加权)
|
||
/// 2. jieba 合并(空格分隔)— 发现跨标签拆分的人名(["杨晨","晨Yome"])
|
||
/// 3. 最佳候选人名 → 写入 person_name,从标签中移除关联标签
|
||
///
|
||
/// 返回 (脏集, 从标签中移除的人名标签数)
|
||
pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) {
|
||
let albums = db.get_all_albums_full().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut person_like_removed = 0usize;
|
||
|
||
let known_persons: HashSet<String> = db.get_all_person_names().await
|
||
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
|
||
|
||
for album in &albums {
|
||
let tags_json = match album.tags.as_deref() {
|
||
Some(t) if !t.is_empty() && t != "[]" => t,
|
||
_ => continue,
|
||
};
|
||
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
|
||
Ok(t) => t, Err(_) => continue,
|
||
};
|
||
|
||
let has_person = album.person_name_original.as_deref()
|
||
.map(|s| !s.is_empty()).unwrap_or(false);
|
||
|
||
// 快速判断是否需要处理
|
||
let has_known_tag = old_tags.iter().any(|t| known_persons.contains(&t.to_lowercase()));
|
||
let has_like_tag = !has_person && old_tags.iter().any(|t|
|
||
title_parser::is_likely_person_name(t) && !title_parser::is_excluded_name(t));
|
||
if has_person && !has_known_tag && !has_like_tag { continue; }
|
||
|
||
// ── 第一层:逐个标签收集候选人名 ──
|
||
let mut known_person_found: Option<String> = None;
|
||
let mut person_like_tags: Vec<String> = Vec::new();
|
||
|
||
for tag in &old_tags {
|
||
let tag_lower = tag.to_lowercase();
|
||
if known_persons.contains(&tag_lower) {
|
||
known_person_found = Some(tag.clone());
|
||
continue;
|
||
}
|
||
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
|
||
person_like_tags.push(tag.clone());
|
||
continue;
|
||
}
|
||
}
|
||
|
||
// 从 person_like_tags 中选最优(中文加权)
|
||
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
|
||
if let Some(cn) = title_parser::extract_cn(t) {
|
||
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
|
||
} else { t.len() }
|
||
});
|
||
|
||
// ── 第二层:空格连接所有标签,用 jieba 提取人名 ──
|
||
let combined = old_tags.join(" ");
|
||
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
|
||
|
||
// ── 确定最终人名 ──
|
||
let final_person = known_person_found.or_else(|| {
|
||
match (&jieba_name, &best_layer1) {
|
||
(Some(j), Some(b)) => {
|
||
// jieba 结果更完整 → 用 jieba
|
||
let j_len: usize = j.len();
|
||
let b_len: usize = b.len();
|
||
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
|
||
}
|
||
(Some(j), None) => Some(j.clone()),
|
||
(None, Some(b)) => Some(b.clone()),
|
||
(None, None) => None,
|
||
}
|
||
});
|
||
|
||
if let Some(ref person_name) = final_person {
|
||
// ── 从标签中移除所有人名相关标签 ──
|
||
let mut changed = false;
|
||
let mut clean_tags: Vec<String> = Vec::new();
|
||
let person_lower = person_name.to_lowercase();
|
||
let (primary_cleaned, _) = title_parser::split_person_name_with_aliases(person_name);
|
||
let primary_lower = primary_cleaned.to_lowercase();
|
||
|
||
for tag in &old_tags {
|
||
let tag_lower = tag.to_lowercase();
|
||
let should_remove =
|
||
tag_lower == person_lower // 完全匹配
|
||
|| tag_lower == primary_lower // 中文核心匹配
|
||
|| (tag.len() >= 2 && tag.chars().all(|c| c.is_ascii_alphabetic())
|
||
&& !tag_lower.starts_with("cos")
|
||
&& person_lower.contains(&tag_lower)) // 英文后缀
|
||
|| (tag_lower.len() >= 2 && person_lower.contains(&tag_lower)); // 人名包含
|
||
|
||
if should_remove {
|
||
person_like_removed += 1;
|
||
changed = true;
|
||
continue;
|
||
}
|
||
clean_tags.push(tag.clone());
|
||
}
|
||
|
||
if changed || !has_person {
|
||
// ── 只清理标签中的人名,不写入 person 字段 ──
|
||
// 人名提取统一由 Phase 1 从标题解析完成,保证一个相册只有一个 pid
|
||
let tags_json_new = serde_json::to_string(&clean_tags).unwrap_or_else(|_| "[]".to_string());
|
||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.filter(|pair| !known_persons.contains(&pair[0].to_lowercase()))
|
||
.collect();
|
||
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||
db.update_album_tags(&album.id, &tags_json_new, &words_json_new).await;
|
||
|
||
dirty.add(&album.id);
|
||
}
|
||
}
|
||
}
|
||
|
||
if person_like_removed > 0 {
|
||
println!("[管线·PhaseA1] 从标签中检测并提取人名:移除 {} 个人名标签,写入 {} 个相册的 person 字段",
|
||
person_like_removed, dirty.len());
|
||
} else {
|
||
println!("[管线·PhaseA1] 检测标签中的人名:无异常");
|
||
}
|
||
(dirty, person_like_removed)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase A2:清理 persons 表中的非人物
|
||
// ════════════════════════════════════════════════
|
||
|
||
pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) {
|
||
let persons = db.get_all_persons_with_counts().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut deleted = 0usize;
|
||
|
||
for (pid, name, _albums, _images) in &persons {
|
||
if title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name) {
|
||
let album_list = db.get_person_albums(*pid).await;
|
||
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
|
||
for aid in &album_ids { dirty.add(aid); }
|
||
println!("[管线·PhaseA2] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||
db.update_album_person_by_person_id(*pid).await;
|
||
db.delete_person_completely(*pid).await;
|
||
deleted += 1;
|
||
}
|
||
}
|
||
|
||
if deleted > 0 {
|
||
println!("[管线·PhaseA2] 清理完成:删除 {} 个非人物,标记 {} 个脏相册", deleted, dirty.len());
|
||
} else {
|
||
println!("[管线·PhaseA2] 清理完成:无非人物需要删除");
|
||
}
|
||
(dirty, deleted)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase A2B:清理 persons 表中与停用词重叠的人物
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// Phase A2B:检查 persons 表中的人名是否在停用词词典中。
|
||
///
|
||
/// 当用户在词典中添加了新的停用词,而这些词恰好已被作为人物创建时,
|
||
/// 需要删除该人物并释放对应的相册,让它们在后续 Phase 1 中重新解析。
|
||
/// 这样可以防止停用词(如 "photo"、"model" 等通用词)被错误识别为人物。
|
||
pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) {
|
||
let persons = db.get_all_persons_with_counts().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut deleted = 0usize;
|
||
|
||
for (pid, name, _albums, _images) in &persons {
|
||
// 使用 tag_dict 的 is_excluded_person 进行判断(含包含匹配)
|
||
let is_excluded = tag_dict::get_tag_dict().is_excluded_person(name);
|
||
|
||
// 也检查中文核心(如 "秀人" 是排除词,"秀人网" 也会包含)
|
||
let cn_check = title_parser::extract_cn(name)
|
||
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
|
||
.unwrap_or(false);
|
||
|
||
if is_excluded || cn_check {
|
||
let album_list = db.get_person_albums(*pid).await;
|
||
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
|
||
for aid in &album_ids { dirty.add(aid); }
|
||
println!("[管线·PhaseA2B] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||
// 清空相册的 person 字段
|
||
db.update_album_person_by_person_id(*pid).await;
|
||
// 彻底删除该人物
|
||
db.delete_person_completely(*pid).await;
|
||
deleted += 1;
|
||
}
|
||
}
|
||
|
||
if deleted > 0 {
|
||
println!("[管线·PhaseA2B] 清理完成:移除 {} 个被排除的人物,标记 {} 个脏相册", deleted, dirty.len());
|
||
} else {
|
||
println!("[管线·PhaseA2B] 清理完成:无被排除的人物");
|
||
}
|
||
(dirty, deleted)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase A3:收集新相册 + 半成品
|
||
// ════════════════════════════════════════════════
|
||
|
||
pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
|
||
let albums = db.get_all_albums_full().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut new_count = 0usize;
|
||
let mut partial_count = 0usize;
|
||
|
||
for album in &albums {
|
||
let tags_empty = album.tags.as_deref()
|
||
.map(|t| t.is_empty() || t == "[]" || t == "null").unwrap_or(true);
|
||
if tags_empty { dirty.add(&album.id); new_count += 1; continue; }
|
||
|
||
let has_person = album.person_name_original.as_deref()
|
||
.map(|s| !s.is_empty()).unwrap_or(false);
|
||
if !has_person { dirty.add(&album.id); partial_count += 1; }
|
||
}
|
||
|
||
println!("[管线·PhaseA3] 收集相册:{} 个全新,{} 个半成品,脏集合计 {} 个", new_count, partial_count, dirty.len());
|
||
dirty
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase 1:增量解析 — 只处理脏集中的相册
|
||
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
|
||
// └─ Phase 1b:批量写库(IO 密集,一个事务)
|
||
|
||
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
|
||
if dirty_set.is_empty() {
|
||
println!("[管线·Phase1] 增量解析跳过:脏集为空");
|
||
return (0, 0);
|
||
}
|
||
|
||
let albums = db.get_all_albums_full().await;
|
||
|
||
// ── Phase 1a:并行解析(CPU bound,rayon) ──
|
||
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
|
||
.filter(|a| dirty_set.contains(&a.id))
|
||
.collect();
|
||
|
||
let skipped = dirty_albums.iter().filter(|a| {
|
||
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
|
||
}).count();
|
||
let total = dirty_albums.len();
|
||
|
||
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
|
||
.par_iter()
|
||
.filter_map(|album| {
|
||
let title = album.title.as_deref()?;
|
||
if title.is_empty() { return None; }
|
||
Some(title_parser::parse_album_title(&album.id, title))
|
||
})
|
||
.collect();
|
||
|
||
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
|
||
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
|
||
|
||
// ── Phase 1b:批量写入 ──
|
||
let _guard = db.write_lock.lock().await;
|
||
let mut tx = db.pool.begin().await.expect("begin tx");
|
||
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
|
||
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
|
||
});
|
||
tx.commit().await.expect("commit tx");
|
||
|
||
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
|
||
total, parsed.len(), skipped, person_count, tag_count);
|
||
(person_count, tag_count)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase C:全量重计 — 确保所有 person 的计数准确
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// Phase C:全量重计所有人物相册和图片计数
|
||
///
|
||
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
|
||
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
|
||
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
|
||
pub async fn phase_recount_all(db: &Database) -> usize {
|
||
let persons = db.get_all_persons_with_counts().await;
|
||
let mut fixed = 0usize;
|
||
for (pid, name, old_albums, _) in &persons {
|
||
let actual = sqlx::query_scalar::<_, i64>(
|
||
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
|
||
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
|
||
if actual != *old_albums {
|
||
db.update_person_counts(*pid).await;
|
||
fixed += 1;
|
||
}
|
||
}
|
||
if fixed > 0 {
|
||
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
|
||
} else {
|
||
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
|
||
}
|
||
fixed
|
||
}
|
||
// ════════════════════════════════════════════════
|
||
|
||
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
|
||
let result = person_merger::execute_merge(db).await;
|
||
if result.persons_merged > 0 {
|
||
println!("[管线·Phase2] 人物合并完成:分析 {} 组,合并 {} 人,迁移 {} 个别名",
|
||
result.groups_analyzed, result.persons_merged, result.aliases_moved);
|
||
} else {
|
||
println!("[管线·Phase2] 人物合并完成:无需要合并的人物");
|
||
}
|
||
(result.persons_merged, result.aliases_moved)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Phase B:后校验标签 — 用合并后的人名检查
|
||
// ════════════════════════════════════════════════
|
||
|
||
pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
|
||
let albums = db.get_all_albums_full().await;
|
||
let all_person_names: HashSet<String> = db.get_all_person_names().await
|
||
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
|
||
let all_aliases: HashSet<String> = db.get_all_aliases().await
|
||
.into_iter().map(|(a, _)| a.to_lowercase()).collect();
|
||
let person_or_alias: HashSet<String> = all_person_names.into_iter().chain(all_aliases).collect();
|
||
|
||
if person_or_alias.is_empty() {
|
||
println!("[管线·PhaseB] 后校验标签完成:无人物数据,跳过");
|
||
return 0;
|
||
}
|
||
|
||
let mut tag_removed = 0usize;
|
||
let mut updates: Vec<(String, String, String)> = Vec::new();
|
||
|
||
for album in &albums {
|
||
let tags_json = match album.tags.as_deref() {
|
||
Some(t) if !t.is_empty() && t != "[]" => t, _ => continue,
|
||
};
|
||
let old_tags: Vec<String> = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, };
|
||
|
||
let mut album_removed = 0usize;
|
||
let good_tags: Vec<String> = old_tags.into_iter().filter(|t| {
|
||
if person_or_alias.contains(&t.to_lowercase()) { album_removed += 1; false } else { true }
|
||
}).collect();
|
||
|
||
if album_removed == 0 { continue; }
|
||
tag_removed += album_removed;
|
||
|
||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||
.unwrap_or_default().into_iter().filter(|pair| !person_or_alias.contains(&pair[0].to_lowercase())).collect();
|
||
|
||
let tags_json_new = serde_json::to_string(&good_tags).unwrap_or_else(|_| "[]".to_string());
|
||
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||
updates.push((album.id.clone(), tags_json_new, words_json_new));
|
||
}
|
||
|
||
// ── 批量写入:单个事务 ──
|
||
if !updates.is_empty() {
|
||
let _guard = db.write_lock.lock().await;
|
||
if let Ok(mut tx) = db.pool.begin().await {
|
||
for (album_id, tags_json, all_words_json) in &updates {
|
||
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
|
||
}
|
||
tx.commit().await.ok();
|
||
}
|
||
}
|
||
|
||
if tag_removed > 0 {
|
||
println!("[管线·PhaseB] 后校验标签完成:移除 {} 个人名/别名", tag_removed);
|
||
} else {
|
||
println!("[管线·PhaseB] 后校验标签完成:无需要修复的标签");
|
||
}
|
||
tag_removed
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// 合并阶段 — 减少全库扫描次数
|
||
// ════════════════════════════════════════════════
|
||
|
||
/// 合并预处理结果
|
||
#[derive(Debug, Clone, Default)]
|
||
struct PreprocessResult {
|
||
tags_cleaned: usize,
|
||
short_tags_removed: usize,
|
||
person_like_tags_removed: usize,
|
||
new_albums_found: usize,
|
||
dirty: DirtySet,
|
||
}
|
||
|
||
/// 合并预处理:一次相册扫描完成 4 件事
|
||
/// 1. 清洗标签标点
|
||
/// 2. 移除短标签
|
||
/// 3. 检测并移除标签中的人名
|
||
/// 4. 收集新相册/半成品
|
||
///
|
||
/// 所有写库操作在单个事务中完成,避免逐条事务的 WAL fsync 开销。
|
||
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
|
||
let albums = db.get_all_albums_full().await;
|
||
|
||
let known_persons: HashSet<String> = db.get_all_person_names().await
|
||
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
|
||
|
||
let mut result = PreprocessResult::default();
|
||
// 收集需要更新的相册变更:(album_id, tags_json, all_words_json)
|
||
let mut updates: Vec<(String, String, String)> = Vec::new();
|
||
|
||
for album in &albums {
|
||
let tags_json = album.tags.as_deref();
|
||
let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false);
|
||
let has_person = album.person_name_original.as_deref()
|
||
.map(|s| !s.is_empty()).unwrap_or(false);
|
||
|
||
// ── 新相册/半成品检测 ──
|
||
if !has_tags || !has_person {
|
||
result.dirty.add(&album.id);
|
||
if !has_tags {
|
||
result.new_albums_found += 1;
|
||
}
|
||
}
|
||
|
||
// ── 没有标签就不需要清洗/检测 ──
|
||
if !has_tags { continue; }
|
||
|
||
let old_tags: Vec<String> = match serde_json::from_str(tags_json.unwrap()) {
|
||
Ok(t) => t, Err(_) => continue,
|
||
};
|
||
if old_tags.is_empty() { continue; }
|
||
|
||
// ── 三步标签处理合并到一次遍历 ──
|
||
let mut changed = false;
|
||
let mut new_tags: Vec<String> = Vec::new();
|
||
let mut person_like_removed_this = 0usize;
|
||
|
||
// 先找到最优人名(用于后续从标签中移除人名相关)
|
||
// 第一层:逐个标签收集候选人名
|
||
let mut known_person_found: Option<String> = None;
|
||
let mut person_like_tags: Vec<String> = Vec::new();
|
||
|
||
for tag in &old_tags {
|
||
let tag_lower = tag.to_lowercase();
|
||
if known_persons.contains(&tag_lower) {
|
||
known_person_found = Some(tag.clone());
|
||
continue;
|
||
}
|
||
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
|
||
person_like_tags.push(tag.clone());
|
||
continue;
|
||
}
|
||
}
|
||
|
||
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
|
||
if let Some(cn) = title_parser::extract_cn(t) {
|
||
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
|
||
} else { t.len() }
|
||
});
|
||
|
||
let combined = old_tags.join(" ");
|
||
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
|
||
|
||
let final_person = known_person_found.or_else(|| {
|
||
match (&jieba_name, &best_layer1) {
|
||
(Some(j), Some(b)) => {
|
||
let j_len: usize = j.len();
|
||
let b_len: usize = b.len();
|
||
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
|
||
}
|
||
(Some(j), None) => Some(j.clone()),
|
||
(None, Some(b)) => Some(b.clone()),
|
||
(None, None) => None,
|
||
}
|
||
});
|
||
|
||
let person_lower = final_person.as_ref().map(|n| n.to_lowercase());
|
||
let primary_lower = final_person.as_ref().and_then(|n| {
|
||
let (p, _) = title_parser::split_person_name_with_aliases(n);
|
||
if p.is_empty() { None } else { Some(p.to_lowercase()) }
|
||
});
|
||
|
||
for tag in old_tags {
|
||
// Step 1: 清洗标点
|
||
let cleaned = match tag_extractor::clean_tag(&tag) {
|
||
Some(t) => t,
|
||
None => { changed = true; continue; }
|
||
};
|
||
if cleaned != tag { changed = true; }
|
||
|
||
// Step 2: 短标签过滤
|
||
if is_short_tag(&cleaned) {
|
||
changed = true;
|
||
result.short_tags_removed += 1;
|
||
continue;
|
||
}
|
||
|
||
// Step 3: 人名标签过滤
|
||
let cleaned_lower = cleaned.to_lowercase();
|
||
let mut is_person_tag = false;
|
||
if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) {
|
||
is_person_tag =
|
||
cleaned_lower == *pl
|
||
|| cleaned_lower == *pr
|
||
|| (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic())
|
||
&& !cleaned_lower.starts_with("cos")
|
||
&& pl.contains(&cleaned_lower))
|
||
|| (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower));
|
||
}
|
||
|
||
if is_person_tag {
|
||
person_like_removed_this += 1;
|
||
changed = true;
|
||
continue;
|
||
}
|
||
|
||
new_tags.push(cleaned);
|
||
}
|
||
|
||
result.person_like_tags_removed += person_like_removed_this;
|
||
|
||
if !changed { continue; }
|
||
result.tags_cleaned += 1;
|
||
result.dirty.add(&album.id);
|
||
|
||
// 收集变更,等下批量写
|
||
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
|
||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.filter_map(|pair| {
|
||
let cleaned = tag_extractor::clean_tag(&pair[0])?;
|
||
if is_short_tag(&cleaned) { return None; }
|
||
if known_persons.contains(&pair[0].to_lowercase()) { return None; }
|
||
Some([cleaned, pair[1].clone()])
|
||
})
|
||
.collect();
|
||
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||
updates.push((album.id.clone(), tags_json_new, all_words_json_new));
|
||
}
|
||
|
||
// ── 批量写入:单个事务 ──
|
||
if !updates.is_empty() {
|
||
let _guard = db.write_lock.lock().await;
|
||
let mut tx = match db.pool.begin().await {
|
||
Ok(t) => t,
|
||
Err(e) => {
|
||
eprintln!("[管线·Phase①] 开启事务失败: {}", e);
|
||
return result;
|
||
}
|
||
};
|
||
|
||
for (album_id, tags_json, all_words_json) in &updates {
|
||
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
|
||
}
|
||
|
||
if let Err(e) = tx.commit().await {
|
||
eprintln!("[管线·Phase①] 提交事务失败: {}", e);
|
||
}
|
||
}
|
||
|
||
println!("[管线·Phase①] 合并预处理完成:清洗{}相册,移除短标签{},移除人名标签{},新相册{},脏集合计{}",
|
||
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed,
|
||
result.new_albums_found, result.dirty.len());
|
||
|
||
result
|
||
}
|
||
|
||
/// 合并人物清理:一次人物遍历完成 2 件事
|
||
/// 1. 清理非人物
|
||
/// 2. 清理停用词重叠人物
|
||
async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) {
|
||
let persons = db.get_all_persons_with_counts().await;
|
||
let mut dirty = DirtySet::new();
|
||
let mut non_persons_deleted = 0usize;
|
||
let mut stop_word_persons_removed = 0usize;
|
||
|
||
for (pid, name, _albums, _images) in &persons {
|
||
// 判断是否应该删除
|
||
let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name);
|
||
|
||
let is_stop_word = {
|
||
let excluded = tag_dict::get_tag_dict().is_excluded_person(name);
|
||
let cn_check = title_parser::extract_cn(name)
|
||
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
|
||
.unwrap_or(false);
|
||
excluded || cn_check
|
||
};
|
||
|
||
if !is_non_person && !is_stop_word { continue; }
|
||
|
||
let album_list = db.get_person_albums(*pid).await;
|
||
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
|
||
for aid in &album_ids { dirty.add(aid); }
|
||
|
||
if is_stop_word {
|
||
println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||
stop_word_persons_removed += 1;
|
||
} else {
|
||
println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||
non_persons_deleted += 1;
|
||
}
|
||
|
||
db.update_album_person_by_person_id(*pid).await;
|
||
db.delete_person_completely(*pid).await;
|
||
}
|
||
|
||
if non_persons_deleted > 0 || stop_word_persons_removed > 0 {
|
||
println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}",
|
||
non_persons_deleted, stop_word_persons_removed, dirty.len());
|
||
} else {
|
||
println!("[管线·Phase②] 人物清理完成:无异常");
|
||
}
|
||
|
||
(dirty, non_persons_deleted, stop_word_persons_removed)
|
||
}
|
||
|
||
// ════════════════════════════════════════════════
|
||
// Pipeline 一键执行
|
||
// ════════════════════════════════════════════════
|
||
|
||
#[derive(Debug, Clone, Default)]
|
||
pub struct PipelineResult {
|
||
pub tags_cleaned: usize, // Phase ①
|
||
pub short_tags_removed: usize, // Phase ①
|
||
pub person_like_tags_removed: usize, // Phase ①
|
||
pub dirty_albums_detected: usize, // 总脏集
|
||
pub non_persons_deleted: usize, // Phase ②
|
||
pub stop_word_persons_removed: usize, // Phase ②
|
||
pub new_albums_found: usize, // Phase ①
|
||
pub persons_updated: usize, // Phase ③
|
||
pub tags_updated: usize, // Phase ③
|
||
pub persons_merged: usize, // Phase ④
|
||
pub aliases_moved: usize, // Phase ④
|
||
pub tags_removed_person: usize, // Phase ⑤
|
||
}
|
||
|
||
/// 一键执行精简管线(5 阶段,2-3 次全库扫描)
|
||
///
|
||
/// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描
|
||
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
|
||
/// Phase ③ 增量解析脏集 → 只扫脏集
|
||
/// Phase ④ 人物合并 + 重计 → 1次人物扫描
|
||
/// Phase ⑤ 后校验标签中的人名
|
||
pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
||
// Phase ①:合并预处理(1 次相册扫描)
|
||
let pre = phase_preprocess_combined(db).await;
|
||
let tags_cleaned = pre.tags_cleaned;
|
||
let short_tags_removed = pre.short_tags_removed;
|
||
let person_like_tags_removed = pre.person_like_tags_removed;
|
||
let new_albums_found = pre.new_albums_found;
|
||
|
||
// Phase ②:合并人物清理(1 次人物扫描)
|
||
let (dirty_persons, non_persons_deleted, stop_word_persons_removed) =
|
||
phase_clean_persons_combined(db).await;
|
||
|
||
// 合并脏集
|
||
let mut master_dirty = pre.dirty;
|
||
master_dirty.extend(dirty_persons.into_inner());
|
||
let dirty_albums_detected = master_dirty.len();
|
||
|
||
// Phase ③:增量解析
|
||
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
||
|
||
// Phase ④:人物合并(内部会做包含关系 + 核心合并)
|
||
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
||
|
||
// Phase C:全量重计(合并后修正计数)
|
||
let _recounted = phase_recount_all(db).await;
|
||
|
||
// Phase ⑤:后校验
|
||
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
||
|
||
PipelineResult {
|
||
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
|
||
non_persons_deleted, stop_word_persons_removed, new_albums_found,
|
||
persons_updated, tags_updated, persons_merged,
|
||
aliases_moved, tags_removed_person,
|
||
}
|
||
}
|