update code level

This commit is contained in:
lychang
2026-07-20 07:35:26 +08:00
parent 5e6b27f26d
commit a33ef65bd7
6 changed files with 674 additions and 184 deletions

View File

@@ -2,14 +2,12 @@
//!
//! 核心架构:脏集追踪 + 增量解析
//!
//! Phase 0 清洗标签标点
//! Phase A1 检测标签中的人名并直接提取到 person 字段 → 加入脏集
//! Phase A2 清理非人物 → 加入脏集
//! Phase A3 收集新相册/半成品 → 加入脏集
//! Phase 1 解析脏集相册(增量解析,不扫全库)
//! Phase C 全量重计所有 person 的相册计数
//! Phase 2 人物合并
//! Phase B 后校验标签中的人名
//! 精简后流程(6 阶段,2 次全库扫描):
//! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描
//! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描
//! Phase ③ 增量解析脏集相册 → 只扫脏集
//! Phase ④ 人物合并 + 重计 → 1次人物扫描
//! Phase ⑤ 后校验标签中的人名 → 可优化
//!
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
@@ -27,6 +25,7 @@ use rayon::prelude::*;
// ════════════════════════════════════════════════
/// 脏集:记录哪些相册需要重新解析
#[derive(Debug, Clone)]
pub struct DirtySet {
ids: HashSet<String>,
}
@@ -41,6 +40,10 @@ impl DirtySet {
pub fn into_inner(self) -> HashSet<String> { self.ids }
}
impl Default for DirtySet {
fn default() -> Self { Self::new() }
}
// ════════════════════════════════════════════════
// Phase 0:清洗已有标签中的头尾标点
// ════════════════════════════════════════════════
@@ -93,6 +96,78 @@ pub async fn phase_clean_tags(db: &Database) -> usize {
cleaned_count
}
// ════════════════════════════════════════════════
// Phase 0B:移除过短的标签(少于2个汉字/字母)
// ════════════════════════════════════════════════
/// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个
fn is_short_tag(tag: &str) -> bool {
let mut valid_count = 0usize;
for ch in tag.chars() {
if ch.is_ascii_alphabetic() {
valid_count += 1;
} else if ('\u{4e00}'..='\u{9fff}').contains(&ch) {
valid_count += 1;
}
// 数字、符号等不算有效字符
}
valid_count < 2
}
/// Phase 0B:扫描所有标签,移除过短的标签(有效字符 < 2),
/// 受影响的相册加入脏集等待重新解析。
pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) {
let albums = db.get_all_albums_full().await;
let mut dirty = DirtySet::new();
let mut removed_total = 0usize;
let mut removed_samples: HashSet<String> = HashSet::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t,
_ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
let mut changed = false;
let mut new_tags = Vec::new();
for tag in old_tags {
if is_short_tag(&tag) {
removed_samples.insert(tag);
changed = true;
removed_total += 1;
} else {
new_tags.push(tag);
}
}
if !changed { continue; }
dirty.add(&album.id);
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter(|pair| !is_short_tag(&pair[0]))
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
if removed_total > 0 {
let samples: Vec<_> = removed_samples.iter().take(10).collect();
println!("[管线·Phase0B] 短标签清理完成:移除 {} 个短标签,影响 {} 个相册,示例: {:?}",
removed_total, dirty.len(), samples);
} else {
println!("[管线·Phase0B] 短标签清理完成:无短标签");
}
(dirty, removed_total)
}
// ════════════════════════════════════════════════
// Phase A1:检测标签中的人名并直接提取到 person 字段
// ════════════════════════════════════════════════
@@ -465,48 +540,278 @@ pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
}
// ════════════════════════════════════════════════
// Pipeline 一键执行(7 阶段)
// 合并阶段 — 减少全库扫描次数
// ════════════════════════════════════════════════
/// 合并预处理结果
#[derive(Debug, Clone, Default)]
struct PreprocessResult {
tags_cleaned: usize,
short_tags_removed: usize,
person_like_tags_removed: usize,
new_albums_found: usize,
dirty: DirtySet,
}
/// 合并预处理:一次相册扫描完成 4 件事
/// 1. 清洗标签标点
/// 2. 移除短标签
/// 3. 检测并移除标签中的人名
/// 4. 收集新相册/半成品
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
let albums = db.get_all_albums_full().await;
let known_persons: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
let mut result = PreprocessResult::default();
for album in &albums {
let tags_json = album.tags.as_deref();
let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false);
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
// ── 新相册/半成品检测 ──
if !has_tags || !has_person {
result.dirty.add(&album.id);
if !has_tags {
result.new_albums_found += 1;
}
}
// ── 没有标签就不需要清洗/检测 ──
if !has_tags { continue; }
let old_tags: Vec<String> = match serde_json::from_str(tags_json.unwrap()) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
// ── 三步标签处理合并到一次遍历 ──
let mut changed = false;
let mut new_tags: Vec<String> = Vec::new();
let mut person_like_removed_this = 0usize;
// 先找到最优人名(用于后续从标签中移除人名相关)
// 第一层:逐个标签收集候选人名
let mut known_person_found: Option<String> = None;
let mut person_like_tags: Vec<String> = Vec::new();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
if known_persons.contains(&tag_lower) {
known_person_found = Some(tag.clone());
continue;
}
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
person_like_tags.push(tag.clone());
continue;
}
}
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
if let Some(cn) = title_parser::extract_cn(t) {
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
} else { t.len() }
});
let combined = old_tags.join(" ");
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
let final_person = known_person_found.or_else(|| {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),
(None, None) => None,
}
});
let person_lower = final_person.as_ref().map(|n| n.to_lowercase());
let primary_lower = final_person.as_ref().and_then(|n| {
let (p, _) = title_parser::split_person_name_with_aliases(n);
if p.is_empty() { None } else { Some(p.to_lowercase()) }
});
for tag in old_tags {
// Step 1: 清洗标点
let cleaned = match tag_extractor::clean_tag(&tag) {
Some(t) => t,
None => { changed = true; continue; }
};
if cleaned != tag { changed = true; }
// Step 2: 短标签过滤
if is_short_tag(&cleaned) {
changed = true;
result.short_tags_removed += 1;
continue;
}
// Step 3: 人名标签过滤
let cleaned_lower = cleaned.to_lowercase();
let mut is_person_tag = false;
if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) {
is_person_tag =
cleaned_lower == *pl
|| cleaned_lower == *pr
|| (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic())
&& !cleaned_lower.starts_with("cos")
&& pl.contains(&cleaned_lower))
|| (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower));
}
if is_person_tag {
person_like_removed_this += 1;
changed = true;
continue;
}
new_tags.push(cleaned);
}
result.person_like_tags_removed += person_like_removed_this;
if !changed { continue; }
result.tags_cleaned += 1;
result.dirty.add(&album.id);
// 写回
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter_map(|pair| {
let cleaned = tag_extractor::clean_tag(&pair[0])?;
if is_short_tag(&cleaned) { return None; }
if known_persons.contains(&pair[0].to_lowercase()) { return None; }
Some([cleaned, pair[1].clone()])
})
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
}
println!("[管线·Phase①] 合并预处理完成:清洗{}相册,移除短标签{},移除人名标签{},新相册{},脏集合计{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed,
result.new_albums_found, result.dirty.len());
result
}
/// 合并人物清理:一次人物遍历完成 2 件事
/// 1. 清理非人物
/// 2. 清理停用词重叠人物
async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut non_persons_deleted = 0usize;
let mut stop_word_persons_removed = 0usize;
for (pid, name, _albums, _images) in &persons {
// 判断是否应该删除
let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name);
let is_stop_word = {
let excluded = tag_dict::get_tag_dict().is_excluded_person(name);
let cn_check = title_parser::extract_cn(name)
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
.unwrap_or(false);
excluded || cn_check
};
if !is_non_person && !is_stop_word { continue; }
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
if is_stop_word {
println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
stop_word_persons_removed += 1;
} else {
println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
non_persons_deleted += 1;
}
db.update_album_person_by_person_id(*pid).await;
db.delete_person_completely(*pid).await;
}
if non_persons_deleted > 0 || stop_word_persons_removed > 0 {
println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}",
non_persons_deleted, stop_word_persons_removed, dirty.len());
} else {
println!("[管线·Phase②] 人物清理完成:无异常");
}
(dirty, non_persons_deleted, stop_word_persons_removed)
}
// ════════════════════════════════════════════════
// Pipeline 一键执行
// ════════════════════════════════════════════════
#[derive(Debug, Clone, Default)]
pub struct PipelineResult {
pub tags_cleaned: usize, // Phase 0
pub person_like_tags_removed: usize, // Phase A1
pub dirty_albums_detected: usize, // A1+A2+A2B+A3
pub non_persons_deleted: usize, // Phase A2
pub stop_word_persons_removed: usize, // Phase A2B
pub new_albums_found: usize, // Phase A3
pub persons_updated: usize, // Phase 1
pub tags_updated: usize, // Phase 1
pub persons_merged: usize, // Phase 2
pub aliases_moved: usize, // Phase 2
pub tags_removed_person: usize, // Phase B
pub tags_cleaned: usize, // Phase ①
pub short_tags_removed: usize, // Phase ①
pub person_like_tags_removed: usize, // Phase ①
pub dirty_albums_detected: usize, // 总脏集
pub non_persons_deleted: usize, // Phase ②
pub stop_word_persons_removed: usize, // Phase ②
pub new_albums_found: usize, // Phase ①
pub persons_updated: usize, // Phase ③
pub tags_updated: usize, // Phase ③
pub persons_merged: usize, // Phase ④
pub aliases_moved: usize, // Phase ④
pub tags_removed_person: usize, // Phase ⑤
}
/// 一键执行完整管线(8 阶段顺序串行)
/// 一键执行精简管线(5 阶段,2-3 次全库扫描)
///
/// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
/// Phase ③ 增量解析脏集 → 只扫脏集
/// Phase ④ 人物合并 + 重计 → 1次人物扫描
/// Phase ⑤ 后校验标签中的人名
pub async fn run_pipeline(db: &Database) -> PipelineResult {
let tags_cleaned = phase_clean_tags(db).await;
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
let (dirty_a2, non_persons_deleted) = phase_clean_persons(db).await;
let (dirty_a2b, stop_word_persons_removed) = phase_clean_stop_word_persons(db).await;
let dirty_a3 = phase_collect_new_albums(db).await;
let new_albums_found = dirty_a3.len();
// Phase ①:合并预处理(1 次相册扫描)
let pre = phase_preprocess_combined(db).await;
let tags_cleaned = pre.tags_cleaned;
let short_tags_removed = pre.short_tags_removed;
let person_like_tags_removed = pre.person_like_tags_removed;
let new_albums_found = pre.new_albums_found;
let mut master_dirty = DirtySet::new();
master_dirty.extend(dirty_a1.into_inner());
master_dirty.extend(dirty_a2.into_inner());
master_dirty.extend(dirty_a2b.into_inner());
master_dirty.extend(dirty_a3.into_inner());
// Phase ②:合并人物清理(1 次人物扫描)
let (dirty_persons, non_persons_deleted, stop_word_persons_removed) =
phase_clean_persons_combined(db).await;
// 合并脏集
let mut master_dirty = pre.dirty;
master_dirty.extend(dirty_persons.into_inner());
let dirty_albums_detected = master_dirty.len();
// Phase ③:增量解析
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
// Phase C:全量重计(在合并前修正所有计数,让合并看到准确数值)
let _recounted = phase_recount_all(db).await;
// Phase ④:人物合并(内部会做包含关系 + 核心合并)
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
// Phase C:全量重计(合并后修正计数)
let _recounted = phase_recount_all(db).await;
// Phase ⑤:后校验
let tags_removed_person = phase_clean_tags_after_parse(db).await;
PipelineResult {
tags_cleaned, person_like_tags_removed, dirty_albums_detected,
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
non_persons_deleted, stop_word_persons_removed, new_albums_found,
persons_updated, tags_updated, persons_merged,
aliases_moved, tags_removed_person,