update code level
This commit is contained in:
@@ -2,14 +2,12 @@
|
||||
//!
|
||||
//! 核心架构:脏集追踪 + 增量解析
|
||||
//!
|
||||
//! Phase 0 清洗标签标点
|
||||
//! Phase A1 检测标签中的人名并直接提取到 person 字段 → 加入脏集
|
||||
//! Phase A2 清理非人物 → 加入脏集
|
||||
//! Phase A3 收集新相册/半成品 → 加入脏集
|
||||
//! Phase 1 解析脏集相册(增量解析,不扫全库)
|
||||
//! Phase C 全量重计所有 person 的相册计数
|
||||
//! Phase 2 人物合并
|
||||
//! Phase B 后校验标签中的人名
|
||||
//! 精简后流程(6 阶段,2 次全库扫描):
|
||||
//! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描
|
||||
//! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描
|
||||
//! Phase ③ 增量解析脏集相册 → 只扫脏集
|
||||
//! Phase ④ 人物合并 + 重计 → 1次人物扫描
|
||||
//! Phase ⑤ 后校验标签中的人名 → 可优化
|
||||
//!
|
||||
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
|
||||
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
|
||||
@@ -27,6 +25,7 @@ use rayon::prelude::*;
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
/// 脏集:记录哪些相册需要重新解析
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DirtySet {
|
||||
ids: HashSet<String>,
|
||||
}
|
||||
@@ -41,6 +40,10 @@ impl DirtySet {
|
||||
pub fn into_inner(self) -> HashSet<String> { self.ids }
|
||||
}
|
||||
|
||||
impl Default for DirtySet {
|
||||
fn default() -> Self { Self::new() }
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Phase 0:清洗已有标签中的头尾标点
|
||||
// ════════════════════════════════════════════════
|
||||
@@ -93,6 +96,78 @@ pub async fn phase_clean_tags(db: &Database) -> usize {
|
||||
cleaned_count
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Phase 0B:移除过短的标签(少于2个汉字/字母)
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
/// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个
|
||||
fn is_short_tag(tag: &str) -> bool {
|
||||
let mut valid_count = 0usize;
|
||||
for ch in tag.chars() {
|
||||
if ch.is_ascii_alphabetic() {
|
||||
valid_count += 1;
|
||||
} else if ('\u{4e00}'..='\u{9fff}').contains(&ch) {
|
||||
valid_count += 1;
|
||||
}
|
||||
// 数字、符号等不算有效字符
|
||||
}
|
||||
valid_count < 2
|
||||
}
|
||||
|
||||
/// Phase 0B:扫描所有标签,移除过短的标签(有效字符 < 2),
|
||||
/// 受影响的相册加入脏集等待重新解析。
|
||||
pub async fn phase_remove_short_tags(db: &Database) -> (DirtySet, usize) {
|
||||
let albums = db.get_all_albums_full().await;
|
||||
let mut dirty = DirtySet::new();
|
||||
let mut removed_total = 0usize;
|
||||
let mut removed_samples: HashSet<String> = HashSet::new();
|
||||
|
||||
for album in &albums {
|
||||
let tags_json = match album.tags.as_deref() {
|
||||
Some(t) if !t.is_empty() && t != "[]" => t,
|
||||
_ => continue,
|
||||
};
|
||||
let old_tags: Vec<String> = match serde_json::from_str(tags_json) {
|
||||
Ok(t) => t, Err(_) => continue,
|
||||
};
|
||||
if old_tags.is_empty() { continue; }
|
||||
|
||||
let mut changed = false;
|
||||
let mut new_tags = Vec::new();
|
||||
for tag in old_tags {
|
||||
if is_short_tag(&tag) {
|
||||
removed_samples.insert(tag);
|
||||
changed = true;
|
||||
removed_total += 1;
|
||||
} else {
|
||||
new_tags.push(tag);
|
||||
}
|
||||
}
|
||||
if !changed { continue; }
|
||||
|
||||
dirty.add(&album.id);
|
||||
|
||||
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
|
||||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|pair| !is_short_tag(&pair[0]))
|
||||
.collect();
|
||||
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||||
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
|
||||
}
|
||||
|
||||
if removed_total > 0 {
|
||||
let samples: Vec<_> = removed_samples.iter().take(10).collect();
|
||||
println!("[管线·Phase0B] 短标签清理完成:移除 {} 个短标签,影响 {} 个相册,示例: {:?}",
|
||||
removed_total, dirty.len(), samples);
|
||||
} else {
|
||||
println!("[管线·Phase0B] 短标签清理完成:无短标签");
|
||||
}
|
||||
(dirty, removed_total)
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Phase A1:检测标签中的人名并直接提取到 person 字段
|
||||
// ════════════════════════════════════════════════
|
||||
@@ -465,48 +540,278 @@ pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Pipeline 一键执行(7 阶段)
|
||||
// 合并阶段 — 减少全库扫描次数
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
/// 合并预处理结果
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct PreprocessResult {
|
||||
tags_cleaned: usize,
|
||||
short_tags_removed: usize,
|
||||
person_like_tags_removed: usize,
|
||||
new_albums_found: usize,
|
||||
dirty: DirtySet,
|
||||
}
|
||||
|
||||
/// 合并预处理:一次相册扫描完成 4 件事
|
||||
/// 1. 清洗标签标点
|
||||
/// 2. 移除短标签
|
||||
/// 3. 检测并移除标签中的人名
|
||||
/// 4. 收集新相册/半成品
|
||||
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
|
||||
let albums = db.get_all_albums_full().await;
|
||||
|
||||
let known_persons: HashSet<String> = db.get_all_person_names().await
|
||||
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
|
||||
|
||||
let mut result = PreprocessResult::default();
|
||||
|
||||
for album in &albums {
|
||||
let tags_json = album.tags.as_deref();
|
||||
let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false);
|
||||
let has_person = album.person_name_original.as_deref()
|
||||
.map(|s| !s.is_empty()).unwrap_or(false);
|
||||
|
||||
// ── 新相册/半成品检测 ──
|
||||
if !has_tags || !has_person {
|
||||
result.dirty.add(&album.id);
|
||||
if !has_tags {
|
||||
result.new_albums_found += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ── 没有标签就不需要清洗/检测 ──
|
||||
if !has_tags { continue; }
|
||||
|
||||
let old_tags: Vec<String> = match serde_json::from_str(tags_json.unwrap()) {
|
||||
Ok(t) => t, Err(_) => continue,
|
||||
};
|
||||
if old_tags.is_empty() { continue; }
|
||||
|
||||
// ── 三步标签处理合并到一次遍历 ──
|
||||
let mut changed = false;
|
||||
let mut new_tags: Vec<String> = Vec::new();
|
||||
let mut person_like_removed_this = 0usize;
|
||||
|
||||
// 先找到最优人名(用于后续从标签中移除人名相关)
|
||||
// 第一层:逐个标签收集候选人名
|
||||
let mut known_person_found: Option<String> = None;
|
||||
let mut person_like_tags: Vec<String> = Vec::new();
|
||||
|
||||
for tag in &old_tags {
|
||||
let tag_lower = tag.to_lowercase();
|
||||
if known_persons.contains(&tag_lower) {
|
||||
known_person_found = Some(tag.clone());
|
||||
continue;
|
||||
}
|
||||
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
|
||||
person_like_tags.push(tag.clone());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
|
||||
if let Some(cn) = title_parser::extract_cn(t) {
|
||||
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
|
||||
} else { t.len() }
|
||||
});
|
||||
|
||||
let combined = old_tags.join(" ");
|
||||
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
|
||||
|
||||
let final_person = known_person_found.or_else(|| {
|
||||
match (&jieba_name, &best_layer1) {
|
||||
(Some(j), Some(b)) => {
|
||||
let j_len: usize = j.len();
|
||||
let b_len: usize = b.len();
|
||||
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
|
||||
}
|
||||
(Some(j), None) => Some(j.clone()),
|
||||
(None, Some(b)) => Some(b.clone()),
|
||||
(None, None) => None,
|
||||
}
|
||||
});
|
||||
|
||||
let person_lower = final_person.as_ref().map(|n| n.to_lowercase());
|
||||
let primary_lower = final_person.as_ref().and_then(|n| {
|
||||
let (p, _) = title_parser::split_person_name_with_aliases(n);
|
||||
if p.is_empty() { None } else { Some(p.to_lowercase()) }
|
||||
});
|
||||
|
||||
for tag in old_tags {
|
||||
// Step 1: 清洗标点
|
||||
let cleaned = match tag_extractor::clean_tag(&tag) {
|
||||
Some(t) => t,
|
||||
None => { changed = true; continue; }
|
||||
};
|
||||
if cleaned != tag { changed = true; }
|
||||
|
||||
// Step 2: 短标签过滤
|
||||
if is_short_tag(&cleaned) {
|
||||
changed = true;
|
||||
result.short_tags_removed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Step 3: 人名标签过滤
|
||||
let cleaned_lower = cleaned.to_lowercase();
|
||||
let mut is_person_tag = false;
|
||||
if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) {
|
||||
is_person_tag =
|
||||
cleaned_lower == *pl
|
||||
|| cleaned_lower == *pr
|
||||
|| (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic())
|
||||
&& !cleaned_lower.starts_with("cos")
|
||||
&& pl.contains(&cleaned_lower))
|
||||
|| (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower));
|
||||
}
|
||||
|
||||
if is_person_tag {
|
||||
person_like_removed_this += 1;
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
new_tags.push(cleaned);
|
||||
}
|
||||
|
||||
result.person_like_tags_removed += person_like_removed_this;
|
||||
|
||||
if !changed { continue; }
|
||||
result.tags_cleaned += 1;
|
||||
result.dirty.add(&album.id);
|
||||
|
||||
// 写回
|
||||
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
|
||||
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
|
||||
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|pair| {
|
||||
let cleaned = tag_extractor::clean_tag(&pair[0])?;
|
||||
if is_short_tag(&cleaned) { return None; }
|
||||
if known_persons.contains(&pair[0].to_lowercase()) { return None; }
|
||||
Some([cleaned, pair[1].clone()])
|
||||
})
|
||||
.collect();
|
||||
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
|
||||
db.update_album_tags(&album.id, &tags_json_new, &all_words_json_new).await;
|
||||
}
|
||||
|
||||
println!("[管线·Phase①] 合并预处理完成:清洗{}相册,移除短标签{},移除人名标签{},新相册{},脏集合计{}",
|
||||
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed,
|
||||
result.new_albums_found, result.dirty.len());
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// 合并人物清理:一次人物遍历完成 2 件事
|
||||
/// 1. 清理非人物
|
||||
/// 2. 清理停用词重叠人物
|
||||
async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) {
|
||||
let persons = db.get_all_persons_with_counts().await;
|
||||
let mut dirty = DirtySet::new();
|
||||
let mut non_persons_deleted = 0usize;
|
||||
let mut stop_word_persons_removed = 0usize;
|
||||
|
||||
for (pid, name, _albums, _images) in &persons {
|
||||
// 判断是否应该删除
|
||||
let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name);
|
||||
|
||||
let is_stop_word = {
|
||||
let excluded = tag_dict::get_tag_dict().is_excluded_person(name);
|
||||
let cn_check = title_parser::extract_cn(name)
|
||||
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
|
||||
.unwrap_or(false);
|
||||
excluded || cn_check
|
||||
};
|
||||
|
||||
if !is_non_person && !is_stop_word { continue; }
|
||||
|
||||
let album_list = db.get_person_albums(*pid).await;
|
||||
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
|
||||
for aid in &album_ids { dirty.add(aid); }
|
||||
|
||||
if is_stop_word {
|
||||
println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||||
stop_word_persons_removed += 1;
|
||||
} else {
|
||||
println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
|
||||
non_persons_deleted += 1;
|
||||
}
|
||||
|
||||
db.update_album_person_by_person_id(*pid).await;
|
||||
db.delete_person_completely(*pid).await;
|
||||
}
|
||||
|
||||
if non_persons_deleted > 0 || stop_word_persons_removed > 0 {
|
||||
println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}",
|
||||
non_persons_deleted, stop_word_persons_removed, dirty.len());
|
||||
} else {
|
||||
println!("[管线·Phase②] 人物清理完成:无异常");
|
||||
}
|
||||
|
||||
(dirty, non_persons_deleted, stop_word_persons_removed)
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Pipeline 一键执行
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PipelineResult {
|
||||
pub tags_cleaned: usize, // Phase 0
|
||||
pub person_like_tags_removed: usize, // Phase A1
|
||||
pub dirty_albums_detected: usize, // A1+A2+A2B+A3
|
||||
pub non_persons_deleted: usize, // Phase A2
|
||||
pub stop_word_persons_removed: usize, // Phase A2B
|
||||
pub new_albums_found: usize, // Phase A3
|
||||
pub persons_updated: usize, // Phase 1
|
||||
pub tags_updated: usize, // Phase 1
|
||||
pub persons_merged: usize, // Phase 2
|
||||
pub aliases_moved: usize, // Phase 2
|
||||
pub tags_removed_person: usize, // Phase B
|
||||
pub tags_cleaned: usize, // Phase ①
|
||||
pub short_tags_removed: usize, // Phase ①
|
||||
pub person_like_tags_removed: usize, // Phase ①
|
||||
pub dirty_albums_detected: usize, // 总脏集
|
||||
pub non_persons_deleted: usize, // Phase ②
|
||||
pub stop_word_persons_removed: usize, // Phase ②
|
||||
pub new_albums_found: usize, // Phase ①
|
||||
pub persons_updated: usize, // Phase ③
|
||||
pub tags_updated: usize, // Phase ③
|
||||
pub persons_merged: usize, // Phase ④
|
||||
pub aliases_moved: usize, // Phase ④
|
||||
pub tags_removed_person: usize, // Phase ⑤
|
||||
}
|
||||
|
||||
/// 一键执行完整管线(8 阶段顺序串行)
|
||||
/// 一键执行精简管线(5 阶段,2-3 次全库扫描)
|
||||
///
|
||||
/// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描
|
||||
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
|
||||
/// Phase ③ 增量解析脏集 → 只扫脏集
|
||||
/// Phase ④ 人物合并 + 重计 → 1次人物扫描
|
||||
/// Phase ⑤ 后校验标签中的人名
|
||||
pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
||||
let tags_cleaned = phase_clean_tags(db).await;
|
||||
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
|
||||
let (dirty_a2, non_persons_deleted) = phase_clean_persons(db).await;
|
||||
let (dirty_a2b, stop_word_persons_removed) = phase_clean_stop_word_persons(db).await;
|
||||
let dirty_a3 = phase_collect_new_albums(db).await;
|
||||
let new_albums_found = dirty_a3.len();
|
||||
// Phase ①:合并预处理(1 次相册扫描)
|
||||
let pre = phase_preprocess_combined(db).await;
|
||||
let tags_cleaned = pre.tags_cleaned;
|
||||
let short_tags_removed = pre.short_tags_removed;
|
||||
let person_like_tags_removed = pre.person_like_tags_removed;
|
||||
let new_albums_found = pre.new_albums_found;
|
||||
|
||||
let mut master_dirty = DirtySet::new();
|
||||
master_dirty.extend(dirty_a1.into_inner());
|
||||
master_dirty.extend(dirty_a2.into_inner());
|
||||
master_dirty.extend(dirty_a2b.into_inner());
|
||||
master_dirty.extend(dirty_a3.into_inner());
|
||||
// Phase ②:合并人物清理(1 次人物扫描)
|
||||
let (dirty_persons, non_persons_deleted, stop_word_persons_removed) =
|
||||
phase_clean_persons_combined(db).await;
|
||||
|
||||
// 合并脏集
|
||||
let mut master_dirty = pre.dirty;
|
||||
master_dirty.extend(dirty_persons.into_inner());
|
||||
let dirty_albums_detected = master_dirty.len();
|
||||
|
||||
// Phase ③:增量解析
|
||||
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
||||
// Phase C:全量重计(在合并前修正所有计数,让合并看到准确数值)
|
||||
let _recounted = phase_recount_all(db).await;
|
||||
|
||||
// Phase ④:人物合并(内部会做包含关系 + 核心合并)
|
||||
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
||||
|
||||
// Phase C:全量重计(合并后修正计数)
|
||||
let _recounted = phase_recount_all(db).await;
|
||||
|
||||
// Phase ⑤:后校验
|
||||
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
||||
|
||||
PipelineResult {
|
||||
tags_cleaned, person_like_tags_removed, dirty_albums_detected,
|
||||
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
|
||||
non_persons_deleted, stop_word_persons_removed, new_albums_found,
|
||||
persons_updated, tags_updated, persons_merged,
|
||||
aliases_moved, tags_removed_person,
|
||||
|
||||
Reference in New Issue
Block a user