This commit is contained in:
lychang
2026-07-20 04:46:02 +08:00
parent 00af6e8528
commit a6bfe7eda2
15 changed files with 927 additions and 864 deletions

View File

@@ -7,6 +7,7 @@
//! Phase A2 清理非人物 → 加入脏集
//! Phase A3 收集新相册/半成品 → 加入脏集
//! Phase 1 解析脏集相册(增量解析,不扫全库)
//! Phase C 全量重计所有 person 的相册计数
//! Phase 2 人物合并
//! Phase B 后校验标签中的人名
//!
@@ -19,6 +20,7 @@ use crate::engine::tag_extractor;
use crate::engine::person_merger;
use crate::engine::tag_dict;
use std::collections::HashSet;
use rayon::prelude::*;
// ════════════════════════════════════════════════
// 脏集 — 追踪需要重新解析的相册 ID
@@ -324,7 +326,8 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
// ════════════════════════════════════════════════
// Phase 1:增量解析 — 只处理脏集中的相册
// ════════════════════════════════════════════════
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
// └─ Phase 1b:批量写库(IO 密集,一个事务)
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
if dirty_set.is_empty() {
@@ -333,98 +336,70 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
}
let albums = db.get_all_albums_full().await;
let patterns = person_merger::collect_person_patterns(db).await;
let use_patterns = !patterns.is_empty();
let mut person_updated = 0usize;
let mut tag_updated = 0usize;
let mut skipped = 0usize;
for album in &albums {
if !dirty_set.contains(&album.id) { continue; }
let title = album.title.as_deref().unwrap_or("");
if title.is_empty() { skipped += 1; continue; }
// ── Phase 1a:并行解析(CPU bound,rayon) ──
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
.filter(|a| dirty_set.contains(&a.id))
.collect();
let (jieba_person, jieba_tags) = title_parser::extract_person_and_tags(title);
let skipped = dirty_albums.iter().filter(|a| {
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
}).count();
let total = dirty_albums.len();
let best_person = if use_patterns {
person_merger::match_title(title, &patterns)
.map(|(_, name, _)| name).or(jieba_person)
} else {
jieba_person
};
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
Some(title_parser::parse_album_title(&album.id, title))
})
.collect();
// 用事务包裹所有写入,避免逐条 autocommit 导致巨慢
let mut pending_tx: Option<sqlx::Transaction<'_, sqlx::Sqlite>> = None;
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
if let Some(name) = &best_person {
// 尝试用 include 列表扩展为更长的完整匹配(如 "杨晨晨" → "杨晨晨Yome")
let expanded = tag_dict::get_tag_dict()
.find_longest_included_match(name)
.filter(|longer| longer.len() > name.len()); // 仅当更长时替换
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
// ── Phase 1b:批量写入 ──
let _guard = db.write_lock.lock().await;
let mut tx = db.pool.begin().await.expect("begin tx");
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
});
tx.commit().await.expect("commit tx");
let (primary, aliases) = title_parser::split_person_name_with_aliases(effective_name);
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
// 跳过字符串 <= 2 的人名(太短不像人名)
if final_name.chars().count() <= 2 {
continue;
}
let mut tx = db.pool.begin().await.expect("begin tx");
db.update_album_person_with_tx(&mut tx, &album.id, effective_name, &final_name).await;
let pid = db.insert_person_with_tx(&mut tx, &final_name, None).await;
// 先清旧关联:删除该相册的所有旧 person_album 记录
db.delete_album_person_albums_with_tx(&mut tx, &album.id).await;
db.insert_person_album_with_tx(&mut tx, pid, &album.id).await;
// 先清旧别名:删除该人物的所有旧别名,再插入新的
db.delete_person_aliases_with_tx(&mut tx, pid).await;
for (at, an) in &aliases {
if !an.is_empty() && an != &final_name {
db.insert_person_alias_with_tx(&mut tx, pid, at, an).await;
}
}
db.update_person_counts_with_tx(&mut tx, pid).await;
pending_tx = Some(tx);
person_updated += 1;
}
// 确定标签排除用的人名 — 用最完整的名字
// 优先级:person_name_original(完整含英文后缀) > best_person > person_name_cleaned(仅中文)
let exclude_person = album.person_name_original.as_deref()
.filter(|s| !s.is_empty())
.or_else(|| best_person.as_deref())
.or_else(|| album.person_name_cleaned.as_deref().filter(|s| !s.is_empty()));
let mut raw_tags = tag_extractor::extract_tags_from_title(title, exclude_person);
raw_tags.extend(jieba_tags);
let unique = tag_extractor::unique_tags(raw_tags);
let tag_names: Vec<String> = unique.iter().map(|(t, _)| t.clone()).collect();
if !tag_names.is_empty() {
let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string());
let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect();
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
let mut tx = match pending_tx.take() {
Some(t) => t,
None => db.pool.begin().await.expect("begin tx"),
};
db.update_album_tags_with_tx(&mut tx, &album.id, &tags_json, &words_json).await;
tx.commit().await.expect("commit tx");
tag_updated += 1;
} else if let Some(tx) = pending_tx.take() {
// 只有 person 写入,没有标签写入
tx.commit().await.expect("commit tx");
}
}
println!("[管线·Phase1] 增量解析完成:脏集 {} 个,处理 {} 个,跳过 {} 个,更新人名 {} 个,更新标签 {} 个",
dirty_set.len(), person_updated.max(tag_updated), skipped, person_updated, tag_updated);
(person_updated, tag_updated)
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
total, parsed.len(), skipped, person_count, tag_count);
(person_count, tag_count)
}
// ════════════════════════════════════════════════
// Phase 2:人物合并
// Phase C:全量重计 — 确保所有 person 的计数准确
// ════════════════════════════════════════════════
/// Phase C:全量重计所有人物相册和图片计数
///
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
pub async fn phase_recount_all(db: &Database) -> usize {
let persons = db.get_all_persons_with_counts().await;
let mut fixed = 0usize;
for (pid, name, old_albums, _) in &persons {
let actual = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
if actual != *old_albums {
db.update_person_counts(*pid).await;
fixed += 1;
}
}
if fixed > 0 {
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
} else {
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
}
fixed
}
// ════════════════════════════════════════════════
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
@@ -506,7 +481,7 @@ pub struct PipelineResult {
pub tags_removed_person: usize, // Phase B
}
/// 一键执行完整管线(7 阶段顺序串行)
/// 一键执行完整管线(8 阶段顺序串行)
pub async fn run_pipeline(db: &Database) -> PipelineResult {
let tags_cleaned = phase_clean_tags(db).await;
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
@@ -523,6 +498,8 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
let dirty_albums_detected = master_dirty.len();
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
// Phase C:全量重计(在合并前修正所有计数,让合并看到准确数值)
let _recounted = phase_recount_all(db).await;
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
let tags_removed_person = phase_clean_tags_after_parse(db).await;