update
This commit is contained in:
@@ -7,6 +7,7 @@
|
||||
//! Phase A2 清理非人物 → 加入脏集
|
||||
//! Phase A3 收集新相册/半成品 → 加入脏集
|
||||
//! Phase 1 解析脏集相册(增量解析,不扫全库)
|
||||
//! Phase C 全量重计所有 person 的相册计数
|
||||
//! Phase 2 人物合并
|
||||
//! Phase B 后校验标签中的人名
|
||||
//!
|
||||
@@ -19,6 +20,7 @@ use crate::engine::tag_extractor;
|
||||
use crate::engine::person_merger;
|
||||
use crate::engine::tag_dict;
|
||||
use std::collections::HashSet;
|
||||
use rayon::prelude::*;
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// 脏集 — 追踪需要重新解析的相册 ID
|
||||
@@ -324,7 +326,8 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Phase 1:增量解析 — 只处理脏集中的相册
|
||||
// ════════════════════════════════════════════════
|
||||
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
|
||||
// └─ Phase 1b:批量写库(IO 密集,一个事务)
|
||||
|
||||
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
|
||||
if dirty_set.is_empty() {
|
||||
@@ -333,98 +336,70 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
|
||||
}
|
||||
|
||||
let albums = db.get_all_albums_full().await;
|
||||
let patterns = person_merger::collect_person_patterns(db).await;
|
||||
let use_patterns = !patterns.is_empty();
|
||||
let mut person_updated = 0usize;
|
||||
let mut tag_updated = 0usize;
|
||||
let mut skipped = 0usize;
|
||||
|
||||
for album in &albums {
|
||||
if !dirty_set.contains(&album.id) { continue; }
|
||||
let title = album.title.as_deref().unwrap_or("");
|
||||
if title.is_empty() { skipped += 1; continue; }
|
||||
// ── Phase 1a:并行解析(CPU bound,rayon) ──
|
||||
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
|
||||
.filter(|a| dirty_set.contains(&a.id))
|
||||
.collect();
|
||||
|
||||
let (jieba_person, jieba_tags) = title_parser::extract_person_and_tags(title);
|
||||
let skipped = dirty_albums.iter().filter(|a| {
|
||||
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
|
||||
}).count();
|
||||
let total = dirty_albums.len();
|
||||
|
||||
let best_person = if use_patterns {
|
||||
person_merger::match_title(title, &patterns)
|
||||
.map(|(_, name, _)| name).or(jieba_person)
|
||||
} else {
|
||||
jieba_person
|
||||
};
|
||||
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
|
||||
.par_iter()
|
||||
.filter_map(|album| {
|
||||
let title = album.title.as_deref()?;
|
||||
if title.is_empty() { return None; }
|
||||
Some(title_parser::parse_album_title(&album.id, title))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// 用事务包裹所有写入,避免逐条 autocommit 导致巨慢
|
||||
let mut pending_tx: Option<sqlx::Transaction<'_, sqlx::Sqlite>> = None;
|
||||
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
|
||||
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
|
||||
|
||||
if let Some(name) = &best_person {
|
||||
// 尝试用 include 列表扩展为更长的完整匹配(如 "杨晨晨" → "杨晨晨Yome")
|
||||
let expanded = tag_dict::get_tag_dict()
|
||||
.find_longest_included_match(name)
|
||||
.filter(|longer| longer.len() > name.len()); // 仅当更长时替换
|
||||
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
|
||||
// ── Phase 1b:批量写入 ──
|
||||
let _guard = db.write_lock.lock().await;
|
||||
let mut tx = db.pool.begin().await.expect("begin tx");
|
||||
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
|
||||
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
|
||||
});
|
||||
tx.commit().await.expect("commit tx");
|
||||
|
||||
let (primary, aliases) = title_parser::split_person_name_with_aliases(effective_name);
|
||||
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
|
||||
|
||||
// 跳过字符串 <= 2 的人名(太短不像人名)
|
||||
if final_name.chars().count() <= 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut tx = db.pool.begin().await.expect("begin tx");
|
||||
db.update_album_person_with_tx(&mut tx, &album.id, effective_name, &final_name).await;
|
||||
let pid = db.insert_person_with_tx(&mut tx, &final_name, None).await;
|
||||
// 先清旧关联:删除该相册的所有旧 person_album 记录
|
||||
db.delete_album_person_albums_with_tx(&mut tx, &album.id).await;
|
||||
db.insert_person_album_with_tx(&mut tx, pid, &album.id).await;
|
||||
// 先清旧别名:删除该人物的所有旧别名,再插入新的
|
||||
db.delete_person_aliases_with_tx(&mut tx, pid).await;
|
||||
for (at, an) in &aliases {
|
||||
if !an.is_empty() && an != &final_name {
|
||||
db.insert_person_alias_with_tx(&mut tx, pid, at, an).await;
|
||||
}
|
||||
}
|
||||
db.update_person_counts_with_tx(&mut tx, pid).await;
|
||||
pending_tx = Some(tx);
|
||||
person_updated += 1;
|
||||
}
|
||||
|
||||
// 确定标签排除用的人名 — 用最完整的名字
|
||||
// 优先级:person_name_original(完整含英文后缀) > best_person > person_name_cleaned(仅中文)
|
||||
let exclude_person = album.person_name_original.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.or_else(|| best_person.as_deref())
|
||||
.or_else(|| album.person_name_cleaned.as_deref().filter(|s| !s.is_empty()));
|
||||
|
||||
let mut raw_tags = tag_extractor::extract_tags_from_title(title, exclude_person);
|
||||
raw_tags.extend(jieba_tags);
|
||||
let unique = tag_extractor::unique_tags(raw_tags);
|
||||
let tag_names: Vec<String> = unique.iter().map(|(t, _)| t.clone()).collect();
|
||||
|
||||
if !tag_names.is_empty() {
|
||||
let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string());
|
||||
let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect();
|
||||
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
|
||||
let mut tx = match pending_tx.take() {
|
||||
Some(t) => t,
|
||||
None => db.pool.begin().await.expect("begin tx"),
|
||||
};
|
||||
db.update_album_tags_with_tx(&mut tx, &album.id, &tags_json, &words_json).await;
|
||||
tx.commit().await.expect("commit tx");
|
||||
tag_updated += 1;
|
||||
} else if let Some(tx) = pending_tx.take() {
|
||||
// 只有 person 写入,没有标签写入
|
||||
tx.commit().await.expect("commit tx");
|
||||
}
|
||||
}
|
||||
|
||||
println!("[管线·Phase1] 增量解析完成:脏集 {} 个,处理 {} 个,跳过 {} 个,更新人名 {} 个,更新标签 {} 个",
|
||||
dirty_set.len(), person_updated.max(tag_updated), skipped, person_updated, tag_updated);
|
||||
(person_updated, tag_updated)
|
||||
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
|
||||
total, parsed.len(), skipped, person_count, tag_count);
|
||||
(person_count, tag_count)
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════
|
||||
// Phase 2:人物合并
|
||||
// Phase C:全量重计 — 确保所有 person 的计数准确
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
/// Phase C:全量重计所有人物相册和图片计数
|
||||
///
|
||||
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
|
||||
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
|
||||
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
|
||||
pub async fn phase_recount_all(db: &Database) -> usize {
|
||||
let persons = db.get_all_persons_with_counts().await;
|
||||
let mut fixed = 0usize;
|
||||
for (pid, name, old_albums, _) in &persons {
|
||||
let actual = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
|
||||
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
|
||||
if actual != *old_albums {
|
||||
db.update_person_counts(*pid).await;
|
||||
fixed += 1;
|
||||
}
|
||||
}
|
||||
if fixed > 0 {
|
||||
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
|
||||
} else {
|
||||
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
|
||||
}
|
||||
fixed
|
||||
}
|
||||
// ════════════════════════════════════════════════
|
||||
|
||||
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
|
||||
@@ -506,7 +481,7 @@ pub struct PipelineResult {
|
||||
pub tags_removed_person: usize, // Phase B
|
||||
}
|
||||
|
||||
/// 一键执行完整管线(7 阶段顺序串行)
|
||||
/// 一键执行完整管线(8 阶段顺序串行)
|
||||
pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
||||
let tags_cleaned = phase_clean_tags(db).await;
|
||||
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
|
||||
@@ -523,6 +498,8 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
||||
let dirty_albums_detected = master_dirty.len();
|
||||
|
||||
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
||||
// Phase C:全量重计(在合并前修正所有计数,让合并看到准确数值)
|
||||
let _recounted = phase_recount_all(db).await;
|
||||
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
||||
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user