update code level
This commit is contained in:
@@ -460,7 +460,7 @@ Base URL: `http://<host>:<port>/api`
|
|||||||
## 管线
|
## 管线
|
||||||
|
|
||||||
### POST /api/pipeline/run
|
### POST /api/pipeline/run
|
||||||
手动触发完整管线。
|
手动触发完整管线(dict 优先匹配解析)。
|
||||||
|
|
||||||
**参数**:无
|
**参数**:无
|
||||||
|
|
||||||
@@ -478,9 +478,6 @@ Base URL: `http://<host>:<port>/api`
|
|||||||
"new_albums_found": 0,
|
"new_albums_found": 0,
|
||||||
"persons_updated": 0,
|
"persons_updated": 0,
|
||||||
"tags_updated": 0,
|
"tags_updated": 0,
|
||||||
"persons_merged": 0,
|
|
||||||
"aliases_moved": 0,
|
|
||||||
"tags_removed_person": 0,
|
|
||||||
"empty_albums_deleted": 0,
|
"empty_albums_deleted": 0,
|
||||||
"dup_albums_deleted": 0,
|
"dup_albums_deleted": 0,
|
||||||
"search_db_synced": 0,
|
"search_db_synced": 0,
|
||||||
@@ -522,8 +519,6 @@ Base URL: `http://<host>:<port>/api`
|
|||||||
"success": true,
|
"success": true,
|
||||||
"albums_parsed": 100,
|
"albums_parsed": 100,
|
||||||
"persons_updated": 80,
|
"persons_updated": 80,
|
||||||
"persons_merged": 5,
|
|
||||||
"aliases_moved": 3,
|
|
||||||
"non_persons_deleted": 2,
|
"non_persons_deleted": 2,
|
||||||
"stop_word_persons_removed": 1,
|
"stop_word_persons_removed": 1,
|
||||||
"synced_to_search_db": 100,
|
"synced_to_search_db": 100,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
//! 核心引擎层 — 标题解析、人名匹配、标签提取、推荐计算
|
//! 核心引擎层 — 标题解析、标签提取、推荐计算
|
||||||
|
|
||||||
pub mod title_parser;
|
pub mod title_parser;
|
||||||
pub mod person_merger;
|
|
||||||
pub mod tag_dict;
|
pub mod tag_dict;
|
||||||
pub mod tag_extractor;
|
pub mod tag_extractor;
|
||||||
pub mod recommend;
|
pub mod recommend;
|
||||||
@@ -9,7 +8,6 @@ pub mod pipeline;
|
|||||||
pub mod crawler;
|
pub mod crawler;
|
||||||
|
|
||||||
#[allow(unused)] pub use title_parser::*;
|
#[allow(unused)] pub use title_parser::*;
|
||||||
#[allow(unused)] pub use person_merger::*;
|
|
||||||
#[allow(unused)] pub use tag_dict::*;
|
#[allow(unused)] pub use tag_dict::*;
|
||||||
#[allow(unused)] pub use tag_extractor::*;
|
#[allow(unused)] pub use tag_extractor::*;
|
||||||
pub use recommend::*;
|
pub use recommend::*;
|
||||||
|
|||||||
@@ -1,493 +0,0 @@
|
|||||||
//! 人名匹配与合并
|
|
||||||
//! 对应 Python 的 match_person_from_title.py + merge_persons_by_core.py
|
|
||||||
//! 所有函数均为 async,因为底层 Database 操作是异步的
|
|
||||||
|
|
||||||
use crate::db::Database;
|
|
||||||
use crate::engine::title_parser::{extract_cn, is_excluded_name};
|
|
||||||
use std::collections::{HashMap, HashSet};
|
|
||||||
|
|
||||||
/// 匹配模式
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct Pattern {
|
|
||||||
pub text: String,
|
|
||||||
pub person_name: String,
|
|
||||||
pub source: String, // "name" | "alias"
|
|
||||||
pub length: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 匹配结果
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct MatchResult {
|
|
||||||
pub matched: bool,
|
|
||||||
pub method: String,
|
|
||||||
pub person: Option<String>,
|
|
||||||
pub matched_text: Option<String>,
|
|
||||||
pub all_persons: Vec<String>,
|
|
||||||
pub multi_person: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 深度匹配结果
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct DeepMatchResult {
|
|
||||||
pub matched: bool,
|
|
||||||
pub method: String, // "jieba_extract" | "exact_name_match" | "alias_fuzzy_match"
|
|
||||||
pub person: Option<String>,
|
|
||||||
pub matched_text: Option<String>,
|
|
||||||
pub all_persons: Vec<String>,
|
|
||||||
pub multi_person: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Pass1: 收集模式 ──
|
|
||||||
|
|
||||||
/// 收集所有人名和别名,构建匹配模式列表
|
|
||||||
/// 按长度降序排列,确保长名优先匹配
|
|
||||||
pub async fn collect_person_patterns(db: &Database) -> Vec<Pattern> {
|
|
||||||
let mut patterns = Vec::new();
|
|
||||||
|
|
||||||
// 1. 收集所有人名
|
|
||||||
let rows = db.get_all_person_names().await;
|
|
||||||
for (name, _albums) in &rows {
|
|
||||||
if is_excluded_name(name) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
patterns.push(Pattern {
|
|
||||||
text: name.clone(),
|
|
||||||
person_name: name.clone(),
|
|
||||||
source: "name".to_string(),
|
|
||||||
length: name.len(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 收集所有别名
|
|
||||||
let aliases = db.get_all_aliases().await;
|
|
||||||
for (alias_name, person_name) in &aliases {
|
|
||||||
if is_excluded_name(person_name) || is_excluded_name(alias_name) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if alias_name != person_name {
|
|
||||||
patterns.push(Pattern {
|
|
||||||
text: alias_name.clone(),
|
|
||||||
person_name: person_name.clone(),
|
|
||||||
source: "alias".to_string(),
|
|
||||||
length: alias_name.len(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 去重
|
|
||||||
let mut best_map: HashMap<String, Pattern> = HashMap::new();
|
|
||||||
for p in patterns {
|
|
||||||
let key = p.text.to_lowercase();
|
|
||||||
best_map.entry(key).or_insert(p);
|
|
||||||
}
|
|
||||||
let mut patterns: Vec<Pattern> = best_map.into_values().collect();
|
|
||||||
|
|
||||||
// 4. 按长度降序排列
|
|
||||||
patterns.sort_by(|a, b| b.length.cmp(&a.length));
|
|
||||||
|
|
||||||
patterns
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pass1: 简单子串匹配
|
|
||||||
pub fn match_title(title: &str, patterns: &[Pattern]) -> Option<(String, String, String)> {
|
|
||||||
if title.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
for p in patterns {
|
|
||||||
if p.text.len() < 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if title.contains(&p.text) {
|
|
||||||
return Some((p.text.clone(), p.person_name.clone(), p.source.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pass2: 深度匹配(多重策略)
|
|
||||||
pub fn deep_match_title(
|
|
||||||
title: &str,
|
|
||||||
known_names: &HashSet<String>,
|
|
||||||
known_aliases: &HashSet<String>,
|
|
||||||
) -> DeepMatchResult {
|
|
||||||
let mut result = DeepMatchResult {
|
|
||||||
matched: false,
|
|
||||||
method: String::new(),
|
|
||||||
person: None,
|
|
||||||
matched_text: None,
|
|
||||||
all_persons: Vec::new(),
|
|
||||||
multi_person: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
if title.is_empty() {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
let title_clean = title.trim();
|
|
||||||
|
|
||||||
// 策略1: 用正则提取人名
|
|
||||||
let extracted = crate::engine::title_parser::extract_person_name_from_title(title_clean);
|
|
||||||
if let Some(e) = extracted.first() {
|
|
||||||
if known_names.contains(e) {
|
|
||||||
result.matched = true;
|
|
||||||
result.method = "jieba_extract".to_string();
|
|
||||||
result.person = Some(e.clone());
|
|
||||||
result.matched_text = Some(e.clone());
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 策略2: 标题中精确子串匹配已知人名(长名优先)
|
|
||||||
let mut found: Vec<String> = Vec::new();
|
|
||||||
let mut sorted_names: Vec<&String> = known_names.iter().collect();
|
|
||||||
sorted_names.sort_by(|a, b| b.len().cmp(&a.len()));
|
|
||||||
|
|
||||||
for name in &sorted_names {
|
|
||||||
if name.len() < 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if title_clean.contains(name.as_str()) {
|
|
||||||
found.push(name.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found.is_empty() {
|
|
||||||
found.sort_by(|a, b| b.len().cmp(&a.len()));
|
|
||||||
let best = found[0].clone();
|
|
||||||
result.matched = true;
|
|
||||||
result.method = "exact_name_match".to_string();
|
|
||||||
result.person = Some(best.clone());
|
|
||||||
result.matched_text = Some(best);
|
|
||||||
result.all_persons = found.clone();
|
|
||||||
result.multi_person = found.len() > 1;
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 策略3: 别名模糊匹配
|
|
||||||
let title_lower = title_clean.to_lowercase();
|
|
||||||
let mut sorted_aliases: Vec<&String> = known_aliases.iter().collect();
|
|
||||||
sorted_aliases.sort_by(|a, b| b.len().cmp(&a.len()));
|
|
||||||
|
|
||||||
for alias in &sorted_aliases {
|
|
||||||
if alias.len() < 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if title_lower.contains(alias.to_lowercase().as_str()) {
|
|
||||||
result.matched = true;
|
|
||||||
result.method = "alias_fuzzy_match".to_string();
|
|
||||||
result.person = Some(alias.to_string());
|
|
||||||
result.matched_text = Some(alias.to_string());
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 收集所有已知人名和别名
|
|
||||||
pub async fn collect_known_names(db: &Database) -> (HashSet<String>, HashSet<String>) {
|
|
||||||
let mut known_names = HashSet::new();
|
|
||||||
let mut known_aliases = HashSet::new();
|
|
||||||
|
|
||||||
for (name, _) in db.get_all_person_names().await {
|
|
||||||
known_names.insert(name);
|
|
||||||
}
|
|
||||||
for (alias_name, _) in db.get_all_aliases().await {
|
|
||||||
known_aliases.insert(alias_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
(known_names, known_aliases)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 检测含多个人名的相册
|
|
||||||
pub async fn detect_multi_person_titles(
|
|
||||||
db: &Database,
|
|
||||||
known_names: Option<&HashSet<String>>,
|
|
||||||
) -> Vec<(String, String, Vec<String>)> {
|
|
||||||
let names = match known_names {
|
|
||||||
Some(n) => n.clone(),
|
|
||||||
None => collect_known_names(db).await.0,
|
|
||||||
};
|
|
||||||
|
|
||||||
let albums = db.get_all_albums_full().await;
|
|
||||||
let mut multi_person = Vec::new();
|
|
||||||
|
|
||||||
for album in &albums {
|
|
||||||
let title = album.title.as_deref().unwrap_or("");
|
|
||||||
if title.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut found: Vec<String> = Vec::new();
|
|
||||||
let mut sorted_names: Vec<&String> = names.iter().collect();
|
|
||||||
sorted_names.sort_by(|a, b| b.len().cmp(&a.len()));
|
|
||||||
|
|
||||||
for name in &sorted_names {
|
|
||||||
if name.len() < 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if title.contains(name.as_str()) {
|
|
||||||
found.push(name.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if found.len() > 1 {
|
|
||||||
multi_person.push((
|
|
||||||
album.id.clone(),
|
|
||||||
title.to_string(),
|
|
||||||
found,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
multi_person
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── 中文核心合并分析 ──
|
|
||||||
|
|
||||||
/// 分析所有人名的中文核心分组
|
|
||||||
/// 返回: {core: [(id, name, albums, images), ...]}
|
|
||||||
pub async fn analyze_groups(db: &Database) -> HashMap<String, Vec<(i64, String, i64, i64)>> {
|
|
||||||
let persons = db.get_all_persons_with_counts().await;
|
|
||||||
let mut groups: HashMap<String, Vec<(i64, String, i64, i64)>> = HashMap::new();
|
|
||||||
|
|
||||||
for (pid, name, albums, images) in persons {
|
|
||||||
if let Some(cn) = extract_cn(&name) {
|
|
||||||
if cn.len() >= 2 {
|
|
||||||
groups.entry(cn).or_default().push((pid, name, albums, images));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for items in groups.values_mut() {
|
|
||||||
items.sort_by(|a, b| b.2.cmp(&a.2));
|
|
||||||
}
|
|
||||||
|
|
||||||
groups.retain(|_, v| v.len() >= 2);
|
|
||||||
|
|
||||||
groups
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 合并结果统计
|
|
||||||
#[derive(Debug, Clone, Default)]
|
|
||||||
pub struct MergeResult {
|
|
||||||
pub groups_analyzed: usize,
|
|
||||||
pub persons_merged: usize,
|
|
||||||
pub aliases_moved: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 执行合并:扫描所有可合并的人物组,合并到相册数最多的那个
|
|
||||||
///
|
|
||||||
/// 合并分两个阶段:
|
|
||||||
/// Phase 1: 中文核心相同 → 合并到相册数最多的
|
|
||||||
/// Phase 2: 名字包含关系(A包含B,且B更短)→ A合并到B(短名保留)
|
|
||||||
///
|
|
||||||
/// 返回合并统计
|
|
||||||
pub async fn execute_merge(db: &Database) -> MergeResult {
|
|
||||||
let mut result = MergeResult::default();
|
|
||||||
|
|
||||||
// ── Phase 1: 中文核心合并 ──
|
|
||||||
let groups = analyze_groups(db).await;
|
|
||||||
result.groups_analyzed = groups.len();
|
|
||||||
|
|
||||||
for (_core, items) in &groups {
|
|
||||||
if items.len() < 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 第一个(相册数最多)作为目标
|
|
||||||
let dst_id = items[0].0;
|
|
||||||
let dst_name = &items[0].1;
|
|
||||||
|
|
||||||
// 其余的合并到目标
|
|
||||||
for (src_id, src_name, _, _) in &items[1..] {
|
|
||||||
println!("[合并·核心] {} (id={}) → {} (id={})", src_name, src_id, dst_name, dst_id);
|
|
||||||
|
|
||||||
let src_aliases = db.get_person_aliases(*src_id).await;
|
|
||||||
let alias_count_before = src_aliases.len();
|
|
||||||
|
|
||||||
db.merge_person_into(*src_id, dst_id).await;
|
|
||||||
|
|
||||||
result.persons_merged += 1;
|
|
||||||
let moved = alias_count_before + 1;
|
|
||||||
result.aliases_moved += moved;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Phase 2: 包含关系合并 ──
|
|
||||||
// 重新获取当前所有人物(Phase 1 后可能有变化)
|
|
||||||
let mut persons = db.get_all_persons_with_counts().await;
|
|
||||||
|
|
||||||
// 过滤掉排除名单中的
|
|
||||||
persons.retain(|(_, name, _, _)| !is_excluded_name(name));
|
|
||||||
|
|
||||||
// 按名字长度升序排列(短名在前,作为合并目标候选)
|
|
||||||
persons.sort_by(|a, b| a.1.len().cmp(&b.1.len()));
|
|
||||||
|
|
||||||
// 用一个 set 记录已被合并的 id,避免重复操作
|
|
||||||
let mut merged_ids: HashSet<i64> = HashSet::new();
|
|
||||||
|
|
||||||
for i in 0..persons.len() {
|
|
||||||
let (dst_id, dst_name, _dst_albums, _) = &persons[i];
|
|
||||||
if merged_ids.contains(dst_id) { continue; }
|
|
||||||
if dst_name.len() < 2 { continue; }
|
|
||||||
|
|
||||||
// 找所有比 dst_name 长且包含 dst_name 的人
|
|
||||||
let dst_lower = dst_name.to_lowercase();
|
|
||||||
|
|
||||||
for j in (i + 1)..persons.len() {
|
|
||||||
let (src_id, src_name, _, _) = &persons[j];
|
|
||||||
if merged_ids.contains(src_id) { continue; }
|
|
||||||
if src_name.len() <= dst_name.len() { continue; }
|
|
||||||
|
|
||||||
let src_lower = src_name.to_lowercase();
|
|
||||||
if src_lower.contains(&dst_lower) {
|
|
||||||
// 额外保护:dst 的中文核心必须和 src 的中文核心一致,
|
|
||||||
// 或者 dst 完全匹配 src 的开头/结尾,避免误合并(如 "小美" 包含 "小")
|
|
||||||
let dst_cn = extract_cn(dst_name).unwrap_or_default();
|
|
||||||
let src_cn = extract_cn(src_name).unwrap_or_default();
|
|
||||||
let cn_ok = !dst_cn.is_empty() && !src_cn.is_empty() && src_cn.contains(&dst_cn);
|
|
||||||
let starts_with = src_lower.starts_with(&dst_lower);
|
|
||||||
let ends_with = src_lower.ends_with(&dst_lower);
|
|
||||||
|
|
||||||
if !cn_ok && !starts_with && !ends_with {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("[合并·包含] {} (id={}) → {} (id={})", src_name, src_id, dst_name, dst_id);
|
|
||||||
|
|
||||||
let src_aliases = db.get_person_aliases(*src_id).await;
|
|
||||||
let alias_count_before = src_aliases.len();
|
|
||||||
|
|
||||||
db.merge_person_into(*src_id, *dst_id).await;
|
|
||||||
|
|
||||||
merged_ids.insert(*src_id);
|
|
||||||
result.persons_merged += 1;
|
|
||||||
let moved = alias_count_before + 1;
|
|
||||||
result.aliases_moved += moved;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.persons_merged > 0 {
|
|
||||||
println!("[合并] 完成:分析 {} 组,合并 {} 个人物,迁移 {} 个别名",
|
|
||||||
result.groups_analyzed, result.persons_merged, result.aliases_moved);
|
|
||||||
}
|
|
||||||
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 统计匹配分析
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct MatchStats {
|
|
||||||
pub total_albums: usize,
|
|
||||||
pub matched: usize,
|
|
||||||
pub unmatched: usize,
|
|
||||||
pub already_set: usize,
|
|
||||||
pub changed: usize,
|
|
||||||
pub by_source: HashMap<String, usize>,
|
|
||||||
pub name_frequency: HashMap<String, usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 执行 Pass1 匹配分析
|
|
||||||
pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats {
|
|
||||||
let albums = db.get_all_albums_full().await;
|
|
||||||
let mut stats = MatchStats {
|
|
||||||
total_albums: albums.len(),
|
|
||||||
matched: 0,
|
|
||||||
unmatched: 0,
|
|
||||||
already_set: 0,
|
|
||||||
changed: 0,
|
|
||||||
by_source: HashMap::new(),
|
|
||||||
name_frequency: HashMap::new(),
|
|
||||||
};
|
|
||||||
|
|
||||||
for album in &albums {
|
|
||||||
let title = album.title.as_deref().unwrap_or("");
|
|
||||||
let current = ""; // person_name_original 已迁移到 search.db
|
|
||||||
|
|
||||||
if let Some((_text, person, source)) = match_title(title, patterns) {
|
|
||||||
*stats.by_source.entry(source).or_insert(0) += 1;
|
|
||||||
*stats.name_frequency.entry(person.clone()).or_insert(0) += 1;
|
|
||||||
|
|
||||||
if current.is_empty() {
|
|
||||||
stats.matched += 1;
|
|
||||||
} else if current != person {
|
|
||||||
stats.changed += 1;
|
|
||||||
} else {
|
|
||||||
stats.already_set += 1;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
stats.unmatched += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stats
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 深度匹配分析统计
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct DeepMatchStats {
|
|
||||||
pub total_albums: usize,
|
|
||||||
pub pass1_matched: usize,
|
|
||||||
pub pass1_unmatched: usize,
|
|
||||||
pub pass2_jieba_match: usize,
|
|
||||||
pub pass2_exact_match: usize,
|
|
||||||
pub pass2_alias_match: usize,
|
|
||||||
pub pass2_still_unmatched: usize,
|
|
||||||
pub pass2_multi_person: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 执行 Pass2 深度匹配分析
|
|
||||||
pub async fn analyze_deep_matches(db: &Database) -> DeepMatchStats {
|
|
||||||
let (known_names, known_aliases) = collect_known_names(db).await;
|
|
||||||
let albums = db.get_all_albums_full().await;
|
|
||||||
|
|
||||||
let mut stats = DeepMatchStats {
|
|
||||||
total_albums: albums.len(),
|
|
||||||
pass1_matched: 0,
|
|
||||||
pass1_unmatched: 0,
|
|
||||||
pass2_jieba_match: 0,
|
|
||||||
pass2_exact_match: 0,
|
|
||||||
pass2_alias_match: 0,
|
|
||||||
pass2_still_unmatched: 0,
|
|
||||||
pass2_multi_person: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
for album in &albums {
|
|
||||||
let title = album.title.as_deref().unwrap_or("");
|
|
||||||
let current = ""; // person_name_original 已迁移到 search.db
|
|
||||||
|
|
||||||
if title.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if !current.is_empty() && known_names.contains(current) {
|
|
||||||
stats.pass1_matched += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
stats.pass1_unmatched += 1;
|
|
||||||
|
|
||||||
let result = deep_match_title(title, &known_names, &known_aliases);
|
|
||||||
if !result.matched {
|
|
||||||
stats.pass2_still_unmatched += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
match result.method.as_str() {
|
|
||||||
"jieba_extract" => stats.pass2_jieba_match += 1,
|
|
||||||
"exact_name_match" => stats.pass2_exact_match += 1,
|
|
||||||
"alias_fuzzy_match" => stats.pass2_alias_match += 1,
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.multi_person {
|
|
||||||
stats.pass2_multi_person += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stats
|
|
||||||
}
|
|
||||||
@@ -1,20 +1,20 @@
|
|||||||
//! 处理管线 — 批量处理相册元数据
|
//! 处理管线 — 批量处理相册元数据
|
||||||
//!
|
//!
|
||||||
//! 核心架构:脏集追踪 + 增量解析
|
//! 核心架构:脏集追踪 + 增量解析,dict 优先匹配
|
||||||
//!
|
//!
|
||||||
//! 精简后流程(6 阶段,2 次全库扫描):
|
//! 流程:
|
||||||
//! Phase ① 合并预处理(清洗标点/短标签/人名标签/新相册)→ 1次相册扫描
|
//! Phase Z0 删除空相册
|
||||||
//! Phase ② 合并人物清理(非人物/停用词人物) → 1次人物扫描
|
//! Phase Z1 去重同名相册
|
||||||
//! Phase ③ 增量解析脏集相册 → 只扫脏集
|
//! Phase ① 收集新相册(未处理)
|
||||||
//! Phase ④ 人物合并 + 重计 → 1次人物扫描
|
//! Phase ② 合并人物清理(非人物/停用词人物)→ 1次人物扫描
|
||||||
//! Phase ⑤ 后校验标签中的人名 → 可优化
|
//! Phase ③ dict 优先解析脏集相册 → 只扫脏集
|
||||||
|
//! Phase C 全量重计人物计数
|
||||||
//!
|
//!
|
||||||
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
|
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
|
||||||
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
|
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
|
||||||
|
|
||||||
use crate::db::Database;
|
use crate::db::Database;
|
||||||
use crate::engine::title_parser;
|
use crate::engine::title_parser;
|
||||||
use crate::engine::person_merger;
|
|
||||||
use crate::engine::tag_dict;
|
use crate::engine::tag_dict;
|
||||||
use crate::search_db::SearchDb;
|
use crate::search_db::SearchDb;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
@@ -76,16 +76,19 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
|
|||||||
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
|
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
|
||||||
// └─ Phase 1b:批量写库(IO 密集,一个事务)
|
// └─ Phase 1b:批量写库(IO 密集,一个事务)
|
||||||
|
|
||||||
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
|
pub async fn phase_parse_dirty(db: &Database, search_db: &SearchDb, dirty_set: &DirtySet) -> (usize, usize) {
|
||||||
if dirty_set.is_empty() {
|
if dirty_set.is_empty() {
|
||||||
println!("[管线·Phase1] 增量解析跳过:脏集为空");
|
println!("[管线·Phase3] dict 优先解析跳过:脏集为空");
|
||||||
return (0, 0);
|
return (0, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
let dirty_ids: Vec<String> = dirty_set.clone().into_inner().into_iter().collect();
|
let dirty_ids: Vec<String> = dirty_set.clone().into_inner().into_iter().collect();
|
||||||
let albums = db.get_albums_by_ids(&dirty_ids).await;
|
let albums = db.get_albums_by_ids(&dirty_ids).await;
|
||||||
|
|
||||||
// ── Phase 1a:并行解析(CPU bound,rayon) ──
|
// 加载 search.db 的 dict 条目(提供 merged_to 别名解析)
|
||||||
|
let dict_entries = search_db.load_dict_all().await.unwrap_or_default();
|
||||||
|
|
||||||
|
// ── Phase 3a:并行解析(CPU bound,rayon) ──
|
||||||
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
|
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
|
||||||
.filter(|a| dirty_set.contains(&a.id))
|
.filter(|a| dirty_set.contains(&a.id))
|
||||||
.collect();
|
.collect();
|
||||||
@@ -100,22 +103,23 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
|
|||||||
.filter_map(|album| {
|
.filter_map(|album| {
|
||||||
let title = album.title.as_deref()?;
|
let title = album.title.as_deref()?;
|
||||||
if title.is_empty() { return None; }
|
if title.is_empty() { return None; }
|
||||||
Some(title_parser::parse_album_title(&album.id, title))
|
// 使用 dict 优先匹配解析
|
||||||
|
Some(title_parser::parse_album_title_with_dict(&album.id, title, &dict_entries))
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let person_count = parsed.iter().filter(|r| !r.person_names.is_empty()).count();
|
let person_count = parsed.iter().filter(|r| !r.person_names.is_empty()).count();
|
||||||
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
|
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
|
||||||
|
|
||||||
// ── Phase 1b:批量写入 ──
|
// ── Phase 3b:批量写入 ──
|
||||||
let _guard = db.write_lock.lock().await;
|
let _guard = db.write_lock.lock().await;
|
||||||
let mut tx = db.pool.begin().await.expect("begin tx");
|
let mut tx = db.pool.begin().await.expect("begin tx");
|
||||||
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
|
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
|
||||||
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
|
eprintln!("[管线·Phase3b] 批量写入失败: {}", e);
|
||||||
});
|
});
|
||||||
tx.commit().await.expect("commit tx");
|
tx.commit().await.expect("commit tx");
|
||||||
|
|
||||||
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
|
println!("[管线·Phase3] dict 优先解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
|
||||||
total, parsed.len(), skipped, person_count, tag_count);
|
total, parsed.len(), skipped, person_count, tag_count);
|
||||||
(person_count, tag_count)
|
(person_count, tag_count)
|
||||||
}
|
}
|
||||||
@@ -149,29 +153,7 @@ pub async fn phase_recount_all(db: &Database) -> usize {
|
|||||||
fixed
|
fixed
|
||||||
}
|
}
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
|
// Preprocess + Dirty Tracking
|
||||||
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
|
|
||||||
let result = person_merger::execute_merge(db).await;
|
|
||||||
if result.persons_merged > 0 {
|
|
||||||
println!("[管线·Phase2] 人物合并完成:分析 {} 组,合并 {} 人,迁移 {} 个别名",
|
|
||||||
result.groups_analyzed, result.persons_merged, result.aliases_moved);
|
|
||||||
} else {
|
|
||||||
println!("[管线·Phase2] 人物合并完成:无需要合并的人物");
|
|
||||||
}
|
|
||||||
(result.persons_merged, result.aliases_moved)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ════════════════════════════════════════════════
|
|
||||||
// Phase B:后校验标签 — 用合并后的人名检查
|
|
||||||
// ════════════════════════════════════════════════
|
|
||||||
|
|
||||||
pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
|
|
||||||
println!("[管线·PhaseB] 后校验标签完成:已迁移到 search.db,跳过");
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
// ════════════════════════════════════════════════
|
|
||||||
// 合并阶段 — 减少全库扫描次数
|
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
|
|
||||||
/// 合并预处理结果
|
/// 合并预处理结果
|
||||||
@@ -318,22 +300,18 @@ pub struct PipelineResult {
|
|||||||
pub new_albums_found: usize, // Phase ①
|
pub new_albums_found: usize, // Phase ①
|
||||||
pub persons_updated: usize, // Phase ③
|
pub persons_updated: usize, // Phase ③
|
||||||
pub tags_updated: usize, // Phase ③
|
pub tags_updated: usize, // Phase ③
|
||||||
pub persons_merged: usize, // Phase ④
|
|
||||||
pub aliases_moved: usize, // Phase ④
|
|
||||||
pub tags_removed_person: usize, // Phase ⑤
|
|
||||||
pub empty_albums_deleted: usize, // Phase Z0
|
pub empty_albums_deleted: usize, // Phase Z0
|
||||||
pub dup_albums_deleted: usize, // Phase Z1
|
pub dup_albums_deleted: usize, // Phase Z1
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 一键执行精简管线
|
/// 一键执行精简管线(dict 优先匹配)
|
||||||
///
|
///
|
||||||
/// Phase Z0 删除空相册
|
/// Phase Z0 删除空相册
|
||||||
/// Phase Z1 去重同名相册
|
/// Phase Z1 去重同名相册
|
||||||
/// Phase ① 合并预处理(清洗+短标签+人名标签+新相册) → 1次相册扫描
|
/// Phase ① 收集新相册
|
||||||
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
|
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
|
||||||
/// Phase ③ 增量解析脏集 → 只扫脏集
|
/// Phase ③ dict 优先解析脏集 → 只扫脏集
|
||||||
/// Phase ④ 人物合并 + 重计 → 1次人物扫描
|
/// Phase C 全量重计
|
||||||
/// Phase ⑤ 后校验标签中的人名
|
|
||||||
pub async fn run_pipeline(db: &Database, search_db: &SearchDb) -> PipelineResult {
|
pub async fn run_pipeline(db: &Database, search_db: &SearchDb) -> PipelineResult {
|
||||||
// Phase Z0:删除空相册
|
// Phase Z0:删除空相册
|
||||||
let empty_albums_deleted = phase_delete_empty_albums(db).await;
|
let empty_albums_deleted = phase_delete_empty_albums(db).await;
|
||||||
@@ -341,7 +319,7 @@ pub async fn run_pipeline(db: &Database, search_db: &SearchDb) -> PipelineResult
|
|||||||
// Phase Z1:去重同名相册(含删除文件夹和 search.db 数据)
|
// Phase Z1:去重同名相册(含删除文件夹和 search.db 数据)
|
||||||
let dup_albums_deleted = phase_dedup_albums(db, search_db).await;
|
let dup_albums_deleted = phase_dedup_albums(db, search_db).await;
|
||||||
|
|
||||||
// Phase ①:合并预处理(1 次相册扫描)
|
// Phase ①:收集新相册
|
||||||
let pre = phase_preprocess_combined(db).await;
|
let pre = phase_preprocess_combined(db).await;
|
||||||
let tags_cleaned = pre.tags_cleaned;
|
let tags_cleaned = pre.tags_cleaned;
|
||||||
let short_tags_removed = pre.short_tags_removed;
|
let short_tags_removed = pre.short_tags_removed;
|
||||||
@@ -357,23 +335,16 @@ pub async fn run_pipeline(db: &Database, search_db: &SearchDb) -> PipelineResult
|
|||||||
master_dirty.extend(dirty_persons.into_inner());
|
master_dirty.extend(dirty_persons.into_inner());
|
||||||
let dirty_albums_detected = master_dirty.len();
|
let dirty_albums_detected = master_dirty.len();
|
||||||
|
|
||||||
// Phase ③:增量解析
|
// Phase ③:dict 优先解析脏集
|
||||||
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
let (persons_updated, tags_updated) = phase_parse_dirty(db, search_db, &master_dirty).await;
|
||||||
|
|
||||||
// Phase ④:人物合并(内部会做包含关系 + 核心合并)
|
// Phase C:全量重计(修正计数)
|
||||||
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
|
||||||
|
|
||||||
// Phase C:全量重计(合并后修正计数)
|
|
||||||
let _recounted = phase_recount_all(db).await;
|
let _recounted = phase_recount_all(db).await;
|
||||||
|
|
||||||
// Phase ⑤:后校验
|
|
||||||
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
|
||||||
|
|
||||||
PipelineResult {
|
PipelineResult {
|
||||||
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
|
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
|
||||||
non_persons_deleted, stop_word_persons_removed, new_albums_found,
|
non_persons_deleted, stop_word_persons_removed, new_albums_found,
|
||||||
persons_updated, tags_updated, persons_merged,
|
persons_updated, tags_updated,
|
||||||
aliases_moved, tags_removed_person,
|
|
||||||
empty_albums_deleted, dup_albums_deleted,
|
empty_albums_deleted, dup_albums_deleted,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ async fn main() -> std::io::Result<()> {
|
|||||||
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
|
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
|
||||||
if !new_albums.is_empty() {
|
if !new_albums.is_empty() {
|
||||||
info!("[定时] 发现 {} 个新相册,增量解析...", new_albums.len());
|
info!("[定时] 发现 {} 个新相册,增量解析...", new_albums.len());
|
||||||
engine::pipeline::phase_parse_dirty(&timer_db, &new_albums).await;
|
engine::pipeline::phase_parse_dirty(&timer_db, &timer_search_db, &new_albums).await;
|
||||||
info!("[定时] 同步到 search.db...");
|
info!("[定时] 同步到 search.db...");
|
||||||
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
|
engine::pipeline::sync_dirty_albums(&timer_db, &timer_search_db, &new_albums).await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ pub async fn delete_person(
|
|||||||
if !album_ids.is_empty() {
|
if !album_ids.is_empty() {
|
||||||
let mut dirty = engine::pipeline::DirtySet::new();
|
let mut dirty = engine::pipeline::DirtySet::new();
|
||||||
for aid in &album_ids { dirty.add(aid); }
|
for aid in &album_ids { dirty.add(aid); }
|
||||||
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &dirty).await;
|
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &search_db, &dirty).await;
|
||||||
let synced = engine::pipeline::sync_dirty_albums(&db, &search_db, &dirty).await;
|
let synced = engine::pipeline::sync_dirty_albums(&db, &search_db, &dirty).await;
|
||||||
|
|
||||||
// 5. 重载内存词典
|
// 5. 重载内存词典
|
||||||
|
|||||||
@@ -22,19 +22,15 @@ pub async fn pipeline_run(
|
|||||||
"new_albums_found": result.new_albums_found,
|
"new_albums_found": result.new_albums_found,
|
||||||
"persons_updated": result.persons_updated,
|
"persons_updated": result.persons_updated,
|
||||||
"tags_updated": result.tags_updated,
|
"tags_updated": result.tags_updated,
|
||||||
"persons_merged": result.persons_merged,
|
|
||||||
"aliases_moved": result.aliases_moved,
|
|
||||||
"tags_removed_person": result.tags_removed_person,
|
|
||||||
"empty_albums_deleted": result.empty_albums_deleted,
|
"empty_albums_deleted": result.empty_albums_deleted,
|
||||||
"dup_albums_deleted": result.dup_albums_deleted,
|
"dup_albums_deleted": result.dup_albums_deleted,
|
||||||
"search_db_synced": synced,
|
"search_db_synced": synced,
|
||||||
"message": format!(
|
"message": format!(
|
||||||
"管线执行完毕:删除空相册{},去重{},清洗标点{}相册,移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},合并人物{},迁移别名{},后移除人名标签{},同步标签/人物到search.db{}",
|
"管线执行完毕:删除空相册{},去重{},清洗标点{}相册,移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},同步标签/人物到search.db{}",
|
||||||
result.empty_albums_deleted, result.dup_albums_deleted,
|
result.empty_albums_deleted, result.dup_albums_deleted,
|
||||||
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
|
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
|
||||||
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
|
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
|
||||||
result.persons_updated, result.tags_updated,
|
result.persons_updated, result.tags_updated, synced
|
||||||
result.persons_merged, result.aliases_moved, result.tags_removed_person, synced
|
|
||||||
)
|
)
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
@@ -96,15 +92,13 @@ pub async fn pipeline_reparse_persons(
|
|||||||
"success": true,
|
"success": true,
|
||||||
"albums_parsed": result.new_albums_found,
|
"albums_parsed": result.new_albums_found,
|
||||||
"persons_updated": result.persons_updated,
|
"persons_updated": result.persons_updated,
|
||||||
"persons_merged": result.persons_merged,
|
|
||||||
"aliases_moved": result.aliases_moved,
|
|
||||||
"non_persons_deleted": result.non_persons_deleted,
|
"non_persons_deleted": result.non_persons_deleted,
|
||||||
"stop_word_persons_removed": result.stop_word_persons_removed,
|
"stop_word_persons_removed": result.stop_word_persons_removed,
|
||||||
"synced_to_search_db": synced,
|
"synced_to_search_db": synced,
|
||||||
"message": format!(
|
"message": format!(
|
||||||
"人物重新解析完成:重置{}个相册,解析{}个,更新人物{}个,合并{}个,迁移别名{}个,删除非人物{},清除重叠{},同步{}个到search.db",
|
"人物重新解析完成:重置{}个相册,解析{}个,更新人物{}个,删除非人物{},清除重叠{},同步{}个到search.db",
|
||||||
reset_count, result.new_albums_found,
|
reset_count, result.new_albums_found,
|
||||||
result.persons_updated, result.persons_merged, result.aliases_moved,
|
result.persons_updated,
|
||||||
result.non_persons_deleted, result.stop_word_persons_removed, synced,
|
result.non_persons_deleted, result.stop_word_persons_removed, synced,
|
||||||
)
|
)
|
||||||
}))
|
}))
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ pub async fn delete_tag(
|
|||||||
// 3. 重新解析受影响相册
|
// 3. 重新解析受影响相册
|
||||||
let mut dirty = engine::pipeline::DirtySet::new();
|
let mut dirty = engine::pipeline::DirtySet::new();
|
||||||
for aid in &album_ids { dirty.add(aid); }
|
for aid in &album_ids { dirty.add(aid); }
|
||||||
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &dirty).await;
|
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &search_db, &dirty).await;
|
||||||
|
|
||||||
// 4. 同步到 search.db
|
// 4. 同步到 search.db
|
||||||
let synced = engine::pipeline::sync_dirty_albums(&db, &search_db, &dirty).await;
|
let synced = engine::pipeline::sync_dirty_albums(&db, &search_db, &dirty).await;
|
||||||
|
|||||||
Reference in New Issue
Block a user