536 lines
19 KiB
Rust
536 lines
19 KiB
Rust
//! 标题解析 — 统一最大匹配管线（v5）
|
||
//!
|
||
//! 核心思路：一次扫描同时识别 person 和 tags，两者互斥，不会混淆。
|
||
//!
|
||
//! 流程：
|
||
//! 1. 预处理：移除照片数后缀、[工作室]前缀、称呼前缀、编号后缀
|
||
//! 2. 最大匹配切分：在 keymap + include + stop_words 词典中做最长子串匹配
|
||
//! 3. 角色分类：每个片段有明确分类（person / style / outfit / ... / stop / other）
|
||
//! 4. 选人名：优先级 = keymap(person) > include 列表 > 启发式
|
||
//! 5. 其余分类即为标签
|
||
//!
|
||
//! 相比 v4 的改进：
|
||
//! - 中文连续文本也能正确切分（不再依赖空格）
|
||
//! - person 和 tag 在同一 pass 中确定，不会混淆
|
||
//! - 最长匹配优先，避免"性感"和"性感女神"都匹配的问题
|
||
|
||
use regex::Regex;
|
||
use std::collections::HashSet;
|
||
use lazy_static::lazy_static;
|
||
use crate::engine::tag_dict;
|
||
|
||
// ── 正则 ──
|
||
|
||
lazy_static! {
|
||
static ref RE_BRACKET_PREFIX: Regex = Regex::new(r"^\[([^\]]+)\]\s*").unwrap();
|
||
static ref RE_NUMBER_SUFFIX: Regex = Regex::new(r"[\s#]*(?:no\.?\s*|vol\.?\s*|#)?\d+[\s]*$").unwrap();
|
||
static ref RE_PHOTOS_SUFFIX: Regex = Regex::new(r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$").unwrap();
|
||
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
||
static ref RE_ENGLISH_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
|
||
static ref RE_TRAILING_PUNCT: Regex = Regex::new(r"[\s.,;:，。；：\-–—]+$").unwrap();
|
||
static ref RE_PAREN_CONTENT: Regex = Regex::new(r"[（(]([^）)]*)[）)]").unwrap();
|
||
/// 分隔符：空白、常见标点、中英文之间的边界
|
||
static ref RE_SEPARATOR: Regex = Regex::new(r"[\s\-–—_,.;:，。；：、/\\|·]+").unwrap();
|
||
}
|
||
|
||
// ── 预处理 ──
|
||
|
||
/// 统一标题预处理：移除照片数后缀、[工作室] 前缀、称呼前缀、编号后缀
|
||
pub fn preprocess_title(title: &str) -> String {
|
||
let t = title.trim();
|
||
if t.is_empty() { return String::new(); }
|
||
|
||
let mut t = RE_PHOTOS_SUFFIX.replace(t, "").trim().to_string();
|
||
t = RE_BRACKET_PREFIX.replace(&t, "").trim().to_string();
|
||
|
||
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
|
||
if t.starts_with(prefix) {
|
||
t = t[prefix.len()..].trim().to_string();
|
||
break;
|
||
}
|
||
}
|
||
|
||
t = RE_NUMBER_SUFFIX.replace(&t, "").trim().to_string();
|
||
t
|
||
}
|
||
|
||
// ── 切分片段：带分类 ──
|
||
|
||
/// 一个切分片段及其分类
|
||
#[derive(Debug, Clone)]
|
||
struct Segment {
|
||
text: String,
|
||
category: String,
|
||
}
|
||
|
||
// ── 最大匹配切分（核心算法） ──
|
||
|
||
/// 对标题做最大匹配切分，返回带分类的片段列表。
|
||
///
|
||
/// 算法：从左到右扫描，每次尝试最长可能的匹配（max_len 到 1）。
|
||
/// 匹配来源优先级（同一长度下）：
|
||
/// keymap(person) > keymap(其他) > stop_words > include_person
|
||
///
|
||
/// 未匹配到的连续字符作为 "other" 片段保留。
|
||
fn max_match_segment(title: &str) -> Vec<Segment> {
|
||
let dict = tag_dict::get_tag_dict();
|
||
|
||
// 收集所有可匹配的词及分类
|
||
let mut all_words: Vec<(String, String)> = Vec::new();
|
||
|
||
// 1. keymap 中的所有词
|
||
for (word, cat) in dict.get_tag_category_map() {
|
||
all_words.push((word.clone(), cat.clone()));
|
||
}
|
||
|
||
// 2. include 列表中的人名 → "person_include"
|
||
for name in dict.get_included_persons() {
|
||
if !dict.get_tag_category_map().contains_key(name) {
|
||
all_words.push((name.clone(), "person_include".to_string()));
|
||
}
|
||
}
|
||
|
||
// 3. 停用词 → "stop"
|
||
for word in dict.get_stop_words() {
|
||
all_words.push((word.clone(), "stop".to_string()));
|
||
}
|
||
|
||
// 计算最大词长
|
||
let max_len = all_words.iter().map(|(w, _)| w.chars().count()).max().unwrap_or(8);
|
||
|
||
let mut segments: Vec<Segment> = Vec::new();
|
||
let chars: Vec<char> = title.chars().collect();
|
||
let n = chars.len();
|
||
let mut i = 0;
|
||
|
||
while i < n {
|
||
// 跳过分隔符
|
||
let ch = chars[i];
|
||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
|
||
let mut found = false;
|
||
|
||
// 从最长到最短尝试匹配
|
||
let try_max = max_len.min(n - i);
|
||
'outer: for len in (1..=try_max).rev() {
|
||
let candidate: String = chars[i..i + len].iter().collect();
|
||
let candidate_lower = candidate.to_lowercase();
|
||
|
||
// 收集所有匹配该长度的词，按优先级选最好的
|
||
let mut best: Option<&String> = None; // 存储分类
|
||
|
||
for (word, cat) in &all_words {
|
||
if word.to_lowercase() == candidate_lower {
|
||
best = Some(cat);
|
||
break;
|
||
}
|
||
}
|
||
|
||
if let Some(cat) = best {
|
||
segments.push(Segment {
|
||
text: candidate,
|
||
category: cat.clone(),
|
||
});
|
||
i += len;
|
||
found = true;
|
||
break 'outer;
|
||
}
|
||
}
|
||
|
||
if !found {
|
||
// 未匹配到：收集连续的非分隔字符作为 "other"
|
||
let start = i;
|
||
while i < n {
|
||
let ch = chars[i];
|
||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
||
break;
|
||
}
|
||
// 如果下一个字符开始能匹配到词典中的词，也停在这里
|
||
let remaining = n - i;
|
||
if i > start && remaining >= 1 {
|
||
let peek_max = max_len.min(remaining);
|
||
let mut matched = false;
|
||
for len in (1..=peek_max).rev() {
|
||
let peek: String = chars[i..i + len].iter().collect();
|
||
let peek_lower = peek.to_lowercase();
|
||
for (word, _) in &all_words {
|
||
if word.to_lowercase() == peek_lower {
|
||
matched = true;
|
||
break;
|
||
}
|
||
}
|
||
if matched { break; }
|
||
}
|
||
if matched { break; }
|
||
}
|
||
i += 1;
|
||
}
|
||
if i > start {
|
||
let other: String = chars[start..i].iter().collect();
|
||
segments.push(Segment {
|
||
text: other,
|
||
category: "other".to_string(),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
segments
|
||
}
|
||
|
||
// ── 人名启发式判断 ──
|
||
|
||
/// 判断一个词是否"看起来像人名"
|
||
///
|
||
/// 注意：调用方应已排除 keymap 中的非 person 类型词、停用词等。
|
||
pub fn is_likely_person_name(name: &str) -> bool {
|
||
let name = name.trim();
|
||
if name.is_empty() { return false; }
|
||
|
||
let dict = tag_dict::get_tag_dict();
|
||
|
||
// 强制包含名单优先放行
|
||
if dict.is_included_person(name) {
|
||
return true;
|
||
}
|
||
|
||
// 字符数限制
|
||
let char_count = name.chars().count();
|
||
if char_count < 2 { return false; }
|
||
|
||
// keymap 中的非 person 类型词一定不是人名
|
||
if let Some(cat) = dict.classify(name) {
|
||
if cat != "person" { return false; }
|
||
}
|
||
|
||
// 排除已知非人名
|
||
if dict.is_excluded_person(name) { return false; }
|
||
|
||
let name_lower = name.to_lowercase();
|
||
|
||
// 纯数字排除
|
||
if name.chars().all(|c| c.is_ascii_digit()) { return false; }
|
||
|
||
// 含数字太多的排除
|
||
let digit_count = name.chars().filter(|c| c.is_ascii_digit()).count();
|
||
if digit_count * 2 > name.len() { return false; }
|
||
|
||
// 纯英文且过长排除
|
||
let has_cn = RE_CHINESE.is_match(name);
|
||
if !has_cn && name.len() > 15 { return false; }
|
||
|
||
// 以数字开头排除
|
||
if name.chars().next().map(|c| c.is_ascii_digit()).unwrap_or(false) { return false; }
|
||
|
||
// 含特殊字符过多排除
|
||
let special_count = name.chars().filter(|c| {
|
||
!c.is_alphanumeric() && !RE_CHINESE.is_match(&c.to_string())
|
||
}).count();
|
||
if special_count > 2 { return false; }
|
||
|
||
// 已知后缀排除
|
||
let known_suffix = ["photos", "series", "collection", "edition", "set", "vol", "album"];
|
||
for suffix in known_suffix {
|
||
if name_lower.ends_with(suffix) { return false; }
|
||
}
|
||
|
||
// 全大写且长度 > 6 排除（缩写/工作室名）
|
||
if name.chars().all(|c| c.is_ascii_uppercase()) && name.len() > 6 { return false; }
|
||
|
||
// 中文字符数判断：2-5 个中文字符更像人名
|
||
let cn_count: usize = name.chars().filter(|c| RE_CHINESE.is_match(&c.to_string())).count();
|
||
if has_cn && (cn_count < 2 || cn_count > 5) {
|
||
return false;
|
||
}
|
||
|
||
true
|
||
}
|
||
|
||
// ── 核心解析入口 ──
|
||
|
||
/// 统一相册标题解析入口（v5 最大匹配版本）
|
||
///
|
||
/// 流程：预处理 → 最大匹配切分 → 选人 → 其余为标签
|
||
///
|
||
/// 返回 ParseResult 可直接用于批量写库。
|
||
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
|
||
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
|
||
|
||
let title = title.trim();
|
||
if title.is_empty() { return result; }
|
||
|
||
let cleaned = preprocess_title(title);
|
||
if cleaned.is_empty() { return result; }
|
||
|
||
// Step 1: 最大匹配切分
|
||
let segments = max_match_segment(&cleaned);
|
||
|
||
// Step 2: 从 segments 中选人（优先级最高的）
|
||
let mut person_candidates: Vec<(String, u8)> = Vec::new(); // (name, priority)
|
||
|
||
for seg in &segments {
|
||
match seg.category.as_str() {
|
||
"person" => {
|
||
person_candidates.push((seg.text.clone(), 0)); // 优先级 0 = 最高
|
||
}
|
||
"person_include" => {
|
||
person_candidates.push((seg.text.clone(), 1)); // 优先级 1
|
||
}
|
||
"other" => {
|
||
if is_likely_person_name(&seg.text) {
|
||
person_candidates.push((seg.text.clone(), 2)); // 优先级 2 = 启发式
|
||
}
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
// 按优先级排序，取最优
|
||
person_candidates.sort_by_key(|(_, p)| *p);
|
||
|
||
if let Some((name, _)) = person_candidates.first() {
|
||
let (primary, _aliases) = split_person_name_with_aliases(name);
|
||
let final_name = if primary.is_empty() { name.clone() } else { primary };
|
||
|
||
if final_name.chars().count() >= 2 {
|
||
result.person_name = Some(final_name);
|
||
}
|
||
}
|
||
|
||
// Step 3: 收集标签（非 person / stop / other 类的片段）
|
||
let mut tags: Vec<(String, String)> = Vec::new();
|
||
let person_lower = result.person_name.as_ref().map(|n| n.to_lowercase());
|
||
|
||
for seg in &segments {
|
||
let cat = &seg.category;
|
||
|
||
// 跳过停用词
|
||
if cat == "stop" { continue; }
|
||
|
||
// 跳过 person 相关
|
||
if cat == "person" || cat == "person_include" { continue; }
|
||
|
||
// 如果是 other 类，检查是否被选为人名
|
||
if cat == "other" {
|
||
if let Some(ref pl) = person_lower {
|
||
if seg.text.to_lowercase().contains(pl) || pl.contains(&seg.text.to_lowercase()) {
|
||
continue;
|
||
}
|
||
}
|
||
// other 类也做个简单过滤：太短、纯数字等跳过
|
||
if seg.text.chars().count() < 2 { continue; }
|
||
if seg.text.chars().all(|c| c.is_ascii_digit()) { continue; }
|
||
// 太短的英文 other 也跳过（噪声）
|
||
if !RE_CHINESE.is_match(&seg.text) && seg.text.len() < 3 { continue; }
|
||
}
|
||
|
||
// 有效分类标签
|
||
let display_cat = if cat == "other" { "other".to_string() } else { cat.clone() };
|
||
tags.push((seg.text.clone(), display_cat));
|
||
}
|
||
|
||
// 去重（小写）
|
||
let mut seen = HashSet::new();
|
||
tags.retain(|(t, _)| seen.insert(t.to_lowercase()));
|
||
|
||
result.tags = tags;
|
||
result
|
||
}
|
||
|
||
// ── 人名清洗工具函数 ──
|
||
|
||
/// 提取中文核心（开头的连续中文字符）
|
||
pub fn extract_cn(name: &str) -> Option<String> {
|
||
let re = Regex::new(r"^([\u{4e00}-\u{9fff}]+)").unwrap();
|
||
re.captures(name).map(|c| c[1].to_string())
|
||
}
|
||
|
||
/// 清理人名：移除前缀、括号内容、末尾标点
|
||
pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
|
||
let mut aliases = Vec::new();
|
||
let mut name = raw_name.to_string();
|
||
|
||
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
|
||
if name.starts_with(prefix) {
|
||
aliases.push(("prefix".to_string(), prefix.to_string()));
|
||
name = name[prefix.len()..].trim().to_string();
|
||
break;
|
||
}
|
||
}
|
||
|
||
let mut cleaned = String::new();
|
||
let mut last_end = 0;
|
||
for cap in RE_PAREN_CONTENT.captures_iter(&name) {
|
||
let m = cap.get(0).unwrap();
|
||
let content = cap.get(1).map_or("", |c| c.as_str());
|
||
cleaned.push_str(&name[last_end..m.start()]);
|
||
last_end = m.end();
|
||
if !content.is_empty() {
|
||
if RE_CHINESE.is_match(content) {
|
||
aliases.push(("nickname".to_string(), content.to_string()));
|
||
} else {
|
||
aliases.push(("english".to_string(), content.to_string()));
|
||
}
|
||
}
|
||
}
|
||
cleaned.push_str(&name[last_end..]);
|
||
let name = RE_TRAILING_PUNCT.replace(&cleaned.trim(), "").trim().to_string();
|
||
|
||
aliases.push(("original".to_string(), raw_name.trim().to_string()));
|
||
|
||
if let Some(caps) = RE_ENGLISH_SUFFIX.captures(&name) {
|
||
let chinese_part = caps[1].to_string();
|
||
let english_part = caps[2].to_string();
|
||
if !english_part.is_empty() {
|
||
aliases.push(("chinese".to_string(), chinese_part));
|
||
aliases.push(("english".to_string(), english_part));
|
||
}
|
||
}
|
||
|
||
if name.is_empty() { (raw_name.to_string(), aliases) } else { (name, aliases) }
|
||
}
|
||
|
||
/// 拆分人名为「中文主名 + 别名列表」
|
||
pub fn split_person_name_with_aliases(raw_name: &str) -> (String, Vec<(String, String)>) {
|
||
let (cleaned, mut aliases) = clean_person_name(raw_name);
|
||
let mut primary_name = cleaned.clone();
|
||
|
||
if let Some(cn_part) = extract_cn(&cleaned) {
|
||
if !cn_part.is_empty() && cn_part != cleaned {
|
||
primary_name = cn_part.clone();
|
||
if let Some(rest) = cleaned.strip_prefix(&cn_part) {
|
||
let english_suffix = rest.trim().to_string();
|
||
if !english_suffix.is_empty() {
|
||
let already_has = aliases.iter().any(|(_, a)| a == &english_suffix);
|
||
if !already_has {
|
||
aliases.push(("english".to_string(), english_suffix));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if cleaned != primary_name {
|
||
let already_has = aliases.iter().any(|(_, a)| a == &cleaned);
|
||
if !already_has {
|
||
aliases.push(("cleaned".to_string(), cleaned));
|
||
}
|
||
}
|
||
|
||
(primary_name, aliases)
|
||
}
|
||
|
||
/// 判断是否为工作室/平台名（需排除）
|
||
pub fn is_excluded_name(name: &str) -> bool {
|
||
tag_dict::get_tag_dict().is_excluded_person(name)
|
||
}
|
||
|
||
/// 判断后缀是否有效
|
||
pub fn is_valid_suffix(suffix: &str) -> bool {
|
||
if suffix.is_empty() { return true; }
|
||
!RE_CHINESE.is_match(suffix)
|
||
}
|
||
|
||
/// 标签归一化
|
||
pub fn normalize_tag(tag: &str) -> String {
|
||
tag.trim().to_lowercase()
|
||
}
|
||
|
||
// ── 兼容函数（供 pipeline / person_merger 调用） ──
|
||
|
||
/// 从标题中提取人物名（兼容函数）
|
||
pub fn extract_person_name_from_title(title: &str) -> Option<String> {
|
||
parse_album_title("", title).person_name
|
||
}
|
||
|
||
/// 从标题中提取人物名和标签（兼容函数）
|
||
pub fn extract_person_and_tags(title: &str) -> (Option<String>, Vec<(String, String)>) {
|
||
let result = parse_album_title("", title);
|
||
(result.person_name, result.tags)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn test_parse_album_title_with_person() {
|
||
let r = parse_album_title("id1", "杨晨晨Yome 私房写真");
|
||
assert_eq!(r.person_name.as_deref(), Some("杨晨晨"));
|
||
assert!(!r.tags.is_empty());
|
||
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
|
||
assert!(tag_names.contains(&"私房"), "应包含 '私房' 标签");
|
||
assert!(tag_names.contains(&"写真"), "应包含 '写真' 标签");
|
||
}
|
||
|
||
#[test]
|
||
fn test_parse_album_title_no_person() {
|
||
let r = parse_album_title("id2", "浴室 可爱 性感 写真");
|
||
assert!(r.person_name.is_none());
|
||
assert!(!r.tags.is_empty());
|
||
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
|
||
assert!(tag_names.contains(&"浴室"));
|
||
assert!(tag_names.contains(&"性感"));
|
||
assert!(tag_names.contains(&"写真"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_parse_album_title_with_prefix() {
|
||
let r = parse_album_title("id3", "美女模特 杨晨晨 私房");
|
||
assert_eq!(r.person_name.as_deref(), Some("杨晨晨"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_parse_continuous_chinese() {
|
||
// 连续中文：杨晨晨私房写真 — 应能切分出人名和标签
|
||
let r = parse_album_title("id4", "杨晨晨私房写真");
|
||
assert_eq!(r.person_name.as_deref(), Some("杨晨晨"));
|
||
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
|
||
assert!(tag_names.contains(&"私房"), "应包含 '私房'");
|
||
assert!(tag_names.contains(&"写真"), "应包含 '写真'");
|
||
}
|
||
|
||
#[test]
|
||
fn test_extract_cn() {
|
||
assert_eq!(extract_cn("杨晨晨Yome").unwrap(), "杨晨晨");
|
||
assert_eq!(extract_cn("王馨瑶yanni").unwrap(), "王馨瑶");
|
||
assert!(extract_cn("Yome").is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn test_split_person_name_with_aliases() {
|
||
let (name, aliases) = split_person_name_with_aliases("杨晨晨Yome");
|
||
assert_eq!(name, "杨晨晨");
|
||
let alias_names: Vec<&str> = aliases.iter().map(|(_, a)| a.as_str()).collect();
|
||
assert!(alias_names.contains(&"Yome"));
|
||
|
||
let (name2, _) = split_person_name_with_aliases("杨晨晨");
|
||
assert_eq!(name2, "杨晨晨");
|
||
|
||
let (name3, _) = split_person_name_with_aliases("Yome");
|
||
assert_eq!(name3, "Yome");
|
||
}
|
||
|
||
#[test]
|
||
fn test_person_not_in_tags() {
|
||
// 人名不应出现在标签中
|
||
let r = parse_album_title("id5", "杨晨晨 日系 比基尼");
|
||
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
|
||
assert!(!tag_names.contains(&"杨晨晨"), "人名 '杨晨晨' 不应出现在标签中");
|
||
assert!(tag_names.contains(&"日系"));
|
||
assert!(tag_names.contains(&"比基尼"));
|
||
}
|
||
|
||
#[test]
|
||
fn test_stop_words_removed() {
|
||
// 停用词不应出现在标签或人名中
|
||
let r = parse_album_title("id6", "杨晨晨 photos collection");
|
||
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
|
||
assert!(!tag_names.contains(&"photos"), "停用词 'photos' 不应在标签中");
|
||
assert!(!tag_names.contains(&"collection"), "停用词 'collection' 不应在标签中");
|
||
}
|
||
}
|