update code level
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
//!
|
||||
//! 流程:
|
||||
//! 1. 预处理:移除照片数后缀、[工作室]前缀、称呼前缀、编号后缀
|
||||
//! 2. 最大匹配切分:在 keymap + include + stop_words 词典中做最长子串匹配
|
||||
//! 2. 切分:中文最大匹配 + 英文按空格切分
|
||||
//! 3. 角色分类:每个片段有明确分类(person / style / outfit / ... / stop / other)
|
||||
//! 4. 选人名:优先级 = keymap(person) > include 列表 > 启发式
|
||||
//! 5. 其余分类即为标签
|
||||
@@ -13,6 +13,7 @@
|
||||
//! - 中文连续文本也能正确切分(不再依赖空格)
|
||||
//! - person 和 tag 在同一 pass 中确定,不会混淆
|
||||
//! - 最长匹配优先,避免"性感"和"性感女神"都匹配的问题
|
||||
//! - 英文只按空格切分,不逐字匹配,不移除英文介词等停用词
|
||||
|
||||
use regex::Regex;
|
||||
use std::collections::HashSet;
|
||||
@@ -63,15 +64,23 @@ struct Segment {
|
||||
category: String,
|
||||
}
|
||||
|
||||
// ── 最大匹配切分(核心算法) ──
|
||||
// ── 切分核心算法 ──
|
||||
|
||||
/// 对标题做最大匹配切分,返回带分类的片段列表。
|
||||
/// 判断字符是否是中文字符
|
||||
fn is_chinese_char(ch: char) -> bool {
|
||||
('\u{4e00}'..='\u{9fff}').contains(&ch)
|
||||
}
|
||||
|
||||
/// 对标题做切分,返回带分类的片段列表。
|
||||
///
|
||||
/// 算法:从左到右扫描,每次尝试最长可能的匹配(max_len 到 1)。
|
||||
/// 匹配来源优先级(同一长度下):
|
||||
/// 算法分两路:
|
||||
/// - 中文:从左到右最大匹配(逐字尝试,最长优先)
|
||||
/// - 英文/数字:按空格/分隔符直接切分为完整单词,查词典分类
|
||||
///
|
||||
/// 匹配来源优先级:
|
||||
/// keymap(person) > keymap(其他) > stop_words > include_person
|
||||
///
|
||||
/// 未匹配到的连续字符作为 "other" 片段保留。
|
||||
/// 未匹配到的作为 "other" 片段保留。
|
||||
fn max_match_segment(title: &str) -> Vec<Segment> {
|
||||
let dict = tag_dict::get_tag_dict();
|
||||
|
||||
@@ -95,87 +104,91 @@ fn max_match_segment(title: &str) -> Vec<Segment> {
|
||||
all_words.push((word.clone(), "stop".to_string()));
|
||||
}
|
||||
|
||||
// 计算最大词长
|
||||
// 计算最大词长(中文最大匹配用)
|
||||
let max_len = all_words.iter().map(|(w, _)| w.chars().count()).max().unwrap_or(8);
|
||||
|
||||
// 快速查找函数
|
||||
let find_category = |word: &str| -> Option<String> {
|
||||
let word_lower = word.to_lowercase();
|
||||
for (w, cat) in &all_words {
|
||||
if w.to_lowercase() == word_lower {
|
||||
return Some(cat.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
};
|
||||
|
||||
let mut segments: Vec<Segment> = Vec::new();
|
||||
let chars: Vec<char> = title.chars().collect();
|
||||
let n = chars.len();
|
||||
let mut i = 0;
|
||||
|
||||
while i < n {
|
||||
// 跳过分隔符
|
||||
let ch = chars[i];
|
||||
|
||||
// 跳过分隔符
|
||||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut found = false;
|
||||
|
||||
// 从最长到最短尝试匹配
|
||||
let try_max = max_len.min(n - i);
|
||||
'outer: for len in (1..=try_max).rev() {
|
||||
let candidate: String = chars[i..i + len].iter().collect();
|
||||
let candidate_lower = candidate.to_lowercase();
|
||||
|
||||
// 收集所有匹配该长度的词,按优先级选最好的
|
||||
let mut best: Option<&String> = None; // 存储分类
|
||||
|
||||
for (word, cat) in &all_words {
|
||||
if word.to_lowercase() == candidate_lower {
|
||||
best = Some(cat);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(cat) = best {
|
||||
segments.push(Segment {
|
||||
text: candidate,
|
||||
category: cat.clone(),
|
||||
});
|
||||
i += len;
|
||||
found = true;
|
||||
break 'outer;
|
||||
// ── 英文/数字:直接按分隔符切分整个单词,不逐字匹配 ──
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
let start = i;
|
||||
while i < n && chars[i].is_ascii_alphanumeric() {
|
||||
i += 1;
|
||||
}
|
||||
let word: String = chars[start..i].iter().collect();
|
||||
let cat = find_category(&word).unwrap_or_else(|| "other".to_string());
|
||||
segments.push(Segment { text: word, category: cat });
|
||||
continue;
|
||||
}
|
||||
|
||||
if !found {
|
||||
// 未匹配到:收集连续的非分隔字符作为 "other"
|
||||
let start = i;
|
||||
while i < n {
|
||||
let ch = chars[i];
|
||||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
||||
break;
|
||||
// ── 中文:最大匹配 ──
|
||||
if is_chinese_char(ch) {
|
||||
let mut found = false;
|
||||
let try_max = max_len.min(n - i);
|
||||
|
||||
'outer: for len in (1..=try_max).rev() {
|
||||
let candidate: String = chars[i..i + len].iter().collect();
|
||||
if let Some(cat) = find_category(&candidate) {
|
||||
segments.push(Segment { text: candidate, category: cat });
|
||||
i += len;
|
||||
found = true;
|
||||
break 'outer;
|
||||
}
|
||||
// 如果下一个字符开始能匹配到词典中的词,也停在这里
|
||||
let remaining = n - i;
|
||||
if i > start && remaining >= 1 {
|
||||
let peek_max = max_len.min(remaining);
|
||||
let mut matched = false;
|
||||
for len in (1..=peek_max).rev() {
|
||||
let peek: String = chars[i..i + len].iter().collect();
|
||||
let peek_lower = peek.to_lowercase();
|
||||
for (word, _) in &all_words {
|
||||
if word.to_lowercase() == peek_lower {
|
||||
}
|
||||
|
||||
if !found {
|
||||
// 未匹配:收集连续中文字符作为 other
|
||||
let start = i;
|
||||
while i < n && is_chinese_char(chars[i]) {
|
||||
// 前瞻:如果下一个位置开始能匹配词典,就停在这里
|
||||
if i > start {
|
||||
let remaining = n - i;
|
||||
let peek_max = max_len.min(remaining);
|
||||
let mut matched = false;
|
||||
for plen in (1..=peek_max).rev() {
|
||||
let peek: String = chars[i..i + plen].iter().collect();
|
||||
if find_category(&peek).is_some() {
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if matched { break; }
|
||||
}
|
||||
if matched { break; }
|
||||
i += 1;
|
||||
}
|
||||
if i > start {
|
||||
let other: String = chars[start..i].iter().collect();
|
||||
segments.push(Segment { text: other, category: "other".to_string() });
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if i > start {
|
||||
let other: String = chars[start..i].iter().collect();
|
||||
segments.push(Segment {
|
||||
text: other,
|
||||
category: "other".to_string(),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 其他字符跳过
|
||||
i += 1;
|
||||
}
|
||||
|
||||
segments
|
||||
@@ -253,7 +266,7 @@ pub fn is_likely_person_name(name: &str) -> bool {
|
||||
|
||||
/// 统一相册标题解析入口(v5 最大匹配版本)
|
||||
///
|
||||
/// 流程:预处理 → 最大匹配切分 → 选人 → 其余为标签
|
||||
/// 流程:预处理 → 切分 → 选人 → 其余为标签
|
||||
///
|
||||
/// 返回 ParseResult 可直接用于批量写库。
|
||||
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
|
||||
@@ -265,7 +278,7 @@ pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::P
|
||||
let cleaned = preprocess_title(title);
|
||||
if cleaned.is_empty() { return result; }
|
||||
|
||||
// Step 1: 最大匹配切分
|
||||
// Step 1: 切分
|
||||
let segments = max_match_segment(&cleaned);
|
||||
|
||||
// Step 2: 从 segments 中选人(优先级最高的)
|
||||
|
||||
Reference in New Issue
Block a user