update code level
This commit is contained in:
@@ -4,7 +4,7 @@
|
|||||||
//!
|
//!
|
||||||
//! 流程:
|
//! 流程:
|
||||||
//! 1. 预处理:移除照片数后缀、[工作室]前缀、称呼前缀、编号后缀
|
//! 1. 预处理:移除照片数后缀、[工作室]前缀、称呼前缀、编号后缀
|
||||||
//! 2. 最大匹配切分:在 keymap + include + stop_words 词典中做最长子串匹配
|
//! 2. 切分:中文最大匹配 + 英文按空格切分
|
||||||
//! 3. 角色分类:每个片段有明确分类(person / style / outfit / ... / stop / other)
|
//! 3. 角色分类:每个片段有明确分类(person / style / outfit / ... / stop / other)
|
||||||
//! 4. 选人名:优先级 = keymap(person) > include 列表 > 启发式
|
//! 4. 选人名:优先级 = keymap(person) > include 列表 > 启发式
|
||||||
//! 5. 其余分类即为标签
|
//! 5. 其余分类即为标签
|
||||||
@@ -13,6 +13,7 @@
|
|||||||
//! - 中文连续文本也能正确切分(不再依赖空格)
|
//! - 中文连续文本也能正确切分(不再依赖空格)
|
||||||
//! - person 和 tag 在同一 pass 中确定,不会混淆
|
//! - person 和 tag 在同一 pass 中确定,不会混淆
|
||||||
//! - 最长匹配优先,避免"性感"和"性感女神"都匹配的问题
|
//! - 最长匹配优先,避免"性感"和"性感女神"都匹配的问题
|
||||||
|
//! - 英文只按空格切分,不逐字匹配,不移除英文介词等停用词
|
||||||
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
@@ -63,15 +64,23 @@ struct Segment {
|
|||||||
category: String,
|
category: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 最大匹配切分(核心算法) ──
|
// ── 切分核心算法 ──
|
||||||
|
|
||||||
/// 对标题做最大匹配切分,返回带分类的片段列表。
|
/// 判断字符是否是中文字符
|
||||||
|
fn is_chinese_char(ch: char) -> bool {
|
||||||
|
('\u{4e00}'..='\u{9fff}').contains(&ch)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 对标题做切分,返回带分类的片段列表。
|
||||||
///
|
///
|
||||||
/// 算法:从左到右扫描,每次尝试最长可能的匹配(max_len 到 1)。
|
/// 算法分两路:
|
||||||
/// 匹配来源优先级(同一长度下):
|
/// - 中文:从左到右最大匹配(逐字尝试,最长优先)
|
||||||
|
/// - 英文/数字:按空格/分隔符直接切分为完整单词,查词典分类
|
||||||
|
///
|
||||||
|
/// 匹配来源优先级:
|
||||||
/// keymap(person) > keymap(其他) > stop_words > include_person
|
/// keymap(person) > keymap(其他) > stop_words > include_person
|
||||||
///
|
///
|
||||||
/// 未匹配到的连续字符作为 "other" 片段保留。
|
/// 未匹配到的作为 "other" 片段保留。
|
||||||
fn max_match_segment(title: &str) -> Vec<Segment> {
|
fn max_match_segment(title: &str) -> Vec<Segment> {
|
||||||
let dict = tag_dict::get_tag_dict();
|
let dict = tag_dict::get_tag_dict();
|
||||||
|
|
||||||
@@ -95,87 +104,91 @@ fn max_match_segment(title: &str) -> Vec<Segment> {
|
|||||||
all_words.push((word.clone(), "stop".to_string()));
|
all_words.push((word.clone(), "stop".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// 计算最大词长
|
// 计算最大词长(中文最大匹配用)
|
||||||
let max_len = all_words.iter().map(|(w, _)| w.chars().count()).max().unwrap_or(8);
|
let max_len = all_words.iter().map(|(w, _)| w.chars().count()).max().unwrap_or(8);
|
||||||
|
|
||||||
|
// 快速查找函数
|
||||||
|
let find_category = |word: &str| -> Option<String> {
|
||||||
|
let word_lower = word.to_lowercase();
|
||||||
|
for (w, cat) in &all_words {
|
||||||
|
if w.to_lowercase() == word_lower {
|
||||||
|
return Some(cat.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
let mut segments: Vec<Segment> = Vec::new();
|
let mut segments: Vec<Segment> = Vec::new();
|
||||||
let chars: Vec<char> = title.chars().collect();
|
let chars: Vec<char> = title.chars().collect();
|
||||||
let n = chars.len();
|
let n = chars.len();
|
||||||
let mut i = 0;
|
let mut i = 0;
|
||||||
|
|
||||||
while i < n {
|
while i < n {
|
||||||
// 跳过分隔符
|
|
||||||
let ch = chars[i];
|
let ch = chars[i];
|
||||||
|
|
||||||
|
// 跳过分隔符
|
||||||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
||||||
i += 1;
|
i += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut found = false;
|
// ── 英文/数字:直接按分隔符切分整个单词,不逐字匹配 ──
|
||||||
|
if ch.is_ascii_alphanumeric() {
|
||||||
// 从最长到最短尝试匹配
|
let start = i;
|
||||||
let try_max = max_len.min(n - i);
|
while i < n && chars[i].is_ascii_alphanumeric() {
|
||||||
'outer: for len in (1..=try_max).rev() {
|
i += 1;
|
||||||
let candidate: String = chars[i..i + len].iter().collect();
|
|
||||||
let candidate_lower = candidate.to_lowercase();
|
|
||||||
|
|
||||||
// 收集所有匹配该长度的词,按优先级选最好的
|
|
||||||
let mut best: Option<&String> = None; // 存储分类
|
|
||||||
|
|
||||||
for (word, cat) in &all_words {
|
|
||||||
if word.to_lowercase() == candidate_lower {
|
|
||||||
best = Some(cat);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(cat) = best {
|
|
||||||
segments.push(Segment {
|
|
||||||
text: candidate,
|
|
||||||
category: cat.clone(),
|
|
||||||
});
|
|
||||||
i += len;
|
|
||||||
found = true;
|
|
||||||
break 'outer;
|
|
||||||
}
|
}
|
||||||
|
let word: String = chars[start..i].iter().collect();
|
||||||
|
let cat = find_category(&word).unwrap_or_else(|| "other".to_string());
|
||||||
|
segments.push(Segment { text: word, category: cat });
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found {
|
// ── 中文:最大匹配 ──
|
||||||
// 未匹配到:收集连续的非分隔字符作为 "other"
|
if is_chinese_char(ch) {
|
||||||
let start = i;
|
let mut found = false;
|
||||||
while i < n {
|
let try_max = max_len.min(n - i);
|
||||||
let ch = chars[i];
|
|
||||||
if RE_SEPARATOR.is_match(&ch.to_string()) {
|
'outer: for len in (1..=try_max).rev() {
|
||||||
break;
|
let candidate: String = chars[i..i + len].iter().collect();
|
||||||
|
if let Some(cat) = find_category(&candidate) {
|
||||||
|
segments.push(Segment { text: candidate, category: cat });
|
||||||
|
i += len;
|
||||||
|
found = true;
|
||||||
|
break 'outer;
|
||||||
}
|
}
|
||||||
// 如果下一个字符开始能匹配到词典中的词,也停在这里
|
}
|
||||||
let remaining = n - i;
|
|
||||||
if i > start && remaining >= 1 {
|
if !found {
|
||||||
let peek_max = max_len.min(remaining);
|
// 未匹配:收集连续中文字符作为 other
|
||||||
let mut matched = false;
|
let start = i;
|
||||||
for len in (1..=peek_max).rev() {
|
while i < n && is_chinese_char(chars[i]) {
|
||||||
let peek: String = chars[i..i + len].iter().collect();
|
// 前瞻:如果下一个位置开始能匹配词典,就停在这里
|
||||||
let peek_lower = peek.to_lowercase();
|
if i > start {
|
||||||
for (word, _) in &all_words {
|
let remaining = n - i;
|
||||||
if word.to_lowercase() == peek_lower {
|
let peek_max = max_len.min(remaining);
|
||||||
|
let mut matched = false;
|
||||||
|
for plen in (1..=peek_max).rev() {
|
||||||
|
let peek: String = chars[i..i + plen].iter().collect();
|
||||||
|
if find_category(&peek).is_some() {
|
||||||
matched = true;
|
matched = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if matched { break; }
|
if matched { break; }
|
||||||
}
|
}
|
||||||
if matched { break; }
|
i += 1;
|
||||||
|
}
|
||||||
|
if i > start {
|
||||||
|
let other: String = chars[start..i].iter().collect();
|
||||||
|
segments.push(Segment { text: other, category: "other".to_string() });
|
||||||
}
|
}
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
if i > start {
|
|
||||||
let other: String = chars[start..i].iter().collect();
|
|
||||||
segments.push(Segment {
|
|
||||||
text: other,
|
|
||||||
category: "other".to_string(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 其他字符跳过
|
||||||
|
i += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
segments
|
segments
|
||||||
@@ -253,7 +266,7 @@ pub fn is_likely_person_name(name: &str) -> bool {
|
|||||||
|
|
||||||
/// 统一相册标题解析入口(v5 最大匹配版本)
|
/// 统一相册标题解析入口(v5 最大匹配版本)
|
||||||
///
|
///
|
||||||
/// 流程:预处理 → 最大匹配切分 → 选人 → 其余为标签
|
/// 流程:预处理 → 切分 → 选人 → 其余为标签
|
||||||
///
|
///
|
||||||
/// 返回 ParseResult 可直接用于批量写库。
|
/// 返回 ParseResult 可直接用于批量写库。
|
||||||
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
|
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
|
||||||
@@ -265,7 +278,7 @@ pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::P
|
|||||||
let cleaned = preprocess_title(title);
|
let cleaned = preprocess_title(title);
|
||||||
if cleaned.is_empty() { return result; }
|
if cleaned.is_empty() { return result; }
|
||||||
|
|
||||||
// Step 1: 最大匹配切分
|
// Step 1: 切分
|
||||||
let segments = max_match_segment(&cleaned);
|
let segments = max_match_segment(&cleaned);
|
||||||
|
|
||||||
// Step 2: 从 segments 中选人(优先级最高的)
|
// Step 2: 从 segments 中选人(优先级最高的)
|
||||||
|
|||||||
Reference in New Issue
Block a user