This commit is contained in:
lychang
2026-07-19 23:56:28 +08:00
parent 34bcd67682
commit 36969a4fa0
16 changed files with 1925 additions and 350 deletions

View File

@@ -96,9 +96,14 @@ lazy_static! {
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
static ref RE_ENGLISH_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
static ref RE_TRAILING_PUNCT: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
/// 人名模式:2-5个中文字符 + 可选英文后缀
/// 人名模式:2-5个中文字符(纯中文,不混杂英文)
/// 中英文混杂的情况由 split_person_name_with_aliases 处理为 主名+别名
static ref RE_CN_NAME: Regex = Regex::new(
r"([\u{4e00}-\u{9fff}]{2,5}(?:[A-Za-z][A-Za-z0-9_]*)?)"
r"([\u{4e00}-\u{9fff}]{2,5})"
).unwrap();
/// 纯英文人名模式:2-15个英文字母,首字母可大写
static ref RE_EN_NAME: Regex = Regex::new(
r"([A-Za-z][A-Za-z]{1,14})"
).unwrap();
/// 常见人名后缀(表明前面是人名)
static ref RE_NAME_CONTEXT: Regex = Regex::new(
@@ -201,6 +206,7 @@ pub fn extract_person_name_from_title(title: &str) -> Option<String> {
}
// 策略C: 直接在整个标题中搜索人名模式
// 先搜纯中文(优先级更高)
for cap in RE_CN_NAME.captures_iter(&t) {
let name = clean_name(&cap[1]);
if is_likely_person_name(&name) {
@@ -208,6 +214,16 @@ pub fn extract_person_name_from_title(title: &str) -> Option<String> {
}
}
// 再搜纯英文
for cap in RE_EN_NAME.captures_iter(&t) {
let name = clean_name(&cap[1]);
// 排除停用词
if STOP_WORDS_SET.contains(&name.to_lowercase().as_str()) { continue; }
if is_likely_person_name(&name) {
return Some(name);
}
}
None
}
@@ -245,13 +261,25 @@ fn extract_person_with_jieba(title: &str) -> Option<String> {
}
/// 从一个文本片段中提取人名
/// 只支持纯中文(2-6字)或纯英文(2-15字),不混杂
/// 中文优先级更高
fn extract_name_from_segment(segment: &str) -> Option<String> {
let seg = segment.trim();
if seg.is_empty() {
return None;
}
// 尝试直接匹配人名模式
// 判断片段是否纯中文
let is_pure_cn = seg.chars().all(|c| RE_CHINESE.is_match(&c.to_string()));
// 判断片段是否纯英文
let is_pure_en = seg.chars().all(|c| c.is_ascii_alphabetic());
// 策略1: 纯中文片段(2-6字)
if is_pure_cn && seg.len() >= 2 && seg.len() <= 6 {
return Some(clean_name(seg));
}
// 策略2: 尝试匹配纯中文人名模式
if let Some(cap) = RE_CN_NAME.captures(seg) {
let name = clean_name(&cap[1]);
if name.len() >= 2 {
@@ -259,10 +287,20 @@ fn extract_name_from_segment(segment: &str) -> Option<String> {
}
}
// 如果整个片段是纯中文(2-5字),也认为是人名
let is_pure_cn = seg.chars().all(|c| RE_CHINESE.is_match(&c.to_string()));
if is_pure_cn && seg.len() >= 2 && seg.len() <= 6 {
return Some(clean_name(seg));
// 策略3: 纯英文片段(2-15字)
if is_pure_en && seg.len() >= 2 && seg.len() <= 15 {
// 过滤停用词
if !STOP_WORDS_SET.contains(&seg.to_lowercase().as_str()) {
return Some(clean_name(seg));
}
}
// 策略4: 尝试匹配纯英文人名模式
if let Some(cap) = RE_EN_NAME.captures(seg) {
let name = clean_name(&cap[1]);
if name.len() >= 2 && !STOP_WORDS_SET.contains(&name.to_lowercase().as_str()) {
return Some(name);
}
}
None
@@ -482,13 +520,18 @@ pub fn extract_person_and_tags(title: &str) -> (Option<String>, Vec<(String, Str
}
/// 从 jieba 分词结果中提取人名
/// 返回完整的原始人名(可能含英文后缀),供 split_person_name_with_aliases 拆分为 主名+别名
/// 例如 jieba 拆出 "水淼/nr Aqua/eng" → 返回 "水淼Aqua"
fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
let mut candidates: Vec<(String, usize)> = Vec::new();
for tag in tags {
for (i, tag) in tags.iter().enumerate() {
let word = tag.word.trim();
if word.len() < 2 { continue; }
// 过滤纯标点
if is_punctuation_only(word) { continue; }
let priority = match tag.tag {
"nr" => 0, // 人名(最高优先级)
"nrfg" => 1, // 古体人名
@@ -499,7 +542,33 @@ fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
};
if is_likely_person_name(word) {
candidates.push((word.to_string(), priority));
// 如果是中文人名,检查后面是否紧跟英文词(如 水淼/nr Aqua/eng)
// 拼装成完整原始名,供 split_person_name_with_aliases 拆分为 主名+别名
let mut full_name = word.to_string();
let is_chinese = word.chars().any(|c| RE_CHINESE.is_match(&c.to_string()));
if is_chinese {
let mut j = i + 1;
while j < tags.len() {
let next_word = tags[j].word.trim();
let next_tag = tags[j].tag;
if next_word.is_empty() { j += 1; continue; }
// 英文词(eng)或纯英文字母,且不是停用词
if next_tag == "eng" ||
(next_word.chars().all(|c| c.is_ascii_alphabetic()) && next_word.len() >= 2 && next_word.len() <= 15) {
let lower = next_word.to_lowercase();
// 跳过停用词(如 photos、cosplay 等停在停用词之前)
let stop_words: std::collections::HashSet<&str> = STOP_WORDS.iter().copied().collect();
if !stop_words.contains(lower.as_str()) && !EXCLUDED_LOWER.contains(&lower) {
full_name.push_str(next_word);
j += 1;
continue;
}
}
break;
}
}
candidates.push((full_name, priority));
}
}
@@ -507,6 +576,18 @@ fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
candidates.into_iter().next().map(|(name, _)| clean_name(&name))
}
/// 判断字符串是否全为标点符号(不含任何字、字母或数字)
fn is_punctuation_only(s: &str) -> bool {
if s.is_empty() { return true; }
// 检查是否包含任意"有效字符"(中文字、字母、数字)
!s.chars().any(|c| {
c.is_ascii_alphanumeric()
|| ('\u{4e00}'..='\u{9fff}').contains(&c)
|| ('\u{3400}'..='\u{4dbf}').contains(&c)
|| ('\u{f900}'..='\u{faff}').contains(&c)
})
}
/// 从 jieba 分词结果中提取标签(排除人名)
fn extract_tags_from_jieba_result(
tags: &[jieba_rs::Tag],
@@ -529,6 +610,13 @@ fn extract_tags_from_jieba_result(
}
if let Some(cn) = extract_cn(n) {
v.push(cn.to_lowercase());
// 中英文混合名的英文后缀也要排除(如 "水淼Aqua" → "aqua")
if let Some(rest) = n.strip_prefix(&cn) {
let suffix = rest.trim();
if !suffix.is_empty() {
v.push(suffix.to_lowercase());
}
}
}
v
})
@@ -540,6 +628,9 @@ fn extract_tags_from_jieba_result(
let word_lower = word.to_lowercase();
// 纯标点/符号跳过
if is_punctuation_only(word) { continue; }
// 纯数字跳过
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
@@ -558,14 +649,14 @@ fn extract_tags_from_jieba_result(
"n" | "ng" => "noun",
"nz" => "proper",
"ns" => "location",
"nt" => "organization",
"vn" => "noun_verb",
"a" | "ad" | "an" => "adj",
"v" | "vd" | "vg" => "verb",
"eng" => "english",
"eng" => "other",
"x" => "other",
// 排除的词性
// 排除的词性(这些不应该作为标签)
"nr" | "nrfg" | "nrt" => continue, // 人名
"nt" => continue, // 机构团体名(不是标签)
"vn" => continue, // 名动词(分类模糊)
"a" | "ad" | "an" => continue, // 形容词(不是标签)
"v" | "vd" | "vg" => continue, // 动词(不是标签)
"m" | "mq" => continue,
"q" | "qv" | "qt" => continue,
"r" | "rr" | "rz" => continue,
@@ -583,8 +674,6 @@ fn extract_tags_from_jieba_result(
_ => continue,
};
if category == "english" && word.len() < 3 { continue; }
result.push((word.to_string(), category.to_string()));
}
@@ -650,10 +739,13 @@ mod tests {
#[test]
fn test_extract_person_name() {
assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨Yome");
assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶yanni");
// 纯中文人名(中文优先级更高,英文后缀作为别名处理)
assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨");
assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶");
assert_eq!(extract_person_name_from_title("[XiuRen] 杨晨晨 性感").unwrap(), "杨晨晨");
assert_eq!(extract_person_name_from_title("美女模特 杨晨晨 私房").unwrap(), "杨晨晨");
// 纯英文人名也支持
assert!(extract_person_name_from_title("Yome sexy photos").is_some());
}
#[test]