update
This commit is contained in:
@@ -96,9 +96,14 @@ lazy_static! {
|
||||
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
||||
static ref RE_ENGLISH_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
|
||||
static ref RE_TRAILING_PUNCT: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
|
||||
/// 人名模式:2-5个中文字符 + 可选英文后缀
|
||||
/// 人名模式:2-5个中文字符(纯中文,不混杂英文)
|
||||
/// 中英文混杂的情况由 split_person_name_with_aliases 处理为 主名+别名
|
||||
static ref RE_CN_NAME: Regex = Regex::new(
|
||||
r"([\u{4e00}-\u{9fff}]{2,5}(?:[A-Za-z][A-Za-z0-9_]*)?)"
|
||||
r"([\u{4e00}-\u{9fff}]{2,5})"
|
||||
).unwrap();
|
||||
/// 纯英文人名模式:2-15个英文字母,首字母可大写
|
||||
static ref RE_EN_NAME: Regex = Regex::new(
|
||||
r"([A-Za-z][A-Za-z]{1,14})"
|
||||
).unwrap();
|
||||
/// 常见人名后缀(表明前面是人名)
|
||||
static ref RE_NAME_CONTEXT: Regex = Regex::new(
|
||||
@@ -201,6 +206,7 @@ pub fn extract_person_name_from_title(title: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
// 策略C: 直接在整个标题中搜索人名模式
|
||||
// 先搜纯中文(优先级更高)
|
||||
for cap in RE_CN_NAME.captures_iter(&t) {
|
||||
let name = clean_name(&cap[1]);
|
||||
if is_likely_person_name(&name) {
|
||||
@@ -208,6 +214,16 @@ pub fn extract_person_name_from_title(title: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
// 再搜纯英文
|
||||
for cap in RE_EN_NAME.captures_iter(&t) {
|
||||
let name = clean_name(&cap[1]);
|
||||
// 排除停用词
|
||||
if STOP_WORDS_SET.contains(&name.to_lowercase().as_str()) { continue; }
|
||||
if is_likely_person_name(&name) {
|
||||
return Some(name);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
@@ -245,13 +261,25 @@ fn extract_person_with_jieba(title: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
/// 从一个文本片段中提取人名
|
||||
/// 只支持纯中文(2-6字)或纯英文(2-15字),不混杂
|
||||
/// 中文优先级更高
|
||||
fn extract_name_from_segment(segment: &str) -> Option<String> {
|
||||
let seg = segment.trim();
|
||||
if seg.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 尝试直接匹配人名模式
|
||||
// 判断片段是否纯中文
|
||||
let is_pure_cn = seg.chars().all(|c| RE_CHINESE.is_match(&c.to_string()));
|
||||
// 判断片段是否纯英文
|
||||
let is_pure_en = seg.chars().all(|c| c.is_ascii_alphabetic());
|
||||
|
||||
// 策略1: 纯中文片段(2-6字)
|
||||
if is_pure_cn && seg.len() >= 2 && seg.len() <= 6 {
|
||||
return Some(clean_name(seg));
|
||||
}
|
||||
|
||||
// 策略2: 尝试匹配纯中文人名模式
|
||||
if let Some(cap) = RE_CN_NAME.captures(seg) {
|
||||
let name = clean_name(&cap[1]);
|
||||
if name.len() >= 2 {
|
||||
@@ -259,10 +287,20 @@ fn extract_name_from_segment(segment: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
// 如果整个片段是纯中文(2-5字),也认为是人名
|
||||
let is_pure_cn = seg.chars().all(|c| RE_CHINESE.is_match(&c.to_string()));
|
||||
if is_pure_cn && seg.len() >= 2 && seg.len() <= 6 {
|
||||
return Some(clean_name(seg));
|
||||
// 策略3: 纯英文片段(2-15字)
|
||||
if is_pure_en && seg.len() >= 2 && seg.len() <= 15 {
|
||||
// 过滤停用词
|
||||
if !STOP_WORDS_SET.contains(&seg.to_lowercase().as_str()) {
|
||||
return Some(clean_name(seg));
|
||||
}
|
||||
}
|
||||
|
||||
// 策略4: 尝试匹配纯英文人名模式
|
||||
if let Some(cap) = RE_EN_NAME.captures(seg) {
|
||||
let name = clean_name(&cap[1]);
|
||||
if name.len() >= 2 && !STOP_WORDS_SET.contains(&name.to_lowercase().as_str()) {
|
||||
return Some(name);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
@@ -482,13 +520,18 @@ pub fn extract_person_and_tags(title: &str) -> (Option<String>, Vec<(String, Str
|
||||
}
|
||||
|
||||
/// 从 jieba 分词结果中提取人名
|
||||
/// 返回完整的原始人名(可能含英文后缀),供 split_person_name_with_aliases 拆分为 主名+别名
|
||||
/// 例如 jieba 拆出 "水淼/nr Aqua/eng" → 返回 "水淼Aqua"
|
||||
fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
|
||||
let mut candidates: Vec<(String, usize)> = Vec::new();
|
||||
|
||||
for tag in tags {
|
||||
for (i, tag) in tags.iter().enumerate() {
|
||||
let word = tag.word.trim();
|
||||
if word.len() < 2 { continue; }
|
||||
|
||||
// 过滤纯标点
|
||||
if is_punctuation_only(word) { continue; }
|
||||
|
||||
let priority = match tag.tag {
|
||||
"nr" => 0, // 人名(最高优先级)
|
||||
"nrfg" => 1, // 古体人名
|
||||
@@ -499,7 +542,33 @@ fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
|
||||
};
|
||||
|
||||
if is_likely_person_name(word) {
|
||||
candidates.push((word.to_string(), priority));
|
||||
// 如果是中文人名,检查后面是否紧跟英文词(如 水淼/nr Aqua/eng)
|
||||
// 拼装成完整原始名,供 split_person_name_with_aliases 拆分为 主名+别名
|
||||
let mut full_name = word.to_string();
|
||||
let is_chinese = word.chars().any(|c| RE_CHINESE.is_match(&c.to_string()));
|
||||
if is_chinese {
|
||||
let mut j = i + 1;
|
||||
while j < tags.len() {
|
||||
let next_word = tags[j].word.trim();
|
||||
let next_tag = tags[j].tag;
|
||||
if next_word.is_empty() { j += 1; continue; }
|
||||
// 英文词(eng)或纯英文字母,且不是停用词
|
||||
if next_tag == "eng" ||
|
||||
(next_word.chars().all(|c| c.is_ascii_alphabetic()) && next_word.len() >= 2 && next_word.len() <= 15) {
|
||||
let lower = next_word.to_lowercase();
|
||||
// 跳过停用词(如 photos、cosplay 等停在停用词之前)
|
||||
let stop_words: std::collections::HashSet<&str> = STOP_WORDS.iter().copied().collect();
|
||||
if !stop_words.contains(lower.as_str()) && !EXCLUDED_LOWER.contains(&lower) {
|
||||
full_name.push_str(next_word);
|
||||
j += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
candidates.push((full_name, priority));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -507,6 +576,18 @@ fn extract_person_from_tags(tags: &[jieba_rs::Tag]) -> Option<String> {
|
||||
candidates.into_iter().next().map(|(name, _)| clean_name(&name))
|
||||
}
|
||||
|
||||
/// 判断字符串是否全为标点符号(不含任何字、字母或数字)
|
||||
fn is_punctuation_only(s: &str) -> bool {
|
||||
if s.is_empty() { return true; }
|
||||
// 检查是否包含任意"有效字符"(中文字、字母、数字)
|
||||
!s.chars().any(|c| {
|
||||
c.is_ascii_alphanumeric()
|
||||
|| ('\u{4e00}'..='\u{9fff}').contains(&c)
|
||||
|| ('\u{3400}'..='\u{4dbf}').contains(&c)
|
||||
|| ('\u{f900}'..='\u{faff}').contains(&c)
|
||||
})
|
||||
}
|
||||
|
||||
/// 从 jieba 分词结果中提取标签(排除人名)
|
||||
fn extract_tags_from_jieba_result(
|
||||
tags: &[jieba_rs::Tag],
|
||||
@@ -529,6 +610,13 @@ fn extract_tags_from_jieba_result(
|
||||
}
|
||||
if let Some(cn) = extract_cn(n) {
|
||||
v.push(cn.to_lowercase());
|
||||
// 中英文混合名的英文后缀也要排除(如 "水淼Aqua" → "aqua")
|
||||
if let Some(rest) = n.strip_prefix(&cn) {
|
||||
let suffix = rest.trim();
|
||||
if !suffix.is_empty() {
|
||||
v.push(suffix.to_lowercase());
|
||||
}
|
||||
}
|
||||
}
|
||||
v
|
||||
})
|
||||
@@ -540,6 +628,9 @@ fn extract_tags_from_jieba_result(
|
||||
|
||||
let word_lower = word.to_lowercase();
|
||||
|
||||
// 纯标点/符号跳过
|
||||
if is_punctuation_only(word) { continue; }
|
||||
|
||||
// 纯数字跳过
|
||||
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
|
||||
|
||||
@@ -558,14 +649,14 @@ fn extract_tags_from_jieba_result(
|
||||
"n" | "ng" => "noun",
|
||||
"nz" => "proper",
|
||||
"ns" => "location",
|
||||
"nt" => "organization",
|
||||
"vn" => "noun_verb",
|
||||
"a" | "ad" | "an" => "adj",
|
||||
"v" | "vd" | "vg" => "verb",
|
||||
"eng" => "english",
|
||||
"eng" => "other",
|
||||
"x" => "other",
|
||||
// 排除的词性
|
||||
// 排除的词性(这些不应该作为标签)
|
||||
"nr" | "nrfg" | "nrt" => continue, // 人名
|
||||
"nt" => continue, // 机构团体名(不是标签)
|
||||
"vn" => continue, // 名动词(分类模糊)
|
||||
"a" | "ad" | "an" => continue, // 形容词(不是标签)
|
||||
"v" | "vd" | "vg" => continue, // 动词(不是标签)
|
||||
"m" | "mq" => continue,
|
||||
"q" | "qv" | "qt" => continue,
|
||||
"r" | "rr" | "rz" => continue,
|
||||
@@ -583,8 +674,6 @@ fn extract_tags_from_jieba_result(
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
if category == "english" && word.len() < 3 { continue; }
|
||||
|
||||
result.push((word.to_string(), category.to_string()));
|
||||
}
|
||||
|
||||
@@ -650,10 +739,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_extract_person_name() {
|
||||
assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨Yome");
|
||||
assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶yanni");
|
||||
// 纯中文人名(中文优先级更高,英文后缀作为别名处理)
|
||||
assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨");
|
||||
assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶");
|
||||
assert_eq!(extract_person_name_from_title("[XiuRen] 杨晨晨 性感").unwrap(), "杨晨晨");
|
||||
assert_eq!(extract_person_name_from_title("美女模特 杨晨晨 私房").unwrap(), "杨晨晨");
|
||||
// 纯英文人名也支持
|
||||
assert!(extract_person_name_from_title("Yome sexy photos").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user