update

src/engine/title_parser.rs

@@ -0,0 +1,285 @@
+//! 标题解析、人名提取、标签提取
+//! 对应 Python 的 tag_index.py + clean_person_names.py 的核心逻辑
+
+use regex::Regex;
+use std::collections::HashSet;
+use lazy_static::lazy_static;
+
+// ── 常量 ──
+
+/// 模特称呼前缀（移除用）
+const MODEL_TITLE_PREFIX: &[&str] = &[
+    "美女模特", "性感女神", "清纯美女", "气质美女", "可爱女孩",
+    "甜美少女", "时尚达人", "女神", "美女", "模特", "小姐姐",
+    "小仙女", "御姐", "萝莉", "辣妹", "潮人",
+];
+
+/// 停用词（标签过滤用）
+const STOP_WORDS: &[&str] = &[
+    "写真", "套图", "图片", "照片", "摄影", "拍摄", "风格",
+    "系列", "作品", "专辑", "主题", "人像", "艺术", "唯美",
+    "诱惑", "性感", "迷人", "魅力", "时尚", "潮流", "高清",
+    "原版", "无水印", "私房", "内衣", "比基尼", "泳装",
+    "旗袍", "制服", "丝袜", "美腿", "美胸", "翘臀",
+    "cosplay", "cos", "角色扮演", "动漫", "游戏",
+    "classic", "exclusive", "premium", "collection",
+    "photo", "photos", "video", "videos", "set", "vol",
+    "no", "番号", "编号", "期号",
+];
+
+/// 保留词性（仅用于 jieba 分词场景，这里做占位符）
+const _KEEP_POS: &[&str] = &["n", "nr", "ns", "nz", "v", "vn", "a", "an", "eng", "x"];
+
+/// 工作室/平台排除名
+const EXCLUDED_NAMES: &[&str] = &[
+    "Pure Media", "XiuRen秀人网", "尤蜜", "魅妍社", "花漾",
+    "瑞丝馆", "撸图", "情欲萝莉", "蜜桃社", "tgod",
+    "MiStar", "BLUECAKE", "HuaYang", "YouMi", "秀人",
+];
+
+lazy_static! {
+    /// 编译后的正则
+    static ref RE_COSPLAY: Regex = Regex::new(
+        r"(?i)^(.+?)\s+cos(play)?\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
+    ).unwrap();
+    static ref RE_COSPLAY_BY: Regex = Regex::new(
+        r"(?i)^(.+?)\s+cos(play)?\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
+    ).unwrap();
+    static ref RE_BRACKET_PREFIX: Regex = Regex::new(r"^\[([^\]]+)\]\s*").unwrap();
+    static ref RE_NUMBER_SUFFIX: Regex = Regex::new(r"[\s#]*(?:no\.?\s*|vol\.?\s*|#)?\d+[\s]*$").unwrap();
+    static ref RE_PAREN_CONTENT: Regex = Regex::new(r"[（(]([^）)]*)[）)]").unwrap();
+    static ref RE_SPLIT_DASH: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap();
+    static ref RE_PHOTOS_SUFFIX: Regex = Regex::new(r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$").unwrap();
+    static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
+    static ref RE_ENGLISH_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
+    static ref RE_TRAILING_PUNCT: Regex = Regex::new(r"[\s.,;:，。；：\-–—]+$").unwrap();
+
+    /// 停用词集合
+    static ref STOP_WORDS_SET: HashSet<&'static str> = {
+        let mut s = HashSet::new();
+        for w in STOP_WORDS { s.insert(*w); }
+        s
+    };
+
+    /// 排除名集合（小写）
+    static ref EXCLUDED_LOWER: HashSet<String> = {
+        let mut s = HashSet::new();
+        for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); }
+        s.extend(["youmi", "huayang", "bluecake", "mistar", "xiuren", "秀人"].map(|s| s.to_string()));
+        s
+    };
+}
+
+// ── 人名提取 ──
+
+/// 从标题中提取模特/人物名称
+/// 对应 Python tag_index.extract_person_name_from_title
+pub fn extract_person_name_from_title(title: &str) -> Option<String> {
+    let title = title.trim();
+    if title.is_empty() {
+        return None;
+    }
+
+    // 清理末尾的 "N photos" 等
+    let title = RE_PHOTOS_SUFFIX.replace(title, "").trim().to_string();
+
+    // 1. cosplay 格式: "Name cosplay Character – Series"
+    if let Some(caps) = RE_COSPLAY_BY.captures(&title) {
+        return Some(caps[3].trim().to_string());
+    }
+    if let Some(caps) = RE_COSPLAY.captures(&title) {
+        return Some(caps[1].trim().to_string());
+    }
+
+    // 2. 中文摄影工作室格式
+    let title = RE_BRACKET_PREFIX.replace(&title, "").trim().to_string();
+
+    // 移除模特称呼前缀
+    let mut title = title;
+    for prefix in MODEL_TITLE_PREFIX {
+        if title.starts_with(prefix) {
+            title = title[prefix.len()..].trim().to_string();
+            break;
+        }
+    }
+
+    // 移除编号后缀
+    let title = RE_NUMBER_SUFFIX.replace(&title, "").trim().to_string();
+
+    // 用 `–—\-` 分割，取第一部分作为人名
+    let parts: Vec<&str> = RE_SPLIT_DASH.split(&title).collect();
+    let name = parts[0].trim().to_string();
+
+    if name.len() >= 2 {
+        Some(name)
+    } else {
+        None
+    }
+}
+
+/// 提取中文核心（开头的连续中文字符）
+/// 对应 Python merge_persons_by_core.extract_cn
+pub fn extract_cn(name: &str) -> Option<String> {
+    let re = Regex::new(r"^([\u{4e00}-\u{9fff}]+)").unwrap();
+    re.captures(name).map(|c| c[1].to_string())
+}
+
+/// 判断后缀是否有效（不含中文=有效英文后缀）
+/// 对应 Python merge_persons_by_core.is_valid_suffix
+pub fn is_valid_suffix(suffix: &str) -> bool {
+    if suffix.is_empty() {
+        return true;
+    }
+    !RE_CHINESE.is_match(suffix)
+}
+
+/// 判断是否为工作室/平台名（需排除）
+pub fn is_excluded_name(name: &str) -> bool {
+    EXCLUDED_LOWER.contains(&name.to_lowercase())
+}
+
+// ── 人名清洗 ──
+
+/// 清洗人名：移除前缀、括号内容、末尾标点
+/// 返回 (cleaned_name, aliases)
+/// 对应 Python clean_person_names.clean_person_name
+pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
+    let mut aliases = Vec::new();
+    let mut name = raw_name.to_string();
+
+    // 1. 移除前缀
+    for prefix in MODEL_TITLE_PREFIX {
+        if name.starts_with(prefix) {
+            aliases.push(("prefix".to_string(), prefix.to_string()));
+            name = name[prefix.len()..].trim().to_string();
+            break;
+        }
+    }
+
+    // 2. 移除末尾括号内容
+    let mut cleaned = String::new();
+    let mut last_end = 0;
+    for cap in RE_PAREN_CONTENT.captures_iter(&name) {
+        let m = cap.get(0).unwrap();
+        let content = cap.get(1).map_or("", |c| c.as_str());
+        cleaned.push_str(&name[last_end..m.start()]);
+        last_end = m.end();
+
+        if !content.is_empty() {
+            if RE_CHINESE.is_match(content) {
+                aliases.push(("nickname".to_string(), content.to_string()));
+            } else {
+                aliases.push(("english".to_string(), content.to_string()));
+            }
+        }
+    }
+    cleaned.push_str(&name[last_end..]);
+    let name = cleaned.trim().to_string();
+
+    // 3. 清理末尾标点
+    let name = RE_TRAILING_PUNCT.replace(&name, "").trim().to_string();
+
+    // 4. 从清洗后名字提取英文名别名
+    if let Some(caps) = RE_ENGLISH_SUFFIX.captures(&name) {
+        let chinese_part = caps[1].to_string();
+        let english_part = caps[2].to_string();
+        if !english_part.is_empty() {
+            aliases.push(("chinese".to_string(), chinese_part));
+            aliases.push(("english".to_string(), english_part));
+        }
+    }
+
+    if name.is_empty() {
+        (raw_name.to_string(), aliases)
+    } else {
+        (name, aliases)
+    }
+}
+
+/// 获取归并键（小写化+移除空格和标点）
+pub fn get_person_key(name: &str) -> String {
+    name.to_lowercase()
+        .chars()
+        .filter(|c| !c.is_whitespace() && !c.is_ascii_punctuation())
+        .collect()
+}
+
+// ── 标签提取 ──
+
+/// 从标题中提取标签（排除人名部分）
+/// 返回 Vec<(tag, pos)>
+/// 对应 Python tag_index.extract_tags_from_title
+#[allow(dead_code)]
+pub fn extract_tags_from_title_legacy(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
+    let mut title = title.to_string();
+
+    // 清理末尾的 "N photos" 等
+    title = RE_PHOTOS_SUFFIX.replace(&title, "").to_string();
+
+    // 移除人名
+    if let Some(name) = person_name {
+        title = title.replace(name, " ");
+        for part in name.split_whitespace() {
+            title = title.replace(part, " ");
+        }
+    }
+
+    // 移除 [工作室] 前缀
+    title = RE_BRACKET_PREFIX.replace(&title, "").to_string();
+
+    // 按空白分割，过滤停用词和短词
+    let mut tags = Vec::new();
+    let mut seen = HashSet::new();
+
+    for word in title.split_whitespace() {
+        let word = word.trim().to_lowercase();
+        if word.len() < 2 {
+            continue;
+        }
+        if STOP_WORDS_SET.contains(word.as_str()) {
+            continue;
+        }
+        if word.chars().all(|c| c.is_ascii_digit() || c == '.') {
+            continue;
+        }
+        if seen.contains(&word) {
+            continue;
+        }
+        seen.insert(word.clone());
+        tags.push((word, "x".to_string()));
+    }
+
+    tags
+}
+
+/// 标签归一化
+pub fn normalize_tag(tag: &str) -> String {
+    tag.trim().to_lowercase()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_person_name() {
+        assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨Yome");
+        assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶yanni");
+        assert_eq!(extract_person_name_from_title("[XiuRen] 杨晨晨 性感").unwrap(), "杨晨晨");
+        assert_eq!(extract_person_name_from_title("美女模特 杨晨晨 私房").unwrap(), "杨晨晨");
+    }
+
+    #[test]
+    fn test_extract_cn() {
+        assert_eq!(extract_cn("杨晨晨Yome").unwrap(), "杨晨晨");
+        assert_eq!(extract_cn("王馨瑶yanni").unwrap(), "王馨瑶");
+        assert!(extract_cn("Yome").is_none());
+    }
+
+    #[test]
+    fn test_clean_person_name() {
+        let (cleaned, aliases) = clean_person_name("美女模特 杨晨晨Yome");
+        assert_eq!(cleaned, "美女模特 杨晨晨Yome");
+        assert!(!aliases.is_empty());
+    }
+}