This commit is contained in:
lychang
2026-07-20 05:23:31 +08:00
parent a6bfe7eda2
commit 390b548935
6 changed files with 1236 additions and 1736 deletions

View File

@@ -128,12 +128,13 @@ pub fn deep_match_title(
let title_clean = title.trim();
// 策略1: 用正则提取人名
if let Some(extracted) = crate::engine::title_parser::extract_person_name_from_title(title_clean) {
if known_names.contains(&extracted) {
let extracted = crate::engine::title_parser::extract_person_name_from_title(title_clean);
if let Some(ref e) = extracted {
if known_names.contains(e) {
result.matched = true;
result.method = "jieba_extract".to_string();
result.person = Some(extracted.clone());
result.matched_text = Some(extracted);
result.person = Some(e.clone());
result.matched_text = Some(e.clone());
return result;
}
}

View File

@@ -166,7 +166,9 @@ pub async fn phase_detect_person_in_tags(db: &Database) -> (DirtySet, usize) {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
// jieba 结果更完整 → 用 jieba
if j.len() > b.len() || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),

View File

@@ -1,15 +1,13 @@
//! 标签词典 — 从外部 JSON 文件加载分类标签、停用词、人物规则
//!
//! 词典文件格式(dict/tags.json v2):
//! 词典文件格式(dict/tags.json v4):
//! ```json
//! {
//! "version": 2,
//! "tags": { "style": {"words": [...]}, "outfit": {...}, ... },
//! "version": 4,
//! "metadata": { "style": {"label":"风格","color_bg":"...","color_fg":"..."}, ... },
//! "keymap": { "日系": "style", "比基尼": "outfit", ... },
//! "stop_words": [...],
//! "person_rules": {
//! "prefixes": ["美女模特", ...],
//! "excluded": ["XiuRen", ...]
//! }
//! "person_rules": { "prefixes": [...], "excluded": [...], "include": [...] }
//! }
//! ```
@@ -19,50 +17,45 @@ use serde::Deserialize;
use lazy_static::lazy_static;
use std::sync::RwLock;
// ── JSON 反序列化结构 ──
// ── JSON 反序列化结构 (v4) ──
/// 词典文件的顶层结构 (v3)
/// v4 分类元数据(label + 颜色)
#[derive(Debug, Clone, Deserialize)]
pub struct CategoryMeta {
#[serde(default)]
pub label: String,
#[serde(default = "default_color_bg")]
pub color_bg: String,
#[serde(default = "default_color_fg")]
pub color_fg: String,
}
fn default_color_bg() -> String { "#f1f5f9".to_string() }
fn default_color_fg() -> String { "#64748b".to_string() }
/// 词典文件的顶层结构 (v4)
#[derive(Debug, Clone, Deserialize)]
pub struct TagDictFile {
pub version: u32,
#[allow(unused)]
pub description: Option<String>,
/// v3: 分类元数据 { "style": {"label":..., "color_bg":..., ...} }
/// v4: 分类元数据 { "style": {"label":"风格","color_bg":"...","color_fg":"..."}, ... }
#[serde(default)]
pub categories: HashMap<String, TagCategoryMeta>,
/// v3: 分类词表 { "style": ["日系", ...], "outfit": [...], ... }
pub metadata: HashMap<String, CategoryMeta>,
/// v4: 扁平化匹配串→分类名 { "日系": "style", "比基尼": "outfit", ... }
#[serde(default)]
pub keymap: HashMap<String, String>,
/// 停用词列表
#[serde(default)]
pub tags: HashMap<String, Vec<String>>,
pub stop_words: Vec<String>,
/// 人物称呼前缀(顶层字段,从 JSON 文件读取)
#[serde(default)]
pub person_rules: Option<PersonRulesDef>,
}
/// 分类元数据(v3 分离到 categories 字段)
#[derive(Debug, Clone, Deserialize)]
pub struct TagCategoryMeta {
#[serde(default)]
pub label: Option<String>,
#[serde(default)]
pub color_bg: Option<String>,
#[serde(default)]
pub color_fg: Option<String>,
}
/// 人物规则定义
#[derive(Debug, Clone, Deserialize)]
pub struct PersonRulesDef {
#[serde(default)]
pub prefixes: Vec<String>,
#[serde(default)]
pub excluded: Vec<String>,
#[serde(default)]
pub include: Vec<String>,
pub person_prefixes: Vec<String>,
}
// ── 运行时词典结构 ──
/// 分类信息
/// 分类信息(运行时使用,兼容旧版 API)
#[derive(Debug, Clone)]
pub struct CategoryInfo {
pub name: String,
@@ -75,9 +68,9 @@ pub struct CategoryInfo {
/// 运行时标签词典
#[derive(Debug, Clone)]
pub struct TagDict {
/// 分类名称 → 分类详情
/// 分类名称 → 分类详情(兼容旧版 API)
pub categories: HashMap<String, CategoryInfo>,
/// 标签 → 分类名称(查找加速)
/// 标签 → 分类名称(兼容旧版 API,基于 keymap 构建)
pub tag_to_category: HashMap<String, String>,
/// 停用词集合
pub stop_words: HashSet<String>,
@@ -87,6 +80,11 @@ pub struct TagDict {
pub excluded_person_names: HashSet<String>,
/// 强制包含的人物名(已知模特/演员名,只要出现就优先识别为人名)
pub included_person_names: HashSet<String>,
// ── v4 新字段 ──
/// 扁平化匹配串 → 分类名(v4 keymap)
pub keymap: HashMap<String, String>,
/// 分类元数据(v4 metadata)
pub metadata: HashMap<String, CategoryMeta>,
}
impl Default for TagDict {
@@ -98,6 +96,8 @@ impl Default for TagDict {
person_prefixes: Vec::new(),
excluded_person_names: HashSet::new(),
included_person_names: HashSet::new(),
keymap: HashMap::new(),
metadata: HashMap::new(),
};
dict.fill_builtin();
dict
@@ -133,55 +133,76 @@ impl TagDict {
}
};
let (person_prefixes, excluded_person_names, included_person_names) = match file.person_rules {
Some(rules) => (
rules.prefixes,
rules.excluded.into_iter().collect(),
rules.include.into_iter().collect(),
),
None => (Vec::new(), HashSet::new(), HashSet::new()),
};
// ── 人物前缀直接从顶层字段读取 ──
let person_prefixes = file.person_prefixes.clone();
let mut dict = TagDict {
categories: HashMap::new(),
tag_to_category: HashMap::new(),
stop_words: file.stop_words.into_iter().collect(),
person_prefixes,
excluded_person_names,
included_person_names,
excluded_person_names: HashSet::new(),
included_person_names: HashSet::new(),
keymap: HashMap::new(),
metadata: HashMap::new(),
};
// ── 解析分类元数据(v3 从 categories 字段读取) ──
for (cat_name, meta) in &file.categories {
let label = meta.label.clone().unwrap_or_else(|| cat_name.clone());
let color_bg = meta.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string());
let color_fg = meta.color_fg.clone().unwrap_or_else(|| "#64748b".to_string());
// ── 加载 metadata(分类元数据) ──
dict.metadata = file.metadata.clone();
let words = file.tags.get(cat_name).cloned().unwrap_or_default();
for word in &words {
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
}
// ── 加载 keymap(扁平化匹配串→分类名) ──
dict.keymap = file.keymap.clone();
// ── 构建 tag_to_category(基于 keymap) ──
for (word, cat_name) in &file.keymap {
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
}
// ── 构建 categories(基于 metadata + keymap 提取 words) ──
// 从 keymap 中按分类收集所有词
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &file.keymap {
words_by_cat.entry(cat_name.clone())
.or_default()
.push(word.clone());
}
// 按 metadata 中的分类顺序构建
for (cat_name, meta) in &file.metadata {
let words = words_by_cat.remove(cat_name).unwrap_or_default();
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label,
color_bg,
color_fg,
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
// keymap 中有但 metadata 中没有的分类(兜底)
for (cat_name, words) in words_by_cat {
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label: cat_name.clone(),
color_bg: "#f1f5f9".to_string(),
color_fg: "#64748b".to_string(),
words,
});
}
// ── 标签词汇自动加入排除人名 ──
// 所有 tags 中的词汇,都自动被视为非人名
for words in file.tags.values() {
for word in words {
// ── 从 keymap 构建排除人名和包含人名 ──
// person 类型的 key 加入 included_person_names
// 其他类型(style/outfit/scene/body/mood/topic/position/location/brand)加入 excluded_person_names
for (word, cat_name) in &file.keymap {
if cat_name == "person" {
dict.included_person_names.insert(word.to_lowercase());
} else {
dict.excluded_person_names.insert(word.to_lowercase());
}
}
if !path.as_os_str().is_empty() {
println!("[词典] v{} 已加载: {} 分类, {} 标签词, {} 停用词, {} 前缀, 排除+标签共 {} 词",
file.version, dict.categories.len(), dict.tag_to_category.len(),
println!("[词典] v{} 已加载: {} 分类, {} 标签词(keymap), {} 停用词, {} 前缀, 排除+标签共 {} 词",
file.version, dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len());
}
@@ -193,7 +214,7 @@ impl TagDict {
*self = Self::load();
}
/// 获取所有分类名称(固定顺序)
/// 获取所有分类名称(固定顺序:9 个标准分类优先)
pub fn get_category_order(&self) -> Vec<&str> {
let preferred = ["style", "outfit", "scene", "body", "mood", "topic", "position", "location", "brand"];
let mut result: Vec<&str> = preferred.iter()
@@ -228,11 +249,21 @@ impl TagDict {
}
}
/// 判断一个词属于哪个已知分类
/// 判断一个词属于哪个已知分类(基于 tag_to_category,由 keymap 构建)
pub fn classify(&self, word: &str) -> Option<&str> {
self.tag_to_category.get(&word.to_lowercase()).map(|s| s.as_str())
}
/// 在 keymap 中查找 word,返回分类名(与 classify 相同,v4 新增 API)
pub fn match_keymap(&self, word: &str) -> Option<&str> {
self.keymap.get(&word.to_lowercase()).map(|s| s.as_str())
}
/// 获取 v4 元数据
pub fn get_metadata(&self) -> &HashMap<String, CategoryMeta> {
&self.metadata
}
/// 判断是否是停用词
pub fn is_stop_word(&self, word: &str) -> bool {
self.stop_words.contains(&word.to_lowercase())
@@ -267,8 +298,6 @@ impl TagDict {
return true;
}
}
// 反向包含:person_name 包含在排除名中(如排除 "秀人",人名 "秀人网")
// 实际上上面的 contains 已经覆盖了
false
}
@@ -314,47 +343,112 @@ impl TagDict {
/// 内置回退
fn fill_builtin(&mut self) {
let builtin = vec![
("style", "风格", "#f0f9ff", "#0284c7", vec![
"日系", "韩系", "复古", "甜美", "性感", "清新", "冷艳",
"优雅", "知性", "可爱", "俏皮", "温柔",
"时尚", "文艺", "古典",
]),
("outfit", "服装", "#fdf2f8", "#db2777", vec![
"比基尼", "泳装", "旗袍", "制服", "丝袜", "内衣",
"连衣裙", "短裙", "长裙", "牛仔裤", "热裤",
"吊带", "抹胸", "蕾丝", "镂空",
"高跟鞋",
]),
("scene", "场景", "#f0fdf4", "#15803d", vec![
"室内", "户外", "浴室", "床上",
"海边", "沙滩", "泳池", "花园",
]),
("body", "身体", "#fef3c7", "#b45309", vec![
"美腿", "美胸", "翘臀", "玉足", "美背", "香肩",
]),
("mood", "情绪", "#faf5ff", "#7c3aed", vec![
"清纯", "诱惑", "唯美", "清新", "冷艳", "妩媚",
]),
("topic", "主题", "#f0f9ff", "#0369a1", vec![
"写真", "私房", "街拍", "旅拍", "婚纱照",
]),
("position", "姿势", "#fefce8", "#a16207", vec![
"站立", "坐姿", "躺姿", "跪姿", "侧卧",
]),
("location", "地点", "#f0fdfa", "#0f766e", vec![
"日本", "韩国", "中国", "泰国",
"东京", "首尔", "北京", "上海",
"japan", "korea", "china",
]),
("brand", "品牌", "#fff7ed", "#c2410c", vec![
"Victoria's Secret", "维密",
"Calvin Klein", "CK",
"Gucci", "Chanel",
"Louis Vuitton",
]),
// ── 元数据(与 v4 JSON metadata 同构) ──
let builtin_metadata: Vec<( &str, &str, &str, &str)> = vec![
("style", "风格", "#f0f9ff", "#0284c7"),
("outfit", "服装", "#fdf2f8", "#db2777"),
("scene", "场景", "#f0fdf4", "#15803d"),
("body", "身体", "#fef3c7", "#b45309"),
("mood", "情绪", "#faf5ff", "#7c3aed"),
("topic", "主题", "#f0f9ff", "#0369a1"),
("position", "姿势", "#fefce8", "#a16207"),
("location", "地点", "#f0fdfa", "#0f766e"),
("brand", "品牌", "#fff7ed", "#c2410c"),
];
for (name, label, bg, fg) in &builtin_metadata {
self.metadata.insert(name.to_string(), CategoryMeta {
label: label.to_string(),
color_bg: bg.to_string(),
color_fg: fg.to_string(),
});
}
// ── keymap ──
let builtin_keymap: Vec<(&str, &str)> = vec![
// style
("日系", "style"), ("韩系", "style"), ("复古", "style"),
("甜美", "style"), ("性感", "style"), ("清新", "style"),
("冷艳", "style"), ("优雅", "style"), ("知性", "style"),
("可爱", "style"), ("俏皮", "style"), ("温柔", "style"),
("时尚", "style"), ("文艺", "style"), ("古典", "style"),
("elegant", "style"), ("fashion", "style"), ("japanese", "style"),
("korean", "style"), ("retro", "style"), ("sexy", "style"),
("vintage", "style"),
// outfit
("比基尼", "outfit"), ("泳装", "outfit"), ("旗袍", "outfit"),
("制服", "outfit"), ("丝袜", "outfit"), ("内衣", "outfit"),
("连衣裙", "outfit"), ("短裙", "outfit"), ("长裙", "outfit"),
("牛仔裤", "outfit"), ("热裤", "outfit"), ("吊带", "outfit"),
("抹胸", "outfit"), ("蕾丝", "outfit"), ("镂空", "outfit"),
("高跟鞋", "outfit"), ("T恤", "outfit"),
// scene
("室内", "scene"), ("户外", "scene"), ("浴室", "scene"),
("床上", "scene"), ("海边", "scene"), ("沙滩", "scene"),
("泳池", "scene"), ("花园", "scene"),
// body
("美腿", "body"), ("美胸", "body"), ("翘臀", "body"),
("玉足", "body"), ("美背", "body"), ("香肩", "body"),
("ass", "body"), ("back", "body"), ("breasts", "body"),
("feet", "body"), ("legs", "body"), ("shoulders", "body"),
("skin", "body"), ("tattoo", "body"), ("waist", "body"),
// mood
("清纯", "mood"), ("诱惑", "mood"), ("唯美", "mood"),
("妩媚", "mood"), ("beautiful", "mood"), ("charming", "mood"),
("cool", "mood"), ("cute", "mood"), ("fresh", "mood"),
("gentle", "mood"), ("lazy", "mood"), ("pure", "mood"),
("seductive", "mood"), ("shy", "mood"), ("sweet", "mood"),
("wild", "mood"),
// topic
("写真", "topic"), ("私房", "topic"), ("街拍", "topic"),
("旅拍", "topic"), ("婚纱照", "topic"),
("cosplay", "topic"), ("photoshoot", "topic"),
("portrait", "topic"), ("selfie", "topic"), ("travel", "topic"),
("wedding", "topic"),
// position
("站立", "position"), ("坐姿", "position"), ("躺姿", "position"),
("跪姿", "position"), ("侧卧", "position"),
("kneeling", "position"), ("lying", "position"),
("sitting", "position"), ("standing", "position"),
// location
("日本", "location"), ("韩国", "location"), ("中国", "location"),
("泰国", "location"), ("东京", "location"), ("首尔", "location"),
("北京", "location"), ("上海", "location"),
("japan", "location"), ("korea", "location"), ("china", "location"),
("bali", "location"), ("bangkok", "location"),
("phuket", "location"), ("seoul", "location"),
("thailand", "location"), ("tokyo", "location"),
// brand
("Victoria's Secret", "brand"), ("维密", "brand"),
("Calvin Klein", "brand"), ("CK", "brand"),
("Gucci", "brand"), ("Chanel", "brand"),
("Louis Vuitton", "brand"),
];
for (word, cat) in &builtin_keymap {
self.keymap.insert(word.to_string(), cat.to_string());
self.tag_to_category.insert(word.to_lowercase(), cat.to_string());
}
// ── categories(基于 metadata + keymap 派生) ──
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &self.keymap {
words_by_cat.entry(cat_name.clone())
.or_default()
.push(word.clone());
}
for (name, meta) in &self.metadata {
let words = words_by_cat.remove(name).unwrap_or_default();
self.categories.insert(name.clone(), CategoryInfo {
name: name.clone(),
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
// ── 停用词 ──
let stop_words: Vec<&str> = vec![
"photos", "photo", "pictures", "picture", "images", "image",
"video", "videos", "cosplay", "cos",
@@ -367,43 +461,40 @@ impl TagDict {
"美女", "模特",
"高清", "原版", "无水印", "番号", "编号",
];
for w in &stop_words { self.stop_words.insert(w.to_string()); }
// ── 人物前缀 ──
let person_prefixes: Vec<&str> = vec![
"美女模特", "性感女神", "清纯美女", "气质美女", "可爱女孩",
"甜美少女", "时尚达人", "女神", "美女", "模特", "小姐姐",
"小仙女", "御姐", "萝莉", "辣妹", "潮人",
];
self.person_prefixes = person_prefixes.iter().map(|s| s.to_string()).collect();
let excluded: Vec<&str> = vec![
// ── 排除人名(从 keymap 非 person 类型派生)──
for (word, cat) in &builtin_keymap {
if cat == &"person" {
self.included_person_names.insert(word.to_lowercase());
} else {
self.excluded_person_names.insert(word.to_lowercase());
}
}
// 额外内置排除(keymap 中未覆盖的工作室名)
let extra_excluded: Vec<&str> = vec![
"XiuRen", "xiuren", "秀人", "秀人网",
"MFStar", "MiiTao", "蜜桃社",
"YouMi", "尤蜜", "HuaYang", "花洋",
"FeiLin", "Feiling", "飞灵", "MFModel",
"LeiLei", "Tristana", "Av",
];
for n in &extra_excluded { self.excluded_person_names.insert(n.to_string()); }
let included: Vec<&str> = vec![
// ── 包含人名(从 keymap person 类型已派生,再加几个默认) ──
let default_included: Vec<&str> = vec![
"杨晨晨", "水淼", "轩美宣", "王馨瑶",
"雪儿", "曼曼", "佳佳", "小乔",
];
for w in &stop_words { self.stop_words.insert(w.to_string()); }
self.person_prefixes = person_prefixes.iter().map(|s| s.to_string()).collect();
for n in &excluded { self.excluded_person_names.insert(n.to_string()); }
for n in &included { self.included_person_names.insert(n.to_string()); }
for (name, label, bg, fg, words) in builtin {
let ws: Vec<String> = words.iter().map(|w| w.to_string()).collect();
for w in &ws {
self.tag_to_category.insert(w.to_lowercase(), name.to_string());
// 标签词自动加入排除人名
self.excluded_person_names.insert(w.to_lowercase());
}
self.categories.insert(name.to_string(), CategoryInfo {
name: name.to_string(), label: label.to_string(),
color_bg: bg.to_string(), color_fg: fg.to_string(), words: ws,
});
}
for n in &default_included { self.included_person_names.insert(n.to_string()); }
}
}
@@ -423,8 +514,8 @@ pub fn reload_tag_dict() -> String {
let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染");
dict.reload();
format!(
"词典已重载: {} 分类, {} 标签, {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.tag_to_category.len(),
"词典已重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)
@@ -434,8 +525,8 @@ pub fn reload_tag_dict() -> String {
pub fn get_tag_dict_stats() -> String {
let dict = get_tag_dict();
format!(
"标签词典: {} 分类, {} 标签, {} 停用词, {} 前缀, {} 排除人名, {} 包含人名 | 文件: dict/tags.json",
dict.categories.len(), dict.tag_to_category.len(),
"标签词典: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名 | 文件: dict/tags.json",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)

View File

@@ -1,10 +1,10 @@
//! 标签提取器 — 从标题中提取分类明确的标签
//!
//! 标签分类体系(外部词典 dict/tags.json 定义):
//! style outfit scene body mood topic position location brand
//! 标签分类体系(外部词典 dict/tags.json v4 定义):
//! style outfit scene body mood topic position location brand person
//!
//! 核心设计:不直接调用 JIEBA,jieba 标签由外部(title_parser::parse_album_title)
//! 传入,避免重复分词。Tag extractor 只做:词典匹配 + jieba 结果过滤 + 正则兜底。
//! 核心设计:纯词典 keymap substring 匹配 + 正则兜底,无 jieba 依赖。
//! Person 类型由 title_parser 单独处理,此处不纳入标签。
use regex::Regex;
use lazy_static::lazy_static;
@@ -12,9 +12,9 @@ use std::collections::HashSet;
use crate::engine::tag_dict;
lazy_static! {
/// 匹配英文单词、数字、中文字符(用于正则兜底分词)
static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
/// 提取标签中的有效部分(去头尾标点)
/// 去除头尾标点符号
static ref TAG_CORE_RE: Regex = Regex::new(
r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+$"#
).unwrap();
@@ -31,8 +31,8 @@ pub fn is_punctuation_only(s: &str) -> bool {
})
}
/// 清洗单个标签:去除头尾标点符号
/// 清洗后如果只剩下标点或为空,则返回 None
/// 清洗单个标签:去除头尾标点符号。
/// 清洗后如果只剩下标点或为空,则返回 None。
pub fn clean_tag(tag: &str) -> Option<String> {
let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string();
if cleaned.is_empty() || is_punctuation_only(&cleaned) || cleaned.len() < 2 {
@@ -42,18 +42,15 @@ pub fn clean_tag(tag: &str) -> Option<String> {
}
}
/// 获取分类优先级索引(从词典动态获取)
/// 获取分类优先级索引(从词典动态获取)。
fn category_order_index(category: &str) -> usize {
let dict = tag_dict::get_tag_dict();
let order = dict.get_category_order();
order.iter().position(|c| c == &category).unwrap_or(999)
}
// ═══════════════════════════════════════════════════════
// 标签分类(非 jieba)
// ═══════════════════════════════════════════════════════
/// 判断一个词属于哪个分类(在词典中查找)
/// 判断一个词属于哪个分类(在词典 keymap 中查找)。
/// 如果在 keymap 中找不到,返回 "other"。
pub fn classify_tag(word: &str) -> String {
let dict = tag_dict::get_tag_dict();
dict.classify(word)
@@ -61,23 +58,29 @@ pub fn classify_tag(word: &str) -> String {
.unwrap_or_else(|| "other".to_string())
}
/// 获取标签的中文分类名
/// 获取标签的中文分类名。
pub fn get_category_label(category: &str) -> String {
tag_dict::get_tag_dict().get_category_label(category)
}
/// 获取标签分类的颜色
/// 获取标签分类的颜色 (bg, fg)。
pub fn get_category_color(category: &str) -> (String, String) {
tag_dict::get_tag_dict().get_category_color(category)
}
// ═══════════════════════════════════════════════════════
// 步骤 A:词典标签匹配(substring,无 jieba)
// 词典 keymap substring 匹配(最长匹配优先)
// ═══════════════════════════════════════════════════════
/// 步骤 A — 从标题中通过词典 substring 匹配提取分类标签。
/// 已被已有标签(seen)或人名命中的词不再重复匹配。
/// 不调用 jieba。
/// 从标题中通过词典 keymap substring 匹配提取分类标签,**最长匹配优先**。
///
/// 策略:
/// 1. 先从 `title_lower` 中移除 `person_name` 中的词(防止人名命中标签)
/// 2. keymap 中所有标签按长度降序排列(最长的先匹配)
/// 3. 匹配到某个标签后,将该标签从 `title_lower` 中**替换为空格**,
/// 防止其子串/短标签再次命中,实现最长匹配覆盖
/// 4. 跳过已被 `seen` 包含的词
/// 5. 跳过 `person` 类型的标签(由 title_parser 处理)
pub fn extract_dict_tags(
title: &str,
person_name: &[String],
@@ -96,77 +99,37 @@ pub fn extract_dict_tags(
let mut sorted: Vec<(String, String)> = tag_category_map.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
// 长度降序排列:保证最长匹配优先
sorted.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
for (tag, category) in &sorted {
if title_lower.contains(tag.as_str()) && !seen.contains(tag) {
seen.insert(tag.clone());
tags.push((tag.clone(), category.clone()));
}
// 跳过 person 类型(由 title_parser 单独处理)
if category == "person" { continue; }
if seen.contains(tag) { continue; }
if !title_lower.contains(tag.as_str()) { continue; }
seen.insert(tag.clone());
tags.push((tag.clone(), category.clone()));
// 将匹配到的长标签从标题中移除(替换为空格),
// 避免其子串/短标签再次命中,实现最长匹配覆盖
title_lower = title_lower.replace(tag.as_str(), " ");
}
tags
}
// ═══════════════════════════════════════════════════════
// 步骤 B:jieba 名词标签提取(依赖外部 jieba result)
// 正则兜底标签提取
// ═══════════════════════════════════════════════════════
/// 步骤 B — 从 jieba 分词结果中提取名词标签。
/// 已被 seen 中的词跳过。不重复调用 JIEBA。
pub fn extract_jieba_noun_tags(
jieba_tags: &[jieba_rs::Tag],
person_name: &[String],
stop_words: &HashSet<String>,
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let mut tags = Vec::new();
for tag in jieba_tags {
let word = tag.word.trim();
if word.len() < 2 { continue; }
let word_lower = word.to_lowercase();
if is_punctuation_only(word) { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if stop_words.contains(&word_lower) { continue; }
if seen.contains(&word_lower) { continue; }
// 排除人名
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
continue;
}
let pos = tag.tag;
let category = match pos {
"n" | "ng" => "noun",
"nz" => "proper",
"ns" => "location",
"eng" => "other",
"x" => "other",
"nr" | "nrfg" | "nrt" => continue, // 人名(已单独提取)
"nt" | "vn" | "a" | "ad" | "an"
| "v" | "vd" | "vg"
| "m" | "mq" | "q" | "qv" | "qt"
| "r" | "rr" | "rz"
| "c" | "p"
| "u" | "ul" | "uv" | "ud" | "ug" | "uj"
| "d" | "e" | "y" | "o" | "w"
| "t" | "tg" | "s" | "f"
| "b" | "bl" | "z" => continue,
_ => continue,
};
seen.insert(word_lower);
tags.push((word.to_string(), category.to_string()));
}
tags
}
// ═══════════════════════════════════════════════════════
// 步骤 C:正则兜底标签(无 jieba)
// ═══════════════════════════════════════════════════════
/// 步骤 C — 正则分词提取剩余关键词(兜底)。不调 jieba。
/// 正则分词提取剩余关键词(兜底),全部归为 "other" 类。
///
/// 过滤规则:
/// - 长度 < 2、纯标点、纯数字 → 跳过
/// - 停用词 → 跳过
/// - 已被 `seen` 包含的词 → 跳过
/// - 人名 → 跳过
/// - 已在 keymap 中分类过的词 → 跳过(避免重复)
pub fn extract_regex_tags(
title: &str,
person_name: &[String],
@@ -186,7 +149,7 @@ pub fn extract_regex_tags(
if stop_words.contains(&word_lower) { continue; }
if seen.contains(&word_lower) { continue; }
// 排除人名
// 排除人名相关部分
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
continue;
}
@@ -202,21 +165,33 @@ pub fn extract_regex_tags(
}
// ═══════════════════════════════════════════════════════
// extract_tags_from_title(保持对外兼容,内部调 A+B+C)
// extract_tags_from_title(完整流程:词典匹配 → 正则兜底)
// ═══════════════════════════════════════════════════════
/// 从标题中提取分类标签(完整流程)。
///
/// 三部曲:词典匹配 → jieba 名词提取 → 正则兜底。
/// jieba 部分使用外部传入的 jieba result,避免重复分词。
/// 两步流程:
/// 1. **词典 keymap substring 匹配** — 在 `title_lower` 中搜索所有 keymap 中的 key,
/// 最长匹配优先,匹配后从标题中移除已匹配文本,避免短标签重复命中。
/// 跳过 person 类型标签。
/// 2. **正则兜底** — 用 WORD_RE 提取剩余单词,归为 "other" 类。
///
/// 过滤规则:
/// - 跳过停用词
/// - 跳过 person 类型(由 title_parser 处理)
/// - 跳过人名部分
///
/// # 参数
/// - `title` — 原始标题
/// - `person_name` — 已提取的人名(可选),用于从标题中排除,防止命中标签
pub fn extract_tags_from_title(
title: &str,
person_name: Option<&str>,
jieba_tags: &[jieba_rs::Tag],
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let stop_words = dict.get_stop_words();
// 构建人名排除列表:人名本身 + 按空白切分的各部分
let person_exclude: Vec<String> = person_name
.map(|n| {
let mut v = vec![n.to_lowercase()];
@@ -228,22 +203,50 @@ pub fn extract_tags_from_title(
.unwrap_or_default();
let mut seen = HashSet::new();
let mut tags = Vec::new();
// A: 词典匹配
// Step 1: 词典 keymap substring 匹配(最长匹配优先)
tags.extend(extract_dict_tags(title, &person_exclude, &mut seen));
// B: jieba 名词(用外部传入的 jieba result)
tags.extend(extract_jieba_noun_tags(jieba_tags, &person_exclude, stop_words, &mut seen));
// C: 正则兜底
// Step 2: 正则兜底(剩余未匹配的词,归 "other" 类)
tags.extend(extract_regex_tags(title, &person_exclude, stop_words, &mut seen));
tags
}
/// 标签去重并排序(分类标签优先)
/// 纯 keymap substring 匹配(无正则兜底),用于快速标签提取。
///
/// 只做词典匹配,不做正则兜底。适合在不需要"其他"类标签的场景下使用。
///
/// # 参数
/// - `title` — 原始标题
/// - `person_name` — 已提取的人名(可选),用于从标题中排除人名部分
pub fn extract_tags_with_keymap(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
let person_exclude: Vec<String> = person_name
.map(|n| {
let mut v = vec![n.to_lowercase()];
for part in n.split_whitespace() {
v.push(part.to_lowercase());
}
v
})
.unwrap_or_default();
let mut seen = HashSet::new();
extract_dict_tags(title, &person_exclude, &mut seen)
}
// ═══════════════════════════════════════════════════════
// 标签去重、排序、过滤
// ═══════════════════════════════════════════════════════
/// 标签去重并排序(分类标签优先,按词典中的 category_order 排序)。
///
/// 去重逻辑:
/// 1. 先按分类优先级排序(style < outfit < scene < ... < other)
/// 2. 逐个清洗标签(去头尾标点)
/// 3. 过滤坏标签(纯数字、纯标点、停用词等)
/// 4. 小写去重
pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
let mut seen = HashSet::new();
let mut result = Vec::new();
@@ -269,7 +272,14 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
result
}
/// 判断一个标签是否"坏标签"
/// 判断一个标签是否"坏标签"(应被过滤掉)。
///
/// 规则:
/// - 空串或长度 < 2
/// - 纯数字
/// - 纯标点符号
/// - clean_tag 后变短(说明含不可见标点)
/// - 停用词
pub fn is_bad_tag(tag: &str) -> bool {
let tag = tag.trim();
if tag.is_empty() || tag.len() < 2 { return true; }
@@ -289,23 +299,40 @@ mod tests {
#[test]
fn test_extract_tags() {
// 杨晨晨Yome 是 person 类型,由 title_parser 处理,不应出现在标签中
// photos 是停用词,不应出现在标签中
let tags = extract_tags_from_title(
"XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'",
Some("杨晨晨Yome"),
&[],
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!names.contains(&"photos"));
assert!(!names.contains(&"杨晨晨"));
assert!(!names.contains(&"photos"), "tags 不应包含停用词 'photos'");
assert!(!names.contains(&"杨晨晨"), "tags 不应包含 person 类型词 '杨晨晨'");
// 验证 keymap 中的词正确匹配
assert!(names.contains(&"dress"), "tags 应包含 keymap 中的 'dress'");
}
#[test]
fn test_longest_match_priority() {
// "连衣裙" (3字) 在内建词典 outfit 中。标题包含 "连衣裙",
// 匹配后无子串可再匹配。
let tags = extract_tags_from_title(
"粉色连衣裙 2025",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(names.contains(&"连衣裙"), "'连衣裙' 应在内建词典的结果中");
// "连衣裙" 被移除后,其子串不会再次匹配
assert_eq!(names.iter().filter(|t| **t == "连衣裙").count(), 1);
}
#[test]
fn test_classify_tag() {
// v4 JSON keymap 中 "日系":"style"
assert_eq!(classify_tag("日系"), "style");
assert_eq!(classify_tag("比基尼"), "outfit");
assert_eq!(classify_tag("浴室"), "scene");
assert_eq!(classify_tag("美腿"), "body");
assert_eq!(classify_tag("清纯"), "mood");
assert_eq!(classify_tag("写真"), "topic");
assert_eq!(classify_tag("unknown"), "other");
}
@@ -316,4 +343,31 @@ mod tests {
assert_eq!(bg, "#f0f9ff");
assert_eq!(fg, "#0284c7");
}
#[test]
fn test_extract_tags_with_keymap() {
// 纯 keymap 匹配,不包含正则兜底的词
let tags = extract_tags_with_keymap(
"日系 比基尼 甜美 沙滩 2025",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(names.contains(&"日系"), "日系 应在 keymap 匹配结果中");
assert!(names.contains(&"比基尼"), "比基尼 应在 keymap 匹配结果中");
assert!(names.contains(&"沙滩"), "沙滩 应在 keymap 匹配结果中");
// "2025" 不在 keymap 中,不应出现(纯 keymap 无正则兜底)
assert!(!names.contains(&"2025"), "纯 keymap 匹配不应包含 '2025'");
}
#[test]
fn test_person_type_excluded_from_dict() {
// 杨晨晨 在 v4 keymap 中是 person 类型,不应被提取为标签
let tags = extract_tags_with_keymap(
"杨晨晨 日系 比基尼",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!names.contains(&"杨晨晨"), "person 类型的 '杨晨晨' 不应出现在标签中");
assert!(names.contains(&"日系"), "非 person 类型词应正常匹配");
}
}

File diff suppressed because it is too large Load Diff