From 5b803cc1783164e92732ade3fa64c708774fae9d Mon Sep 17 00:00:00 2001 From: lychang Date: Mon, 20 Jul 2026 01:33:44 +0800 Subject: [PATCH] update --- dict/tags.json | 312 ++++++++++++++++++++++++++++++++++++ src/db/connection.rs | 15 ++ src/engine/mod.rs | 2 + src/engine/pipeline.rs | 53 +++++- src/engine/tag_dict.rs | 309 +++++++++++++++++++++++++++++++++++ src/engine/tag_extractor.rs | 237 ++++++++------------------- src/main.rs | 36 ++++- src/routes/dict.rs | 37 +++++ src/routes/mod.rs | 4 +- 9 files changed, 821 insertions(+), 184 deletions(-) create mode 100644 dict/tags.json create mode 100644 src/engine/tag_dict.rs create mode 100644 src/routes/dict.rs diff --git a/dict/tags.json b/dict/tags.json new file mode 100644 index 0000000..a52f455 --- /dev/null +++ b/dict/tags.json @@ -0,0 +1,312 @@ +{ + "version": 1, + "description": "标签词典 — 支持 style/outfit/scene/body/mood/topic 六大分类", + "tags": { + "style": { + "label": "风格", + "color_bg": "#f0f9ff", + "color_fg": "#0284c7", + "words": [ + "日系", + "韩系", + "森系", + "复古", + "甜美", + "性感", + "清新", + "冷艳", + "优雅", + "知性", + "可爱", + "俏皮", + "温柔", + "妩媚", + "野性", + "清纯", + "时尚", + "潮流", + "街头", + "学院", + "文艺", + "古典", + "民族", + "异域", + "japanese", + "korean", + "retro", + "vintage", + "cute", + "sexy", + "elegant", + "fresh", + "sweet", + "wild", + "pure", + "fashion" + ] + }, + "outfit": { + "label": "服装", + "color_bg": "#fdf2f8", + "color_fg": "#db2777", + "words": [ + "比基尼", + "泳装", + "泳衣", + "旗袍", + "制服", + "丝袜", + "内衣", + "文胸", + "包臀裙", + "紧身裙", + "超短裙", + "连衣裙", + "半身裙", + "短裙", + "长裙", + "牛仔裤", + "热裤", + "短裤", + "休闲裤", + "西装裤", + "T恤", + "衬衫", + "毛衣", + "针织衫", + "外套", + "风衣", + "大衣", + "西装", + "吊带", + "抹胸", + "露背", + "透视", + "蕾丝", + "镂空", + "开叉", + "高跟鞋", + "运动鞋", + "凉鞋", + "拖鞋", + "靴子", + "bikini", + "swimsuit", + "stockings", + "lingerie", + "dress", + "skirt", + "jeans", + "shirt", + "sweater", + "coat", + "heels" + ] + }, + "scene": { + "label": "场景", + "color_bg": "#f0fdf4", + "color_fg": "#15803d", + "words": [ + "室内", + "户外", + "街拍", + "浴室", + "浴缸", + "床上", + "床照", + "沙发", + "厨房", + "阳台", + "窗边", + "楼梯", + "走廊", + "电梯", + "车内", + "驾驶座", + "海边", + "沙滩", + "泳池", + "温泉", + "森林", + "草地", + "花园", + "公园", + "街道", + "巷弄", + "建筑", + "天台", + "屋顶", + "咖啡馆", + "餐厅", + "酒店", + "indoor", + "outdoor", + "street", + "beach", + "pool", + "bathroom", + "bedroom", + "kitchen", + "balcony", + "car", + "cafe", + "hotel" + ] + }, + "body": { + "label": "身体", + "color_bg": "#fef3c7", + "color_fg": "#b45309", + "words": [ + "美腿", + "美胸", + "翘臀", + "玉足", + "美背", + "香肩", + "蛮腰", + "细腰", + "长腿", + "酥胸", + "丰臀", + "白皙", + "肌肤", + "纹身", + "legs", + "breasts", + "ass", + "feet", + "back", + "shoulders", + "waist", + "skin", + "tattoo" + ] + }, + "mood": { + "label": "情绪", + "color_bg": "#faf5ff", + "color_fg": "#7c3aed", + "words": [ + "清纯", + "诱惑", + "唯美", + "清新", + "冷艳", + "妩媚", + "娇羞", + "俏皮", + "温柔", + "野性", + "冷傲", + "热情", + "慵懒", + "迷离", + "甜美", + "知性", + "pure", + "seductive", + "beautiful", + "fresh", + "cool", + "charming", + "shy", + "cute", + "gentle", + "wild", + "lazy", + "sweet" + ] + }, + "topic": { + "label": "主题", + "color_bg": "#f0f9ff", + "color_fg": "#0369a1", + "words": [ + "写真", + "私房", + "街拍", + "旅拍", + "婚纱照", + "艺术照", + "生活照", + "自拍", + "跟拍", + "约拍", + "商拍", + "photoshoot", + "portrait", + "selfie", + "travel", + "wedding" + ] + } + }, + "stop_words": [ + "photos", + "photo", + "pictures", + "picture", + "images", + "image", + "video", + "videos", + "cosplay", + "cos", + "and", + "the", + "of", + "in", + "a", + "an", + "is", + "to", + "for", + "part", + "vol", + "collection", + "special", + "extra", + "bonus", + "new", + "old", + "selfie", + "official", + "original", + "preview", + "patreon", + "with", + "without", + "set", + "gallery", + "album", + "page", + "issue", + "model", + "idol", + "star", + "actor", + "actress", + "singer", + "合集", + "系列", + "作品", + "原版", + "无水印", + "番号", + "编号", + "site", + "web", + "online", + "digital", + "exclusive", + "premium", + "classic", + "photography", + "shoot", + "shot", + "no", + "性感美女" + ] +} diff --git a/src/db/connection.rs b/src/db/connection.rs index ca29f8e..9c53e0d 100644 --- a/src/db/connection.rs +++ b/src/db/connection.rs @@ -51,6 +51,21 @@ impl Database { Ok(db) } + /// 执行 WAL checkpoint(TRUNCATE)，将 WAL 完全合并回主文件并清理 -wal / -shm 文件 + /// + /// 应在服务优雅关闭时调用，确保数据库文件处于干净一致的状态， + /// 避免下次启动时 WAL 恢复带来的额外开销。 + pub async fn wal_checkpoint(&self) { + let _guard = self.write_lock.lock().await; + // wal_checkpoint 返回一行 (busy, log, checkpointed)，只关注结果忽略返回值 + match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)") + .execute(&self.pool).await + { + Ok(_) => println!("[DB] WAL checkpoint(TRUNCATE) 完成，数据库文件已合并"), + Err(e) => eprintln!("[DB] WAL checkpoint 失败: {}", e), + } + } + /// 确保 image_count 字段存在 async fn ensure_image_count_field(&self) -> Result<(), sqlx::Error> { let has_col = sqlx::query_scalar::<_, i64>( diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 7d18a2e..9582e96 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -2,6 +2,7 @@ pub mod title_parser; pub mod person_merger; +pub mod tag_dict; pub mod tag_extractor; pub mod recommend; pub mod pipeline; @@ -9,6 +10,7 @@ pub mod crawler; #[allow(unused)] pub use title_parser::*; #[allow(unused)] pub use person_merger::*; +#[allow(unused)] pub use tag_dict::*; #[allow(unused)] pub use tag_extractor::*; pub use recommend::*; #[allow(unused)] pub use pipeline::*; diff --git a/src/engine/pipeline.rs b/src/engine/pipeline.rs index 0e367b4..f39f537 100644 --- a/src/engine/pipeline.rs +++ b/src/engine/pipeline.rs @@ -17,6 +17,7 @@ use crate::db::Database; use crate::engine::title_parser; use crate::engine::tag_extractor; use crate::engine::person_merger; +use crate::engine::tag_dict; use std::collections::HashSet; // ════════════════════════════════════════════════ @@ -273,6 +274,51 @@ pub async fn phase_clean_persons(db: &Database) -> (DirtySet, usize) { (dirty, deleted) } +// ════════════════════════════════════════════════ +// Phase A2B：清理 persons 表中与停用词重叠的人物 +// ════════════════════════════════════════════════ + +/// Phase A2B：检查 persons 表中的人名是否在停用词词典中。 +/// +/// 当用户在词典中添加了新的停用词，而这些词恰好已被作为人物创建时， +/// 需要删除该人物并释放对应的相册，让它们在后续 Phase 1 中重新解析。 +/// 这样可以防止停用词（如 "photo"、"model" 等通用词）被错误识别为人物。 +pub async fn phase_clean_stop_word_persons(db: &Database) -> (DirtySet, usize) { + let persons = db.get_all_persons_with_counts().await; + let dict = tag_dict::get_tag_dict(); + let mut dirty = DirtySet::new(); + let mut deleted = 0usize; + + for (pid, name, _albums, _images) in &persons { + // 检查人名本身是否在停用词中 + let in_stop_words = dict.is_stop_word(name) + || dict.is_stop_word(&name.to_lowercase()); + // 检查中文核心是否在停用词中（例如 "杨晨晨" 不在，但 "photo" 可能在） + let cn_in_stop_words = title_parser::extract_cn(name) + .map(|cn| dict.is_stop_word(&cn)) + .unwrap_or(false); + + if in_stop_words || cn_in_stop_words { + let album_list = db.get_person_albums(*pid).await; + let album_ids: Vec = album_list.into_iter().map(|a| a.id.to_string()).collect(); + for aid in &album_ids { dirty.add(aid); } + println!("[管线·PhaseA2B] 停用词命中人物：{} (id={})，释放 {} 个相册", name, pid, album_ids.len()); + // 清空相册的 person 字段 + db.update_album_person_by_person_id(*pid).await; + // 彻底删除该人物 + db.delete_person_completely(*pid).await; + deleted += 1; + } + } + + if deleted > 0 { + println!("[管线·PhaseA2B] 清理完成：移除 {} 个与停用词重叠的人物，标记 {} 个脏相册", deleted, dirty.len()); + } else { + println!("[管线·PhaseA2B] 清理完成：无与停用词重叠的人物"); + } + (dirty, deleted) +} + // ════════════════════════════════════════════════ // Phase A3：收集新相册 + 半成品 // ════════════════════════════════════════════════ @@ -454,8 +500,9 @@ pub async fn phase_clean_tags_after_parse(db: &Database) -> usize { pub struct PipelineResult { pub tags_cleaned: usize, // Phase 0 pub person_like_tags_removed: usize, // Phase A1 - pub dirty_albums_detected: usize, // A1+A2+A3 + pub dirty_albums_detected: usize, // A1+A2+A2B+A3 pub non_persons_deleted: usize, // Phase A2 + pub stop_word_persons_removed: usize, // Phase A2B pub new_albums_found: usize, // Phase A3 pub persons_updated: usize, // Phase 1 pub tags_updated: usize, // Phase 1 @@ -469,12 +516,14 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult { let tags_cleaned = phase_clean_tags(db).await; let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await; let (dirty_a2, non_persons_deleted) = phase_clean_persons(db).await; + let (dirty_a2b, stop_word_persons_removed) = phase_clean_stop_word_persons(db).await; let dirty_a3 = phase_collect_new_albums(db).await; let new_albums_found = dirty_a3.len(); let mut master_dirty = DirtySet::new(); master_dirty.extend(dirty_a1.into_inner()); master_dirty.extend(dirty_a2.into_inner()); + master_dirty.extend(dirty_a2b.into_inner()); master_dirty.extend(dirty_a3.into_inner()); let dirty_albums_detected = master_dirty.len(); @@ -484,7 +533,7 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult { PipelineResult { tags_cleaned, person_like_tags_removed, dirty_albums_detected, - non_persons_deleted, new_albums_found, + non_persons_deleted, stop_word_persons_removed, new_albums_found, persons_updated, tags_updated, persons_merged, aliases_moved, tags_removed_person, } diff --git a/src/engine/tag_dict.rs b/src/engine/tag_dict.rs new file mode 100644 index 0000000..1e73a14 --- /dev/null +++ b/src/engine/tag_dict.rs @@ -0,0 +1,309 @@ +//! 标签词典 — 从外部 JSON 文件加载分类标签、停用词、颜色配置 +//! +//! 结构： +//! - `TagDict` 是唯一的运行时词典实例，通过 `lazy_static!` 全局可用 +//! - 词典文件路径默认 `dict/tags.json`，可通过 `load()` 或 `reload()` 加载 +//! - 提供 `get_category()` / `get_all_categories()` / `is_stop_word()` / `classify()` 等查询接口 +//! - `reload()` 可热重载，词典变更后无需重启服务即可生效 + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use serde::Deserialize; +use lazy_static::lazy_static; +use std::sync::RwLock; + +// ── JSON 反序列化结构 ── + +/// 词典文件的顶层结构 +#[derive(Debug, Clone, Deserialize)] +pub struct TagDictFile { + pub version: u32, + #[allow(unused)] + pub description: Option, + pub tags: HashMap, + pub stop_words: Vec, +} + +/// 分类定义 +#[derive(Debug, Clone, Deserialize)] +pub struct TagCategoryDef { + #[allow(unused)] + pub label: Option, + #[allow(unused)] + pub color_bg: Option, + #[allow(unused)] + pub color_fg: Option, + pub words: Vec, +} + +// ── 运行时词典结构 ── + +/// 分类信息 +#[derive(Debug, Clone)] +pub struct CategoryInfo { + pub name: String, + pub label: String, + pub color_bg: String, + pub color_fg: String, + pub words: Vec, +} + +/// 运行时标签词典 +#[derive(Debug, Clone)] +pub struct TagDict { + /// 分类名称 → 分类详情 + pub categories: HashMap, + /// 标签 → 分类名称（查找加速） + pub tag_to_category: HashMap, + /// 停用词集合 + pub stop_words: HashSet, +} + +impl Default for TagDict { + fn default() -> Self { + let mut dict = TagDict { + categories: HashMap::new(), + tag_to_category: HashMap::new(), + stop_words: HashSet::new(), + }; + // 内建回退：即使 JSON 文件不存在也能工作 + dict.fill_builtin(); + dict + } +} + +impl TagDict { + /// 从默认路径加载词典（`dict/tags.json`） + pub fn load() -> Self { + Self::load_from("dict/tags.json") + } + + /// 从指定路径加载词典 + pub fn load_from(path: impl AsRef) -> Self { + let path = path.as_ref(); + let content = match std::fs::read_to_string(path) { + Ok(c) => c, + Err(e) => { + eprintln!("[词典] 警告: 无法读取词典文件 {:?}: {}，使用内建回退", path, e); + let mut d = TagDict::default(); + d.fill_builtin(); + return d; + } + }; + + let file: TagDictFile = match serde_json::from_str(&content) { + Ok(f) => f, + Err(e) => { + eprintln!("[词典] 警告: 解析词典文件 {:?} 失败: {}，使用内建回退", path, e); + let mut d = TagDict::default(); + d.fill_builtin(); + return d; + } + }; + + let mut dict = TagDict { + categories: HashMap::new(), + tag_to_category: HashMap::new(), + stop_words: file.stop_words.into_iter().collect(), + }; + + for (cat_name, def) in file.tags { + let label = def.label.clone().unwrap_or_else(|| cat_name.clone()); + let color_bg = def.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string()); + let color_fg = def.color_fg.clone().unwrap_or_else(|| "#64748b".to_string()); + + for word in &def.words { + dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone()); + } + + dict.categories.insert(cat_name.clone(), CategoryInfo { + name: cat_name, + label, + color_bg, + color_fg, + words: def.words, + }); + } + + println!("[词典] 已加载 {} 个分类, {} 个标签, {} 个停用词", + dict.categories.len(), dict.tag_to_category.len(), dict.stop_words.len()); + + dict + } + + /// 热重载词典（从默认路径重新读取） + pub fn reload(&mut self) { + *self = Self::load(); + } + + /// 获取所有分类名称（按固定顺序返回，保证前端显示一致性） + pub fn get_category_order(&self) -> Vec<&str> { + let preferred = ["style", "outfit", "scene", "body", "mood", "topic"]; + let mut result: Vec<&str> = preferred.iter() + .filter(|c| self.categories.contains_key(**c)) + .map(|c| *c) + .collect(); + // 补上自定义分类 + for name in self.categories.keys() { + if !result.contains(&name.as_str()) { + result.push(name); + } + } + result + } + + /// 获取分类信息（如果存在） + pub fn get_category(&self, name: &str) -> Option<&CategoryInfo> { + self.categories.get(name) + } + + /// 获取分类的中文标签 + pub fn get_category_label(&self, name: &str) -> String { + self.categories.get(name) + .map(|c| c.label.clone()) + .unwrap_or_else(|| "其他".to_string()) + } + + /// 获取分类的颜色 + pub fn get_category_color(&self, name: &str) -> (String, String) { + match self.categories.get(name) { + Some(c) => (c.color_bg.clone(), c.color_fg.clone()), + None => ("#f1f5f9".to_string(), "#64748b".to_string()), + } + } + + /// 判断一个词是否属于某个已知分类，如果是则返回分类名 + pub fn classify(&self, word: &str) -> Option<&str> { + self.tag_to_category.get(&word.to_lowercase()).map(|s| s.as_str()) + } + + /// 判断一个词是否是停用词 + pub fn is_stop_word(&self, word: &str) -> bool { + self.stop_words.contains(&word.to_lowercase()) + } + + /// 获取所有标签到分类的映射（用于匹配加速） + pub fn get_tag_category_map(&self) -> &HashMap { + &self.tag_to_category + } + + /// 获取停用词集合 + pub fn get_stop_words(&self) -> &HashSet { + &self.stop_words + } + + /// 内置回退：当 JSON 文件不可用时使用硬编码数据 + fn fill_builtin(&mut self) { + let builtin_categories: Vec<(&str, &str, &str, &str, Vec<&str>)> = vec![ + ("style", "风格", "#f0f9ff", "#0284c7", vec![ + "日系", "韩系", "森系", "复古", "甜美", "性感", "清新", "冷艳", + "优雅", "知性", "可爱", "俏皮", "温柔", "妩媚", "野性", "清纯", + "时尚", "潮流", "街头", "学院", "文艺", "古典", "民族", "异域", + "japanese", "korean", "retro", "vintage", "cute", "sexy", + "elegant", "fresh", "sweet", "wild", "pure", "fashion", + ]), + ("outfit", "服装", "#fdf2f8", "#db2777", vec![ + "比基尼", "泳装", "泳衣", "旗袍", "制服", "丝袜", "内衣", "文胸", + "包臀裙", "紧身裙", "超短裙", "连衣裙", "半身裙", "短裙", "长裙", + "牛仔裤", "热裤", "短裤", "休闲裤", "西装裤", + "T恤", "衬衫", "毛衣", "针织衫", "外套", "风衣", "大衣", "西装", + "吊带", "抹胸", "露背", "透视", "蕾丝", "镂空", "开叉", + "高跟鞋", "运动鞋", "凉鞋", "拖鞋", "靴子", + "bikini", "swimsuit", "stockings", "lingerie", "dress", + "skirt", "jeans", "shirt", "sweater", "coat", "heels", + ]), + ("scene", "场景", "#f0fdf4", "#15803d", vec![ + "室内", "户外", "街拍", "浴室", "浴缸", "床上", "床照", "沙发", + "厨房", "阳台", "窗边", "楼梯", "走廊", "电梯", "车内", "驾驶座", + "海边", "沙滩", "泳池", "温泉", "森林", "草地", "花园", "公园", + "街道", "巷弄", "建筑", "天台", "屋顶", "咖啡馆", "餐厅", "酒店", + "indoor", "outdoor", "street", "beach", "pool", "bathroom", + "bedroom", "kitchen", "balcony", "car", "cafe", "hotel", + ]), + ("body", "身体", "#fef3c7", "#b45309", vec![ + "美腿", "美胸", "翘臀", "玉足", "美背", "香肩", "蛮腰", "细腰", + "长腿", "酥胸", "丰臀", "白皙", "肌肤", "纹身", + "legs", "breasts", "ass", "feet", "back", "shoulders", "waist", + "skin", "tattoo", + ]), + ("mood", "情绪", "#faf5ff", "#7c3aed", vec![ + "清纯", "诱惑", "唯美", "清新", "冷艳", "妩媚", "娇羞", "俏皮", + "温柔", "野性", "冷傲", "热情", "慵懒", "迷离", "甜美", "知性", + "pure", "seductive", "beautiful", "fresh", "cool", "charming", + "shy", "cute", "gentle", "wild", "lazy", "sweet", + ]), + ("topic", "主题", "#f0f9ff", "#0369a1", vec![ + "写真", "私房", "街拍", "旅拍", "婚纱照", "艺术照", "生活照", + "自拍", "跟拍", "约拍", "商拍", + "photoshoot", "portrait", "selfie", "travel", "wedding", + ]), + ]; + + let stop_words: Vec<&str> = vec![ + "photos", "photo", "pictures", "picture", "images", "image", "video", "videos", + "cosplay", "cos", "and", "the", "of", "in", "a", "an", "is", "to", "for", + "part", "vol", "collection", "special", "extra", "bonus", + "new", "old", "selfie", "official", "original", "preview", "patreon", + "with", "without", "set", "gallery", "album", "page", "issue", + "model", "idol", "star", "actor", "actress", "singer", + "秀人", "网", "图", "套", "合集", "系列", "作品", "美女", "模特", + "女神", "小姐姐", "性感", "高清", "原版", "无水印", "番号", "编号", + "site", "web", "online", "digital", "exclusive", "premium", + "classic", "photography", "shoot", "shot", + ]; + + for word in &stop_words { + self.stop_words.insert(word.to_string()); + } + + for (name, label, bg, fg, words) in builtin_categories { + let word_strings: Vec = words.iter().map(|w| w.to_string()).collect(); + for word in &word_strings { + self.tag_to_category.insert(word.to_lowercase(), name.to_string()); + } + self.categories.insert(name.to_string(), CategoryInfo { + name: name.to_string(), + label: label.to_string(), + color_bg: bg.to_string(), + color_fg: fg.to_string(), + words: word_strings, + }); + } + } +} + +// ── lazy_static 全局词典实例 ── + +lazy_static! { + /// 全局词典实例。首次访问时自动从默认路径加载。 + /// 可在运行时通过 `reload_tag_dict()` 热重载。 + static ref TAG_DICT: RwLock = RwLock::new(TagDict::load()); +} + +/// 获取全局词典的只读引用 +pub fn get_tag_dict() -> std::sync::RwLockReadGuard<'static, TagDict> { + TAG_DICT.read().expect("tag_dict 锁被污染") +} + +/// 热重载标签词典（从默认路径重新读取） +pub fn reload_tag_dict() -> String { + let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染"); + dict.reload(); + format!( + "词典已重载：{} 个分类, {} 个标签, {} 个停用词", + dict.categories.len(), + dict.tag_to_category.len(), + dict.stop_words.len(), + ) +} + +/// 获取词典的统计信息字符串 +pub fn get_tag_dict_stats() -> String { + let dict = get_tag_dict(); + format!( + "标签词典: {} 个分类, {} 个标签, {} 个停用词 | 文件: dict/tags.json", + dict.categories.len(), + dict.tag_to_category.len(), + dict.stop_words.len(), + ) +} diff --git a/src/engine/tag_extractor.rs b/src/engine/tag_extractor.rs index 33d7023..17e518b 100644 --- a/src/engine/tag_extractor.rs +++ b/src/engine/tag_extractor.rs @@ -1,6 +1,6 @@ //! 标签提取器 — 从标题中提取分类明确、格式美观的标签 //! -//! 标签分类体系： +//! 标签分类体系（外部词典 dict/tags.json 定义）： //! - style: 风格标签（日系/韩系/森系/复古/甜美/性感等） //! - outfit: 服装标签（比基尼/泳装/旗袍/制服/丝袜/内衣等） //! - scene: 场景标签（室内/户外/街拍/浴室/床上/车内等） @@ -8,12 +8,15 @@ //! - mood: 情绪标签（清纯/诱惑/唯美/清新/冷艳等） //! - topic: 主题标签（写真/私房/街拍/旅拍等） //! - other: 其他关键词 +//! +//! 词典通过 `dict/tags.json` 加载，支持运行时热重载（POST /api/dict/reload） use regex::Regex; use lazy_static::lazy_static; use std::collections::HashSet; use jieba_rs::Jieba; use crate::engine::title_parser; +use crate::engine::tag_dict; lazy_static! { static ref JIEBA: Jieba = Jieba::new(); @@ -53,117 +56,46 @@ pub fn clean_tag(tag: &str) -> Option { } } -// ── 标签词库（按分类） ── - -/// 风格标签 -const STYLE_TAGS: &[&str] = &[ - "日系", "韩系", "森系", "复古", "甜美", "性感", "清新", "冷艳", - "优雅", "知性", "可爱", "俏皮", "温柔", "妩媚", "野性", "清纯", - "时尚", "潮流", "街头", "学院", "文艺", "古典", "民族", "异域", - "japanese", "korean", "retro", "vintage", "cute", "sexy", - "elegant", "fresh", "sweet", "wild", "pure", "fashion", -]; - -/// 服装标签 -const OUTFIT_TAGS: &[&str] = &[ - "比基尼", "泳装", "泳衣", "旗袍", "制服", "丝袜", "内衣", "文胸", - "包臀裙", "紧身裙", "超短裙", "连衣裙", "半身裙", "短裙", "长裙", - "牛仔裤", "热裤", "短裤", "休闲裤", "西装裤", - "T恤", "衬衫", "毛衣", "针织衫", "外套", "风衣", "大衣", "西装", - "吊带", "抹胸", "露背", "透视", "蕾丝", "镂空", "开叉", - "高跟鞋", "运动鞋", "凉鞋", "拖鞋", "靴子", - "bikini", "swimsuit", "stockings", "lingerie", "dress", - "skirt", "jeans", "shirt", "sweater", "coat", "heels", -]; - -/// 场景标签 -const SCENE_TAGS: &[&str] = &[ - "室内", "户外", "街拍", "浴室", "浴缸", "床上", "床照", "沙发", - "厨房", "阳台", "窗边", "楼梯", "走廊", "电梯", "车内", "驾驶座", - "海边", "沙滩", "泳池", "温泉", "森林", "草地", "花园", "公园", - "街道", "巷弄", "建筑", "天台", "屋顶", "咖啡馆", "餐厅", "酒店", - "indoor", "outdoor", "street", "beach", "pool", "bathroom", - "bedroom", "kitchen", "balcony", "car", "cafe", "hotel", -]; - -/// 身体标签 -const BODY_TAGS: &[&str] = &[ - "美腿", "美胸", "翘臀", "玉足", "美背", "香肩", "蛮腰", "细腰", - "长腿", "酥胸", "丰臀", "白皙", "肌肤", "纹身", - "legs", "breasts", "ass", "feet", "back", "shoulders", "waist", - "skin", "tattoo", -]; - -/// 情绪标签 -const MOOD_TAGS: &[&str] = &[ - "清纯", "诱惑", "唯美", "清新", "冷艳", "妩媚", "娇羞", "俏皮", - "温柔", "野性", "冷傲", "热情", "慵懒", "迷离", "甜美", "知性", - "pure", "seductive", "beautiful", "fresh", "cool", "charming", - "shy", "cute", "gentle", "wild", "lazy", "sweet", -]; - -/// 主题标签 -const TOPIC_TAGS: &[&str] = &[ - "写真", "私房", "街拍", "旅拍", "婚纱照", "艺术照", "生活照", - "自拍", "跟拍", "约拍", "商拍", - "photoshoot", "portrait", "selfie", "travel", "wedding", -]; - -/// 停用词（完全排除，不做标签） -const STOP_WORDS: &[&str] = &[ - "photos", "photo", "pictures", "picture", "images", "image", "video", "videos", - "cosplay", "cos", "and", "the", "of", "in", "a", "an", "is", "to", "for", - "part", "vol", "collection", "special", "extra", "bonus", - "new", "old", "selfie", "official", "original", "preview", "patreon", - "with", "without", "set", "gallery", "album", "page", "issue", - "model", "idol", "star", "actor", "actress", "singer", - "秀人", "网", "图", "套", "合集", "系列", "作品", "美女", "模特", - "女神", "小姐姐", "性感", "高清", "原版", "无水印", "番号", "编号", - "site", "web", "online", "digital", "exclusive", "premium", - "classic", "photography", "shoot", "shot", -]; - -/// 全局停用词（标签提取时也排除） -fn get_stop_words_set() -> HashSet<&'static str> { - let mut s = HashSet::new(); - for w in STOP_WORDS { s.insert(*w); } - s +/// 通过外部词典判断一个词属于哪个分类 +/// 先在词典中查找，如果在词典中未找到则返回 "other" +fn classify_tag(word: &str) -> String { + tag_dict::get_tag_dict().classify(word) + .map(|s| s.to_string()) + .unwrap_or_else(|| "other".to_string()) } -/// 所有分类词库（用于快速匹配分类） -fn get_tag_categories() -> Vec<(&'static [&'static str], &'static str)> { +/// 获取分类的优先级索引（用于排序） +fn category_order_index(category: &str) -> usize { + // 使用词典中的顺序，预定义分类在前，自定义在后 + match category { + "style" => 0, + "outfit" => 1, + "scene" => 2, + "body" => 3, + "mood" => 4, + "topic" => 5, + "noun" => 6, + "proper" => 7, + "location" => 8, + _ => 999, + } +} + +/// 获取所有分类（按词典定义的顺序） +fn get_category_order_list() -> Vec<&'static str> { vec![ - (STYLE_TAGS, "style"), - (OUTFIT_TAGS, "outfit"), - (SCENE_TAGS, "scene"), - (BODY_TAGS, "body"), - (MOOD_TAGS, "mood"), - (TOPIC_TAGS, "topic"), + "style", "outfit", "scene", "body", "mood", "topic", + "noun", "proper", "location", "other" ] } -/// 判断一个词属于哪个分类 -fn classify_tag(word: &str) -> &'static str { - let word_lower = word.to_lowercase(); - for (tags, category) in get_tag_categories() { - for tag in tags { - if tag.to_lowercase() == word_lower { - return category; - } - } - } - "other" -} - -// ── 核心函数 ── - /// 用 jieba 分词提取名词标签（排除人名） /// 提取词性为 n(名词)、nz(其它专名)、ns(地名)、vn(名动词) 的词 /// 但排除 nr(人名) 类，因为人名已单独提取 fn extract_tags_with_jieba( title: &str, person_name: Option<&str>, - stop_words: &HashSet<&'static str>, + stop_words: &HashSet, seen: &HashSet, ) -> Vec<(String, String)> { let mut tags = Vec::new(); @@ -181,13 +113,6 @@ fn extract_tags_with_jieba( // 也提取中文核心名用于排除 if let Some(cn) = title_parser::extract_cn(n) { v.push(cn.to_lowercase()); - // 中英文混合名的英文后缀也要排除（如 "水淼Aqua" → "aqua"） - if let Some(rest) = n.strip_prefix(&cn) { - let suffix = rest.trim(); - if !suffix.is_empty() { - v.push(suffix.to_lowercase()); - } - } } v }) @@ -253,10 +178,14 @@ fn extract_tags_with_jieba( } /// 从标题中提取分类标签 +/// +/// 使用外部词典（dict/tags.json）中的分类词库进行匹配， +/// 词典支持运行时热重载，修改后无需重启服务。 pub fn extract_tags_from_title( title: &str, person_name: Option<&str>, ) -> Vec<(String, String)> { + let dict = tag_dict::get_tag_dict(); let mut tags = Vec::new(); let mut title_lower = title.to_lowercase(); @@ -267,34 +196,30 @@ pub fn extract_tags_from_title( for part in name.split_whitespace() { title_lower = title_lower.replace(&part.to_lowercase(), " "); } - // 中英文混合名：人名的英文后缀也从 title 中移除（如 "水淼Aqua" → "aqua"） - if let Some(cn) = title_parser::extract_cn(name) { - if let Some(rest) = name.strip_prefix(&cn) { - let suffix = rest.trim().to_lowercase(); - if !suffix.is_empty() { - title_lower = title_lower.replace(&suffix, " "); - } - } - } } - let stop_words = get_stop_words_set(); + let stop_words = dict.get_stop_words(); + let person_name_lower = person_name.map(|n| n.to_lowercase()); let mut seen = HashSet::new(); - // 第一步：优先匹配分类词库中的词（保证高质量标签） - for (tag_list, category) in get_tag_categories() { - for tag in tag_list { - let tag_lower = tag.to_lowercase(); - if title_lower.contains(&tag_lower) && !seen.contains(&tag_lower) { - seen.insert(tag_lower); - tags.push((tag.to_string(), category.to_string())); - } + // 第一步：优先匹配外部词典中的词（保证高质量标签） + let tag_category_map = dict.get_tag_category_map(); + // 按标签长度降序匹配，避免短标签覆盖长标签 + let mut sorted_tags: Vec<(String, String)> = tag_category_map.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + sorted_tags.sort_by(|a, b| b.0.len().cmp(&a.0.len())); + + for (tag, category) in &sorted_tags { + if title_lower.contains(tag.as_str()) && !seen.contains(tag) { + seen.insert(tag.clone()); + tags.push((tag.clone(), category.clone())); } } // 第二步：用 jieba 分词提取名词标签（比正则更准确） - let jieba_tags = extract_tags_with_jieba(title, person_name, &stop_words, &seen); + let jieba_tags = extract_tags_with_jieba(title, person_name, stop_words, &seen); for (tag, category) in jieba_tags { let key = tag.to_lowercase(); if !seen.contains(&key) { @@ -312,24 +237,14 @@ pub fn extract_tags_from_title( if word.len() < 2 { continue; } if is_punctuation_only(word) { continue; } if word.chars().all(|c| c.is_ascii_digit()) { continue; } - if stop_words.contains(word_lower.as_str()) { continue; } + if dict.is_stop_word(word_lower.as_str()) { continue; } if let Some(ref pn) = person_name_lower { if word_lower.contains(pn.as_str()) { continue; } } if seen.contains(&word_lower) { continue; } // 已经在分类词库中匹配过的跳过 - let mut already_matched = false; - for (tag_list, _) in get_tag_categories() { - for tag in tag_list { - if tag.to_lowercase() == word_lower { - already_matched = true; - break; - } - } - if already_matched { break; } - } - if already_matched { continue; } + if dict.classify(word_lower.as_str()).is_some() { continue; } seen.insert(word_lower.clone()); @@ -344,16 +259,10 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> { let mut seen = HashSet::new(); let mut result = Vec::new(); - // 分类排序权重：预设词库分类最优先，然后是 jieba 提取的名词类 - let category_order: Vec<&str> = vec![ - "style", "outfit", "scene", "body", "mood", "topic", - "noun", "proper", "location", "other" - ]; - let mut sorted_tags = tags; sorted_tags.sort_by(|a, b| { - let idx_a = category_order.iter().position(|&c| c == a.1).unwrap_or(999); - let idx_b = category_order.iter().position(|&c| c == b.1).unwrap_or(999); + let idx_a = category_order_index(&a.1); + let idx_b = category_order_index(&b.1); idx_a.cmp(&idx_b) }); @@ -385,41 +294,21 @@ pub fn is_bad_tag(tag: &str) -> bool { if let Some(cleaned) = clean_tag(tag) { if cleaned != tag { return true; } } - let stop_words = get_stop_words_set(); - if stop_words.contains(&tag.to_lowercase().as_str()) { return true; } + let dict = tag_dict::get_tag_dict(); + if dict.is_stop_word(tag.to_lowercase().as_str()) { return true; } false } /// 获取标签的中文分类名（用于前端展示） -pub fn get_category_label(category: &str) -> &'static str { - match category { - "style" => "风格", - "outfit" => "服装", - "scene" => "场景", - "body" => "身体", - "mood" => "情绪", - "topic" => "主题", - "noun" => "名词", - "proper" => "专名", - "location" => "地名", - _ => "其他", - } +pub fn get_category_label(category: &str) -> String { + let dict = tag_dict::get_tag_dict(); + dict.get_category_label(category) } /// 获取标签分类的颜色（用于前端标签着色） -pub fn get_category_color(category: &str) -> (&'static str, &'static str) { - match category { - "style" => ("#f0f9ff", "#0284c7"), // 浅蓝底 + 深蓝字 - "outfit" => ("#fdf2f8", "#db2777"), // 浅粉底 + 深粉字 - "scene" => ("#f0fdf4", "#15803d"), // 浅绿底 + 深绿字 - "body" => ("#fef3c7", "#b45309"), // 浅黄底 + 深黄字 - "mood" => ("#faf5ff", "#7c3aed"), // 浅紫底 + 深紫字 - "topic" => ("#f0f9ff", "#0369a1"), // 浅蓝底 + 蓝字 - "noun" => ("#eff6ff", "#1d4ed8"), // 蓝灰底 + 深蓝字 - "proper" => ("#e0e7ff", "#4338ca"), // 靛蓝底 + 深靛字 - "location"=> ("#ecfdf5", "#047857"), // 翠绿底 + 深绿字 - _ => ("#f1f5f9", "#64748b"), // 默认 - } +pub fn get_category_color(category: &str) -> (String, String) { + let dict = tag_dict::get_tag_dict(); + dict.get_category_color(category) } #[cfg(test)] diff --git a/src/main.rs b/src/main.rs index 2c63520..ba45df2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -67,9 +67,9 @@ async fn main() -> std::io::Result<()> { // ② 全量解析 + 人物合并 info!("[启动管线] ② 全量解析 + 人物合并..."); let result = engine::pipeline::run_pipeline(&boot_db).await; - info!("[启动管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", + info!("[启动管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected, - result.non_persons_deleted, result.new_albums_found, + result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found, result.persons_updated, result.tags_updated, result.persons_merged, result.aliases_moved, result.tags_removed_person); @@ -90,9 +90,9 @@ async fn main() -> std::io::Result<()> { // ① 全量解析 + 人物合并（不重复 import） info!("[定时管线] ① 全量解析 + 人物合并..."); let result = engine::pipeline::run_pipeline(&timer_db).await; - info!("[定时管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", + info!("[定时管线] 完成：清洗标点{}，检测人名标签{}，脏集{}相册，删除非人物{}，清除停用词重叠人物{}，发现新相册{}，更新人名{}，更新标签{}，合并人物{}，迁移别名{}，后移除人名标签{}", result.tags_cleaned, result.person_like_tags_removed, result.dirty_albums_detected, - result.non_persons_deleted, result.new_albums_found, + result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found, result.persons_updated, result.tags_updated, result.persons_merged, result.aliases_moved, result.tags_removed_person); @@ -137,11 +137,16 @@ async fn main() -> std::io::Result<()> { println!(" - GET /api/migration/stats - 迁移状态"); println!(" - POST /api/migration/run?path=persons.db - 执行迁移"); println!(" - GET /api/queue/stats - 队列状态"); + println!("词典管理:"); + println!(" - GET /api/dict/stats - 词典状态"); + println!(" - POST /api/dict/reload - 热重载词典"); println!("按 Ctrl+C 停止服务"); fs::create_dir_all(&config.storage.images_dir).unwrap_or(()); - HttpServer::new(move || { + // 在 move 闭包之前 clone 一份 db 用于关闭信号处理 + let shutdown_db = Arc::clone(&db); + let server = HttpServer::new(move || { App::new() .wrap(middleware::Compress::default()) .wrap(middleware::Logger::default()) @@ -183,6 +188,9 @@ async fn main() -> std::io::Result<()> { .route("/api/pipeline/run", web::post().to(routes::pipeline_run)) // 队列 API .route("/api/queue/stats", web::get().to(routes::get_queue_stats)) + // 词典 API + .route("/api/dict/stats", web::get().to(routes::dict_stats)) + .route("/api/dict/reload", web::post().to(routes::dict_reload)) // 静态文件 .service( afs::Files::new("/assets", "./assets") @@ -209,8 +217,22 @@ async fn main() -> std::io::Result<()> { })) }) .bind(config.server_address())? - .run() - .await + .run(); + + // ── 信号监听：捕获 Ctrl+C / SIGINT，优雅关闭 ── + let server_handle = server.handle(); + tokio::spawn(async move { + tokio::signal::ctrl_c().await.expect("监听关闭信号失败"); + println!("\n[关闭] 收到关闭信号，正在优雅停止服务器..."); + // 停止接受新请求，等待正在处理中的请求完成（最多 10s） + server_handle.stop(true).await; + // 服务器已停止 → 合并 WAL + println!("[关闭] 服务已停止，正在合并数据库 WAL..."); + shutdown_db.wal_checkpoint().await; + println!("[关闭] 数据库文件已合并，再见！"); + }); + + server.await } /// 从 DB 重新加载相册数据到内存缓存 diff --git a/src/routes/dict.rs b/src/routes/dict.rs new file mode 100644 index 0000000..61036f6 --- /dev/null +++ b/src/routes/dict.rs @@ -0,0 +1,37 @@ +//! 标签词典管理 API — 查看/热重载 + +use actix_web::{HttpResponse, Responder}; + +use crate::engine::tag_dict; + +/// GET /api/dict/stats — 查看词典状态 +pub async fn dict_stats() -> impl Responder { + let stats = tag_dict::get_tag_dict_stats(); + let dict = tag_dict::get_tag_dict(); + let mut categories: Vec = Vec::new(); + for name in dict.get_category_order() { + if let Some(cat) = dict.get_category(name) { + categories.push(serde_json::json!({ + "name": name, + "label": cat.label, + "count": cat.words.len(), + })); + } + } + HttpResponse::Ok().json(serde_json::json!({ + "stats": stats, + "categories": categories, + "total_tags": dict.get_tag_category_map().len(), + "total_stop_words": dict.get_stop_words().len(), + })) +} + +/// POST /api/dict/reload — 热重载词典 +pub async fn dict_reload() -> impl Responder { + let msg = tag_dict::reload_tag_dict(); + println!("[API] 词典热重载: {}", msg); + HttpResponse::Ok().json(serde_json::json!({ + "success": true, + "message": msg, + })) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 007867c..b07fcbb 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -9,6 +9,7 @@ pub mod recommend; pub mod queue; pub mod migration; pub mod pipeline; +pub mod dict; pub use album::*; pub use image::*; @@ -18,4 +19,5 @@ pub use tag::*; pub use recommend::*; pub use queue::*; pub use migration::*; -pub use pipeline::*; \ No newline at end of file +pub use pipeline::*; +pub use dict::*; \ No newline at end of file