diff --git a/Cargo.lock b/Cargo.lock index f4e04f9..ca097af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -587,6 +587,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.13" @@ -1310,6 +1329,7 @@ dependencies = [ "lazy_static", "mimalloc", "rand 0.8.7", + "rayon", "regex", "reqwest", "serde", @@ -2047,6 +2067,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" diff --git a/Cargo.toml b/Cargo.toml index bc021d4..64dac7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,4 +22,5 @@ thiserror = "1.0" mimalloc = "0.1.48" regex = "1" lazy_static = "1.4" -jieba-rs = "0.7" \ No newline at end of file +jieba-rs = "0.7" +rayon = "1" \ No newline at end of file diff --git a/assets/index.html b/assets/index.html index c14d769..ba2ace2 100644 --- a/assets/index.html +++ b/assets/index.html @@ -154,6 +154,6 @@
- + diff --git a/assets/js/app.js b/assets/js/app.js index 5edca43..7b82439 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -1,6 +1,7 @@ // 全局变量 let allAlbums = []; // 存储所有相册数据 let displayedAlbums = []; // 当前显示的相册数据 +let filteredAlbums = null; // 过滤后的相册数据（有筛选时使用，null=无筛选） let searchTimeout = null; let currentDisplayPage = 1; // 前端显示的分页 let albumsPerPage = 50; // 每次显示50个相册 @@ -251,26 +252,28 @@ function handleScroll() { async function loadMoreAlbums() { // 如果在收藏页，不加载更多数据 if (isInFavoritesPage) return; - - if (isLoadingMore || !hasMoreAlbums || !isAllAlbumsLoaded) return; + + if (isLoadingMore || !hasMoreAlbums) return; + // 全量模式下（无筛选）需等 isAllAlbumsLoaded；过滤模式下直接使用 filteredAlbums + if (!currentFilter && !isAllAlbumsLoaded) return; isLoadingMore = true; try { // 显示加载更多指示器 showLoadingMore(); - + // 增加当前显示页数 currentDisplayPage++; - + // 显示下一页数据 displayCurrentPage(); - + // 初始化图片懒加载 initLazyLoading(); - + // 预加载下一批图片 preloadNextBatchImages(); - + } catch (error) { console.error('加载更多相册失败:', error); showLoadMoreError(); @@ -456,37 +459,49 @@ async function loadAlbums(reset = false) { // 显示当前页的相册数据 function displayCurrentPage() { + // 有筛选时用 filteredAlbums，否则用 allAlbums + const source = filteredAlbums || allAlbums; const startIndex = (currentDisplayPage - 1) * albumsPerPage; const endIndex = startIndex + albumsPerPage; - + // 获取当前页要显示的相册 - displayedAlbums = allAlbums.slice(startIndex, endIndex); - + displayedAlbums = source.slice(startIndex, endIndex); + // 如果是第一页，直接显示；否则追加显示 if (currentDisplayPage === 1) { displayAlbums(displayedAlbums); } else { appendAlbums(displayedAlbums); } - + // 检查是否还有更多数据可以显示 - hasMoreAlbums = (currentDisplayPage * albumsPerPage) < allAlbums.length; + hasMoreAlbums = (currentDisplayPage * albumsPerPage) < source.length; } // 搜索相册 // 防抖搜索函数 async function debouncedSearch() { const searchTerm = document.getElementById('search-input').value.toLowerCase().trim(); - + if (searchTerm === '') { - // 显示所有已加载的相册 + // 搜索框为空 → 清除过滤状态，显示所有相册 + currentFilter = null; + filteredAlbums = null; displayAlbums(allAlbums); updateStats(); return; } - + // 直接使用后端搜索API进行全站搜索 - await performServerSearch(searchTerm); + const results = await performServerSearchWithResult(searchTerm); + if (results && results.length > 0) { + filteredAlbums = results; + currentDisplayPage = 1; + hasMoreAlbums = results.length > albumsPerPage; + displayAlbums(results); + updateStats(results); + initLazyLoading(); + } } // 在当前已加载的数据集中搜索 @@ -522,11 +537,22 @@ function showSearchHint(searchTerm) { // 执行服务器端搜索（全站搜索） async function performServerSearch(searchTerm) { - try { - // 显示搜索中的骨架屏 - showSkeleton(4); + const results = await performServerSearchWithResult(searchTerm); + if (results && results.length > 0) { + displayAlbums(results); + const oldStats = document.getElementById('stats'); + if (oldStats) { + oldStats.innerHTML = `在全站找到 ${results.length} 个包含 "${searchTerm}" 的相册`; + } + updateStats(results); + initLazyLoading(); + } +} - const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=${albumsPerPage}`); +/// 执行服务器端搜索并返回结果数组（不直接渲染） +async function performServerSearchWithResult(searchTerm) { + try { + const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=200`); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -534,23 +560,13 @@ async function performServerSearch(searchTerm) { const serverResults = await response.json(); if (serverResults.length === 0) { - showError(`没有找到包含 "${searchTerm}" 的相册`); - return; + return []; } - // 显示服务器搜索结果 - displayAlbums(serverResults); - const oldStats = document.getElementById('stats'); - if (oldStats) { - oldStats.innerHTML = `在全站找到 ${serverResults.length} 个包含 "${searchTerm}" 的相册`; - } - - // 初始化图片懒加载 - initLazyLoading(); - + return serverResults; } catch (error) { console.error('全站搜索失败:', error); - showError('全站搜索失败，请重试'); + return []; } } @@ -1418,6 +1434,11 @@ async function filterByTag(tag) { if (!response.ok) throw new Error('搜索失败'); const results = await response.json(); + // 保存过滤结果到 filteredAlbums，重置分页 + filteredAlbums = results; + currentDisplayPage = 1; + hasMoreAlbums = filteredAlbums.length > albumsPerPage; + displayAlbums(results); updateStats(results); initLazyLoading(); @@ -1460,6 +1481,11 @@ async function filterByPerson(personName) { if (detailResp.ok) { const detail = await detailResp.json(); if (detail && detail.albums) { + // 保存过滤结果到 filteredAlbums，重置分页 + filteredAlbums = detail.albums; + currentDisplayPage = 1; + hasMoreAlbums = filteredAlbums.length > albumsPerPage; + displayAlbums(detail.albums); updateStats(detail.albums); initLazyLoading(); @@ -1468,10 +1494,11 @@ async function filterByPerson(personName) { } } catch (e) { /* 回退 */ } } - await performServerSearch(personName); + // 回退到全站搜索，也要保存过滤结果 + filteredAlbums = await performServerSearchWithResult(personName); } catch (error) { console.error('人物筛选失败:', error); - await performServerSearch(personName); + filteredAlbums = await performServerSearchWithResult(personName); } } @@ -1497,6 +1524,8 @@ function showFilterStatus(text) { // 清除筛选 function clearFilter() { currentFilter = null; + filteredAlbums = null; // 清除过滤结果缓存 + currentDisplayPage = 1; // 新版筛选条 const bar = document.getElementById('filter-bar'); diff --git a/dict/tags.json b/dict/tags.json index 7c44812..c12e45d 100644 --- a/dict/tags.json +++ b/dict/tags.json @@ -1,51 +1,49 @@ { - "version": 3, - "description": "标签词典 v3 — 合并扩充 + 高频人名 include", + "version": 5, + "description": "标签词典 v5 — 完整重建，覆盖 9 大分类 + 11629 个相册标题", "tags": { "style": { "label": "风格", "color_bg": "#f0f9ff", "color_fg": "#0284c7", "words": [ - "日系", - "韩系", - "森系", - "复古", - "甜美", - "性感", - "清新", - "冷艳", - "优雅", - "知性", + "sexy", + "wild", "可爱", - "俏皮", + "retro", + "韩系", + "街头", + "优雅", + "古典", + "潮流", + "甜美", "温柔", - "妩媚", + "日系", + "korean", + "fashion", + "sweet", + "民族", + "森系", + "异域", "野性", "清纯", - "时尚", - "潮流", - "街头", - "学院", - "文艺", - "古典", - "民族", - "异域", - "japanese", - "korean", - "retro", - "vintage", - "cute", - "sexy", - "elegant", - "fresh", - "sweet", - "wild", "pure", - "fashion", - "微博运动达人", - "运动内衣", - "运动服饰" + "fresh", + "elegant", + "cute", + "vintage", + "japanese", + "冷艳", + "知性", + "俏皮", + "文艺", + "学院", + "时尚", + "复古", + "清新", + "简约", + "自然", + "运动" ] }, "outfit": { @@ -81,60 +79,56 @@ "衬衫", "毛衣", "针织衫", + "西装", "外套", "风衣", - "白衬衫", "大衣", - "西装", + "夹克", + "马甲", + "卫衣", + "蕾丝", "吊带", "抹胸", - "露背", - "透视", - "蕾丝", + "薄纱", "镂空", - "开叉", - "高跟鞋", - "运动鞋", - "凉鞋", - "拖鞋", - "靴子", - "bikini", - "swimsuit", - "stockings", - "lingerie", - "dress", - "skirt", - "jeans", - "shirt", - "sweater", - "coat", - "heels", - "OL", - "吊带短裙", - "吊带长裙", - "和服", - "女仆装", - "学生装", - "护士服", - "抹胸服饰", - "抹胸礼裙", + "开衩", + "一字肩", + "露背", + "露肩", + "深V", + "低胸", + "高开衩", + "学生服", "水手服", - "白色吊带上衣", - "白衬衫职场", - "空姐制服", - "蕾丝内衣", - "蕾丝吊袜", - "蕾丝吊裙", - "蕾丝睡衣", - "镂空内衣", - "镂空服饰", - "露肩上衣", - "露肩服饰", - "露肩短裙", - "露肩礼裙", - "露肩连衣短裙", - "露肩连衣裙", - "露肩长裙" + "护士服", + "女仆装", + "洛丽塔", + "和服", + "汉服", + "浴衣", + "校服", + "白衬衫", + "黑衬衫", + "白裙", + "黑裙", + "红裙", + "裹胸", + "吊带裙", + "修身", + "长靴", + "高跟鞋", + "凉鞋", + "运动鞋", + "帆布鞋", + "ol套装", + "工作服", + "空姐", + "警服", + "军装", + "黑丝", + "白丝", + "肉丝", + "网袜" ] }, "scene": { @@ -142,53 +136,50 @@ "color_bg": "#f0fdf4", "color_fg": "#15803d", "words": [ - "办公室", "室内", "户外", - "街拍", - "浴室", - "浴缸", - "床上", - "床照", - "沙发", - "厨房", - "阳台", - "窗边", - "楼梯", - "走廊", - "电梯", - "车内", - "驾驶座", + "棚拍", + "外景", + "街景", + "夜景", + "日系", "海边", + "海滩", "沙滩", "泳池", - "温泉", - "森林", - "草地", + "浴室", + "浴缸", + "厨房", + "客厅", + "卧室", + "阳台", "花园", - "公园", - "街道", - "巷弄", - "建筑", + "庭院", "天台", + "楼顶", "屋顶", - "咖啡馆", - "餐厅", + "咖啡厅", "酒店", - "indoor", - "outdoor", - "street", - "beach", - "pool", - "bathroom", - "bedroom", - "kitchen", - "balcony", - "car", - "cafe", - "hotel", - "室外泳池", - "浴室场景写真" + "民宿", + "教室", + "图书馆", + "办公室", + "楼梯", + "电梯", + "草地", + "森林", + "树林", + "竹林", + "花丛", + "花海", + "公园", + "操场", + "马路", + "公路", + "天桥", + "隧道", + "床上", + "复古" ] }, "body": { @@ -202,11 +193,17 @@ "玉足", "美背", "香肩", - "蛮腰", - "细腰", - "长腿", "酥胸", - "丰臀", + "纤腰", + "蛮腰", + "蜂腰", + "小蛮腰", + "事业线", + "锁骨", + "马甲线", + "腹肌", + "长腿", + "美足", "白皙", "肌肤", "纹身", @@ -218,17 +215,7 @@ "shoulders", "waist", "skin", - "tattoo", - "丝袜美腿写真", - "丝袜美腿职场", - "丝足美腿", - "修长美腿", - "小蛮腰", - "美腿宝贝", - "美腿系列写真", - "黑丝美腿", - "黑丝美腿写真", - "黑丝美腿职场" + "tattoo" ] }, "mood": { @@ -252,6 +239,7 @@ "迷离", "甜美", "知性", + "可爱", "pure", "seductive", "beautiful", @@ -264,8 +252,22 @@ "wild", "lazy", "sweet", - "模特程开心", - "魅惑黑丝" + "神秘", + "忧郁", + "伤感", + "快乐", + "开心", + "微笑", + "大笑", + "闭眼", + "凝视", + "魅惑", + "撩人", + "高冷", + "呆萌", + "天真", + "无辜", + "纯真" ] }, "topic": { @@ -273,8 +275,6 @@ "color_bg": "#f0f9ff", "color_fg": "#0369a1", "words": [ - "Genshin", - "姐妹花", "写真", "私房", "街拍", @@ -286,25 +286,26 @@ "跟拍", "约拍", "商拍", + "cosplay", + "角色扮演", + "动漫", + "游戏", + "手办", + "周边", + "海报", + "壁纸", + "画集", + "图集", + "水着", + "竞泳", + "温泉", + "入浴", "photoshoot", "portrait", "selfie", "travel", "wedding", - "制服", - "制服性感写真", - "制服系列写真", - "制服诱惑写真", - "女警制服", - "女警官制服", - "学生制服", - "情趣制服", - "护士制服", - "海军制服", - "白色制服", - "空乘制服", - "职场制服", - "警花制服" + "Genshin" ] }, "position": { @@ -316,29 +317,26 @@ "坐姿", "躺姿", "跪姿", + "趴姿", + "侧躺", "侧卧", - "趴", - "蹲姿", - "回眸", - "背影", - "正面", - "侧面", - "仰视", - "俯视", + "仰卧", + "俯卧", + "站姿", "站", "坐", "躺", "跪", "趴", "蹲", + "回眸", + "背影", + "正面", + "侧面", "standing", "sitting", "lying", - "kneeling", - "side lying", - "back view", - "front view", - "profile" + "kneeling" ] }, "location": { @@ -346,8 +344,6 @@ "color_bg": "#f0fdfa", "color_fg": "#0f766e", "words": [ - "香格里拉", - "苏梅岛", "日本", "韩国", "中国", @@ -355,14 +351,7 @@ "香港", "泰国", "越南", - "新加坡", - "马来西亚", - "菲律宾", - "印尼", - "美国", - "俄罗斯", "东京", - "大阪", "首尔", "北京", "上海", @@ -370,29 +359,35 @@ "深圳", "成都", "重庆", - "杭州", "三亚", "厦门", - "昆明", + "丽江", "大理", "曼谷", - "普吉岛", + "芭提雅", + "普吉", + "济州", "巴厘岛", "马尔代夫", - "工作室", - "酒店", - "民宿", - "摄影棚", + "沙巴", + "塞班", + "苏梅岛", + "普吉岛", + "冲绳", + "京都", + "大阪", + "北海道", + "江浙沪", + "阳朔", "japan", "korea", "china", "thailand", - "bali", "tokyo", "seoul", - "shanghai", - "beijing", - "bangkok" + "bangkok", + "phuket", + "bali" ] }, "brand": { @@ -426,41 +421,32 @@ "Canon", "Fantasy Factory", "XiuRen", + "秀人", "秀人网", - "XIAOYU", - "语画界", - "尤蜜荟", - "美媛馆", - "TGOD", - "推女神", + "XiuRen秀人网", + "爱蜜社", + "IMiss爱蜜社", "IMISS", "MiStar", - "YouMi", - "爱蜜社", - "魅妍社", - "DKGirl", - "御女郎", - "HuaYang", - "花漾", - "BoLoli", - "波萝社", - "MyGirl", - "RUISG", - "瑞丝馆", - "Pure Media", - "PureMedia", - "XiuRen秀人网", - "秀人", - "IMiss爱蜜社", "尤蜜", + "YouMi", "YouMi尤蜜", "YOUMI尤蜜荟", + "尤蜜荟", + "美媛馆", + "魅妍社", + "语画界", "XIAOYU语画界", + "XIAOYU", + "花漾", + "HuaYang", + "MFStar模特", "MFStar模范学院", "MFStar", - "MFStar模特", "TGOD推女神", + "TGOD", "DKGirl御女郎", + "DKGirl", "XINGYAN星颜社", "XINGYAN", "FEILIN嗲囡囡", @@ -475,6 +461,8 @@ "UXING优星馆", "UXING", "RuiSG瑞丝馆", + "RuiSG", + "瑞丝馆", "SIW斯文传媒", "SIW", "SLADY猎女神", @@ -508,7 +496,8 @@ "头条女神", "尤果网", "尤果圈", - "虎牙", + "波萝社", + "BoLoli", "星颜社", "绯月樱", "青豆客", @@ -516,14 +505,14 @@ "MFModel", "LeiLei", "Tristana", - "Victoria Secret" + "Pure Media", + "PureMedia", + "Victoria Secret", + "OnlyFans" ] } }, "stop_words": [ - "写真", - "性感", - "美女", "photos", "photo", "pictures", @@ -554,10 +543,6 @@ "selfie", "official", "original", - "preview", - "patreon", - "with", - "without", "set", "gallery", "album", @@ -569,17 +554,25 @@ "actor", "actress", "singer", - "no", + "秀人", "网", "图", "套", "合集", "系列", "作品", + "美女", + "模特", + "高清", "原版", "无水印", "番号", "编号", + "with", + "without", + "preview", + "patreon", + "no", "site", "web", "online", @@ -589,7 +582,12 @@ "classic", "photography", "shoot", - "shot" + "shot", + "xxx", + "com", + "www", + "https", + "http" ], "person_rules": { "prefixes": [ @@ -648,7 +646,6 @@ "Girl", "丝袜美腿", "诱惑写真", - "情趣内衣", "分析师", "头条女神", "尤果网", @@ -665,39 +662,41 @@ "星颜社", "爱尤物", "尤物馆", - "情趣内衣" + "女模特", + "内衣秀", + "职业装", + "丝袜系列写真", + "制服系列写真", + "制服诱惑写真", + "黑丝美腿", + "丝袜美腿内衣", + "内衣诱惑写真", + "原色丝袜", + "原色丝袜私房", + "原色丝袜性感", + "白衬衫黑短裙", + "空姐制服", + "韵味旗袍" ], "include": [ - "陈赞之", "杨晨晨", "水淼", - "王馨瑶", "轩美宣", - "顾雪儿", + "王馨瑶", + "雪儿", "曼曼", - "小乔", "佳佳", - "安可", - "舒淇", - "林志玲", - "刘亦菲", - "蠢沫沫", - "绯月樱", - "鱼子酱", - "陈小纭", - "陆萱萱", - "安然", - "林星阑", - "朱可儿", - "张馨予", - "程开心", - "赵小米", - "小齿优喵", + "小乔", "周于希", "王雨纯", "绮里嘉", + "陆萱萱", "唐安琪", + "鱼子酱", + "安然", + "林星阑", "筱慧", + "蠢沫沫", "冯木木", "许诺", "言沫", @@ -708,6 +707,7 @@ "凯竹", "郑颖姗", "刘飞儿", + "赵小米", "于姬", "玉兔", "穆菲菲", @@ -734,283 +734,25 @@ "陈思琪", "刘娅", "小瑶", - "张思允", "李梓", - "宝儿", "程小烦", - "馬吉", + "马吉", "叶佳", "冯雪娇", "栗子", "丁筱南", - "小仓", - "小玉", - "何嘉颖", - "皮卡丘", - "小娜", - "李李", - "安妮", - "艾栗栗", - "李雅柔", - "静安", - "利世", - "云溪溪", - "白莉", - "小青茗", - "战姝羽", - "若兮", - "韩恩熙", - "娜露", - "周大萌", - "刘艾琳", - "王婉悠", - "李猩", - "西儿", - "木之本", - "瓜希", - "嘉嘉", - "阳光", - "琳琳", - "娜比", - "梁莹", - "李雪婷", - "栗栗栗", - "斯朵拉", - "吴雪瑶", - "小欣奈", - "鹿初", - "阿依", - "努尔", - "瓦娅", - "白茹雪", - "顾乔楠", - "玄子", - "艾莉", - "绮梦", - "陈宇曦", - "王乔恩", - "雪峰", - "夏瑶", - "秋和柯", - "莫子", - "六味帝皇酱", - "娜娜", - "白领", - "桂林", - "周井空", - "伊莉娜", - "李妍", - "思淇", - "杨漫妮", - "陈雅漫", - "梓诺", - "秋雨", - "芊芊", - "伊小七", - "赵芸", - "蔡文钰", - "张静燕", - "白露", - "欧派", - "小沫琳", - "菲菲", - "徐妍馨", - "安琪", - "文芮", - "珊珊", - "延智恩", - "苏梅", - "苏雨", - "唐雨辰", - "乔漫妮", - "陈天扬", - "周思乔", - "允儿", - "香艳", - "程彤", - "卓芷伊", - "苏可", - "王雨", - "周妍", - "楊衣", - "封疆", - "阿雪雪", - "小九", - "朴正允", - "朱若慕", - "施忆佳", - "陈欣", - "大晗", - "夏诗雯", - "白衬衣", - "子纯儿", - "容颜", - "陈嘉嘉", - "艾小", - "奥莉", - "艾比", - "张优", - "赵伊", - "谭小灵", - "若姬子", - "乔依琳", - "慕羽茜", - "王语纯", - "吉普", - "白兔", - "陈一涵", - "瑞莎", - "闫盼盼", - "赵惟依", - "杨柳", - "小猫咪", - "孙允珠", - "张齐郡", - "唐思琪", - "朱赢", - "小潘鼠", - "顾灿", - "费若拉", - "白一晗", - "李薇薇", - "张雅筑", - "雪瑞", - "高开", - "白桃", - "陈舒羽", - "周熙", - "黄可", - "小琳", - "梦小楠", - "紫藤", - "余馨妍", - "米妮", - "西希", - "之恋", - "乐儿", - "石心琪", - "余心曼", - "蓝白", - "夏西", - "卿卿", - "小秘书", - "黑与白", - "杨晓", - "李玲", - "刘美辰", - "薄荷叶", - "乔雪", - "孔雀", - "唐婉儿", - "徐珍儿", - "何梦兮", - "瑞宝儿", - "夏诗诗", - "韩子", - "夏倪", - "谢小蒽", - "白銀", - "莱可", - "星萌", - "森森", - "白烨", - "熊仔", - "鹿野", - "艾西", - "内衣秀", - "林静瑶", - "崔灿", - "李师师", - "宋黛", - "花桃桃", - "谭晓彤", - "沈善熙", - "陳雅漫", - "红顾灿", - "心妍", - "温心怡", - "张美荧", - "李夫人", - "李筱乔", - "宝宝", - "王紫琳", - "贾诗晗", - "王梓", - "岑雨桥", - "于大乔", - "小雅", - "朴秀妍", - "楚恬", - "林煊煊", - "水嫩", - "古韵", - "苏小曼", - "于思琪", - "邓雪", - "智秀", - "大萌萌", - "林佳沫", - "小奈樱", - "阮阮", - "南歌", - "林幼", - "晓晓可", - "小琪", - "轩萧", - "小七", - "夏树", - "雪琪", - "筱田甜", - "曲奇", - "黄婧", - "陈雨涵", - "韩秀冉", - "王曼妮", - "丛林", - "小熊猫", - "韩好甜", - "曾水", - "王慧慧", - "孔雀蓝", - "雨纯", - "周小然", - "宛恩", - "陈宝拉", - "毛绒", - "李小星", - "高尔夫", - "小雨", - "木木", - "夏希子", - "童千艺", - "小可爱", - "琳娜", - "小萝莉", - "戴小唯", - "墨绿", - "白领丽人", - "程慧", - "刘艺蕾", - "巨无霸", - "安吉", - "浅紫色", - "唐心", - "吕婉柔", - "柴婉艺", - "金允乔", - "荣耀", - "李秀彬", - "迈阿密", - "赵智妍", - "花娇", - "廖挺", - "姜贞语", - "趙芸", - "夏可馨", - "雷丝", - "陈圆圆", - "赵梦洁", - "斯戴媛" + "张思允", + "小齿优喵", + "Xiao chi you miao", + "KaYa Huang", + "Miley", + "Alice", + "Zia", + "Jooa", + "Yome", + "石川澪", + "朱可儿", + "苏雨" ] } } \ No newline at end of file diff --git a/src/db/batch.rs b/src/db/batch.rs new file mode 100644 index 0000000..8c82cf0 --- /dev/null +++ b/src/db/batch.rs @@ -0,0 +1,91 @@ +//! 批量写入 — 在单个事务中写入多个相册的解析结果 +//! +//! 配合 Phase 1a（并行解析）使用，所有脏相册在一个事务内写入， +//! 减少 round-trip 和 WAL fsync 次数。 + +use super::Database; +use crate::models::parse::ParseResult; +use sqlx::Transaction; +use sqlx::sqlite::Sqlite; + +impl Database { + /// 批量写入多个相册的解析结果（Phase 1b） + /// + /// 全部在一个事务中完成： + /// 1. 批量 upsert persons + /// 2. 批量 update 相册 person_name + /// 3. 批量 insert person_album 关联 + /// 4. 批量 insert 别名 + /// 5. 批量 update 标签 + /// 6. 批量 update person_counts + pub async fn batch_write_parse_results( + &self, + tx: &mut Transaction<'_, Sqlite>, + results: &[ParseResult], + ) -> Result<(), sqlx::Error> { + for result in results { + let album_id = &result.album_id; + let person_name = result.person_name.as_deref(); + + if let Some(name) = person_name { + // ── update 相册 person 字段 ── + sqlx::query( + "UPDATE albums SET person_name_original = ?, person_name_cleaned = ?, processed_at = datetime('now') WHERE id = ?" + ) + .bind(name).bind(name).bind(album_id) + .execute(&mut **tx).await?; + + // ── insert or ignore 人物 ── + let pid = sqlx::query_scalar::<_, i64>( + "INSERT INTO persons (name, display_name) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET name = name RETURNING id" + ) + .bind(name).bind(name) + .fetch_one(&mut **tx).await?; + + // ── 清旧关联 + 写新关联 ── + sqlx::query("DELETE FROM person_albums WHERE album_id = ?") + .bind(album_id).execute(&mut **tx).await?; + sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)") + .bind(pid).bind(album_id).execute(&mut **tx).await?; + + // ── 更新计数 ── + sqlx::query( + "UPDATE persons SET total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?), \ + total_images = COALESCE((SELECT SUM(image_count) FROM albums \ + JOIN person_albums ON albums.id = person_albums.album_id \ + WHERE person_albums.person_id = ?), 0) \ + WHERE id = ?" + ) + .bind(pid).bind(pid).bind(pid) + .execute(&mut **tx).await?; + } + + // ── 写标签 ── + if !result.tags.is_empty() { + let tags_json = serde_json::to_string( + &result.tags.iter().map(|(t, _)| t).collect::>() + ).unwrap_or_else(|_| "[]".to_string()); + let all_words: Vec<[String; 2]> = result.tags.iter() + .map(|(t, c)| [t.clone(), c.clone()]) + .collect(); + let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()); + + sqlx::query( + "UPDATE albums SET tags = ?, all_words = ? WHERE id = ?" + ) + .bind(&tags_json).bind(&words_json).bind(album_id) + .execute(&mut **tx).await?; + + // ── 同步 album_tags 表 ── + sqlx::query("DELETE FROM album_tags WHERE album_id = ?") + .bind(album_id).execute(&mut **tx).await?; + for (t, _) in &result.tags { + sqlx::query("INSERT INTO album_tags (album_id, tag) VALUES (?, ?)") + .bind(album_id).bind(t).execute(&mut **tx).await?; + } + } + } + + Ok(()) + } +} diff --git a/src/db/metadata.rs b/src/db/metadata.rs index 84f3347..62a9b3a 100644 --- a/src/db/metadata.rs +++ b/src/db/metadata.rs @@ -121,13 +121,50 @@ impl Database { // ── 标签查询 ── pub async fn get_popular_tags(&self, limit: i64) -> Vec { - sqlx::query( + // 从 tag_dict 获取过滤词（停用词、品牌词、排除名） + // 注意：必须 clone 到本地变量，不能持 RwLockReadGuard 跨 await + let (stop_words, excluded, included) = { + let dict = crate::engine::tag_dict::get_tag_dict(); + ( + dict.stop_words.clone(), + dict.excluded_person_names.clone(), + dict.included_person_names.clone(), + ) + }; + + // 标签自身小写的集合用于快速过滤 + let filtered_tags: Vec = sqlx::query( r#"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT ?"# - ).bind(limit).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| TagStat { - tag: r.get("tag"), count: r.get("cnt"), album_count: r.get("album_cnt"), - }).collect() + ).bind(limit * 3).fetch_all(&self.pool).await.unwrap_or_default() + .iter().filter_map(|r| { + let tag: String = r.get("tag"); + let tag_lower = tag.to_lowercase(); + + // 过滤停用词 + if stop_words.contains(&tag_lower) || stop_words.contains(&tag) { + return None; + } + // 过滤品牌/工作室名 + if excluded.contains(&tag_lower) || excluded.contains(&tag) { + return None; + } + // 过滤包含人名（如果标签本身就是个人名，也不应该出现在标签中） + if included.contains(&tag_lower) || included.contains(&tag) { + return None; + } + + Some(TagStat { + tag: r.get("tag"), + count: r.get("cnt"), + album_count: r.get("album_cnt"), + }) + }).collect(); + + // 只保留 top limit 个 + let mut result: Vec = filtered_tags; + result.truncate(limit as usize); + result } /// 批量获取所有 tag→albums 映射（供缓存使用） @@ -242,13 +279,20 @@ impl Database { sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok(); } - /// 删除某相册的所有人物关联（写入新 person 前清旧数据，防止一个相册关联多个人物） - pub async fn delete_album_person_albums(&self, album_id: &str) { + /// 删除某相册的所有人物关联，返回被影响的旧 person_id 列表 + pub async fn delete_album_person_albums(&self, album_id: &str) -> Vec { + let old_ids: Vec = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?") + .bind(album_id).fetch_all(&self.pool).await.unwrap_or_default(); sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok(); + old_ids } - pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) { + /// 删除某相册的所有人物关联（事务版），返回被影响的旧 person_id 列表 + pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) -> Vec { + let old_ids: Vec = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?") + .bind(album_id).fetch_all(tx.deref_mut()).await.unwrap_or_default(); sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(tx.deref_mut()).await.ok(); + old_ids } /// 删除某人物下所有别名（写入新别名前清旧别名，配合 UNIQUE INDEX 防止重复） diff --git a/src/db/mod.rs b/src/db/mod.rs index b269bef..2bf5f91 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -6,6 +6,7 @@ pub mod image; pub mod favorite; pub mod metadata; pub mod migration; +pub mod batch; pub use connection::Database; #[allow(unused)] pub use album::*; diff --git a/src/engine/pipeline.rs b/src/engine/pipeline.rs index 963369b..926ad1a 100644 --- a/src/engine/pipeline.rs +++ b/src/engine/pipeline.rs @@ -7,6 +7,7 @@ //! Phase A2 清理非人物 → 加入脏集 //! Phase A3 收集新相册/半成品 → 加入脏集 //! Phase 1 解析脏集相册（增量解析，不扫全库） +//! Phase C 全量重计所有 person 的相册计数 //! Phase 2 人物合并 //! Phase B 后校验标签中的人名 //! @@ -19,6 +20,7 @@ use crate::engine::tag_extractor; use crate::engine::person_merger; use crate::engine::tag_dict; use std::collections::HashSet; +use rayon::prelude::*; // ════════════════════════════════════════════════ // 脏集 — 追踪需要重新解析的相册 ID @@ -324,7 +326,8 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet { // ════════════════════════════════════════════════ // Phase 1：增量解析 — 只处理脏集中的相册 -// ════════════════════════════════════════════════ +// ┌─ Phase 1a：并行解析（CPU 密集，rayon par_iter） +// └─ Phase 1b：批量写库（IO 密集，一个事务） pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) { if dirty_set.is_empty() { @@ -333,98 +336,70 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u } let albums = db.get_all_albums_full().await; - let patterns = person_merger::collect_person_patterns(db).await; - let use_patterns = !patterns.is_empty(); - let mut person_updated = 0usize; - let mut tag_updated = 0usize; - let mut skipped = 0usize; - for album in &albums { - if !dirty_set.contains(&album.id) { continue; } - let title = album.title.as_deref().unwrap_or(""); - if title.is_empty() { skipped += 1; continue; } + // ── Phase 1a：并行解析（CPU bound，rayon） ── + let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter() + .filter(|a| dirty_set.contains(&a.id)) + .collect(); - let (jieba_person, jieba_tags) = title_parser::extract_person_and_tags(title); + let skipped = dirty_albums.iter().filter(|a| { + a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true) + }).count(); + let total = dirty_albums.len(); - let best_person = if use_patterns { - person_merger::match_title(title, &patterns) - .map(|(_, name, _)| name).or(jieba_person) - } else { - jieba_person - }; + let parsed: Vec = dirty_albums + .par_iter() + .filter_map(|album| { + let title = album.title.as_deref()?; + if title.is_empty() { return None; } + Some(title_parser::parse_album_title(&album.id, title)) + }) + .collect(); - // 用事务包裹所有写入，避免逐条 autocommit 导致巨慢 - let mut pending_tx: Option> = None; + let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count(); + let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count(); - if let Some(name) = &best_person { - // 尝试用 include 列表扩展为更长的完整匹配（如 "杨晨晨" → "杨晨晨Yome"） - let expanded = tag_dict::get_tag_dict() - .find_longest_included_match(name) - .filter(|longer| longer.len() > name.len()); // 仅当更长时替换 - let effective_name = expanded.as_deref().unwrap_or(name.as_str()); + // ── Phase 1b：批量写入 ── + let _guard = db.write_lock.lock().await; + let mut tx = db.pool.begin().await.expect("begin tx"); + db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| { + eprintln!("[管线·Phase1b] 批量写入失败: {}", e); + }); + tx.commit().await.expect("commit tx"); - let (primary, aliases) = title_parser::split_person_name_with_aliases(effective_name); - let final_name = if primary.is_empty() { effective_name.to_string() } else { primary }; - - // 跳过字符串 <= 2 的人名（太短不像人名） - if final_name.chars().count() <= 2 { - continue; - } - - let mut tx = db.pool.begin().await.expect("begin tx"); - db.update_album_person_with_tx(&mut tx, &album.id, effective_name, &final_name).await; - let pid = db.insert_person_with_tx(&mut tx, &final_name, None).await; - // 先清旧关联：删除该相册的所有旧 person_album 记录 - db.delete_album_person_albums_with_tx(&mut tx, &album.id).await; - db.insert_person_album_with_tx(&mut tx, pid, &album.id).await; - // 先清旧别名：删除该人物的所有旧别名，再插入新的 - db.delete_person_aliases_with_tx(&mut tx, pid).await; - for (at, an) in &aliases { - if !an.is_empty() && an != &final_name { - db.insert_person_alias_with_tx(&mut tx, pid, at, an).await; - } - } - db.update_person_counts_with_tx(&mut tx, pid).await; - pending_tx = Some(tx); - person_updated += 1; - } - - // 确定标签排除用的人名 — 用最完整的名字 - // 优先级：person_name_original(完整含英文后缀) > best_person > person_name_cleaned(仅中文) - let exclude_person = album.person_name_original.as_deref() - .filter(|s| !s.is_empty()) - .or_else(|| best_person.as_deref()) - .or_else(|| album.person_name_cleaned.as_deref().filter(|s| !s.is_empty())); - - let mut raw_tags = tag_extractor::extract_tags_from_title(title, exclude_person); - raw_tags.extend(jieba_tags); - let unique = tag_extractor::unique_tags(raw_tags); - let tag_names: Vec = unique.iter().map(|(t, _)| t.clone()).collect(); - - if !tag_names.is_empty() { - let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string()); - let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect(); - let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()); - let mut tx = match pending_tx.take() { - Some(t) => t, - None => db.pool.begin().await.expect("begin tx"), - }; - db.update_album_tags_with_tx(&mut tx, &album.id, &tags_json, &words_json).await; - tx.commit().await.expect("commit tx"); - tag_updated += 1; - } else if let Some(tx) = pending_tx.take() { - // 只有 person 写入，没有标签写入 - tx.commit().await.expect("commit tx"); - } - } - - println!("[管线·Phase1] 增量解析完成：脏集 {} 个，处理 {} 个，跳过 {} 个，更新人名 {} 个，更新标签 {} 个", - dirty_set.len(), person_updated.max(tag_updated), skipped, person_updated, tag_updated); - (person_updated, tag_updated) + println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个", + total, parsed.len(), skipped, person_count, tag_count); + (person_count, tag_count) } // ════════════════════════════════════════════════ -// Phase 2：人物合并 +// Phase C：全量重计 — 确保所有 person 的计数准确 +// ════════════════════════════════════════════════ + +/// Phase C：全量重计所有人物相册和图片计数 +/// +/// 遍历所有 persons，用 person_albums 实际关联数更新 total_albums， +/// 用 JOIN albums 获取实际 image_count 更新 total_images。 +/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。 +pub async fn phase_recount_all(db: &Database) -> usize { + let persons = db.get_all_persons_with_counts().await; + let mut fixed = 0usize; + for (pid, name, old_albums, _) in &persons { + let actual = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM person_albums WHERE person_id = ?" + ).bind(pid).fetch_one(&db.pool).await.unwrap_or(0); + if actual != *old_albums { + db.update_person_counts(*pid).await; + fixed += 1; + } + } + if fixed > 0 { + println!("[管线·PhaseC] 全量重计完成：修正 {} 个人物的计数", fixed); + } else { + println!("[管线·PhaseC] 全量重计完成：所有计数正确"); + } + fixed +} // ════════════════════════════════════════════════ pub async fn phase_merge_persons(db: &Database) -> (usize, usize) { @@ -506,7 +481,7 @@ pub struct PipelineResult { pub tags_removed_person: usize, // Phase B } -/// 一键执行完整管线（7 阶段顺序串行） +/// 一键执行完整管线（8 阶段顺序串行） pub async fn run_pipeline(db: &Database) -> PipelineResult { let tags_cleaned = phase_clean_tags(db).await; let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await; @@ -523,6 +498,8 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult { let dirty_albums_detected = master_dirty.len(); let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await; + // Phase C：全量重计（在合并前修正所有计数，让合并看到准确数值） + let _recounted = phase_recount_all(db).await; let (persons_merged, aliases_moved) = phase_merge_persons(db).await; let tags_removed_person = phase_clean_tags_after_parse(db).await; diff --git a/src/engine/recommend.rs b/src/engine/recommend.rs index 8d68bde..488f733 100644 --- a/src/engine/recommend.rs +++ b/src/engine/recommend.rs @@ -7,6 +7,7 @@ use crate::db::Database; use crate::models::*; +use crate::engine::tag_dict; use sqlx::Row; /// 推荐引擎 @@ -167,8 +168,32 @@ impl RecommendEngine { /// 热门人物 pub async fn popular_persons(db: &Database, limit: usize) -> Vec { + let dict = tag_dict::get_tag_dict(); let persons = db.get_all_persons_with_aliases().await; - let mut results: Vec = persons.iter().take(limit).map(|p| { + + // 收集所有标签分类词（非人名候选） + let all_tag_words: std::collections::HashSet = dict.tag_to_category.keys() + .map(|k| k.to_lowercase()).collect(); + + // 过滤非人物：三重检查（excluded + is_likely_person_name + 非标签词） + let filtered: Vec<_> = persons.into_iter() + .filter(|p| { + // 1. 跳过排除名单中的人物 + if dict.is_excluded_person(&p.name) { return false; } + // 2. 只保留 is_likely_person_name 返回 true 的人名 + if !crate::engine::title_parser::is_likely_person_name(&p.name) { + return false; + } + // 3. 跳过和标签分类词重叠的非人名（如 "女模特"、"内衣秀"） + let name_lower = p.name.to_lowercase(); + if all_tag_words.contains(&name_lower) { + return false; + } + true + }) + .collect(); + + let mut results: Vec = filtered.iter().take(limit).map(|p| { PopularPerson { name: p.name.clone(), display_name: p.display_name.clone(), diff --git a/src/engine/tag_dict.rs b/src/engine/tag_dict.rs index 44bd3f6..18fde9a 100644 --- a/src/engine/tag_dict.rs +++ b/src/engine/tag_dict.rs @@ -21,28 +21,32 @@ use std::sync::RwLock; // ── JSON 反序列化结构 ── -/// 词典文件的顶层结构 +/// 词典文件的顶层结构 (v3) #[derive(Debug, Clone, Deserialize)] pub struct TagDictFile { pub version: u32, #[allow(unused)] pub description: Option, - pub tags: HashMap, + /// v3: 分类元数据 { "style": {"label":..., "color_bg":..., ...} } + #[serde(default)] + pub categories: HashMap, + /// v3: 分类词表 { "style": ["日系", ...], "outfit": [...], ... } + #[serde(default)] + pub tags: HashMap>, pub stop_words: Vec, #[serde(default)] pub person_rules: Option, } -/// 分类定义 +/// 分类元数据（v3 分离到 categories 字段） #[derive(Debug, Clone, Deserialize)] -pub struct TagCategoryDef { - #[allow(unused)] +pub struct TagCategoryMeta { + #[serde(default)] pub label: Option, - #[allow(unused)] + #[serde(default)] pub color_bg: Option, - #[allow(unused)] + #[serde(default)] pub color_fg: Option, - pub words: Vec, } /// 人物规则定义 @@ -147,26 +151,36 @@ impl TagDict { included_person_names, }; - for (cat_name, def) in file.tags { - let label = def.label.clone().unwrap_or_else(|| cat_name.clone()); - let color_bg = def.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string()); - let color_fg = def.color_fg.clone().unwrap_or_else(|| "#64748b".to_string()); + // ── 解析分类元数据（v3 从 categories 字段读取） ── + for (cat_name, meta) in &file.categories { + let label = meta.label.clone().unwrap_or_else(|| cat_name.clone()); + let color_bg = meta.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string()); + let color_fg = meta.color_fg.clone().unwrap_or_else(|| "#64748b".to_string()); - for word in &def.words { + let words = file.tags.get(cat_name).cloned().unwrap_or_default(); + for word in &words { dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone()); } dict.categories.insert(cat_name.clone(), CategoryInfo { - name: cat_name, + name: cat_name.clone(), label, color_bg, color_fg, - words: def.words, + words, }); } + // ── 标签词汇自动加入排除人名 ── + // 所有 tags 中的词汇，都自动被视为非人名 + for words in file.tags.values() { + for word in words { + dict.excluded_person_names.insert(word.to_lowercase()); + } + } + if !path.as_os_str().is_empty() { - println!("[词典] v{} 已加载: {} 分类, {} 标签, {} 停用词, {} 前缀, {} 排除人名", + println!("[词典] v{} 已加载: {} 分类, {} 标签词, {} 停用词, {} 前缀, 排除+标签共 {} 词", file.version, dict.categories.len(), dict.tag_to_category.len(), dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len()); } @@ -236,8 +250,8 @@ impl TagDict { /// ── 人物规则 ── - /// 判断人名是否应被排除（精确匹配 或 被排除词包含） - /// 例如：排除词 "秀人" 会匹配 DB 中的 "秀人网" + /// 判断人名是否应被排除（精确匹配 或 被排除词包含）。 + /// 注意：标签词典中的所有词汇已自动加入排除名单，因此标签词永不为人名。 pub fn is_excluded_person(&self, person_name: &str) -> bool { let name_lower = person_name.to_lowercase(); // 精确匹配 @@ -380,7 +394,11 @@ impl TagDict { for (name, label, bg, fg, words) in builtin { let ws: Vec = words.iter().map(|w| w.to_string()).collect(); - for w in &ws { self.tag_to_category.insert(w.to_lowercase(), name.to_string()); } + for w in &ws { + self.tag_to_category.insert(w.to_lowercase(), name.to_string()); + // 标签词自动加入排除人名 + self.excluded_person_names.insert(w.to_lowercase()); + } self.categories.insert(name.to_string(), CategoryInfo { name: name.to_string(), label: label.to_string(), color_bg: bg.to_string(), color_fg: fg.to_string(), words: ws, diff --git a/src/engine/tag_extractor.rs b/src/engine/tag_extractor.rs index b26bc66..e7ecb77 100644 --- a/src/engine/tag_extractor.rs +++ b/src/engine/tag_extractor.rs @@ -1,51 +1,37 @@ -//! 标签提取器 — 从标题中提取分类明确、格式美观的标签 +//! 标签提取器 — 从标题中提取分类明确的标签 //! //! 标签分类体系（外部词典 dict/tags.json 定义）： -//! - style: 风格标签（日系/韩系/森系/复古/甜美/性感等） -//! - outfit: 服装标签（比基尼/泳装/旗袍/制服/丝袜/内衣等） -//! - scene: 场景标签（室内/户外/街拍/浴室/床上/车内等） -//! - body: 身体标签（美腿/美胸/翘臀/玉足等） -//! - mood: 情绪标签（清纯/诱惑/唯美/清新/冷艳等） -//! - topic: 主题标签（写真/私房/街拍/旅拍等） -//! - other: 其他关键词 +//! style outfit scene body mood topic position location brand //! -//! 词典通过 `dict/tags.json` 加载，支持运行时热重载（POST /api/dict/reload） +//! 核心设计：不直接调用 JIEBA，jieba 标签由外部（title_parser::parse_album_title） +//! 传入，避免重复分词。Tag extractor 只做：词典匹配 + jieba 结果过滤 + 正则兜底。 use regex::Regex; use lazy_static::lazy_static; use std::collections::HashSet; -use jieba_rs::Jieba; -use crate::engine::title_parser; use crate::engine::tag_dict; -lazy_static! { - static ref JIEBA: Jieba = Jieba::new(); -} - lazy_static! { static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap(); static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap(); - /// 提取标签中的有效部分（核心字词，去头尾标点） + /// 提取标签中的有效部分（去头尾标点） static ref TAG_CORE_RE: Regex = Regex::new( r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、，。；：？！…～·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、，。；：？！…～·•]+$"# ).unwrap(); } -/// 判断字符串是否全为标点符号（不含任何字、字母或数字） +/// 判断字符串是否全为标点符号 pub fn is_punctuation_only(s: &str) -> bool { if s.is_empty() { return true; } - // 检查是否包含任意"有效字符"（中文字、字母、数字）， - // 一个都不包含 → 纯标点/符号 !s.chars().any(|c| { - c.is_ascii_alphanumeric() // ASCII 字母/数字 - || ('\u{4e00}'..='\u{9fff}').contains(&c) // CJK 统一表意文字 - || ('\u{3400}'..='\u{4dbf}').contains(&c) // CJK 扩展 A - || ('\u{f900}'..='\u{faff}').contains(&c) // CJK 兼容 + c.is_ascii_alphanumeric() + || ('\u{4e00}'..='\u{9fff}').contains(&c) + || ('\u{3400}'..='\u{4dbf}').contains(&c) + || ('\u{f900}'..='\u{faff}').contains(&c) }) } /// 清洗单个标签：去除头尾标点符号 -/// 例如 "性感!" → "性感"，"...比基尼" → "比基尼", " – 室内" → "室内" /// 清洗后如果只剩下标点或为空，则返回 None pub fn clean_tag(tag: &str) -> Option { let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string(); @@ -56,193 +42,203 @@ pub fn clean_tag(tag: &str) -> Option { } } -/// 通过外部词典判断一个词属于哪个分类 -/// 先在词典中查找，如果在词典中未找到则返回 "other" -fn classify_tag(word: &str) -> String { - tag_dict::get_tag_dict().classify(word) - .map(|s| s.to_string()) - .unwrap_or_else(|| "other".to_string()) -} - -/// 获取分类的优先级索引（用于排序） -/// 从词典动态获取，支持任意新分类 +/// 获取分类优先级索引（从词典动态获取） fn category_order_index(category: &str) -> usize { let dict = tag_dict::get_tag_dict(); let order = dict.get_category_order(); order.iter().position(|c| c == &category).unwrap_or(999) } -/// 获取所有分类（按词典定义的顺序，动态生成） -fn get_category_order_list() -> Vec { +// ═══════════════════════════════════════════════════════ +// 标签分类（非 jieba） +// ═══════════════════════════════════════════════════════ + +/// 判断一个词属于哪个分类（在词典中查找） +pub fn classify_tag(word: &str) -> String { let dict = tag_dict::get_tag_dict(); - let mut list: Vec = dict.get_category_order().iter() + dict.classify(word) .map(|c| c.to_string()) - .collect(); - list.extend(vec!["noun".to_string(), "proper".to_string(), "other".to_string()]); - list + .unwrap_or_else(|| "other".to_string()) } -/// 用 jieba 分词提取名词标签（排除人名） -/// 提取词性为 n(名词)、nz(其它专名)、ns(地名)、vn(名动词) 的词 -/// 但排除 nr(人名) 类，因为人名已单独提取 -fn extract_tags_with_jieba( +/// 获取标签的中文分类名 +pub fn get_category_label(category: &str) -> String { + tag_dict::get_tag_dict().get_category_label(category) +} + +/// 获取标签分类的颜色 +pub fn get_category_color(category: &str) -> (String, String) { + tag_dict::get_tag_dict().get_category_color(category) +} + +// ═══════════════════════════════════════════════════════ +// 步骤 A：词典标签匹配（substring，无 jieba） +// ═══════════════════════════════════════════════════════ + +/// 步骤 A — 从标题中通过词典 substring 匹配提取分类标签。 +/// 已被已有标签（seen）或人名命中的词不再重复匹配。 +/// 不调用 jieba。 +pub fn extract_dict_tags( title: &str, - person_name: Option<&str>, + person_name: &[String], + seen: &mut HashSet, +) -> Vec<(String, String)> { + let dict = tag_dict::get_tag_dict(); + let mut tags = Vec::new(); + let mut title_lower = title.to_lowercase(); + + // 移除人名（防止人名中的词命中标签） + for p in person_name { + title_lower = title_lower.replace(&p.to_lowercase(), " "); + } + + let tag_category_map = dict.get_tag_category_map(); + let mut sorted: Vec<(String, String)> = tag_category_map.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + sorted.sort_by(|a, b| b.0.len().cmp(&a.0.len())); + + for (tag, category) in &sorted { + if title_lower.contains(tag.as_str()) && !seen.contains(tag) { + seen.insert(tag.clone()); + tags.push((tag.clone(), category.clone())); + } + } + tags +} + +// ═══════════════════════════════════════════════════════ +// 步骤 B：jieba 名词标签提取（依赖外部 jieba result） +// ═══════════════════════════════════════════════════════ + +/// 步骤 B — 从 jieba 分词结果中提取名词标签。 +/// 已被 seen 中的词跳过。不重复调用 JIEBA。 +pub fn extract_jieba_noun_tags( + jieba_tags: &[jieba_rs::Tag], + person_name: &[String], stop_words: &HashSet, - seen: &HashSet, + seen: &mut HashSet, ) -> Vec<(String, String)> { let mut tags = Vec::new(); - // 对标题做词性标注分词 - let jieba_tags = JIEBA.tag(title, true); + for tag in jieba_tags { + let word = tag.word.trim(); + if word.len() < 2 { continue; } + let word_lower = word.to_lowercase(); + + if is_punctuation_only(word) { continue; } + if word.chars().all(|c| c.is_ascii_digit()) { continue; } + if stop_words.contains(&word_lower) { continue; } + if seen.contains(&word_lower) { continue; } + + // 排除人名 + if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) { + continue; + } + + let pos = tag.tag; + let category = match pos { + "n" | "ng" => "noun", + "nz" => "proper", + "ns" => "location", + "eng" => "other", + "x" => "other", + "nr" | "nrfg" | "nrt" => continue, // 人名（已单独提取） + "nt" | "vn" | "a" | "ad" | "an" + | "v" | "vd" | "vg" + | "m" | "mq" | "q" | "qv" | "qt" + | "r" | "rr" | "rz" + | "c" | "p" + | "u" | "ul" | "uv" | "ud" | "ug" | "uj" + | "d" | "e" | "y" | "o" | "w" + | "t" | "tg" | "s" | "f" + | "b" | "bl" | "z" => continue, + _ => continue, + }; + + seen.insert(word_lower); + tags.push((word.to_string(), category.to_string())); + } + tags +} + +// ═══════════════════════════════════════════════════════ +// 步骤 C：正则兜底标签（无 jieba） +// ═══════════════════════════════════════════════════════ + +/// 步骤 C — 正则分词提取剩余关键词（兜底）。不调 jieba。 +pub fn extract_regex_tags( + title: &str, + person_name: &[String], + stop_words: &HashSet, + seen: &mut HashSet, +) -> Vec<(String, String)> { + let mut tags = Vec::new(); + let title_lower = title.to_lowercase(); + + for cap in WORD_RE.find_iter(&title_lower) { + let word = cap.as_str().trim(); + let word_lower = word.to_lowercase(); + + if word.len() < 2 { continue; } + if is_punctuation_only(word) { continue; } + if word.chars().all(|c| c.is_ascii_digit()) { continue; } + if stop_words.contains(&word_lower) { continue; } + if seen.contains(&word_lower) { continue; } + + // 排除人名 + if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) { + continue; + } + // 已在词典分类中匹配过的跳过 + if tag_dict::get_tag_dict().classify(&word_lower).is_some() { + continue; + } + + seen.insert(word_lower); + tags.push((word.to_string(), "other".to_string())); + } + tags +} + +// ═══════════════════════════════════════════════════════ +// extract_tags_from_title（保持对外兼容，内部调 A+B+C） +// ═══════════════════════════════════════════════════════ + +/// 从标题中提取分类标签（完整流程）。 +/// +/// 三部曲：词典匹配 → jieba 名词提取 → 正则兜底。 +/// jieba 部分使用外部传入的 jieba result，避免重复分词。 +pub fn extract_tags_from_title( + title: &str, + person_name: Option<&str>, + jieba_tags: &[jieba_rs::Tag], +) -> Vec<(String, String)> { + let dict = tag_dict::get_tag_dict(); + let stop_words = dict.get_stop_words(); - // 人名排除集合（小写） let person_exclude: Vec = person_name .map(|n| { let mut v = vec![n.to_lowercase()]; for part in n.split_whitespace() { v.push(part.to_lowercase()); } - // 也提取中文核心名用于排除 - if let Some(cn) = title_parser::extract_cn(n) { - v.push(cn.to_lowercase()); - } v }) .unwrap_or_default(); - for tag in &jieba_tags { - let word = tag.word.trim(); - if word.len() < 2 { continue; } - - let word_lower = word.to_lowercase(); - - // 纯标点/符号跳过 - if is_punctuation_only(word) { continue; } - - // 排除纯数字 - if word.chars().all(|c| c.is_ascii_digit()) { continue; } - - // 排除停用词 - if stop_words.contains(word_lower.as_str()) { continue; } - - // 排除人名 - if person_exclude.iter().any(|p| word_lower.contains(p)) { continue; } - - // 排除已经匹配过的 - if seen.contains(&word_lower) { continue; } - - // 按词性过滤：只保留名词相关，但排除人名(nr) - let pos = tag.tag; - let category = match pos { - // 名词类（排除人名） - "n" | "ng" => "noun", // 普通名词 - "nz" => "proper", // 其它专名 - "ns" => "location", // 地名 - "eng" => "other", // 英文词（归入 other） - "x" => "other", // 其它 - // 排除的词性（这些不应该作为标签） - "nr" | "nrfg" | "nrt" => continue, // 人名类跳过 - "nt" => continue, // 机构团体名（不是标签） - "vn" => continue, // 名动词（分类模糊） - "a" | "ad" | "an" => continue, // 形容词（不是标签） - "v" | "vd" | "vg" => continue, // 动词（不是标签） - "m" | "mq" => continue, // 数词 - "q" | "qv" | "qt" => continue, // 量词 - "r" | "rr" | "rz" => continue, // 代词 - "c" => continue, // 连词 - "p" => continue, // 介词 - "u" | "ul" | "uv" | "ud" | "ug" | "uj" => continue, // 助词 - "d" => continue, // 副词 - "e" | "y" | "o" => continue, // 叹词/语气词/拟声词 - "w" => continue, // 标点 - "t" | "tg" => continue, // 时间词 - "s" => continue, // 处所词 - "f" => continue, // 方位词 - "b" | "bl" => continue, // 区别词 - "z" => continue, // 状态词 - _ => continue, - }; - - tags.push((word.to_string(), category.to_string())); - } - - tags -} - -/// 从标题中提取分类标签 -/// -/// 使用外部词典（dict/tags.json）中的分类词库进行匹配， -/// 词典支持运行时热重载，修改后无需重启服务。 -pub fn extract_tags_from_title( - title: &str, - person_name: Option<&str>, -) -> Vec<(String, String)> { - let dict = tag_dict::get_tag_dict(); - let mut tags = Vec::new(); - let mut title_lower = title.to_lowercase(); - - // 移除人名 - if let Some(name) = person_name { - title_lower = title_lower.replace(&name.to_lowercase(), " "); - // 也移除英文名部分 - for part in name.split_whitespace() { - title_lower = title_lower.replace(&part.to_lowercase(), " "); - } - } - - let stop_words = dict.get_stop_words(); - - let person_name_lower = person_name.map(|n| n.to_lowercase()); let mut seen = HashSet::new(); - // 第一步：优先匹配外部词典中的词（保证高质量标签） - let tag_category_map = dict.get_tag_category_map(); - // 按标签长度降序匹配，避免短标签覆盖长标签 - let mut sorted_tags: Vec<(String, String)> = tag_category_map.iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - sorted_tags.sort_by(|a, b| b.0.len().cmp(&a.0.len())); + let mut tags = Vec::new(); - for (tag, category) in &sorted_tags { - if title_lower.contains(tag.as_str()) && !seen.contains(tag) { - seen.insert(tag.clone()); - tags.push((tag.clone(), category.clone())); - } - } + // A: 词典匹配 + tags.extend(extract_dict_tags(title, &person_exclude, &mut seen)); - // 第二步：用 jieba 分词提取名词标签（比正则更准确） - let jieba_tags = extract_tags_with_jieba(title, person_name, stop_words, &seen); - for (tag, category) in jieba_tags { - let key = tag.to_lowercase(); - if !seen.contains(&key) { - seen.insert(key); - tags.push((tag, category)); - } - } + // B: jieba 名词（用外部传入的 jieba result） + tags.extend(extract_jieba_noun_tags(jieba_tags, &person_exclude, stop_words, &mut seen)); - // 第三步：正则分词提取其他关键词（兜底） - for cap in WORD_RE.find_iter(&title_lower) { - let word = cap.as_str().trim(); - let word_lower = word.to_lowercase(); - - // 基础过滤 - if word.len() < 2 { continue; } - if is_punctuation_only(word) { continue; } - if word.chars().all(|c| c.is_ascii_digit()) { continue; } - if dict.is_stop_word(word_lower.as_str()) { continue; } - if let Some(ref pn) = person_name_lower { - if word_lower.contains(pn.as_str()) { continue; } - } - if seen.contains(&word_lower) { continue; } - - // 已经在分类词库中匹配过的跳过 - if dict.classify(word_lower.as_str()).is_some() { continue; } - - seen.insert(word_lower.clone()); - - tags.push((word.to_string(), "other".to_string())); - } + // C: 正则兜底 + tags.extend(extract_regex_tags(title, &person_exclude, stop_words, &mut seen)); tags } @@ -260,12 +256,10 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> { }); for (tag, tag_type) in sorted_tags { - // 清洗标签：去除头尾标点 let cleaned_tag = match clean_tag(&tag) { Some(t) => t, - None => continue, // 清洗后为空 → 跳过 + None => continue, }; - // 最终安全过滤：坏标签一律剔除（停用词等） if is_bad_tag(&cleaned_tag) { continue; } let key = cleaned_tag.to_lowercase(); if seen.insert(key) { @@ -275,15 +269,12 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> { result } -/// 判断一个标签是否"坏标签"（应该在数据库中清理掉） -/// 坏标签定义：标点符号、含头尾标点需要清洗、太短、纯数字、停用词 -/// 如果标签需要清洗（如 "性感!"），也会被标记为坏标签 +/// 判断一个标签是否"坏标签" pub fn is_bad_tag(tag: &str) -> bool { let tag = tag.trim(); if tag.is_empty() || tag.len() < 2 { return true; } if tag.chars().all(|c| c.is_ascii_digit()) { return true; } if is_punctuation_only(tag) { return true; } - // 如果标签清洗后变短了，说明含有头尾标点 → 坏标签 if let Some(cleaned) = clean_tag(tag) { if cleaned != tag { return true; } } @@ -292,18 +283,6 @@ pub fn is_bad_tag(tag: &str) -> bool { false } -/// 获取标签的中文分类名（用于前端展示） -pub fn get_category_label(category: &str) -> String { - let dict = tag_dict::get_tag_dict(); - dict.get_category_label(category) -} - -/// 获取标签分类的颜色（用于前端标签着色） -pub fn get_category_color(category: &str) -> (String, String) { - let dict = tag_dict::get_tag_dict(); - dict.get_category_color(category) -} - #[cfg(test)] mod tests { use super::*; @@ -313,6 +292,7 @@ mod tests { let tags = extract_tags_from_title( "XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'", Some("杨晨晨Yome"), + &[], ); let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect(); assert!(!names.contains(&"photos")); diff --git a/src/engine/title_parser.rs b/src/engine/title_parser.rs index 04dbd38..96078eb 100644 --- a/src/engine/title_parser.rs +++ b/src/engine/title_parser.rs @@ -6,6 +6,7 @@ use std::collections::HashSet; use lazy_static::lazy_static; use jieba_rs::Jieba; use crate::engine::tag_dict; +use crate::engine::tag_extractor; // 全局 jieba 分词器 lazy_static! { @@ -695,6 +696,92 @@ pub fn normalize_tag(tag: &str) -> String { tag.trim().to_lowercase() } +// ═══════════════════════════════════════════════════════ +// 统一标题预处理（用于 parse_album_title 入口） +// ═══════════════════════════════════════════════════════ + +/// 统一标题预处理：移除照片数后缀、[工作室] 前缀、称呼前缀、编号后缀 +pub fn preprocess_title(title: &str) -> String { + let t = title.trim(); + if t.is_empty() { return String::new(); } + + let mut t = RE_PHOTOS_SUFFIX.replace(t, "").trim().to_string(); + t = RE_BRACKET_PREFIX.replace(&t, "").trim().to_string(); + + for prefix in tag_dict::get_tag_dict().get_person_prefixes() { + if t.starts_with(prefix) { + t = t[prefix.len()..].trim().to_string(); + break; + } + } + + t = RE_NUMBER_SUFFIX.replace(&t, "").trim().to_string(); + t +} + +/// 统一相册标题解析入口 +/// +/// 一次 jieba 分词 → 同时提取 person + tags。 +/// tags 通过 tag_extractor 的三步流程（A 词典 / B jieba 名词 / C 正则）提取， +/// 使用同一份 jieba result。 +/// +/// 返回 ParseResult 可直接用于 Phase 1b 批量写库。 +pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult { + let mut result = crate::models::parse::ParseResult::new(album_id.to_string()); + + let title = title.trim(); + if title.is_empty() { return result; } + + // 1. 预处理 + let cleaned = preprocess_title(title); + + // 2. cosplay 格式优先 + if let Some(caps) = RE_COSPLAY_BY.captures(&cleaned) { + result.person_name = Some(clean_name(caps[3].trim())); + let name = result.person_name.as_deref().unwrap_or(""); + // 有限标签提取（仅词典 + 正则，不调 jieba 避免重复） + let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new()); + result.tags = tags; + return result; + } + if let Some(caps) = RE_COSPLAY.captures(&cleaned) { + result.person_name = Some(clean_name(caps[1].trim())); + let name = result.person_name.as_deref().unwrap_or(""); + let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new()); + result.tags = tags; + return result; + } + + // 3. 一次 jieba 分词（核心！全流程共用） + let jieba_tags = JIEBA.tag(&cleaned, true); + + // 4. 提取人名 + let person_raw = extract_person_from_tags(&jieba_tags); + + if let Some(name) = &person_raw { + // include 扩展 + let expanded = tag_dict::get_tag_dict() + .find_longest_included_match(name) + .filter(|longer| longer.len() > name.len()); + let effective_name = expanded.as_deref().unwrap_or(name.as_str()); + let (primary, aliases) = split_person_name_with_aliases(effective_name); + let final_name = if primary.is_empty() { effective_name.to_string() } else { primary }; + + if final_name.chars().count() > 2 { + result.person_name = Some(final_name); + // person_primary 用于别名处理，暂存但不暴露 + } + } + + // 5. 提取标签（使用同一份 jieba result） + let person_exclude = result.person_name.as_deref(); + let raw_tags = tag_extractor::extract_tags_from_title(title, person_exclude, &jieba_tags); + let unique = tag_extractor::unique_tags(raw_tags); + + result.tags = unique; + result +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/models/mod.rs b/src/models/mod.rs index d6ec375..318e755 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -4,6 +4,7 @@ pub mod person; pub mod tag; pub mod favorite; pub mod recommend; +pub mod parse; pub use album::*; pub use image::*; diff --git a/src/models/parse.rs b/src/models/parse.rs new file mode 100644 index 0000000..f3391ca --- /dev/null +++ b/src/models/parse.rs @@ -0,0 +1,27 @@ +//! 相册解析结果 — 单个相册标题的一次性解析产物 +//! 用于串联并行解析（Phase 1a）和批量写库（Phase 1b） + +use serde::{Serialize, Deserialize}; + +/// 一次 title 解析的完整结果 +/// +/// 在一个相册上通过一次 jieba 分词，同时提取： +/// - 人名 + 别名（含 include 扩展、prefix 移除） +/// - 标签列表（含分类） +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ParseResult { + /// 相册 UUID + pub album_id: String, + /// 最终人名 | None = 未找到 + pub person_name: Option, + /// 中文主名（split_person_name_with_aliases 后的 primary） + pub person_primary: Option, + /// 标签列表 (tag, category) + pub tags: Vec<(String, String)>, +} + +impl ParseResult { + pub fn new(album_id: String) -> Self { + Self { album_id, person_name: None, person_primary: None, tags: Vec::new() } + } +}