admin/rust-imager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

update

Browse Source
This commit is contained in:
lychang
2026-07-20 04:46:02 +08:00
parent 00af6e8528
commit a6bfe7eda2
15 changed files with 927 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Cargo.lock generated
View File

@@ -587,6 +587,25 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.13"
@@ -1310,6 +1329,7 @@ dependencies = [
"lazy_static",
"mimalloc",
"rand 0.8.7",
"rayon",
"regex",
"reqwest",
"serde",
@@ -2047,6 +2067,26 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rayon"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"

3
Cargo.toml
View File

@@ -22,4 +22,5 @@ thiserror = "1.0"
mimalloc = "0.1.48"
regex = "1"
lazy_static = "1.4"
jieba-rs = "0.7"
jieba-rs = "0.7"
rayon = "1"

2
assets/index.html
View File

@@ -154,6 +154,6 @@
<!-- 移动端遮罩 -->
<div class="mobile-overlay" id="mobile-overlay" onclick="toggleMobileSidebar()"></div>
<script src="/assets/js/app.js?v=5"></script>
<script src="/assets/js/app.js?v=6"></script>
</body>
</html>

99
assets/js/app.js
View File

@@ -1,6 +1,7 @@
// 全局变量
let allAlbums = []; // 存储所有相册数据
let displayedAlbums = []; // 当前显示的相册数据
let filteredAlbums = null; // 过滤后的相册数据（有筛选时使用，null=无筛选）
let searchTimeout = null;
let currentDisplayPage = 1; // 前端显示的分页
let albumsPerPage = 50; // 每次显示50个相册
@@ -251,26 +252,28 @@ function handleScroll() {
async function loadMoreAlbums() {
// 如果在收藏页，不加载更多数据
if (isInFavoritesPage) return;
if (isLoadingMore || !hasMoreAlbums || !isAllAlbumsLoaded) return;
if (isLoadingMore || !hasMoreAlbums) return;
// 全量模式下（无筛选）需等 isAllAlbumsLoaded；过滤模式下直接使用 filteredAlbums
if (!currentFilter && !isAllAlbumsLoaded) return;
isLoadingMore = true;
try {
// 显示加载更多指示器
showLoadingMore();
// 增加当前显示页数
currentDisplayPage++;
// 显示下一页数据
displayCurrentPage();
// 初始化图片懒加载
initLazyLoading();
// 预加载下一批图片
preloadNextBatchImages();
} catch (error) {
console.error('加载更多相册失败:', error);
showLoadMoreError();
@@ -456,37 +459,49 @@ async function loadAlbums(reset = false) {
// 显示当前页的相册数据
function displayCurrentPage() {
// 有筛选时用 filteredAlbums，否则用 allAlbums
const source = filteredAlbums || allAlbums;
const startIndex = (currentDisplayPage - 1) * albumsPerPage;
const endIndex = startIndex + albumsPerPage;
// 获取当前页要显示的相册
displayedAlbums = allAlbums.slice(startIndex, endIndex);
displayedAlbums = source.slice(startIndex, endIndex);
// 如果是第一页，直接显示；否则追加显示
if (currentDisplayPage === 1) {
displayAlbums(displayedAlbums);
} else {
appendAlbums(displayedAlbums);
}
// 检查是否还有更多数据可以显示
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < allAlbums.length;
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < source.length;
}
// 搜索相册
// 防抖搜索函数
async function debouncedSearch() {
const searchTerm = document.getElementById('search-input').value.toLowerCase().trim();
if (searchTerm === '') {
// 显示所有已加载的相册
// 搜索框为空 → 清除过滤状态，显示所有相册
currentFilter = null;
filteredAlbums = null;
displayAlbums(allAlbums);
updateStats();
return;
}
// 直接使用后端搜索API进行全站搜索
await performServerSearch(searchTerm);
const results = await performServerSearchWithResult(searchTerm);
if (results && results.length > 0) {
filteredAlbums = results;
currentDisplayPage = 1;
hasMoreAlbums = results.length > albumsPerPage;
displayAlbums(results);
updateStats(results);
initLazyLoading();
}
}
// 在当前已加载的数据集中搜索
@@ -522,11 +537,22 @@ function showSearchHint(searchTerm) {
// 执行服务器端搜索（全站搜索）
async function performServerSearch(searchTerm) {
try {
// 显示搜索中的骨架屏
showSkeleton(4);
const results = await performServerSearchWithResult(searchTerm);
if (results && results.length > 0) {
displayAlbums(results);
const oldStats = document.getElementById('stats');
if (oldStats) {
oldStats.innerHTML = `在全站找到 ${results.length} 个包含 "${searchTerm}" 的相册`;
}
updateStats(results);
initLazyLoading();
}
}
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=${albumsPerPage}`);
/// 执行服务器端搜索并返回结果数组（不直接渲染）
async function performServerSearchWithResult(searchTerm) {
try {
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=200`);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
@@ -534,23 +560,13 @@ async function performServerSearch(searchTerm) {
const serverResults = await response.json();
if (serverResults.length === 0) {
showError(`没有找到包含 "${searchTerm}" 的相册`);
return;
return [];
}
// 显示服务器搜索结果
displayAlbums(serverResults);
const oldStats = document.getElementById('stats');
if (oldStats) {
oldStats.innerHTML = `在全站找到 ${serverResults.length} 个包含 "${searchTerm}" 的相册`;
}
// 初始化图片懒加载
initLazyLoading();
return serverResults;
} catch (error) {
console.error('全站搜索失败:', error);
showError('全站搜索失败，请重试');
return [];
}
}
@@ -1418,6 +1434,11 @@ async function filterByTag(tag) {
if (!response.ok) throw new Error('搜索失败');
const results = await response.json();
// 保存过滤结果到 filteredAlbums，重置分页
filteredAlbums = results;
currentDisplayPage = 1;
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
displayAlbums(results);
updateStats(results);
initLazyLoading();
@@ -1460,6 +1481,11 @@ async function filterByPerson(personName) {
if (detailResp.ok) {
const detail = await detailResp.json();
if (detail && detail.albums) {
// 保存过滤结果到 filteredAlbums，重置分页
filteredAlbums = detail.albums;
currentDisplayPage = 1;
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
displayAlbums(detail.albums);
updateStats(detail.albums);
initLazyLoading();
@@ -1468,10 +1494,11 @@ async function filterByPerson(personName) {
}
} catch (e) { /* 回退 */ }
}
await performServerSearch(personName);
// 回退到全站搜索，也要保存过滤结果
filteredAlbums = await performServerSearchWithResult(personName);
} catch (error) {
console.error('人物筛选失败:', error);
await performServerSearch(personName);
filteredAlbums = await performServerSearchWithResult(personName);
}
}
@@ -1497,6 +1524,8 @@ function showFilterStatus(text) {
// 清除筛选
function clearFilter() {
currentFilter = null;
filteredAlbums = null; // 清除过滤结果缓存
currentDisplayPage = 1;
// 新版筛选条
const bar = document.getElementById('filter-bar');

772
dict/tags.json
View File

File diff suppressed because it is too large Load Diff

91
src/db/batch.rs Normal file
View File

@@ -0,0 +1,91 @@
//! 批量写入 — 在单个事务中写入多个相册的解析结果
//!
//! 配合 Phase 1a（并行解析）使用，所有脏相册在一个事务内写入，
//! 减少 round-trip 和 WAL fsync 次数。
use super::Database;
use crate::models::parse::ParseResult;
use sqlx::Transaction;
use sqlx::sqlite::Sqlite;
impl Database {
/// 批量写入多个相册的解析结果（Phase 1b）
///
/// 全部在一个事务中完成：
/// 1. 批量 upsert persons
/// 2. 批量 update 相册 person_name
/// 3. 批量 insert person_album 关联
/// 4. 批量 insert 别名
/// 5. 批量 update 标签
/// 6. 批量 update person_counts
pub async fn batch_write_parse_results(
&self,
tx: &mut Transaction<'_, Sqlite>,
results: &[ParseResult],
) -> Result<(), sqlx::Error> {
for result in results {
let album_id = &result.album_id;
let person_name = result.person_name.as_deref();
if let Some(name) = person_name {
// ── update 相册 person 字段 ──
sqlx::query(
"UPDATE albums SET person_name_original = ?, person_name_cleaned = ?, processed_at = datetime('now') WHERE id = ?"
)
.bind(name).bind(name).bind(album_id)
.execute(&mut **tx).await?;
// ── insert or ignore 人物 ──
let pid = sqlx::query_scalar::<_, i64>(
"INSERT INTO persons (name, display_name) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET name = name RETURNING id"
)
.bind(name).bind(name)
.fetch_one(&mut **tx).await?;
// ── 清旧关联 + 写新关联 ──
sqlx::query("DELETE FROM person_albums WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)")
.bind(pid).bind(album_id).execute(&mut **tx).await?;
// ── 更新计数 ──
sqlx::query(
"UPDATE persons SET total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?), \
total_images = COALESCE((SELECT SUM(image_count) FROM albums \
JOIN person_albums ON albums.id = person_albums.album_id \
WHERE person_albums.person_id = ?), 0) \
WHERE id = ?"
)
.bind(pid).bind(pid).bind(pid)
.execute(&mut **tx).await?;
}
// ── 写标签 ──
if !result.tags.is_empty() {
let tags_json = serde_json::to_string(
&result.tags.iter().map(|(t, _)| t).collect::<Vec<_>>()
).unwrap_or_else(|_| "[]".to_string());
let all_words: Vec<[String; 2]> = result.tags.iter()
.map(|(t, c)| [t.clone(), c.clone()])
.collect();
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
sqlx::query(
"UPDATE albums SET tags = ?, all_words = ? WHERE id = ?"
)
.bind(&tags_json).bind(&words_json).bind(album_id)
.execute(&mut **tx).await?;
// ── 同步 album_tags 表 ──
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;
for (t, _) in &result.tags {
sqlx::query("INSERT INTO album_tags (album_id, tag) VALUES (?, ?)")
.bind(album_id).bind(t).execute(&mut **tx).await?;
}
}
}
Ok(())
}
}

60
src/db/metadata.rs
View File

@@ -121,13 +121,50 @@ impl Database {
// ── 标签查询 ──
pub async fn get_popular_tags(&self, limit: i64) -> Vec<TagStat> {
sqlx::query(
// 从 tag_dict 获取过滤词（停用词、品牌词、排除名）
// 注意：必须 clone 到本地变量，不能持 RwLockReadGuard 跨 await
let (stop_words, excluded, included) = {
let dict = crate::engine::tag_dict::get_tag_dict();
(
dict.stop_words.clone(),
dict.excluded_person_names.clone(),
dict.included_person_names.clone(),
)
};
// 标签自身小写的集合用于快速过滤
let filtered_tags: Vec<TagStat> = sqlx::query(
r#"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT ?"#
).bind(limit).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| TagStat {
tag: r.get("tag"), count: r.get("cnt"), album_count: r.get("album_cnt"),
}).collect()
).bind(limit * 3).fetch_all(&self.pool).await.unwrap_or_default()
.iter().filter_map(|r| {
let tag: String = r.get("tag");
let tag_lower = tag.to_lowercase();
// 过滤停用词
if stop_words.contains(&tag_lower) || stop_words.contains(&tag) {
return None;
}
// 过滤品牌/工作室名
if excluded.contains(&tag_lower) || excluded.contains(&tag) {
return None;
}
// 过滤包含人名（如果标签本身就是个人名，也不应该出现在标签中）
if included.contains(&tag_lower) || included.contains(&tag) {
return None;
}
Some(TagStat {
tag: r.get("tag"),
count: r.get("cnt"),
album_count: r.get("album_cnt"),
})
}).collect();
// 只保留 top limit 个
let mut result: Vec<TagStat> = filtered_tags;
result.truncate(limit as usize);
result
}
/// 批量获取所有 tag→albums 映射（供缓存使用）
@@ -242,13 +279,20 @@ impl Database {
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
}
/// 删除某相册的所有人物关联（写入新 person 前清旧数据，防止一个相册关联多个人物）
pub async fn delete_album_person_albums(&self, album_id: &str) {
/// 删除某相册的所有人物关联，返回被影响的旧 person_id 列表
pub async fn delete_album_person_albums(&self, album_id: &str) -> Vec<i64> {
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
.bind(album_id).fetch_all(&self.pool).await.unwrap_or_default();
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok();
old_ids
}
pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) {
/// 删除某相册的所有人物关联（事务版），返回被影响的旧 person_id 列表
pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) -> Vec<i64> {
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
.bind(album_id).fetch_all(tx.deref_mut()).await.unwrap_or_default();
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(tx.deref_mut()).await.ok();
old_ids
}
/// 删除某人物下所有别名（写入新别名前清旧别名，配合 UNIQUE INDEX 防止重复）

1
src/db/mod.rs
View File

@@ -6,6 +6,7 @@ pub mod image;
pub mod favorite;
pub mod metadata;
pub mod migration;
pub mod batch;
pub use connection::Database;
#[allow(unused)] pub use album::*;

147
src/engine/pipeline.rs
View File

@@ -7,6 +7,7 @@
//! Phase A2 清理非人物 → 加入脏集
//! Phase A3 收集新相册/半成品 → 加入脏集
//! Phase 1 解析脏集相册（增量解析，不扫全库）
//! Phase C 全量重计所有 person 的相册计数
//! Phase 2 人物合并
//! Phase B 后校验标签中的人名
//!
@@ -19,6 +20,7 @@ use crate::engine::tag_extractor;
use crate::engine::person_merger;
use crate::engine::tag_dict;
use std::collections::HashSet;
use rayon::prelude::*;
// ════════════════════════════════════════════════
// 脏集 — 追踪需要重新解析的相册 ID
@@ -324,7 +326,8 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
// ════════════════════════════════════════════════
// Phase 1：增量解析 — 只处理脏集中的相册
// ════════════════════════════════════════════════
// ┌─ Phase 1a：并行解析（CPU 密集，rayon par_iter）
// └─ Phase 1b：批量写库（IO 密集，一个事务）
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
if dirty_set.is_empty() {
@@ -333,98 +336,70 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
}
let albums = db.get_all_albums_full().await;
let patterns = person_merger::collect_person_patterns(db).await;
let use_patterns = !patterns.is_empty();
let mut person_updated = 0usize;
let mut tag_updated = 0usize;
let mut skipped = 0usize;
for album in &albums {
if !dirty_set.contains(&album.id) { continue; }
let title = album.title.as_deref().unwrap_or("");
if title.is_empty() { skipped += 1; continue; }
// ── Phase 1a：并行解析（CPU bound，rayon） ──
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
.filter(|a| dirty_set.contains(&a.id))
.collect();
let (jieba_person, jieba_tags) = title_parser::extract_person_and_tags(title);
let skipped = dirty_albums.iter().filter(|a| {
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
}).count();
let total = dirty_albums.len();
let best_person = if use_patterns {
person_merger::match_title(title, &patterns)
.map(|(_, name, _)| name).or(jieba_person)
} else {
jieba_person
};
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
Some(title_parser::parse_album_title(&album.id, title))
})
.collect();
// 用事务包裹所有写入，避免逐条 autocommit 导致巨慢
let mut pending_tx: Option<sqlx::Transaction<'_, sqlx::Sqlite>> = None;
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
if let Some(name) = &best_person {
// 尝试用 include 列表扩展为更长的完整匹配（如 "杨晨晨" → "杨晨晨Yome"）
let expanded = tag_dict::get_tag_dict()
.find_longest_included_match(name)
.filter(|longer| longer.len() > name.len()); // 仅当更长时替换
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
// ── Phase 1b：批量写入 ──
let _guard = db.write_lock.lock().await;
let mut tx = db.pool.begin().await.expect("begin tx");
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
});
tx.commit().await.expect("commit tx");
let (primary, aliases) = title_parser::split_person_name_with_aliases(effective_name);
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
// 跳过字符串 <= 2 的人名（太短不像人名）
if final_name.chars().count() <= 2 {
continue;
}
let mut tx = db.pool.begin().await.expect("begin tx");
db.update_album_person_with_tx(&mut tx, &album.id, effective_name, &final_name).await;
let pid = db.insert_person_with_tx(&mut tx, &final_name, None).await;
// 先清旧关联：删除该相册的所有旧 person_album 记录
db.delete_album_person_albums_with_tx(&mut tx, &album.id).await;
db.insert_person_album_with_tx(&mut tx, pid, &album.id).await;
// 先清旧别名：删除该人物的所有旧别名，再插入新的
db.delete_person_aliases_with_tx(&mut tx, pid).await;
for (at, an) in &aliases {
if !an.is_empty() && an != &final_name {
db.insert_person_alias_with_tx(&mut tx, pid, at, an).await;
}
}
db.update_person_counts_with_tx(&mut tx, pid).await;
pending_tx = Some(tx);
person_updated += 1;
}
// 确定标签排除用的人名 — 用最完整的名字
// 优先级：person_name_original(完整含英文后缀) > best_person > person_name_cleaned(仅中文)
let exclude_person = album.person_name_original.as_deref()
.filter(|s| !s.is_empty())
.or_else(|| best_person.as_deref())
.or_else(|| album.person_name_cleaned.as_deref().filter(|s| !s.is_empty()));
let mut raw_tags = tag_extractor::extract_tags_from_title(title, exclude_person);
raw_tags.extend(jieba_tags);
let unique = tag_extractor::unique_tags(raw_tags);
let tag_names: Vec<String> = unique.iter().map(|(t, _)| t.clone()).collect();
if !tag_names.is_empty() {
let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string());
let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect();
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
let mut tx = match pending_tx.take() {
Some(t) => t,
None => db.pool.begin().await.expect("begin tx"),
};
db.update_album_tags_with_tx(&mut tx, &album.id, &tags_json, &words_json).await;
tx.commit().await.expect("commit tx");
tag_updated += 1;
} else if let Some(tx) = pending_tx.take() {
// 只有 person 写入，没有标签写入
tx.commit().await.expect("commit tx");
}
}
println!("[管线·Phase1] 增量解析完成：脏集 {} 个，处理 {} 个，跳过 {} 个，更新人名 {} 个，更新标签 {} 个",
dirty_set.len(), person_updated.max(tag_updated), skipped, person_updated, tag_updated);
(person_updated, tag_updated)
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
total, parsed.len(), skipped, person_count, tag_count);
(person_count, tag_count)
}
// ════════════════════════════════════════════════
// Phase 2：人物合并
// Phase C：全量重计 — 确保所有 person 的计数准确
// ════════════════════════════════════════════════
/// Phase C：全量重计所有人物相册和图片计数
///
/// 遍历所有 persons，用 person_albums 实际关联数更新 total_albums，
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
pub async fn phase_recount_all(db: &Database) -> usize {
let persons = db.get_all_persons_with_counts().await;
let mut fixed = 0usize;
for (pid, name, old_albums, _) in &persons {
let actual = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
if actual != *old_albums {
db.update_person_counts(*pid).await;
fixed += 1;
}
}
if fixed > 0 {
println!("[管线·PhaseC] 全量重计完成：修正 {} 个人物的计数", fixed);
} else {
println!("[管线·PhaseC] 全量重计完成：所有计数正确");
}
fixed
}
// ════════════════════════════════════════════════
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
@@ -506,7 +481,7 @@ pub struct PipelineResult {
pub tags_removed_person: usize, // Phase B
}
/// 一键执行完整管线（7 阶段顺序串行）
/// 一键执行完整管线（8 阶段顺序串行）
pub async fn run_pipeline(db: &Database) -> PipelineResult {
let tags_cleaned = phase_clean_tags(db).await;
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
@@ -523,6 +498,8 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
let dirty_albums_detected = master_dirty.len();
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
// Phase C：全量重计（在合并前修正所有计数，让合并看到准确数值）
let _recounted = phase_recount_all(db).await;
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
let tags_removed_person = phase_clean_tags_after_parse(db).await;

27
src/engine/recommend.rs
View File

@@ -7,6 +7,7 @@
use crate::db::Database;
use crate::models::*;
use crate::engine::tag_dict;
use sqlx::Row;
/// 推荐引擎
@@ -167,8 +168,32 @@ impl RecommendEngine {
/// 热门人物
pub async fn popular_persons(db: &Database, limit: usize) -> Vec<PopularPerson> {
let dict = tag_dict::get_tag_dict();
let persons = db.get_all_persons_with_aliases().await;
let mut results: Vec<PopularPerson> = persons.iter().take(limit).map(|p| {
// 收集所有标签分类词（非人名候选）
let all_tag_words: std::collections::HashSet<String> = dict.tag_to_category.keys()
.map(|k| k.to_lowercase()).collect();
// 过滤非人物：三重检查（excluded + is_likely_person_name + 非标签词）
let filtered: Vec<_> = persons.into_iter()
.filter(|p| {
// 1. 跳过排除名单中的人物
if dict.is_excluded_person(&p.name) { return false; }
// 2. 只保留 is_likely_person_name 返回 true 的人名
if !crate::engine::title_parser::is_likely_person_name(&p.name) {
return false;
}
// 3. 跳过和标签分类词重叠的非人名（如 "女模特"、"内衣秀"）
let name_lower = p.name.to_lowercase();
if all_tag_words.contains(&name_lower) {
return false;
}
true
})
.collect();
let mut results: Vec<PopularPerson> = filtered.iter().take(limit).map(|p| {
PopularPerson {
name: p.name.clone(),
display_name: p.display_name.clone(),

56
src/engine/tag_dict.rs
View File

@@ -21,28 +21,32 @@ use std::sync::RwLock;
// ── JSON 反序列化结构 ──
/// 词典文件的顶层结构
/// 词典文件的顶层结构 (v3)
#[derive(Debug, Clone, Deserialize)]
pub struct TagDictFile {
pub version: u32,
#[allow(unused)]
pub description: Option<String>,
pub tags: HashMap<String, TagCategoryDef>,
/// v3: 分类元数据 { "style": {"label":..., "color_bg":..., ...} }
#[serde(default)]
pub categories: HashMap<String, TagCategoryMeta>,
/// v3: 分类词表 { "style": ["日系", ...], "outfit": [...], ... }
#[serde(default)]
pub tags: HashMap<String, Vec<String>>,
pub stop_words: Vec<String>,
#[serde(default)]
pub person_rules: Option<PersonRulesDef>,
}
/// 分类定义
/// 分类元数据（v3 分离到 categories 字段）
#[derive(Debug, Clone, Deserialize)]
pub struct TagCategoryDef {
#[allow(unused)]
pub struct TagCategoryMeta {
#[serde(default)]
pub label: Option<String>,
#[allow(unused)]
#[serde(default)]
pub color_bg: Option<String>,
#[allow(unused)]
#[serde(default)]
pub color_fg: Option<String>,
pub words: Vec<String>,
}
/// 人物规则定义
@@ -147,26 +151,36 @@ impl TagDict {
included_person_names,
};
for (cat_name, def) in file.tags {
let label = def.label.clone().unwrap_or_else(|| cat_name.clone());
let color_bg = def.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string());
let color_fg = def.color_fg.clone().unwrap_or_else(|| "#64748b".to_string());
// ── 解析分类元数据（v3 从 categories 字段读取） ──
for (cat_name, meta) in &file.categories {
let label = meta.label.clone().unwrap_or_else(|| cat_name.clone());
let color_bg = meta.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string());
let color_fg = meta.color_fg.clone().unwrap_or_else(|| "#64748b".to_string());
for word in &def.words {
let words = file.tags.get(cat_name).cloned().unwrap_or_default();
for word in &words {
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
}
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name,
name: cat_name.clone(),
label,
color_bg,
color_fg,
words: def.words,
words,
});
}
// ── 标签词汇自动加入排除人名 ──
// 所有 tags 中的词汇，都自动被视为非人名
for words in file.tags.values() {
for word in words {
dict.excluded_person_names.insert(word.to_lowercase());
}
}
if !path.as_os_str().is_empty() {
println!("[词典] v{} 已加载: {} 分类, {} 标签, {} 停用词, {} 前缀, {} 排除人名",
println!("[词典] v{} 已加载: {} 分类, {} 标签词, {} 停用词, {} 前缀, 排除+标签共 {} 词",
file.version, dict.categories.len(), dict.tag_to_category.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len());
}
@@ -236,8 +250,8 @@ impl TagDict {
/// ── 人物规则 ──
/// 判断人名是否应被排除（精确匹配 或 被排除词包含）
/// 例如：排除词 "秀人" 会匹配 DB 中的 "秀人网"
/// 判断人名是否应被排除（精确匹配 或 被排除词包含）。
/// 注意：标签词典中的所有词汇已自动加入排除名单，因此标签词永不为人名。
pub fn is_excluded_person(&self, person_name: &str) -> bool {
let name_lower = person_name.to_lowercase();
// 精确匹配
@@ -380,7 +394,11 @@ impl TagDict {
for (name, label, bg, fg, words) in builtin {
let ws: Vec<String> = words.iter().map(|w| w.to_string()).collect();
for w in &ws { self.tag_to_category.insert(w.to_lowercase(), name.to_string()); }
for w in &ws {
self.tag_to_category.insert(w.to_lowercase(), name.to_string());
// 标签词自动加入排除人名
self.excluded_person_names.insert(w.to_lowercase());
}
self.categories.insert(name.to_string(), CategoryInfo {
name: name.to_string(), label: label.to_string(),
color_bg: bg.to_string(), color_fg: fg.to_string(), words: ws,

378
src/engine/tag_extractor.rs
View File

@@ -1,51 +1,37 @@
//! 标签提取器 — 从标题中提取分类明确、格式美观的标签
//! 标签提取器 — 从标题中提取分类明确的标签
//!
//! 标签分类体系（外部词典 dict/tags.json 定义）：
//! - style: 风格标签（日系/韩系/森系/复古/甜美/性感等）
//! - outfit: 服装标签（比基尼/泳装/旗袍/制服/丝袜/内衣等）
//! - scene: 场景标签（室内/户外/街拍/浴室/床上/车内等）
//! - body: 身体标签（美腿/美胸/翘臀/玉足等）
//! - mood: 情绪标签（清纯/诱惑/唯美/清新/冷艳等）
//! - topic: 主题标签（写真/私房/街拍/旅拍等）
//! - other: 其他关键词
//! style outfit scene body mood topic position location brand
//!
//! 词典通过 `dict/tags.json` 加载，支持运行时热重载（POST /api/dict/reload）
//! 核心设计：不直接调用 JIEBA，jieba 标签由外部（title_parser::parse_album_title）
//! 传入，避免重复分词。Tag extractor 只做：词典匹配 + jieba 结果过滤 + 正则兜底。
use regex::Regex;
use lazy_static::lazy_static;
use std::collections::HashSet;
use jieba_rs::Jieba;
use crate::engine::title_parser;
use crate::engine::tag_dict;
lazy_static! {
static ref JIEBA: Jieba = Jieba::new();
}
lazy_static! {
static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
/// 提取标签中的有效部分（核心字词，去头尾标点）
/// 提取标签中的有效部分（去头尾标点）
static ref TAG_CORE_RE: Regex = Regex::new(
r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、，。；：？！…～·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、，。；：？！…～·•]+$"#
).unwrap();
}
/// 判断字符串是否全为标点符号（不含任何字、字母或数字）
/// 判断字符串是否全为标点符号
pub fn is_punctuation_only(s: &str) -> bool {
if s.is_empty() { return true; }
// 检查是否包含任意"有效字符"（中文字、字母、数字），
// 一个都不包含 → 纯标点/符号
!s.chars().any(|c| {
c.is_ascii_alphanumeric() // ASCII 字母/数字
|| ('\u{4e00}'..='\u{9fff}').contains(&c) // CJK 统一表意文字
|| ('\u{3400}'..='\u{4dbf}').contains(&c) // CJK 扩展 A
|| ('\u{f900}'..='\u{faff}').contains(&c) // CJK 兼容
c.is_ascii_alphanumeric()
|| ('\u{4e00}'..='\u{9fff}').contains(&c)
|| ('\u{3400}'..='\u{4dbf}').contains(&c)
|| ('\u{f900}'..='\u{faff}').contains(&c)
})
}
/// 清洗单个标签：去除头尾标点符号
/// 例如 "性感!" → "性感"，"...比基尼" → "比基尼", " – 室内" → "室内"
/// 清洗后如果只剩下标点或为空，则返回 None
pub fn clean_tag(tag: &str) -> Option<String> {
let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string();
@@ -56,193 +42,203 @@ pub fn clean_tag(tag: &str) -> Option<String> {
}
}
/// 通过外部词典判断一个词属于哪个分类
/// 先在词典中查找，如果在词典中未找到则返回 "other"
fn classify_tag(word: &str) -> String {
tag_dict::get_tag_dict().classify(word)
.map(|s| s.to_string())
.unwrap_or_else(|| "other".to_string())
}
/// 获取分类的优先级索引（用于排序）
/// 从词典动态获取，支持任意新分类
/// 获取分类优先级索引（从词典动态获取）
fn category_order_index(category: &str) -> usize {
let dict = tag_dict::get_tag_dict();
let order = dict.get_category_order();
order.iter().position(|c| c == &category).unwrap_or(999)
}
/// 获取所有分类（按词典定义的顺序，动态生成）
fn get_category_order_list() -> Vec<String> {
// ═══════════════════════════════════════════════════════
// 标签分类（非 jieba）
// ═══════════════════════════════════════════════════════
/// 判断一个词属于哪个分类（在词典中查找）
pub fn classify_tag(word: &str) -> String {
let dict = tag_dict::get_tag_dict();
let mut list: Vec<String> = dict.get_category_order().iter()
dict.classify(word)
.map(|c| c.to_string())
.collect();
list.extend(vec!["noun".to_string(), "proper".to_string(), "other".to_string()]);
list
.unwrap_or_else(|| "other".to_string())
}
/// 用 jieba 分词提取名词标签（排除人名）
/// 提取词性为 n(名词)、nz(其它专名)、ns(地名)、vn(名动词) 的词
/// 但排除 nr(人名) 类，因为人名已单独提取
fn extract_tags_with_jieba(
/// 获取标签的中文分类名
pub fn get_category_label(category: &str) -> String {
tag_dict::get_tag_dict().get_category_label(category)
}
/// 获取标签分类的颜色
pub fn get_category_color(category: &str) -> (String, String) {
tag_dict::get_tag_dict().get_category_color(category)
}
// ═══════════════════════════════════════════════════════
// 步骤 A：词典标签匹配（substring，无 jieba）
// ═══════════════════════════════════════════════════════
/// 步骤 A — 从标题中通过词典 substring 匹配提取分类标签。
/// 已被已有标签（seen）或人名命中的词不再重复匹配。
/// 不调用 jieba。
pub fn extract_dict_tags(
title: &str,
person_name: Option<&str>,
person_name: &[String],
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let mut tags = Vec::new();
let mut title_lower = title.to_lowercase();
// 移除人名（防止人名中的词命中标签）
for p in person_name {
title_lower = title_lower.replace(&p.to_lowercase(), " ");
}
let tag_category_map = dict.get_tag_category_map();
let mut sorted: Vec<(String, String)> = tag_category_map.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
sorted.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
for (tag, category) in &sorted {
if title_lower.contains(tag.as_str()) && !seen.contains(tag) {
seen.insert(tag.clone());
tags.push((tag.clone(), category.clone()));
}
}
tags
}
// ═══════════════════════════════════════════════════════
// 步骤 B：jieba 名词标签提取（依赖外部 jieba result）
// ═══════════════════════════════════════════════════════
/// 步骤 B — 从 jieba 分词结果中提取名词标签。
/// 已被 seen 中的词跳过。不重复调用 JIEBA。
pub fn extract_jieba_noun_tags(
jieba_tags: &[jieba_rs::Tag],
person_name: &[String],
stop_words: &HashSet<String>,
seen: &HashSet<String>,
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let mut tags = Vec::new();
// 对标题做词性标注分词
let jieba_tags = JIEBA.tag(title, true);
for tag in jieba_tags {
let word = tag.word.trim();
if word.len() < 2 { continue; }
let word_lower = word.to_lowercase();
if is_punctuation_only(word) { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if stop_words.contains(&word_lower) { continue; }
if seen.contains(&word_lower) { continue; }
// 排除人名
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
continue;
}
let pos = tag.tag;
let category = match pos {
"n" | "ng" => "noun",
"nz" => "proper",
"ns" => "location",
"eng" => "other",
"x" => "other",
"nr" | "nrfg" | "nrt" => continue, // 人名（已单独提取）
"nt" | "vn" | "a" | "ad" | "an"
| "v" | "vd" | "vg"
| "m" | "mq" | "q" | "qv" | "qt"
| "r" | "rr" | "rz"
| "c" | "p"
| "u" | "ul" | "uv" | "ud" | "ug" | "uj"
| "d" | "e" | "y" | "o" | "w"
| "t" | "tg" | "s" | "f"
| "b" | "bl" | "z" => continue,
_ => continue,
};
seen.insert(word_lower);
tags.push((word.to_string(), category.to_string()));
}
tags
}
// ═══════════════════════════════════════════════════════
// 步骤 C：正则兜底标签（无 jieba）
// ═══════════════════════════════════════════════════════
/// 步骤 C — 正则分词提取剩余关键词（兜底）。不调 jieba。
pub fn extract_regex_tags(
title: &str,
person_name: &[String],
stop_words: &HashSet<String>,
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let mut tags = Vec::new();
let title_lower = title.to_lowercase();
for cap in WORD_RE.find_iter(&title_lower) {
let word = cap.as_str().trim();
let word_lower = word.to_lowercase();
if word.len() < 2 { continue; }
if is_punctuation_only(word) { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if stop_words.contains(&word_lower) { continue; }
if seen.contains(&word_lower) { continue; }
// 排除人名
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
continue;
}
// 已在词典分类中匹配过的跳过
if tag_dict::get_tag_dict().classify(&word_lower).is_some() {
continue;
}
seen.insert(word_lower);
tags.push((word.to_string(), "other".to_string()));
}
tags
}
// ═══════════════════════════════════════════════════════
// extract_tags_from_title（保持对外兼容，内部调 A+B+C）
// ═══════════════════════════════════════════════════════
/// 从标题中提取分类标签（完整流程）。
///
/// 三部曲：词典匹配 → jieba 名词提取 → 正则兜底。
/// jieba 部分使用外部传入的 jieba result，避免重复分词。
pub fn extract_tags_from_title(
title: &str,
person_name: Option<&str>,
jieba_tags: &[jieba_rs::Tag],
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let stop_words = dict.get_stop_words();
// 人名排除集合（小写）
let person_exclude: Vec<String> = person_name
.map(|n| {
let mut v = vec![n.to_lowercase()];
for part in n.split_whitespace() {
v.push(part.to_lowercase());
}
// 也提取中文核心名用于排除
if let Some(cn) = title_parser::extract_cn(n) {
v.push(cn.to_lowercase());
}
v
})
.unwrap_or_default();
for tag in &jieba_tags {
let word = tag.word.trim();
if word.len() < 2 { continue; }
let word_lower = word.to_lowercase();
// 纯标点/符号跳过
if is_punctuation_only(word) { continue; }
// 排除纯数字
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
// 排除停用词
if stop_words.contains(word_lower.as_str()) { continue; }
// 排除人名
if person_exclude.iter().any(|p| word_lower.contains(p)) { continue; }
// 排除已经匹配过的
if seen.contains(&word_lower) { continue; }
// 按词性过滤：只保留名词相关，但排除人名(nr)
let pos = tag.tag;
let category = match pos {
// 名词类（排除人名）
"n" | "ng" => "noun", // 普通名词
"nz" => "proper", // 其它专名
"ns" => "location", // 地名
"eng" => "other", // 英文词（归入 other）
"x" => "other", // 其它
// 排除的词性（这些不应该作为标签）
"nr" | "nrfg" | "nrt" => continue, // 人名类跳过
"nt" => continue, // 机构团体名（不是标签）
"vn" => continue, // 名动词（分类模糊）
"a" | "ad" | "an" => continue, // 形容词（不是标签）
"v" | "vd" | "vg" => continue, // 动词（不是标签）
"m" | "mq" => continue, // 数词
"q" | "qv" | "qt" => continue, // 量词
"r" | "rr" | "rz" => continue, // 代词
"c" => continue, // 连词
"p" => continue, // 介词
"u" | "ul" | "uv" | "ud" | "ug" | "uj" => continue, // 助词
"d" => continue, // 副词
"e" | "y" | "o" => continue, // 叹词/语气词/拟声词
"w" => continue, // 标点
"t" | "tg" => continue, // 时间词
"s" => continue, // 处所词
"f" => continue, // 方位词
"b" | "bl" => continue, // 区别词
"z" => continue, // 状态词
_ => continue,
};
tags.push((word.to_string(), category.to_string()));
}
tags
}
/// 从标题中提取分类标签
///
/// 使用外部词典（dict/tags.json）中的分类词库进行匹配，
/// 词典支持运行时热重载，修改后无需重启服务。
pub fn extract_tags_from_title(
title: &str,
person_name: Option<&str>,
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let mut tags = Vec::new();
let mut title_lower = title.to_lowercase();
// 移除人名
if let Some(name) = person_name {
title_lower = title_lower.replace(&name.to_lowercase(), " ");
// 也移除英文名部分
for part in name.split_whitespace() {
title_lower = title_lower.replace(&part.to_lowercase(), " ");
}
}
let stop_words = dict.get_stop_words();
let person_name_lower = person_name.map(|n| n.to_lowercase());
let mut seen = HashSet::new();
// 第一步：优先匹配外部词典中的词（保证高质量标签）
let tag_category_map = dict.get_tag_category_map();
// 按标签长度降序匹配，避免短标签覆盖长标签
let mut sorted_tags: Vec<(String, String)> = tag_category_map.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
sorted_tags.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
let mut tags = Vec::new();
for (tag, category) in &sorted_tags {
if title_lower.contains(tag.as_str()) && !seen.contains(tag) {
seen.insert(tag.clone());
tags.push((tag.clone(), category.clone()));
}
}
// A: 词典匹配
tags.extend(extract_dict_tags(title, &person_exclude, &mut seen));
// 第二步：用 jieba 分词提取名词标签（比正则更准确）
let jieba_tags = extract_tags_with_jieba(title, person_name, stop_words, &seen);
for (tag, category) in jieba_tags {
let key = tag.to_lowercase();
if !seen.contains(&key) {
seen.insert(key);
tags.push((tag, category));
}
}
// B: jieba 名词（用外部传入的 jieba result）
tags.extend(extract_jieba_noun_tags(jieba_tags, &person_exclude, stop_words, &mut seen));
// 第三步：正则分词提取其他关键词（兜底）
for cap in WORD_RE.find_iter(&title_lower) {
let word = cap.as_str().trim();
let word_lower = word.to_lowercase();
// 基础过滤
if word.len() < 2 { continue; }
if is_punctuation_only(word) { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if dict.is_stop_word(word_lower.as_str()) { continue; }
if let Some(ref pn) = person_name_lower {
if word_lower.contains(pn.as_str()) { continue; }
}
if seen.contains(&word_lower) { continue; }
// 已经在分类词库中匹配过的跳过
if dict.classify(word_lower.as_str()).is_some() { continue; }
seen.insert(word_lower.clone());
tags.push((word.to_string(), "other".to_string()));
}
// C: 正则兜底
tags.extend(extract_regex_tags(title, &person_exclude, stop_words, &mut seen));
tags
}
@@ -260,12 +256,10 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
});
for (tag, tag_type) in sorted_tags {
// 清洗标签：去除头尾标点
let cleaned_tag = match clean_tag(&tag) {
Some(t) => t,
None => continue, // 清洗后为空 → 跳过
None => continue,
};
// 最终安全过滤：坏标签一律剔除（停用词等）
if is_bad_tag(&cleaned_tag) { continue; }
let key = cleaned_tag.to_lowercase();
if seen.insert(key) {
@@ -275,15 +269,12 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
result
}
/// 判断一个标签是否"坏标签"（应该在数据库中清理掉）
/// 坏标签定义：标点符号、含头尾标点需要清洗、太短、纯数字、停用词
/// 如果标签需要清洗（如 "性感!"），也会被标记为坏标签
/// 判断一个标签是否"坏标签"
pub fn is_bad_tag(tag: &str) -> bool {
let tag = tag.trim();
if tag.is_empty() || tag.len() < 2 { return true; }
if tag.chars().all(|c| c.is_ascii_digit()) { return true; }
if is_punctuation_only(tag) { return true; }
// 如果标签清洗后变短了，说明含有头尾标点 → 坏标签
if let Some(cleaned) = clean_tag(tag) {
if cleaned != tag { return true; }
}
@@ -292,18 +283,6 @@ pub fn is_bad_tag(tag: &str) -> bool {
false
}
/// 获取标签的中文分类名（用于前端展示）
pub fn get_category_label(category: &str) -> String {
let dict = tag_dict::get_tag_dict();
dict.get_category_label(category)
}
/// 获取标签分类的颜色（用于前端标签着色）
pub fn get_category_color(category: &str) -> (String, String) {
let dict = tag_dict::get_tag_dict();
dict.get_category_color(category)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -313,6 +292,7 @@ mod tests {
let tags = extract_tags_from_title(
"XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'",
Some("杨晨晨Yome"),
&[],
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!names.contains(&"photos"));

87
src/engine/title_parser.rs
View File

@@ -6,6 +6,7 @@ use std::collections::HashSet;
use lazy_static::lazy_static;
use jieba_rs::Jieba;
use crate::engine::tag_dict;
use crate::engine::tag_extractor;
// 全局 jieba 分词器
lazy_static! {
@@ -695,6 +696,92 @@ pub fn normalize_tag(tag: &str) -> String {
tag.trim().to_lowercase()
}
// ═══════════════════════════════════════════════════════
// 统一标题预处理（用于 parse_album_title 入口）
// ═══════════════════════════════════════════════════════
/// 统一标题预处理：移除照片数后缀、[工作室] 前缀、称呼前缀、编号后缀
pub fn preprocess_title(title: &str) -> String {
let t = title.trim();
if t.is_empty() { return String::new(); }
let mut t = RE_PHOTOS_SUFFIX.replace(t, "").trim().to_string();
t = RE_BRACKET_PREFIX.replace(&t, "").trim().to_string();
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
if t.starts_with(prefix) {
t = t[prefix.len()..].trim().to_string();
break;
}
}
t = RE_NUMBER_SUFFIX.replace(&t, "").trim().to_string();
t
}
/// 统一相册标题解析入口
///
/// 一次 jieba 分词 → 同时提取 person + tags。
/// tags 通过 tag_extractor 的三步流程（A 词典 / B jieba 名词 / C 正则）提取，
/// 使用同一份 jieba result。
///
/// 返回 ParseResult 可直接用于 Phase 1b 批量写库。
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
let title = title.trim();
if title.is_empty() { return result; }
// 1. 预处理
let cleaned = preprocess_title(title);
// 2. cosplay 格式优先
if let Some(caps) = RE_COSPLAY_BY.captures(&cleaned) {
result.person_name = Some(clean_name(caps[3].trim()));
let name = result.person_name.as_deref().unwrap_or("");
// 有限标签提取（仅词典 + 正则，不调 jieba 避免重复）
let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new());
result.tags = tags;
return result;
}
if let Some(caps) = RE_COSPLAY.captures(&cleaned) {
result.person_name = Some(clean_name(caps[1].trim()));
let name = result.person_name.as_deref().unwrap_or("");
let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new());
result.tags = tags;
return result;
}
// 3. 一次 jieba 分词（核心！全流程共用）
let jieba_tags = JIEBA.tag(&cleaned, true);
// 4. 提取人名
let person_raw = extract_person_from_tags(&jieba_tags);
if let Some(name) = &person_raw {
// include 扩展
let expanded = tag_dict::get_tag_dict()
.find_longest_included_match(name)
.filter(|longer| longer.len() > name.len());
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
let (primary, aliases) = split_person_name_with_aliases(effective_name);
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
if final_name.chars().count() > 2 {
result.person_name = Some(final_name);
// person_primary 用于别名处理，暂存但不暴露
}
}
// 5. 提取标签（使用同一份 jieba result）
let person_exclude = result.person_name.as_deref();
let raw_tags = tag_extractor::extract_tags_from_title(title, person_exclude, &jieba_tags);
let unique = tag_extractor::unique_tags(raw_tags);
result.tags = unique;
result
}
#[cfg(test)]
mod tests {
use super::*;

1
src/models/mod.rs
View File

@@ -4,6 +4,7 @@ pub mod person;
pub mod tag;
pub mod favorite;
pub mod recommend;
pub mod parse;
pub use album::*;
pub use image::*;

27
src/models/parse.rs Normal file
View File

@@ -0,0 +1,27 @@
//! 相册解析结果 — 单个相册标题的一次性解析产物
//! 用于串联并行解析（Phase 1a）和批量写库（Phase 1b）
use serde::{Serialize, Deserialize};
/// 一次 title 解析的完整结果
///
/// 在一个相册上通过一次 jieba 分词，同时提取：
/// - 人名 + 别名（含 include 扩展、prefix 移除）
/// - 标签列表（含分类）
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParseResult {
/// 相册 UUID
pub album_id: String,
/// 最终人名 | None = 未找到
pub person_name: Option<String>,
/// 中文主名（split_person_name_with_aliases 后的 primary）
pub person_primary: Option<String>,
/// 标签列表 (tag, category)
pub tags: Vec<(String, String)>,
}
impl ParseResult {
pub fn new(album_id: String) -> Self {
Self { album_id, person_name: None, person_primary: None, tags: Vec::new() }
}
}