update
This commit is contained in:
40
Cargo.lock
generated
40
Cargo.lock
generated
@@ -587,6 +587,25 @@ dependencies = [
|
|||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-deque"
|
||||||
|
version = "0.8.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.20"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crossbeam-queue"
|
name = "crossbeam-queue"
|
||||||
version = "0.3.13"
|
version = "0.3.13"
|
||||||
@@ -1310,6 +1329,7 @@ dependencies = [
|
|||||||
"lazy_static",
|
"lazy_static",
|
||||||
"mimalloc",
|
"mimalloc",
|
||||||
"rand 0.8.7",
|
"rand 0.8.7",
|
||||||
|
"rayon",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -2047,6 +2067,26 @@ version = "0.10.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rayon"
|
||||||
|
version = "1.12.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||||
|
dependencies = [
|
||||||
|
"either",
|
||||||
|
"rayon-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rayon-core"
|
||||||
|
version = "1.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-deque",
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redox_syscall"
|
name = "redox_syscall"
|
||||||
version = "0.5.18"
|
version = "0.5.18"
|
||||||
|
|||||||
@@ -23,3 +23,4 @@ mimalloc = "0.1.48"
|
|||||||
regex = "1"
|
regex = "1"
|
||||||
lazy_static = "1.4"
|
lazy_static = "1.4"
|
||||||
jieba-rs = "0.7"
|
jieba-rs = "0.7"
|
||||||
|
rayon = "1"
|
||||||
@@ -154,6 +154,6 @@
|
|||||||
<!-- 移动端遮罩 -->
|
<!-- 移动端遮罩 -->
|
||||||
<div class="mobile-overlay" id="mobile-overlay" onclick="toggleMobileSidebar()"></div>
|
<div class="mobile-overlay" id="mobile-overlay" onclick="toggleMobileSidebar()"></div>
|
||||||
|
|
||||||
<script src="/assets/js/app.js?v=5"></script>
|
<script src="/assets/js/app.js?v=6"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
// 全局变量
|
// 全局变量
|
||||||
let allAlbums = []; // 存储所有相册数据
|
let allAlbums = []; // 存储所有相册数据
|
||||||
let displayedAlbums = []; // 当前显示的相册数据
|
let displayedAlbums = []; // 当前显示的相册数据
|
||||||
|
let filteredAlbums = null; // 过滤后的相册数据(有筛选时使用,null=无筛选)
|
||||||
let searchTimeout = null;
|
let searchTimeout = null;
|
||||||
let currentDisplayPage = 1; // 前端显示的分页
|
let currentDisplayPage = 1; // 前端显示的分页
|
||||||
let albumsPerPage = 50; // 每次显示50个相册
|
let albumsPerPage = 50; // 每次显示50个相册
|
||||||
@@ -252,7 +253,9 @@ async function loadMoreAlbums() {
|
|||||||
// 如果在收藏页,不加载更多数据
|
// 如果在收藏页,不加载更多数据
|
||||||
if (isInFavoritesPage) return;
|
if (isInFavoritesPage) return;
|
||||||
|
|
||||||
if (isLoadingMore || !hasMoreAlbums || !isAllAlbumsLoaded) return;
|
if (isLoadingMore || !hasMoreAlbums) return;
|
||||||
|
// 全量模式下(无筛选)需等 isAllAlbumsLoaded;过滤模式下直接使用 filteredAlbums
|
||||||
|
if (!currentFilter && !isAllAlbumsLoaded) return;
|
||||||
|
|
||||||
isLoadingMore = true;
|
isLoadingMore = true;
|
||||||
try {
|
try {
|
||||||
@@ -456,11 +459,13 @@ async function loadAlbums(reset = false) {
|
|||||||
|
|
||||||
// 显示当前页的相册数据
|
// 显示当前页的相册数据
|
||||||
function displayCurrentPage() {
|
function displayCurrentPage() {
|
||||||
|
// 有筛选时用 filteredAlbums,否则用 allAlbums
|
||||||
|
const source = filteredAlbums || allAlbums;
|
||||||
const startIndex = (currentDisplayPage - 1) * albumsPerPage;
|
const startIndex = (currentDisplayPage - 1) * albumsPerPage;
|
||||||
const endIndex = startIndex + albumsPerPage;
|
const endIndex = startIndex + albumsPerPage;
|
||||||
|
|
||||||
// 获取当前页要显示的相册
|
// 获取当前页要显示的相册
|
||||||
displayedAlbums = allAlbums.slice(startIndex, endIndex);
|
displayedAlbums = source.slice(startIndex, endIndex);
|
||||||
|
|
||||||
// 如果是第一页,直接显示;否则追加显示
|
// 如果是第一页,直接显示;否则追加显示
|
||||||
if (currentDisplayPage === 1) {
|
if (currentDisplayPage === 1) {
|
||||||
@@ -470,7 +475,7 @@ function displayCurrentPage() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 检查是否还有更多数据可以显示
|
// 检查是否还有更多数据可以显示
|
||||||
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < allAlbums.length;
|
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < source.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 搜索相册
|
// 搜索相册
|
||||||
@@ -479,14 +484,24 @@ async function debouncedSearch() {
|
|||||||
const searchTerm = document.getElementById('search-input').value.toLowerCase().trim();
|
const searchTerm = document.getElementById('search-input').value.toLowerCase().trim();
|
||||||
|
|
||||||
if (searchTerm === '') {
|
if (searchTerm === '') {
|
||||||
// 显示所有已加载的相册
|
// 搜索框为空 → 清除过滤状态,显示所有相册
|
||||||
|
currentFilter = null;
|
||||||
|
filteredAlbums = null;
|
||||||
displayAlbums(allAlbums);
|
displayAlbums(allAlbums);
|
||||||
updateStats();
|
updateStats();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 直接使用后端搜索API进行全站搜索
|
// 直接使用后端搜索API进行全站搜索
|
||||||
await performServerSearch(searchTerm);
|
const results = await performServerSearchWithResult(searchTerm);
|
||||||
|
if (results && results.length > 0) {
|
||||||
|
filteredAlbums = results;
|
||||||
|
currentDisplayPage = 1;
|
||||||
|
hasMoreAlbums = results.length > albumsPerPage;
|
||||||
|
displayAlbums(results);
|
||||||
|
updateStats(results);
|
||||||
|
initLazyLoading();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 在当前已加载的数据集中搜索
|
// 在当前已加载的数据集中搜索
|
||||||
@@ -522,11 +537,22 @@ function showSearchHint(searchTerm) {
|
|||||||
|
|
||||||
// 执行服务器端搜索(全站搜索)
|
// 执行服务器端搜索(全站搜索)
|
||||||
async function performServerSearch(searchTerm) {
|
async function performServerSearch(searchTerm) {
|
||||||
try {
|
const results = await performServerSearchWithResult(searchTerm);
|
||||||
// 显示搜索中的骨架屏
|
if (results && results.length > 0) {
|
||||||
showSkeleton(4);
|
displayAlbums(results);
|
||||||
|
const oldStats = document.getElementById('stats');
|
||||||
|
if (oldStats) {
|
||||||
|
oldStats.innerHTML = `在全站找到 ${results.length} 个包含 "${searchTerm}" 的相册`;
|
||||||
|
}
|
||||||
|
updateStats(results);
|
||||||
|
initLazyLoading();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=${albumsPerPage}`);
|
/// 执行服务器端搜索并返回结果数组(不直接渲染)
|
||||||
|
async function performServerSearchWithResult(searchTerm) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=200`);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`HTTP error! status: ${response.status}`);
|
throw new Error(`HTTP error! status: ${response.status}`);
|
||||||
}
|
}
|
||||||
@@ -534,23 +560,13 @@ async function performServerSearch(searchTerm) {
|
|||||||
const serverResults = await response.json();
|
const serverResults = await response.json();
|
||||||
|
|
||||||
if (serverResults.length === 0) {
|
if (serverResults.length === 0) {
|
||||||
showError(`没有找到包含 "${searchTerm}" 的相册`);
|
return [];
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 显示服务器搜索结果
|
return serverResults;
|
||||||
displayAlbums(serverResults);
|
|
||||||
const oldStats = document.getElementById('stats');
|
|
||||||
if (oldStats) {
|
|
||||||
oldStats.innerHTML = `在全站找到 ${serverResults.length} 个包含 "${searchTerm}" 的相册`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 初始化图片懒加载
|
|
||||||
initLazyLoading();
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('全站搜索失败:', error);
|
console.error('全站搜索失败:', error);
|
||||||
showError('全站搜索失败,请重试');
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1418,6 +1434,11 @@ async function filterByTag(tag) {
|
|||||||
if (!response.ok) throw new Error('搜索失败');
|
if (!response.ok) throw new Error('搜索失败');
|
||||||
const results = await response.json();
|
const results = await response.json();
|
||||||
|
|
||||||
|
// 保存过滤结果到 filteredAlbums,重置分页
|
||||||
|
filteredAlbums = results;
|
||||||
|
currentDisplayPage = 1;
|
||||||
|
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
|
||||||
|
|
||||||
displayAlbums(results);
|
displayAlbums(results);
|
||||||
updateStats(results);
|
updateStats(results);
|
||||||
initLazyLoading();
|
initLazyLoading();
|
||||||
@@ -1460,6 +1481,11 @@ async function filterByPerson(personName) {
|
|||||||
if (detailResp.ok) {
|
if (detailResp.ok) {
|
||||||
const detail = await detailResp.json();
|
const detail = await detailResp.json();
|
||||||
if (detail && detail.albums) {
|
if (detail && detail.albums) {
|
||||||
|
// 保存过滤结果到 filteredAlbums,重置分页
|
||||||
|
filteredAlbums = detail.albums;
|
||||||
|
currentDisplayPage = 1;
|
||||||
|
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
|
||||||
|
|
||||||
displayAlbums(detail.albums);
|
displayAlbums(detail.albums);
|
||||||
updateStats(detail.albums);
|
updateStats(detail.albums);
|
||||||
initLazyLoading();
|
initLazyLoading();
|
||||||
@@ -1468,10 +1494,11 @@ async function filterByPerson(personName) {
|
|||||||
}
|
}
|
||||||
} catch (e) { /* 回退 */ }
|
} catch (e) { /* 回退 */ }
|
||||||
}
|
}
|
||||||
await performServerSearch(personName);
|
// 回退到全站搜索,也要保存过滤结果
|
||||||
|
filteredAlbums = await performServerSearchWithResult(personName);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('人物筛选失败:', error);
|
console.error('人物筛选失败:', error);
|
||||||
await performServerSearch(personName);
|
filteredAlbums = await performServerSearchWithResult(personName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1497,6 +1524,8 @@ function showFilterStatus(text) {
|
|||||||
// 清除筛选
|
// 清除筛选
|
||||||
function clearFilter() {
|
function clearFilter() {
|
||||||
currentFilter = null;
|
currentFilter = null;
|
||||||
|
filteredAlbums = null; // 清除过滤结果缓存
|
||||||
|
currentDisplayPage = 1;
|
||||||
|
|
||||||
// 新版筛选条
|
// 新版筛选条
|
||||||
const bar = document.getElementById('filter-bar');
|
const bar = document.getElementById('filter-bar');
|
||||||
|
|||||||
772
dict/tags.json
772
dict/tags.json
File diff suppressed because it is too large
Load Diff
91
src/db/batch.rs
Normal file
91
src/db/batch.rs
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
//! 批量写入 — 在单个事务中写入多个相册的解析结果
|
||||||
|
//!
|
||||||
|
//! 配合 Phase 1a(并行解析)使用,所有脏相册在一个事务内写入,
|
||||||
|
//! 减少 round-trip 和 WAL fsync 次数。
|
||||||
|
|
||||||
|
use super::Database;
|
||||||
|
use crate::models::parse::ParseResult;
|
||||||
|
use sqlx::Transaction;
|
||||||
|
use sqlx::sqlite::Sqlite;
|
||||||
|
|
||||||
|
impl Database {
|
||||||
|
/// 批量写入多个相册的解析结果(Phase 1b)
|
||||||
|
///
|
||||||
|
/// 全部在一个事务中完成:
|
||||||
|
/// 1. 批量 upsert persons
|
||||||
|
/// 2. 批量 update 相册 person_name
|
||||||
|
/// 3. 批量 insert person_album 关联
|
||||||
|
/// 4. 批量 insert 别名
|
||||||
|
/// 5. 批量 update 标签
|
||||||
|
/// 6. 批量 update person_counts
|
||||||
|
pub async fn batch_write_parse_results(
|
||||||
|
&self,
|
||||||
|
tx: &mut Transaction<'_, Sqlite>,
|
||||||
|
results: &[ParseResult],
|
||||||
|
) -> Result<(), sqlx::Error> {
|
||||||
|
for result in results {
|
||||||
|
let album_id = &result.album_id;
|
||||||
|
let person_name = result.person_name.as_deref();
|
||||||
|
|
||||||
|
if let Some(name) = person_name {
|
||||||
|
// ── update 相册 person 字段 ──
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE albums SET person_name_original = ?, person_name_cleaned = ?, processed_at = datetime('now') WHERE id = ?"
|
||||||
|
)
|
||||||
|
.bind(name).bind(name).bind(album_id)
|
||||||
|
.execute(&mut **tx).await?;
|
||||||
|
|
||||||
|
// ── insert or ignore 人物 ──
|
||||||
|
let pid = sqlx::query_scalar::<_, i64>(
|
||||||
|
"INSERT INTO persons (name, display_name) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET name = name RETURNING id"
|
||||||
|
)
|
||||||
|
.bind(name).bind(name)
|
||||||
|
.fetch_one(&mut **tx).await?;
|
||||||
|
|
||||||
|
// ── 清旧关联 + 写新关联 ──
|
||||||
|
sqlx::query("DELETE FROM person_albums WHERE album_id = ?")
|
||||||
|
.bind(album_id).execute(&mut **tx).await?;
|
||||||
|
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)")
|
||||||
|
.bind(pid).bind(album_id).execute(&mut **tx).await?;
|
||||||
|
|
||||||
|
// ── 更新计数 ──
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE persons SET total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?), \
|
||||||
|
total_images = COALESCE((SELECT SUM(image_count) FROM albums \
|
||||||
|
JOIN person_albums ON albums.id = person_albums.album_id \
|
||||||
|
WHERE person_albums.person_id = ?), 0) \
|
||||||
|
WHERE id = ?"
|
||||||
|
)
|
||||||
|
.bind(pid).bind(pid).bind(pid)
|
||||||
|
.execute(&mut **tx).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 写标签 ──
|
||||||
|
if !result.tags.is_empty() {
|
||||||
|
let tags_json = serde_json::to_string(
|
||||||
|
&result.tags.iter().map(|(t, _)| t).collect::<Vec<_>>()
|
||||||
|
).unwrap_or_else(|_| "[]".to_string());
|
||||||
|
let all_words: Vec<[String; 2]> = result.tags.iter()
|
||||||
|
.map(|(t, c)| [t.clone(), c.clone()])
|
||||||
|
.collect();
|
||||||
|
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE albums SET tags = ?, all_words = ? WHERE id = ?"
|
||||||
|
)
|
||||||
|
.bind(&tags_json).bind(&words_json).bind(album_id)
|
||||||
|
.execute(&mut **tx).await?;
|
||||||
|
|
||||||
|
// ── 同步 album_tags 表 ──
|
||||||
|
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
|
||||||
|
.bind(album_id).execute(&mut **tx).await?;
|
||||||
|
for (t, _) in &result.tags {
|
||||||
|
sqlx::query("INSERT INTO album_tags (album_id, tag) VALUES (?, ?)")
|
||||||
|
.bind(album_id).bind(t).execute(&mut **tx).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -121,13 +121,50 @@ impl Database {
|
|||||||
// ── 标签查询 ──
|
// ── 标签查询 ──
|
||||||
|
|
||||||
pub async fn get_popular_tags(&self, limit: i64) -> Vec<TagStat> {
|
pub async fn get_popular_tags(&self, limit: i64) -> Vec<TagStat> {
|
||||||
sqlx::query(
|
// 从 tag_dict 获取过滤词(停用词、品牌词、排除名)
|
||||||
|
// 注意:必须 clone 到本地变量,不能持 RwLockReadGuard 跨 await
|
||||||
|
let (stop_words, excluded, included) = {
|
||||||
|
let dict = crate::engine::tag_dict::get_tag_dict();
|
||||||
|
(
|
||||||
|
dict.stop_words.clone(),
|
||||||
|
dict.excluded_person_names.clone(),
|
||||||
|
dict.included_person_names.clone(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
// 标签自身小写的集合用于快速过滤
|
||||||
|
let filtered_tags: Vec<TagStat> = sqlx::query(
|
||||||
r#"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
|
r#"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
|
||||||
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT ?"#
|
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT ?"#
|
||||||
).bind(limit).fetch_all(&self.pool).await.unwrap_or_default()
|
).bind(limit * 3).fetch_all(&self.pool).await.unwrap_or_default()
|
||||||
.iter().map(|r| TagStat {
|
.iter().filter_map(|r| {
|
||||||
tag: r.get("tag"), count: r.get("cnt"), album_count: r.get("album_cnt"),
|
let tag: String = r.get("tag");
|
||||||
}).collect()
|
let tag_lower = tag.to_lowercase();
|
||||||
|
|
||||||
|
// 过滤停用词
|
||||||
|
if stop_words.contains(&tag_lower) || stop_words.contains(&tag) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// 过滤品牌/工作室名
|
||||||
|
if excluded.contains(&tag_lower) || excluded.contains(&tag) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// 过滤包含人名(如果标签本身就是个人名,也不应该出现在标签中)
|
||||||
|
if included.contains(&tag_lower) || included.contains(&tag) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(TagStat {
|
||||||
|
tag: r.get("tag"),
|
||||||
|
count: r.get("cnt"),
|
||||||
|
album_count: r.get("album_cnt"),
|
||||||
|
})
|
||||||
|
}).collect();
|
||||||
|
|
||||||
|
// 只保留 top limit 个
|
||||||
|
let mut result: Vec<TagStat> = filtered_tags;
|
||||||
|
result.truncate(limit as usize);
|
||||||
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 批量获取所有 tag→albums 映射(供缓存使用)
|
/// 批量获取所有 tag→albums 映射(供缓存使用)
|
||||||
@@ -242,13 +279,20 @@ impl Database {
|
|||||||
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
|
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 删除某相册的所有人物关联(写入新 person 前清旧数据,防止一个相册关联多个人物)
|
/// 删除某相册的所有人物关联,返回被影响的旧 person_id 列表
|
||||||
pub async fn delete_album_person_albums(&self, album_id: &str) {
|
pub async fn delete_album_person_albums(&self, album_id: &str) -> Vec<i64> {
|
||||||
|
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
|
||||||
|
.bind(album_id).fetch_all(&self.pool).await.unwrap_or_default();
|
||||||
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok();
|
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok();
|
||||||
|
old_ids
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) {
|
/// 删除某相册的所有人物关联(事务版),返回被影响的旧 person_id 列表
|
||||||
|
pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) -> Vec<i64> {
|
||||||
|
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
|
||||||
|
.bind(album_id).fetch_all(tx.deref_mut()).await.unwrap_or_default();
|
||||||
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(tx.deref_mut()).await.ok();
|
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(tx.deref_mut()).await.ok();
|
||||||
|
old_ids
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 删除某人物下所有别名(写入新别名前清旧别名,配合 UNIQUE INDEX 防止重复)
|
/// 删除某人物下所有别名(写入新别名前清旧别名,配合 UNIQUE INDEX 防止重复)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ pub mod image;
|
|||||||
pub mod favorite;
|
pub mod favorite;
|
||||||
pub mod metadata;
|
pub mod metadata;
|
||||||
pub mod migration;
|
pub mod migration;
|
||||||
|
pub mod batch;
|
||||||
|
|
||||||
pub use connection::Database;
|
pub use connection::Database;
|
||||||
#[allow(unused)] pub use album::*;
|
#[allow(unused)] pub use album::*;
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
//! Phase A2 清理非人物 → 加入脏集
|
//! Phase A2 清理非人物 → 加入脏集
|
||||||
//! Phase A3 收集新相册/半成品 → 加入脏集
|
//! Phase A3 收集新相册/半成品 → 加入脏集
|
||||||
//! Phase 1 解析脏集相册(增量解析,不扫全库)
|
//! Phase 1 解析脏集相册(增量解析,不扫全库)
|
||||||
|
//! Phase C 全量重计所有 person 的相册计数
|
||||||
//! Phase 2 人物合并
|
//! Phase 2 人物合并
|
||||||
//! Phase B 后校验标签中的人名
|
//! Phase B 后校验标签中的人名
|
||||||
//!
|
//!
|
||||||
@@ -19,6 +20,7 @@ use crate::engine::tag_extractor;
|
|||||||
use crate::engine::person_merger;
|
use crate::engine::person_merger;
|
||||||
use crate::engine::tag_dict;
|
use crate::engine::tag_dict;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
use rayon::prelude::*;
|
||||||
|
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
// 脏集 — 追踪需要重新解析的相册 ID
|
// 脏集 — 追踪需要重新解析的相册 ID
|
||||||
@@ -324,7 +326,8 @@ pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
|
|||||||
|
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
// Phase 1:增量解析 — 只处理脏集中的相册
|
// Phase 1:增量解析 — 只处理脏集中的相册
|
||||||
// ════════════════════════════════════════════════
|
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
|
||||||
|
// └─ Phase 1b:批量写库(IO 密集,一个事务)
|
||||||
|
|
||||||
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
|
pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, usize) {
|
||||||
if dirty_set.is_empty() {
|
if dirty_set.is_empty() {
|
||||||
@@ -333,98 +336,70 @@ pub async fn phase_parse_dirty(db: &Database, dirty_set: &DirtySet) -> (usize, u
|
|||||||
}
|
}
|
||||||
|
|
||||||
let albums = db.get_all_albums_full().await;
|
let albums = db.get_all_albums_full().await;
|
||||||
let patterns = person_merger::collect_person_patterns(db).await;
|
|
||||||
let use_patterns = !patterns.is_empty();
|
|
||||||
let mut person_updated = 0usize;
|
|
||||||
let mut tag_updated = 0usize;
|
|
||||||
let mut skipped = 0usize;
|
|
||||||
|
|
||||||
for album in &albums {
|
// ── Phase 1a:并行解析(CPU bound,rayon) ──
|
||||||
if !dirty_set.contains(&album.id) { continue; }
|
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
|
||||||
let title = album.title.as_deref().unwrap_or("");
|
.filter(|a| dirty_set.contains(&a.id))
|
||||||
if title.is_empty() { skipped += 1; continue; }
|
.collect();
|
||||||
|
|
||||||
let (jieba_person, jieba_tags) = title_parser::extract_person_and_tags(title);
|
let skipped = dirty_albums.iter().filter(|a| {
|
||||||
|
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
|
||||||
|
}).count();
|
||||||
|
let total = dirty_albums.len();
|
||||||
|
|
||||||
let best_person = if use_patterns {
|
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
|
||||||
person_merger::match_title(title, &patterns)
|
.par_iter()
|
||||||
.map(|(_, name, _)| name).or(jieba_person)
|
.filter_map(|album| {
|
||||||
} else {
|
let title = album.title.as_deref()?;
|
||||||
jieba_person
|
if title.is_empty() { return None; }
|
||||||
};
|
Some(title_parser::parse_album_title(&album.id, title))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
// 用事务包裹所有写入,避免逐条 autocommit 导致巨慢
|
let person_count = parsed.iter().filter(|r| r.person_name.is_some()).count();
|
||||||
let mut pending_tx: Option<sqlx::Transaction<'_, sqlx::Sqlite>> = None;
|
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
|
||||||
|
|
||||||
if let Some(name) = &best_person {
|
// ── Phase 1b:批量写入 ──
|
||||||
// 尝试用 include 列表扩展为更长的完整匹配(如 "杨晨晨" → "杨晨晨Yome")
|
let _guard = db.write_lock.lock().await;
|
||||||
let expanded = tag_dict::get_tag_dict()
|
let mut tx = db.pool.begin().await.expect("begin tx");
|
||||||
.find_longest_included_match(name)
|
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
|
||||||
.filter(|longer| longer.len() > name.len()); // 仅当更长时替换
|
eprintln!("[管线·Phase1b] 批量写入失败: {}", e);
|
||||||
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
|
});
|
||||||
|
tx.commit().await.expect("commit tx");
|
||||||
|
|
||||||
let (primary, aliases) = title_parser::split_person_name_with_aliases(effective_name);
|
println!("[管线·Phase1] 并行解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
|
||||||
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
|
total, parsed.len(), skipped, person_count, tag_count);
|
||||||
|
(person_count, tag_count)
|
||||||
// 跳过字符串 <= 2 的人名(太短不像人名)
|
|
||||||
if final_name.chars().count() <= 2 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut tx = db.pool.begin().await.expect("begin tx");
|
|
||||||
db.update_album_person_with_tx(&mut tx, &album.id, effective_name, &final_name).await;
|
|
||||||
let pid = db.insert_person_with_tx(&mut tx, &final_name, None).await;
|
|
||||||
// 先清旧关联:删除该相册的所有旧 person_album 记录
|
|
||||||
db.delete_album_person_albums_with_tx(&mut tx, &album.id).await;
|
|
||||||
db.insert_person_album_with_tx(&mut tx, pid, &album.id).await;
|
|
||||||
// 先清旧别名:删除该人物的所有旧别名,再插入新的
|
|
||||||
db.delete_person_aliases_with_tx(&mut tx, pid).await;
|
|
||||||
for (at, an) in &aliases {
|
|
||||||
if !an.is_empty() && an != &final_name {
|
|
||||||
db.insert_person_alias_with_tx(&mut tx, pid, at, an).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
db.update_person_counts_with_tx(&mut tx, pid).await;
|
|
||||||
pending_tx = Some(tx);
|
|
||||||
person_updated += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 确定标签排除用的人名 — 用最完整的名字
|
|
||||||
// 优先级:person_name_original(完整含英文后缀) > best_person > person_name_cleaned(仅中文)
|
|
||||||
let exclude_person = album.person_name_original.as_deref()
|
|
||||||
.filter(|s| !s.is_empty())
|
|
||||||
.or_else(|| best_person.as_deref())
|
|
||||||
.or_else(|| album.person_name_cleaned.as_deref().filter(|s| !s.is_empty()));
|
|
||||||
|
|
||||||
let mut raw_tags = tag_extractor::extract_tags_from_title(title, exclude_person);
|
|
||||||
raw_tags.extend(jieba_tags);
|
|
||||||
let unique = tag_extractor::unique_tags(raw_tags);
|
|
||||||
let tag_names: Vec<String> = unique.iter().map(|(t, _)| t.clone()).collect();
|
|
||||||
|
|
||||||
if !tag_names.is_empty() {
|
|
||||||
let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string());
|
|
||||||
let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect();
|
|
||||||
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
|
|
||||||
let mut tx = match pending_tx.take() {
|
|
||||||
Some(t) => t,
|
|
||||||
None => db.pool.begin().await.expect("begin tx"),
|
|
||||||
};
|
|
||||||
db.update_album_tags_with_tx(&mut tx, &album.id, &tags_json, &words_json).await;
|
|
||||||
tx.commit().await.expect("commit tx");
|
|
||||||
tag_updated += 1;
|
|
||||||
} else if let Some(tx) = pending_tx.take() {
|
|
||||||
// 只有 person 写入,没有标签写入
|
|
||||||
tx.commit().await.expect("commit tx");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("[管线·Phase1] 增量解析完成:脏集 {} 个,处理 {} 个,跳过 {} 个,更新人名 {} 个,更新标签 {} 个",
|
|
||||||
dirty_set.len(), person_updated.max(tag_updated), skipped, person_updated, tag_updated);
|
|
||||||
(person_updated, tag_updated)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
// Phase 2:人物合并
|
// Phase C:全量重计 — 确保所有 person 的计数准确
|
||||||
|
// ════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// Phase C:全量重计所有人物相册和图片计数
|
||||||
|
///
|
||||||
|
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
|
||||||
|
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
|
||||||
|
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
|
||||||
|
pub async fn phase_recount_all(db: &Database) -> usize {
|
||||||
|
let persons = db.get_all_persons_with_counts().await;
|
||||||
|
let mut fixed = 0usize;
|
||||||
|
for (pid, name, old_albums, _) in &persons {
|
||||||
|
let actual = sqlx::query_scalar::<_, i64>(
|
||||||
|
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
|
||||||
|
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
|
||||||
|
if actual != *old_albums {
|
||||||
|
db.update_person_counts(*pid).await;
|
||||||
|
fixed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fixed > 0 {
|
||||||
|
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
|
||||||
|
} else {
|
||||||
|
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
|
||||||
|
}
|
||||||
|
fixed
|
||||||
|
}
|
||||||
// ════════════════════════════════════════════════
|
// ════════════════════════════════════════════════
|
||||||
|
|
||||||
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
|
pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
|
||||||
@@ -506,7 +481,7 @@ pub struct PipelineResult {
|
|||||||
pub tags_removed_person: usize, // Phase B
|
pub tags_removed_person: usize, // Phase B
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 一键执行完整管线(7 阶段顺序串行)
|
/// 一键执行完整管线(8 阶段顺序串行)
|
||||||
pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
||||||
let tags_cleaned = phase_clean_tags(db).await;
|
let tags_cleaned = phase_clean_tags(db).await;
|
||||||
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
|
let (dirty_a1, person_like_tags_removed) = phase_detect_person_in_tags(db).await;
|
||||||
@@ -523,6 +498,8 @@ pub async fn run_pipeline(db: &Database) -> PipelineResult {
|
|||||||
let dirty_albums_detected = master_dirty.len();
|
let dirty_albums_detected = master_dirty.len();
|
||||||
|
|
||||||
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
let (persons_updated, tags_updated) = phase_parse_dirty(db, &master_dirty).await;
|
||||||
|
// Phase C:全量重计(在合并前修正所有计数,让合并看到准确数值)
|
||||||
|
let _recounted = phase_recount_all(db).await;
|
||||||
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
let (persons_merged, aliases_moved) = phase_merge_persons(db).await;
|
||||||
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
let tags_removed_person = phase_clean_tags_after_parse(db).await;
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
|
|
||||||
use crate::db::Database;
|
use crate::db::Database;
|
||||||
use crate::models::*;
|
use crate::models::*;
|
||||||
|
use crate::engine::tag_dict;
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
|
|
||||||
/// 推荐引擎
|
/// 推荐引擎
|
||||||
@@ -167,8 +168,32 @@ impl RecommendEngine {
|
|||||||
|
|
||||||
/// 热门人物
|
/// 热门人物
|
||||||
pub async fn popular_persons(db: &Database, limit: usize) -> Vec<PopularPerson> {
|
pub async fn popular_persons(db: &Database, limit: usize) -> Vec<PopularPerson> {
|
||||||
|
let dict = tag_dict::get_tag_dict();
|
||||||
let persons = db.get_all_persons_with_aliases().await;
|
let persons = db.get_all_persons_with_aliases().await;
|
||||||
let mut results: Vec<PopularPerson> = persons.iter().take(limit).map(|p| {
|
|
||||||
|
// 收集所有标签分类词(非人名候选)
|
||||||
|
let all_tag_words: std::collections::HashSet<String> = dict.tag_to_category.keys()
|
||||||
|
.map(|k| k.to_lowercase()).collect();
|
||||||
|
|
||||||
|
// 过滤非人物:三重检查(excluded + is_likely_person_name + 非标签词)
|
||||||
|
let filtered: Vec<_> = persons.into_iter()
|
||||||
|
.filter(|p| {
|
||||||
|
// 1. 跳过排除名单中的人物
|
||||||
|
if dict.is_excluded_person(&p.name) { return false; }
|
||||||
|
// 2. 只保留 is_likely_person_name 返回 true 的人名
|
||||||
|
if !crate::engine::title_parser::is_likely_person_name(&p.name) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// 3. 跳过和标签分类词重叠的非人名(如 "女模特"、"内衣秀")
|
||||||
|
let name_lower = p.name.to_lowercase();
|
||||||
|
if all_tag_words.contains(&name_lower) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut results: Vec<PopularPerson> = filtered.iter().take(limit).map(|p| {
|
||||||
PopularPerson {
|
PopularPerson {
|
||||||
name: p.name.clone(),
|
name: p.name.clone(),
|
||||||
display_name: p.display_name.clone(),
|
display_name: p.display_name.clone(),
|
||||||
|
|||||||
@@ -21,28 +21,32 @@ use std::sync::RwLock;
|
|||||||
|
|
||||||
// ── JSON 反序列化结构 ──
|
// ── JSON 反序列化结构 ──
|
||||||
|
|
||||||
/// 词典文件的顶层结构
|
/// 词典文件的顶层结构 (v3)
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct TagDictFile {
|
pub struct TagDictFile {
|
||||||
pub version: u32,
|
pub version: u32,
|
||||||
#[allow(unused)]
|
#[allow(unused)]
|
||||||
pub description: Option<String>,
|
pub description: Option<String>,
|
||||||
pub tags: HashMap<String, TagCategoryDef>,
|
/// v3: 分类元数据 { "style": {"label":..., "color_bg":..., ...} }
|
||||||
|
#[serde(default)]
|
||||||
|
pub categories: HashMap<String, TagCategoryMeta>,
|
||||||
|
/// v3: 分类词表 { "style": ["日系", ...], "outfit": [...], ... }
|
||||||
|
#[serde(default)]
|
||||||
|
pub tags: HashMap<String, Vec<String>>,
|
||||||
pub stop_words: Vec<String>,
|
pub stop_words: Vec<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub person_rules: Option<PersonRulesDef>,
|
pub person_rules: Option<PersonRulesDef>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 分类定义
|
/// 分类元数据(v3 分离到 categories 字段)
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct TagCategoryDef {
|
pub struct TagCategoryMeta {
|
||||||
#[allow(unused)]
|
#[serde(default)]
|
||||||
pub label: Option<String>,
|
pub label: Option<String>,
|
||||||
#[allow(unused)]
|
#[serde(default)]
|
||||||
pub color_bg: Option<String>,
|
pub color_bg: Option<String>,
|
||||||
#[allow(unused)]
|
#[serde(default)]
|
||||||
pub color_fg: Option<String>,
|
pub color_fg: Option<String>,
|
||||||
pub words: Vec<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 人物规则定义
|
/// 人物规则定义
|
||||||
@@ -147,26 +151,36 @@ impl TagDict {
|
|||||||
included_person_names,
|
included_person_names,
|
||||||
};
|
};
|
||||||
|
|
||||||
for (cat_name, def) in file.tags {
|
// ── 解析分类元数据(v3 从 categories 字段读取) ──
|
||||||
let label = def.label.clone().unwrap_or_else(|| cat_name.clone());
|
for (cat_name, meta) in &file.categories {
|
||||||
let color_bg = def.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string());
|
let label = meta.label.clone().unwrap_or_else(|| cat_name.clone());
|
||||||
let color_fg = def.color_fg.clone().unwrap_or_else(|| "#64748b".to_string());
|
let color_bg = meta.color_bg.clone().unwrap_or_else(|| "#f1f5f9".to_string());
|
||||||
|
let color_fg = meta.color_fg.clone().unwrap_or_else(|| "#64748b".to_string());
|
||||||
|
|
||||||
for word in &def.words {
|
let words = file.tags.get(cat_name).cloned().unwrap_or_default();
|
||||||
|
for word in &words {
|
||||||
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
|
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
dict.categories.insert(cat_name.clone(), CategoryInfo {
|
dict.categories.insert(cat_name.clone(), CategoryInfo {
|
||||||
name: cat_name,
|
name: cat_name.clone(),
|
||||||
label,
|
label,
|
||||||
color_bg,
|
color_bg,
|
||||||
color_fg,
|
color_fg,
|
||||||
words: def.words,
|
words,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── 标签词汇自动加入排除人名 ──
|
||||||
|
// 所有 tags 中的词汇,都自动被视为非人名
|
||||||
|
for words in file.tags.values() {
|
||||||
|
for word in words {
|
||||||
|
dict.excluded_person_names.insert(word.to_lowercase());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !path.as_os_str().is_empty() {
|
if !path.as_os_str().is_empty() {
|
||||||
println!("[词典] v{} 已加载: {} 分类, {} 标签, {} 停用词, {} 前缀, {} 排除人名",
|
println!("[词典] v{} 已加载: {} 分类, {} 标签词, {} 停用词, {} 前缀, 排除+标签共 {} 词",
|
||||||
file.version, dict.categories.len(), dict.tag_to_category.len(),
|
file.version, dict.categories.len(), dict.tag_to_category.len(),
|
||||||
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len());
|
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len());
|
||||||
}
|
}
|
||||||
@@ -236,8 +250,8 @@ impl TagDict {
|
|||||||
|
|
||||||
/// ── 人物规则 ──
|
/// ── 人物规则 ──
|
||||||
|
|
||||||
/// 判断人名是否应被排除(精确匹配 或 被排除词包含)
|
/// 判断人名是否应被排除(精确匹配 或 被排除词包含)。
|
||||||
/// 例如:排除词 "秀人" 会匹配 DB 中的 "秀人网"
|
/// 注意:标签词典中的所有词汇已自动加入排除名单,因此标签词永不为人名。
|
||||||
pub fn is_excluded_person(&self, person_name: &str) -> bool {
|
pub fn is_excluded_person(&self, person_name: &str) -> bool {
|
||||||
let name_lower = person_name.to_lowercase();
|
let name_lower = person_name.to_lowercase();
|
||||||
// 精确匹配
|
// 精确匹配
|
||||||
@@ -380,7 +394,11 @@ impl TagDict {
|
|||||||
|
|
||||||
for (name, label, bg, fg, words) in builtin {
|
for (name, label, bg, fg, words) in builtin {
|
||||||
let ws: Vec<String> = words.iter().map(|w| w.to_string()).collect();
|
let ws: Vec<String> = words.iter().map(|w| w.to_string()).collect();
|
||||||
for w in &ws { self.tag_to_category.insert(w.to_lowercase(), name.to_string()); }
|
for w in &ws {
|
||||||
|
self.tag_to_category.insert(w.to_lowercase(), name.to_string());
|
||||||
|
// 标签词自动加入排除人名
|
||||||
|
self.excluded_person_names.insert(w.to_lowercase());
|
||||||
|
}
|
||||||
self.categories.insert(name.to_string(), CategoryInfo {
|
self.categories.insert(name.to_string(), CategoryInfo {
|
||||||
name: name.to_string(), label: label.to_string(),
|
name: name.to_string(), label: label.to_string(),
|
||||||
color_bg: bg.to_string(), color_fg: fg.to_string(), words: ws,
|
color_bg: bg.to_string(), color_fg: fg.to_string(), words: ws,
|
||||||
|
|||||||
@@ -1,51 +1,37 @@
|
|||||||
//! 标签提取器 — 从标题中提取分类明确、格式美观的标签
|
//! 标签提取器 — 从标题中提取分类明确的标签
|
||||||
//!
|
//!
|
||||||
//! 标签分类体系(外部词典 dict/tags.json 定义):
|
//! 标签分类体系(外部词典 dict/tags.json 定义):
|
||||||
//! - style: 风格标签(日系/韩系/森系/复古/甜美/性感等)
|
//! style outfit scene body mood topic position location brand
|
||||||
//! - outfit: 服装标签(比基尼/泳装/旗袍/制服/丝袜/内衣等)
|
|
||||||
//! - scene: 场景标签(室内/户外/街拍/浴室/床上/车内等)
|
|
||||||
//! - body: 身体标签(美腿/美胸/翘臀/玉足等)
|
|
||||||
//! - mood: 情绪标签(清纯/诱惑/唯美/清新/冷艳等)
|
|
||||||
//! - topic: 主题标签(写真/私房/街拍/旅拍等)
|
|
||||||
//! - other: 其他关键词
|
|
||||||
//!
|
//!
|
||||||
//! 词典通过 `dict/tags.json` 加载,支持运行时热重载(POST /api/dict/reload)
|
//! 核心设计:不直接调用 JIEBA,jieba 标签由外部(title_parser::parse_album_title)
|
||||||
|
//! 传入,避免重复分词。Tag extractor 只做:词典匹配 + jieba 结果过滤 + 正则兜底。
|
||||||
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use jieba_rs::Jieba;
|
|
||||||
use crate::engine::title_parser;
|
|
||||||
use crate::engine::tag_dict;
|
use crate::engine::tag_dict;
|
||||||
|
|
||||||
lazy_static! {
|
|
||||||
static ref JIEBA: Jieba = Jieba::new();
|
|
||||||
}
|
|
||||||
|
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
|
static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
|
||||||
static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
||||||
/// 提取标签中的有效部分(核心字词,去头尾标点)
|
/// 提取标签中的有效部分(去头尾标点)
|
||||||
static ref TAG_CORE_RE: Regex = Regex::new(
|
static ref TAG_CORE_RE: Regex = Regex::new(
|
||||||
r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+$"#
|
r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+$"#
|
||||||
).unwrap();
|
).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 判断字符串是否全为标点符号(不含任何字、字母或数字)
|
/// 判断字符串是否全为标点符号
|
||||||
pub fn is_punctuation_only(s: &str) -> bool {
|
pub fn is_punctuation_only(s: &str) -> bool {
|
||||||
if s.is_empty() { return true; }
|
if s.is_empty() { return true; }
|
||||||
// 检查是否包含任意"有效字符"(中文字、字母、数字),
|
|
||||||
// 一个都不包含 → 纯标点/符号
|
|
||||||
!s.chars().any(|c| {
|
!s.chars().any(|c| {
|
||||||
c.is_ascii_alphanumeric() // ASCII 字母/数字
|
c.is_ascii_alphanumeric()
|
||||||
|| ('\u{4e00}'..='\u{9fff}').contains(&c) // CJK 统一表意文字
|
|| ('\u{4e00}'..='\u{9fff}').contains(&c)
|
||||||
|| ('\u{3400}'..='\u{4dbf}').contains(&c) // CJK 扩展 A
|
|| ('\u{3400}'..='\u{4dbf}').contains(&c)
|
||||||
|| ('\u{f900}'..='\u{faff}').contains(&c) // CJK 兼容
|
|| ('\u{f900}'..='\u{faff}').contains(&c)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 清洗单个标签:去除头尾标点符号
|
/// 清洗单个标签:去除头尾标点符号
|
||||||
/// 例如 "性感!" → "性感","...比基尼" → "比基尼", " – 室内" → "室内"
|
|
||||||
/// 清洗后如果只剩下标点或为空,则返回 None
|
/// 清洗后如果只剩下标点或为空,则返回 None
|
||||||
pub fn clean_tag(tag: &str) -> Option<String> {
|
pub fn clean_tag(tag: &str) -> Option<String> {
|
||||||
let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string();
|
let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string();
|
||||||
@@ -56,193 +42,203 @@ pub fn clean_tag(tag: &str) -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 通过外部词典判断一个词属于哪个分类
|
/// 获取分类优先级索引(从词典动态获取)
|
||||||
/// 先在词典中查找,如果在词典中未找到则返回 "other"
|
|
||||||
fn classify_tag(word: &str) -> String {
|
|
||||||
tag_dict::get_tag_dict().classify(word)
|
|
||||||
.map(|s| s.to_string())
|
|
||||||
.unwrap_or_else(|| "other".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 获取分类的优先级索引(用于排序)
|
|
||||||
/// 从词典动态获取,支持任意新分类
|
|
||||||
fn category_order_index(category: &str) -> usize {
|
fn category_order_index(category: &str) -> usize {
|
||||||
let dict = tag_dict::get_tag_dict();
|
let dict = tag_dict::get_tag_dict();
|
||||||
let order = dict.get_category_order();
|
let order = dict.get_category_order();
|
||||||
order.iter().position(|c| c == &category).unwrap_or(999)
|
order.iter().position(|c| c == &category).unwrap_or(999)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 获取所有分类(按词典定义的顺序,动态生成)
|
// ═══════════════════════════════════════════════════════
|
||||||
fn get_category_order_list() -> Vec<String> {
|
// 标签分类(非 jieba)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 判断一个词属于哪个分类(在词典中查找)
|
||||||
|
pub fn classify_tag(word: &str) -> String {
|
||||||
let dict = tag_dict::get_tag_dict();
|
let dict = tag_dict::get_tag_dict();
|
||||||
let mut list: Vec<String> = dict.get_category_order().iter()
|
dict.classify(word)
|
||||||
.map(|c| c.to_string())
|
.map(|c| c.to_string())
|
||||||
.collect();
|
.unwrap_or_else(|| "other".to_string())
|
||||||
list.extend(vec!["noun".to_string(), "proper".to_string(), "other".to_string()]);
|
|
||||||
list
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 用 jieba 分词提取名词标签(排除人名)
|
/// 获取标签的中文分类名
|
||||||
/// 提取词性为 n(名词)、nz(其它专名)、ns(地名)、vn(名动词) 的词
|
pub fn get_category_label(category: &str) -> String {
|
||||||
/// 但排除 nr(人名) 类,因为人名已单独提取
|
tag_dict::get_tag_dict().get_category_label(category)
|
||||||
fn extract_tags_with_jieba(
|
}
|
||||||
|
|
||||||
|
/// 获取标签分类的颜色
|
||||||
|
pub fn get_category_color(category: &str) -> (String, String) {
|
||||||
|
tag_dict::get_tag_dict().get_category_color(category)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
// 步骤 A:词典标签匹配(substring,无 jieba)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 步骤 A — 从标题中通过词典 substring 匹配提取分类标签。
|
||||||
|
/// 已被已有标签(seen)或人名命中的词不再重复匹配。
|
||||||
|
/// 不调用 jieba。
|
||||||
|
pub fn extract_dict_tags(
|
||||||
title: &str,
|
title: &str,
|
||||||
person_name: Option<&str>,
|
person_name: &[String],
|
||||||
|
seen: &mut HashSet<String>,
|
||||||
|
) -> Vec<(String, String)> {
|
||||||
|
let dict = tag_dict::get_tag_dict();
|
||||||
|
let mut tags = Vec::new();
|
||||||
|
let mut title_lower = title.to_lowercase();
|
||||||
|
|
||||||
|
// 移除人名(防止人名中的词命中标签)
|
||||||
|
for p in person_name {
|
||||||
|
title_lower = title_lower.replace(&p.to_lowercase(), " ");
|
||||||
|
}
|
||||||
|
|
||||||
|
let tag_category_map = dict.get_tag_category_map();
|
||||||
|
let mut sorted: Vec<(String, String)> = tag_category_map.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.clone()))
|
||||||
|
.collect();
|
||||||
|
sorted.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
|
||||||
|
|
||||||
|
for (tag, category) in &sorted {
|
||||||
|
if title_lower.contains(tag.as_str()) && !seen.contains(tag) {
|
||||||
|
seen.insert(tag.clone());
|
||||||
|
tags.push((tag.clone(), category.clone()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
// 步骤 B:jieba 名词标签提取(依赖外部 jieba result)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 步骤 B — 从 jieba 分词结果中提取名词标签。
|
||||||
|
/// 已被 seen 中的词跳过。不重复调用 JIEBA。
|
||||||
|
pub fn extract_jieba_noun_tags(
|
||||||
|
jieba_tags: &[jieba_rs::Tag],
|
||||||
|
person_name: &[String],
|
||||||
stop_words: &HashSet<String>,
|
stop_words: &HashSet<String>,
|
||||||
seen: &HashSet<String>,
|
seen: &mut HashSet<String>,
|
||||||
) -> Vec<(String, String)> {
|
) -> Vec<(String, String)> {
|
||||||
let mut tags = Vec::new();
|
let mut tags = Vec::new();
|
||||||
|
|
||||||
// 对标题做词性标注分词
|
for tag in jieba_tags {
|
||||||
let jieba_tags = JIEBA.tag(title, true);
|
let word = tag.word.trim();
|
||||||
|
if word.len() < 2 { continue; }
|
||||||
|
let word_lower = word.to_lowercase();
|
||||||
|
|
||||||
|
if is_punctuation_only(word) { continue; }
|
||||||
|
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
|
||||||
|
if stop_words.contains(&word_lower) { continue; }
|
||||||
|
if seen.contains(&word_lower) { continue; }
|
||||||
|
|
||||||
|
// 排除人名
|
||||||
|
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pos = tag.tag;
|
||||||
|
let category = match pos {
|
||||||
|
"n" | "ng" => "noun",
|
||||||
|
"nz" => "proper",
|
||||||
|
"ns" => "location",
|
||||||
|
"eng" => "other",
|
||||||
|
"x" => "other",
|
||||||
|
"nr" | "nrfg" | "nrt" => continue, // 人名(已单独提取)
|
||||||
|
"nt" | "vn" | "a" | "ad" | "an"
|
||||||
|
| "v" | "vd" | "vg"
|
||||||
|
| "m" | "mq" | "q" | "qv" | "qt"
|
||||||
|
| "r" | "rr" | "rz"
|
||||||
|
| "c" | "p"
|
||||||
|
| "u" | "ul" | "uv" | "ud" | "ug" | "uj"
|
||||||
|
| "d" | "e" | "y" | "o" | "w"
|
||||||
|
| "t" | "tg" | "s" | "f"
|
||||||
|
| "b" | "bl" | "z" => continue,
|
||||||
|
_ => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
seen.insert(word_lower);
|
||||||
|
tags.push((word.to_string(), category.to_string()));
|
||||||
|
}
|
||||||
|
tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
// 步骤 C:正则兜底标签(无 jieba)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 步骤 C — 正则分词提取剩余关键词(兜底)。不调 jieba。
|
||||||
|
pub fn extract_regex_tags(
|
||||||
|
title: &str,
|
||||||
|
person_name: &[String],
|
||||||
|
stop_words: &HashSet<String>,
|
||||||
|
seen: &mut HashSet<String>,
|
||||||
|
) -> Vec<(String, String)> {
|
||||||
|
let mut tags = Vec::new();
|
||||||
|
let title_lower = title.to_lowercase();
|
||||||
|
|
||||||
|
for cap in WORD_RE.find_iter(&title_lower) {
|
||||||
|
let word = cap.as_str().trim();
|
||||||
|
let word_lower = word.to_lowercase();
|
||||||
|
|
||||||
|
if word.len() < 2 { continue; }
|
||||||
|
if is_punctuation_only(word) { continue; }
|
||||||
|
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
|
||||||
|
if stop_words.contains(&word_lower) { continue; }
|
||||||
|
if seen.contains(&word_lower) { continue; }
|
||||||
|
|
||||||
|
// 排除人名
|
||||||
|
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// 已在词典分类中匹配过的跳过
|
||||||
|
if tag_dict::get_tag_dict().classify(&word_lower).is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
seen.insert(word_lower);
|
||||||
|
tags.push((word.to_string(), "other".to_string()));
|
||||||
|
}
|
||||||
|
tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
// extract_tags_from_title(保持对外兼容,内部调 A+B+C)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 从标题中提取分类标签(完整流程)。
|
||||||
|
///
|
||||||
|
/// 三部曲:词典匹配 → jieba 名词提取 → 正则兜底。
|
||||||
|
/// jieba 部分使用外部传入的 jieba result,避免重复分词。
|
||||||
|
pub fn extract_tags_from_title(
|
||||||
|
title: &str,
|
||||||
|
person_name: Option<&str>,
|
||||||
|
jieba_tags: &[jieba_rs::Tag],
|
||||||
|
) -> Vec<(String, String)> {
|
||||||
|
let dict = tag_dict::get_tag_dict();
|
||||||
|
let stop_words = dict.get_stop_words();
|
||||||
|
|
||||||
// 人名排除集合(小写)
|
|
||||||
let person_exclude: Vec<String> = person_name
|
let person_exclude: Vec<String> = person_name
|
||||||
.map(|n| {
|
.map(|n| {
|
||||||
let mut v = vec![n.to_lowercase()];
|
let mut v = vec![n.to_lowercase()];
|
||||||
for part in n.split_whitespace() {
|
for part in n.split_whitespace() {
|
||||||
v.push(part.to_lowercase());
|
v.push(part.to_lowercase());
|
||||||
}
|
}
|
||||||
// 也提取中文核心名用于排除
|
|
||||||
if let Some(cn) = title_parser::extract_cn(n) {
|
|
||||||
v.push(cn.to_lowercase());
|
|
||||||
}
|
|
||||||
v
|
v
|
||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
for tag in &jieba_tags {
|
|
||||||
let word = tag.word.trim();
|
|
||||||
if word.len() < 2 { continue; }
|
|
||||||
|
|
||||||
let word_lower = word.to_lowercase();
|
|
||||||
|
|
||||||
// 纯标点/符号跳过
|
|
||||||
if is_punctuation_only(word) { continue; }
|
|
||||||
|
|
||||||
// 排除纯数字
|
|
||||||
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
|
|
||||||
|
|
||||||
// 排除停用词
|
|
||||||
if stop_words.contains(word_lower.as_str()) { continue; }
|
|
||||||
|
|
||||||
// 排除人名
|
|
||||||
if person_exclude.iter().any(|p| word_lower.contains(p)) { continue; }
|
|
||||||
|
|
||||||
// 排除已经匹配过的
|
|
||||||
if seen.contains(&word_lower) { continue; }
|
|
||||||
|
|
||||||
// 按词性过滤:只保留名词相关,但排除人名(nr)
|
|
||||||
let pos = tag.tag;
|
|
||||||
let category = match pos {
|
|
||||||
// 名词类(排除人名)
|
|
||||||
"n" | "ng" => "noun", // 普通名词
|
|
||||||
"nz" => "proper", // 其它专名
|
|
||||||
"ns" => "location", // 地名
|
|
||||||
"eng" => "other", // 英文词(归入 other)
|
|
||||||
"x" => "other", // 其它
|
|
||||||
// 排除的词性(这些不应该作为标签)
|
|
||||||
"nr" | "nrfg" | "nrt" => continue, // 人名类跳过
|
|
||||||
"nt" => continue, // 机构团体名(不是标签)
|
|
||||||
"vn" => continue, // 名动词(分类模糊)
|
|
||||||
"a" | "ad" | "an" => continue, // 形容词(不是标签)
|
|
||||||
"v" | "vd" | "vg" => continue, // 动词(不是标签)
|
|
||||||
"m" | "mq" => continue, // 数词
|
|
||||||
"q" | "qv" | "qt" => continue, // 量词
|
|
||||||
"r" | "rr" | "rz" => continue, // 代词
|
|
||||||
"c" => continue, // 连词
|
|
||||||
"p" => continue, // 介词
|
|
||||||
"u" | "ul" | "uv" | "ud" | "ug" | "uj" => continue, // 助词
|
|
||||||
"d" => continue, // 副词
|
|
||||||
"e" | "y" | "o" => continue, // 叹词/语气词/拟声词
|
|
||||||
"w" => continue, // 标点
|
|
||||||
"t" | "tg" => continue, // 时间词
|
|
||||||
"s" => continue, // 处所词
|
|
||||||
"f" => continue, // 方位词
|
|
||||||
"b" | "bl" => continue, // 区别词
|
|
||||||
"z" => continue, // 状态词
|
|
||||||
_ => continue,
|
|
||||||
};
|
|
||||||
|
|
||||||
tags.push((word.to_string(), category.to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
tags
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 从标题中提取分类标签
|
|
||||||
///
|
|
||||||
/// 使用外部词典(dict/tags.json)中的分类词库进行匹配,
|
|
||||||
/// 词典支持运行时热重载,修改后无需重启服务。
|
|
||||||
pub fn extract_tags_from_title(
|
|
||||||
title: &str,
|
|
||||||
person_name: Option<&str>,
|
|
||||||
) -> Vec<(String, String)> {
|
|
||||||
let dict = tag_dict::get_tag_dict();
|
|
||||||
let mut tags = Vec::new();
|
|
||||||
let mut title_lower = title.to_lowercase();
|
|
||||||
|
|
||||||
// 移除人名
|
|
||||||
if let Some(name) = person_name {
|
|
||||||
title_lower = title_lower.replace(&name.to_lowercase(), " ");
|
|
||||||
// 也移除英文名部分
|
|
||||||
for part in name.split_whitespace() {
|
|
||||||
title_lower = title_lower.replace(&part.to_lowercase(), " ");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let stop_words = dict.get_stop_words();
|
|
||||||
|
|
||||||
let person_name_lower = person_name.map(|n| n.to_lowercase());
|
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
|
|
||||||
// 第一步:优先匹配外部词典中的词(保证高质量标签)
|
let mut tags = Vec::new();
|
||||||
let tag_category_map = dict.get_tag_category_map();
|
|
||||||
// 按标签长度降序匹配,避免短标签覆盖长标签
|
|
||||||
let mut sorted_tags: Vec<(String, String)> = tag_category_map.iter()
|
|
||||||
.map(|(k, v)| (k.clone(), v.clone()))
|
|
||||||
.collect();
|
|
||||||
sorted_tags.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
|
|
||||||
|
|
||||||
for (tag, category) in &sorted_tags {
|
// A: 词典匹配
|
||||||
if title_lower.contains(tag.as_str()) && !seen.contains(tag) {
|
tags.extend(extract_dict_tags(title, &person_exclude, &mut seen));
|
||||||
seen.insert(tag.clone());
|
|
||||||
tags.push((tag.clone(), category.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 第二步:用 jieba 分词提取名词标签(比正则更准确)
|
// B: jieba 名词(用外部传入的 jieba result)
|
||||||
let jieba_tags = extract_tags_with_jieba(title, person_name, stop_words, &seen);
|
tags.extend(extract_jieba_noun_tags(jieba_tags, &person_exclude, stop_words, &mut seen));
|
||||||
for (tag, category) in jieba_tags {
|
|
||||||
let key = tag.to_lowercase();
|
|
||||||
if !seen.contains(&key) {
|
|
||||||
seen.insert(key);
|
|
||||||
tags.push((tag, category));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 第三步:正则分词提取其他关键词(兜底)
|
// C: 正则兜底
|
||||||
for cap in WORD_RE.find_iter(&title_lower) {
|
tags.extend(extract_regex_tags(title, &person_exclude, stop_words, &mut seen));
|
||||||
let word = cap.as_str().trim();
|
|
||||||
let word_lower = word.to_lowercase();
|
|
||||||
|
|
||||||
// 基础过滤
|
|
||||||
if word.len() < 2 { continue; }
|
|
||||||
if is_punctuation_only(word) { continue; }
|
|
||||||
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
|
|
||||||
if dict.is_stop_word(word_lower.as_str()) { continue; }
|
|
||||||
if let Some(ref pn) = person_name_lower {
|
|
||||||
if word_lower.contains(pn.as_str()) { continue; }
|
|
||||||
}
|
|
||||||
if seen.contains(&word_lower) { continue; }
|
|
||||||
|
|
||||||
// 已经在分类词库中匹配过的跳过
|
|
||||||
if dict.classify(word_lower.as_str()).is_some() { continue; }
|
|
||||||
|
|
||||||
seen.insert(word_lower.clone());
|
|
||||||
|
|
||||||
tags.push((word.to_string(), "other".to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
tags
|
tags
|
||||||
}
|
}
|
||||||
@@ -260,12 +256,10 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
for (tag, tag_type) in sorted_tags {
|
for (tag, tag_type) in sorted_tags {
|
||||||
// 清洗标签:去除头尾标点
|
|
||||||
let cleaned_tag = match clean_tag(&tag) {
|
let cleaned_tag = match clean_tag(&tag) {
|
||||||
Some(t) => t,
|
Some(t) => t,
|
||||||
None => continue, // 清洗后为空 → 跳过
|
None => continue,
|
||||||
};
|
};
|
||||||
// 最终安全过滤:坏标签一律剔除(停用词等)
|
|
||||||
if is_bad_tag(&cleaned_tag) { continue; }
|
if is_bad_tag(&cleaned_tag) { continue; }
|
||||||
let key = cleaned_tag.to_lowercase();
|
let key = cleaned_tag.to_lowercase();
|
||||||
if seen.insert(key) {
|
if seen.insert(key) {
|
||||||
@@ -275,15 +269,12 @@ pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 判断一个标签是否"坏标签"(应该在数据库中清理掉)
|
/// 判断一个标签是否"坏标签"
|
||||||
/// 坏标签定义:标点符号、含头尾标点需要清洗、太短、纯数字、停用词
|
|
||||||
/// 如果标签需要清洗(如 "性感!"),也会被标记为坏标签
|
|
||||||
pub fn is_bad_tag(tag: &str) -> bool {
|
pub fn is_bad_tag(tag: &str) -> bool {
|
||||||
let tag = tag.trim();
|
let tag = tag.trim();
|
||||||
if tag.is_empty() || tag.len() < 2 { return true; }
|
if tag.is_empty() || tag.len() < 2 { return true; }
|
||||||
if tag.chars().all(|c| c.is_ascii_digit()) { return true; }
|
if tag.chars().all(|c| c.is_ascii_digit()) { return true; }
|
||||||
if is_punctuation_only(tag) { return true; }
|
if is_punctuation_only(tag) { return true; }
|
||||||
// 如果标签清洗后变短了,说明含有头尾标点 → 坏标签
|
|
||||||
if let Some(cleaned) = clean_tag(tag) {
|
if let Some(cleaned) = clean_tag(tag) {
|
||||||
if cleaned != tag { return true; }
|
if cleaned != tag { return true; }
|
||||||
}
|
}
|
||||||
@@ -292,18 +283,6 @@ pub fn is_bad_tag(tag: &str) -> bool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 获取标签的中文分类名(用于前端展示)
|
|
||||||
pub fn get_category_label(category: &str) -> String {
|
|
||||||
let dict = tag_dict::get_tag_dict();
|
|
||||||
dict.get_category_label(category)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 获取标签分类的颜色(用于前端标签着色)
|
|
||||||
pub fn get_category_color(category: &str) -> (String, String) {
|
|
||||||
let dict = tag_dict::get_tag_dict();
|
|
||||||
dict.get_category_color(category)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -313,6 +292,7 @@ mod tests {
|
|||||||
let tags = extract_tags_from_title(
|
let tags = extract_tags_from_title(
|
||||||
"XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'",
|
"XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'",
|
||||||
Some("杨晨晨Yome"),
|
Some("杨晨晨Yome"),
|
||||||
|
&[],
|
||||||
);
|
);
|
||||||
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
|
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
|
||||||
assert!(!names.contains(&"photos"));
|
assert!(!names.contains(&"photos"));
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ use std::collections::HashSet;
|
|||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
use jieba_rs::Jieba;
|
use jieba_rs::Jieba;
|
||||||
use crate::engine::tag_dict;
|
use crate::engine::tag_dict;
|
||||||
|
use crate::engine::tag_extractor;
|
||||||
|
|
||||||
// 全局 jieba 分词器
|
// 全局 jieba 分词器
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
@@ -695,6 +696,92 @@ pub fn normalize_tag(tag: &str) -> String {
|
|||||||
tag.trim().to_lowercase()
|
tag.trim().to_lowercase()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
// 统一标题预处理(用于 parse_album_title 入口)
|
||||||
|
// ═══════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
/// 统一标题预处理:移除照片数后缀、[工作室] 前缀、称呼前缀、编号后缀
|
||||||
|
pub fn preprocess_title(title: &str) -> String {
|
||||||
|
let t = title.trim();
|
||||||
|
if t.is_empty() { return String::new(); }
|
||||||
|
|
||||||
|
let mut t = RE_PHOTOS_SUFFIX.replace(t, "").trim().to_string();
|
||||||
|
t = RE_BRACKET_PREFIX.replace(&t, "").trim().to_string();
|
||||||
|
|
||||||
|
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
|
||||||
|
if t.starts_with(prefix) {
|
||||||
|
t = t[prefix.len()..].trim().to_string();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t = RE_NUMBER_SUFFIX.replace(&t, "").trim().to_string();
|
||||||
|
t
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 统一相册标题解析入口
|
||||||
|
///
|
||||||
|
/// 一次 jieba 分词 → 同时提取 person + tags。
|
||||||
|
/// tags 通过 tag_extractor 的三步流程(A 词典 / B jieba 名词 / C 正则)提取,
|
||||||
|
/// 使用同一份 jieba result。
|
||||||
|
///
|
||||||
|
/// 返回 ParseResult 可直接用于 Phase 1b 批量写库。
|
||||||
|
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
|
||||||
|
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
|
||||||
|
|
||||||
|
let title = title.trim();
|
||||||
|
if title.is_empty() { return result; }
|
||||||
|
|
||||||
|
// 1. 预处理
|
||||||
|
let cleaned = preprocess_title(title);
|
||||||
|
|
||||||
|
// 2. cosplay 格式优先
|
||||||
|
if let Some(caps) = RE_COSPLAY_BY.captures(&cleaned) {
|
||||||
|
result.person_name = Some(clean_name(caps[3].trim()));
|
||||||
|
let name = result.person_name.as_deref().unwrap_or("");
|
||||||
|
// 有限标签提取(仅词典 + 正则,不调 jieba 避免重复)
|
||||||
|
let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new());
|
||||||
|
result.tags = tags;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
if let Some(caps) = RE_COSPLAY.captures(&cleaned) {
|
||||||
|
result.person_name = Some(clean_name(caps[1].trim()));
|
||||||
|
let name = result.person_name.as_deref().unwrap_or("");
|
||||||
|
let tags = tag_extractor::extract_dict_tags(title, &[name.to_lowercase()], &mut std::collections::HashSet::new());
|
||||||
|
result.tags = tags;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. 一次 jieba 分词(核心!全流程共用)
|
||||||
|
let jieba_tags = JIEBA.tag(&cleaned, true);
|
||||||
|
|
||||||
|
// 4. 提取人名
|
||||||
|
let person_raw = extract_person_from_tags(&jieba_tags);
|
||||||
|
|
||||||
|
if let Some(name) = &person_raw {
|
||||||
|
// include 扩展
|
||||||
|
let expanded = tag_dict::get_tag_dict()
|
||||||
|
.find_longest_included_match(name)
|
||||||
|
.filter(|longer| longer.len() > name.len());
|
||||||
|
let effective_name = expanded.as_deref().unwrap_or(name.as_str());
|
||||||
|
let (primary, aliases) = split_person_name_with_aliases(effective_name);
|
||||||
|
let final_name = if primary.is_empty() { effective_name.to_string() } else { primary };
|
||||||
|
|
||||||
|
if final_name.chars().count() > 2 {
|
||||||
|
result.person_name = Some(final_name);
|
||||||
|
// person_primary 用于别名处理,暂存但不暴露
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. 提取标签(使用同一份 jieba result)
|
||||||
|
let person_exclude = result.person_name.as_deref();
|
||||||
|
let raw_tags = tag_extractor::extract_tags_from_title(title, person_exclude, &jieba_tags);
|
||||||
|
let unique = tag_extractor::unique_tags(raw_tags);
|
||||||
|
|
||||||
|
result.tags = unique;
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ pub mod person;
|
|||||||
pub mod tag;
|
pub mod tag;
|
||||||
pub mod favorite;
|
pub mod favorite;
|
||||||
pub mod recommend;
|
pub mod recommend;
|
||||||
|
pub mod parse;
|
||||||
|
|
||||||
pub use album::*;
|
pub use album::*;
|
||||||
pub use image::*;
|
pub use image::*;
|
||||||
|
|||||||
27
src/models/parse.rs
Normal file
27
src/models/parse.rs
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
//! 相册解析结果 — 单个相册标题的一次性解析产物
|
||||||
|
//! 用于串联并行解析(Phase 1a)和批量写库(Phase 1b)
|
||||||
|
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
|
||||||
|
/// 一次 title 解析的完整结果
|
||||||
|
///
|
||||||
|
/// 在一个相册上通过一次 jieba 分词,同时提取:
|
||||||
|
/// - 人名 + 别名(含 include 扩展、prefix 移除)
|
||||||
|
/// - 标签列表(含分类)
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ParseResult {
|
||||||
|
/// 相册 UUID
|
||||||
|
pub album_id: String,
|
||||||
|
/// 最终人名 | None = 未找到
|
||||||
|
pub person_name: Option<String>,
|
||||||
|
/// 中文主名(split_person_name_with_aliases 后的 primary)
|
||||||
|
pub person_primary: Option<String>,
|
||||||
|
/// 标签列表 (tag, category)
|
||||||
|
pub tags: Vec<(String, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ParseResult {
|
||||||
|
pub fn new(album_id: String) -> Self {
|
||||||
|
Self { album_id, person_name: None, person_primary: None, tags: Vec::new() }
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user