admin/rust-imager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

update code level

Browse Source
This commit is contained in:
lychang
2026-07-20 23:46:39 +08:00
parent 84409207e2
commit 56dd82fb63
6 changed files with 121 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
assets/js/app.js
View File

@@ -1473,7 +1473,8 @@ async function filterByPerson(personName) {
showSkeleton(6);
const response = await fetch(`${API_BASE}/people/search?q=${encodeURIComponent(personName)}`);
if (!response.ok) throw new Error('搜索失败');
const persons = await response.json();
const personData = await response.json();
const persons = personData.persons || personData;
if (persons && persons.length > 0) {
try {

8
migrations/005_cleanup.sql Normal file
View File

@@ -0,0 +1,8 @@
-- 移除不再使用的扩展列（数据已迁移到 search.db）
-- SQLite 3.35.0+ 支持 ALTER TABLE DROP COLUMN
ALTER TABLE albums DROP COLUMN tags;
ALTER TABLE albums DROP COLUMN all_words;
ALTER TABLE albums DROP COLUMN person_name_original;
ALTER TABLE albums DROP COLUMN person_name_cleaned;
ALTER TABLE albums DROP COLUMN raw_title;

30
src/db/album.rs
View File

@@ -167,34 +167,25 @@ impl Database {
}
pub async fn get_all_albums_full(&self) -> Vec<AlbumMeta> {
let has_col = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM pragma_table_info('albums') WHERE name = 'person_name_original'"
).fetch_one(&self.pool).await.unwrap_or(0);
if has_col == 0 { return Vec::new(); }
// 去掉 ORDER BY：管线处理不需要排序，省去排序开销
let rows = sqlx::query(
r#"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count
r#"SELECT id, title, cover_image_id, image_count
FROM albums"#
).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
id: r.get("id"), title: r.get("title"),
person_name_original: r.get("person_name_original"),
person_name_cleaned: r.get("person_name_cleaned"),
tags: r.get("tags"), all_words: r.get("all_words"),
raw_title: r.get("raw_title"), processed_at: r.get("processed_at"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
}).collect()
}
/// 按 ID 列表批量查询相册元数据（用于脏集增量解析，避免全表扫描）
/// 按 ID 列表批量查询相册元数据
pub async fn get_albums_by_ids(&self, ids: &[String]) -> Vec<AlbumMeta> {
if ids.is_empty() { return Vec::new(); }
let placeholders: Vec<String> = (0..ids.len()).map(|_| "?".to_string()).collect();
let sql = format!(
"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count
"SELECT id, title, cover_image_id, image_count
FROM albums WHERE id IN ({})",
placeholders.join(", ")
);
@@ -207,14 +198,19 @@ impl Database {
let rows = query.fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
id: r.get("id"), title: r.get("title"),
person_name_original: r.get("person_name_original"),
person_name_cleaned: r.get("person_name_cleaned"),
tags: r.get("tags"), all_words: r.get("all_words"),
raw_title: r.get("raw_title"), processed_at: r.get("processed_at"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
}).collect()
}
/// 获取所有未处理的相册 ID（processed_at IS NULL），用于管线收集新相册
pub async fn get_unprocessed_album_ids(&self) -> Vec<String> {
let rows = sqlx::query_scalar::<_, String>(
"SELECT id FROM albums WHERE processed_at IS NULL"
).fetch_all(&self.pool).await.unwrap_or_default();
rows
}
// ── 写入 ──
pub async fn create_album(&self, album: CreateAlbum) -> Result<Uuid, sqlx::Error> {

4
src/engine/person_merger.rs
View File

@@ -406,7 +406,7 @@ pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = album.person_name_original.as_deref().unwrap_or("");
let current = ""; // person_name_original 已迁移到 search.db
if let Some((_text, person, source)) = match_title(title, patterns) {
*stats.by_source.entry(source).or_insert(0) += 1;
@@ -458,7 +458,7 @@ pub async fn analyze_deep_matches(db: &Database) -> DeepMatchStats {
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = album.person_name_original.as_deref().unwrap_or("");
let current = ""; // person_name_original 已迁移到 search.db
if title.is_empty() {
continue;

409
src/engine/pipeline.rs
View File

@@ -14,7 +14,6 @@
use crate::db::Database;
use crate::engine::title_parser;
use crate::engine::tag_extractor;
use crate::engine::person_merger;
use crate::engine::tag_dict;
use crate::search_db::SearchDb;
@@ -65,22 +64,10 @@ fn is_short_tag(tag: &str) -> bool {
// ════════════════════════════════════════════════
pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
let albums = db.get_all_albums_full().await;
let ids = db.get_unprocessed_album_ids().await;
let mut dirty = DirtySet::new();
let mut new_count = 0usize;
let mut partial_count = 0usize;
for album in &albums {
let tags_empty = album.tags.as_deref()
.map(|t| t.is_empty() || t == "[]" || t == "null").unwrap_or(true);
if tags_empty { dirty.add(&album.id); new_count += 1; continue; }
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
if !has_person { dirty.add(&album.id); partial_count += 1; }
}
println!("[管线·PhaseA3] 收集相册：{} 个全新，{} 个半成品，脏集合计 {} 个", new_count, partial_count, dirty.len());
for id in &ids { dirty.add(id); }
println!("[管线·PhaseA3] 收集相册：{} 个未处理", dirty.len());
dirty
}
@@ -179,61 +166,8 @@ pub async fn phase_merge_persons(db: &Database) -> (usize, usize) {
// ════════════════════════════════════════════════
pub async fn phase_clean_tags_after_parse(db: &Database) -> usize {
let albums = db.get_all_albums_full().await;
let all_person_names: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
let all_aliases: HashSet<String> = db.get_all_aliases().await
.into_iter().map(|(a, _)| a.to_lowercase()).collect();
let person_or_alias: HashSet<String> = all_person_names.into_iter().chain(all_aliases).collect();
if person_or_alias.is_empty() {
println!("[管线·PhaseB] 后校验标签完成：无人物数据，跳过");
return 0;
}
let mut tag_removed = 0usize;
let mut updates: Vec<(String, String, String)> = Vec::new();
for album in &albums {
let tags_json = match album.tags.as_deref() {
Some(t) if !t.is_empty() && t != "[]" => t, _ => continue,
};
let old_tags: Vec<String> = match serde_json::from_str(tags_json) { Ok(t) => t, Err(_) => continue, };
let mut album_removed = 0usize;
let good_tags: Vec<String> = old_tags.into_iter().filter(|t| {
if person_or_alias.contains(&t.to_lowercase()) { album_removed += 1; false } else { true }
}).collect();
if album_removed == 0 { continue; }
tag_removed += album_removed;
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default().into_iter().filter(|pair| !person_or_alias.contains(&pair[0].to_lowercase())).collect();
let tags_json_new = serde_json::to_string(&good_tags).unwrap_or_else(|_| "[]".to_string());
let words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
updates.push((album.id.clone(), tags_json_new, words_json_new));
}
// ── 批量写入：单个事务 ──
if !updates.is_empty() {
let _guard = db.write_lock.lock().await;
if let Ok(mut tx) = db.pool.begin().await {
for (album_id, tags_json, all_words_json) in &updates {
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
}
tx.commit().await.ok();
}
}
if tag_removed > 0 {
println!("[管线·PhaseB] 后校验标签完成：移除 {} 个人名/别名", tag_removed);
} else {
println!("[管线·PhaseB] 后校验标签完成：无需要修复的标签");
}
tag_removed
println!("[管线·PhaseB] 后校验标签完成：已迁移到 search.db，跳过");
0
}
// ════════════════════════════════════════════════
@@ -250,179 +184,14 @@ struct PreprocessResult {
dirty: DirtySet,
}
/// 合并预处理：一次相册扫描完成 4 件事
/// 1. 清洗标签标点
/// 2. 移除短标签
/// 3. 检测并移除标签中的人名
/// 4. 收集新相册/半成品
///
/// 所有写库操作在单个事务中完成，避免逐条事务的 WAL fsync 开销。
/// 合并预处理：只做收集新相册，不再做标签清洗（已迁移到 search.db）
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
let albums = db.get_all_albums_full().await;
let known_persons: HashSet<String> = db.get_all_person_names().await
.into_iter().map(|(n, _)| n.to_lowercase()).collect();
let mut result = PreprocessResult::default();
// 收集需要更新的相册变更：(album_id, tags_json, all_words_json)
let mut updates: Vec<(String, String, String)> = Vec::new();
for album in &albums {
let tags_json = album.tags.as_deref();
let has_tags = tags_json.map(|t| !t.is_empty() && t != "[]" && t != "null").unwrap_or(false);
let has_person = album.person_name_original.as_deref()
.map(|s| !s.is_empty()).unwrap_or(false);
// ── 新相册/半成品检测 ──
if !has_tags || !has_person {
result.dirty.add(&album.id);
if !has_tags {
result.new_albums_found += 1;
}
}
// ── 没有标签就不需要清洗/检测 ──
if !has_tags { continue; }
let old_tags: Vec<String> = match serde_json::from_str(tags_json.unwrap()) {
Ok(t) => t, Err(_) => continue,
};
if old_tags.is_empty() { continue; }
// ── 三步标签处理合并到一次遍历 ──
let mut changed = false;
let mut new_tags: Vec<String> = Vec::new();
let mut person_like_removed_this = 0usize;
// 先找到最优人名（用于后续从标签中移除人名相关）
// 第一层：逐个标签收集候选人名
let mut known_person_found: Option<String> = None;
let mut person_like_tags: Vec<String> = Vec::new();
for tag in &old_tags {
let tag_lower = tag.to_lowercase();
if known_persons.contains(&tag_lower) {
known_person_found = Some(tag.clone());
continue;
}
if !has_person && title_parser::is_likely_person_name(tag) && !title_parser::is_excluded_name(tag) {
person_like_tags.push(tag.clone());
continue;
}
}
let best_layer1 = person_like_tags.into_iter().max_by_key(|t| {
if let Some(cn) = title_parser::extract_cn(t) {
if cn.len() >= 2 { cn.len() * 10 + t.len() } else { t.len() }
} else { t.len() }
});
let combined = old_tags.join(" ");
let (jieba_name, _) = title_parser::extract_person_and_tags(&combined);
let final_person = known_person_found.or_else(|| {
match (&jieba_name, &best_layer1) {
(Some(j), Some(b)) => {
let j_len: usize = j.len();
let b_len: usize = b.len();
if j_len > b_len || title_parser::is_excluded_name(b) { Some(j.clone()) } else { Some(b.clone()) }
}
(Some(j), None) => Some(j.clone()),
(None, Some(b)) => Some(b.clone()),
(None, None) => None,
}
});
let person_lower = final_person.as_ref().map(|n| n.to_lowercase());
let primary_lower = final_person.as_ref().and_then(|n| {
let (p, _) = title_parser::split_person_name_with_aliases(n);
if p.is_empty() { None } else { Some(p.to_lowercase()) }
});
for tag in old_tags {
// Step 1: 清洗标点
let cleaned = match tag_extractor::clean_tag(&tag) {
Some(t) => t,
None => { changed = true; continue; }
};
if cleaned != tag { changed = true; }
// Step 2: 短标签过滤
if is_short_tag(&cleaned) {
changed = true;
result.short_tags_removed += 1;
continue;
}
// Step 3: 人名标签过滤
let cleaned_lower = cleaned.to_lowercase();
let mut is_person_tag = false;
if let (Some(pl), Some(pr)) = (&person_lower, &primary_lower) {
is_person_tag =
cleaned_lower == *pl
|| cleaned_lower == *pr
|| (cleaned.len() >= 2 && cleaned.chars().all(|c| c.is_ascii_alphabetic())
&& !cleaned_lower.starts_with("cos")
&& pl.contains(&cleaned_lower))
|| (cleaned_lower.len() >= 2 && pl.contains(&cleaned_lower));
}
if is_person_tag {
person_like_removed_this += 1;
changed = true;
continue;
}
new_tags.push(cleaned);
}
result.person_like_tags_removed += person_like_removed_this;
if !changed { continue; }
result.tags_cleaned += 1;
result.dirty.add(&album.id);
// 收集变更，等下批量写
let tags_json_new = serde_json::to_string(&new_tags).unwrap_or_else(|_| "[]".to_string());
let all_words_json = album.all_words.as_deref().unwrap_or("[]");
let all_words_new: Vec<[String; 2]> = serde_json::from_str::<Vec<[String; 2]>>(all_words_json)
.unwrap_or_default()
.into_iter()
.filter_map(|pair| {
let cleaned = tag_extractor::clean_tag(&pair[0])?;
if is_short_tag(&cleaned) { return None; }
if known_persons.contains(&pair[0].to_lowercase()) { return None; }
Some([cleaned, pair[1].clone()])
})
.collect();
let all_words_json_new = serde_json::to_string(&all_words_new).unwrap_or_else(|_| "[]".to_string());
updates.push((album.id.clone(), tags_json_new, all_words_json_new));
}
// ── 批量写入：单个事务 ──
if !updates.is_empty() {
let _guard = db.write_lock.lock().await;
let mut tx = match db.pool.begin().await {
Ok(t) => t,
Err(e) => {
eprintln!("[管线·Phase①] 开启事务失败: {}", e);
return result;
}
};
for (album_id, tags_json, all_words_json) in &updates {
db.update_album_tags_with_tx(&mut tx, album_id, tags_json, all_words_json).await;
}
if let Err(e) = tx.commit().await {
eprintln!("[管线·Phase①] 提交事务失败: {}", e);
}
}
println!("[管线·Phase①] 合并预处理完成：清洗{}相册，移除短标签{}，移除人名标签{}，新相册{}，脏集合计{}",
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed,
let dirty = phase_collect_new_albums(db).await;
result.dirty = dirty;
result.new_albums_found = result.dirty.len();
println!("[管线·Phase①] 合并预处理完成（精简模式）：新相册{}，脏集合计{}",
result.new_albums_found, result.dirty.len());
result
}
@@ -557,9 +326,7 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
for album in &albums {
// ── 1. 写入 albums 表 ──
let cover_id: Option<&str> = None;
// 从主库获取 cover_image_id：因 get_all_albums_full 不返回 cover_image_id，
// 这里用默认值，主库已有的 cover 在下一次增量同步时会补全
let cover_id = album.cover_image_id.as_deref();
let img_count = album.image_count.unwrap_or(0);
if let Err(e) = search_db.upsert_album(&album.id, album.title.as_deref().unwrap_or(""),
cover_id, img_count, None).await {
@@ -568,50 +335,56 @@ pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> us
}
// ── 2. 写入 tags → album_tags ──
let tags_json = album.tags.as_deref().unwrap_or("[]");
if tags_json != "[]" {
if let Ok(tags) = serde_json::from_str::<Vec<(String, String)>>(tags_json) {
search_db.clear_album_tags(&album.id).await.ok();
for (tag_name, category) in &tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
// 从 title 解析 tags，不再依赖 albums 表的 all_words 字段
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
if !parse_result.tags.is_empty() {
search_db.clear_album_tags(&album.id).await.ok();
for (tag_name, category) in &parse_result.tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
}
}
}
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
}
}
}
}
// ── 3. 写入 persons → album_persons ──
// 从 title 解析人物，不再依赖 albums 表的 person_name_cleaned 字段
search_db.clear_album_persons(&album.id).await.ok();
if let Some(ref person_name) = album.person_name_cleaned {
if !person_name.is_empty() {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
if let Some(ref person_name) = parse_result.person_name {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
}
}
}
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同时同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
}
@@ -658,56 +431,62 @@ pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &Dirt
let img_count = album.image_count.unwrap_or(0);
// ── 1. upsert albums 表 ──
if let Err(e) = search_db.upsert_album(&album.id, title, None, img_count, None).await {
if let Err(e) = search_db.upsert_album(&album.id, title, album.cover_image_id.as_deref(), img_count, None).await {
eprintln!("[Sync->search.db·增量] upsert 相册 {} 失败: {}", album.id, e);
continue;
}
// ── 2. 写入 tags → album_tags ──
let tags_json = album.tags.as_deref().unwrap_or("[]");
search_db.clear_album_tags(&album.id).await.ok();
if tags_json != "[]" {
if let Ok(tags) = serde_json::from_str::<Vec<(String, String)>>(tags_json) {
for (tag_name, category) in &tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
// 从 title 解析 tags，不再依赖 albums 表的 all_words 字段
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
if !parse_result.tags.is_empty() {
search_db.clear_album_tags(&album.id).await.ok();
for (tag_name, category) in &parse_result.tags {
let tag_id = match tag_cache.get(tag_name) {
Some(&id) => id,
None => {
match search_db.insert_tag(tag_name, Some(category)).await {
Ok(id) => { tag_cache.insert(tag_name.clone(), id); id }
Err(_) => continue,
}
}
}
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
};
search_db.insert_album_tag(&album.id, tag_id).await.ok();
}
}
}
}
// ── 3. 写入 persons → album_persons ──
// 从 title 解析人物，不再依赖 albums 表的 person_name_cleaned 字段
search_db.clear_album_persons(&album.id).await.ok();
if let Some(ref person_name) = album.person_name_cleaned {
if !person_name.is_empty() {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
if let Some(title) = album.title.as_deref() {
if !title.is_empty() {
let parse_result = title_parser::parse_album_title(&album.id, title);
if let Some(ref person_name) = parse_result.person_name {
let person_id = match person_cache.get(person_name) {
Some(&id) => id,
None => {
match search_db.insert_person(person_name, None).await {
Ok(id) => { person_cache.insert(person_name.clone(), id); id }
Err(_) => continue,
}
}
}
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同时同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
};
search_db.insert_album_person(&album.id, person_id).await.ok();
// 同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db.insert_person_alias(person_id, alias_type, alias_name).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db.insert_person_alias(person_id, "primary", &primary).await.ok();
}
}
}

9
src/models/album.rs
View File

@@ -47,16 +47,11 @@ pub struct AlbumIndex {
pub images: Vec<String>,
}
/// 相册完整元数据（含扩展字段）
/// 相册完整元数据（精简版：仅保留 albums 表原始字段）
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AlbumMeta {
pub id: String,
pub title: Option<String>,
pub person_name_original: Option<String>,
pub person_name_cleaned: Option<String>,
pub tags: Option<String>,
pub all_words: Option<String>,
pub raw_title: Option<String>,
pub processed_at: Option<String>,
pub cover_image_id: Option<String>,
pub image_count: Option<i64>,
}