From 093924ea99f30db92ce273f1f594a6a13890e43b Mon Sep 17 00:00:00 2001 From: lychang Date: Sun, 19 Jul 2026 16:34:04 +0800 Subject: [PATCH] update --- src/bin/migrate.rs | 872 ------------------------------------ src/config.rs | 6 + src/db/mod.rs | 8 +- src/engine/crawler.rs | 2 +- src/engine/mod.rs | 10 +- src/engine/person_merger.rs | 2 +- src/main.rs | 32 +- src/routes/migration.rs | 2 +- src/routes/recommend.rs | 2 +- src/services/mod.rs | 2 +- 10 files changed, 37 insertions(+), 901 deletions(-) delete mode 100644 src/bin/migrate.rs diff --git a/src/bin/migrate.rs b/src/bin/migrate.rs deleted file mode 100644 index cea3935..0000000 --- a/src/bin/migrate.rs +++ /dev/null @@ -1,872 +0,0 @@ -//! 数据库迁移 CLI 工具 -//! -//! 功能: -//! 1. 从老数据库 (persons.db) 读取数据 -//! 2. 对 title 进行解析 (人名提取、标签提取) -//! 3. 生成新的数据库 -//! 4. 验证搜索等功能 -//! -//! 用法: -//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db -//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db --reparse -//! cargo run --bin migrate -- --verify --new-db ./data/imager.db - -use serde::{Deserialize, Serialize}; -use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row}; -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::time::Instant; - -// ============================================================ -// 标题解析模块 (Rust 原生实现) -// ============================================================ - -mod title_parser { - use regex::Regex; - use std::collections::HashSet; - use lazy_static::lazy_static; - - /// 模特称呼前缀 - const MODEL_PREFIXES: &[&str] = &[ - "美女模特", "性感女神", "新人模特", "新人嫩模", "新人妹子", - "气质女神", "气质美女", "性感美女", "宅男女神", "丽质女神", - "丽质美女", "红粉佳人", "美乳女神", "清纯女神", "性感嫩模", - "性感御姐", "美女", "模特", "女神", "御姐", "嫩模", - "新晋女神", "新晋美模", "日本", "日本人气", - ]; - - /// 停用词 - const STOP_WORDS: &[&str] = &[ - "photos", "photo", "phot", "images", "image", "video", "videos", - "picture", "pictures", "pic", "gif", "gifs", "mp4", "webp", - "cosplay", "cos", "set", "part", "collection", "series", - "special", "vol", "no", "no.", "ver", "version", "edition", - "new", "xxx", "preview", "teaser", "sample", - "写真", "套图", "旅拍", "系列", "诱惑", "性感", "美女", - "模特", "女神", "写真集", "首套", "版", "主题", "风格", - "the", "a", "an", "in", "on", "at", "of", "to", "for", - "with", "by", "and", "or", "but", "from", "is", "are", - "this", "that", "these", "those", "it", "its", - "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一", - "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着", - "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "些", - ]; - - /// 工作室排除名 - const EXCLUDED_NAMES: &[&str] = &[ - "xiuren", "秀人", "秀人网", "youmi", "尤蜜", "尤蜜荟", - "xingyan", "星颜", "魅妍社", "mistar", - "mygirl", "美媛馆", "imiss", "爱蜜社", "huayang", "花漾", - "bololi", "波萝社", "feilin", "嗲囡囡", "qingdouke", "青豆客", - "toutiao", "头条女神", "xiaoyu", "语画界", "youmei", "尤美", - "ugirls", "尤果", "尤果网", "mfstar", "模范学院", - "tgod", "推女神", "dgkirl", "御女郎", "candy", "糖果画报", - "wanibooks", "bluecake", "pure media", "puremedia", - ]; - - lazy_static! { - static ref RE_PHOTOS: Regex = Regex::new( - r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$" - ).unwrap(); - static ref RE_COSPLAY: Regex = Regex::new( - r"(?i)^(.+?)\s+cosplay\s+(.+?)(?:\s*[–—\-]\s*(.+))?$" - ).unwrap(); - static ref RE_COSPLAY_BY: Regex = Regex::new( - r"(?i)^(.+?)\s+cosplay\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$" - ).unwrap(); - static ref RE_BRACKET: Regex = Regex::new(r"^[\s]*[\[【][^\]]*[\]】][\s]*").unwrap(); - static ref RE_NUMBER_PREFIX: Regex = Regex::new( - r"(?i)^(?:No\.?\s*\d+|VOL\.?\s*\d+)\s+" - ).unwrap(); - static ref RE_SPLIT: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap(); - static ref RE_PAREN: Regex = Regex::new(r"[（(]([^）)]*)[）)]").unwrap(); - static ref RE_TRAILING: Regex = Regex::new(r"[\s.,;:，。；：\-–—]+$").unwrap(); - static ref RE_EN_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap(); - static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap(); - static ref RE_WORD: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap(); - - static ref STOP_SET: HashSet<&'static str> = { - let mut s = HashSet::new(); - for w in STOP_WORDS { s.insert(*w); } - s - }; - - static ref EXCLUDED_LOWER: HashSet = { - let mut s = HashSet::new(); - for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); } - s - }; - } - - /// 清理标题末尾 - pub fn clean_title_suffix(title: &str) -> String { - RE_PHOTOS.replace(title, "").trim().to_string() - } - - /// 从标题提取人名 - pub fn extract_person_name(title: &str) -> Option { - let clean = clean_title_suffix(title); - if clean.is_empty() { - return None; - } - - // cosplay 格式 - if let Some(caps) = RE_COSPLAY_BY.captures(&clean) { - return Some(caps[2].trim().to_string()); - } - if let Some(caps) = RE_COSPLAY.captures(&clean) { - return Some(caps[1].trim().to_string()); - } - - // 移除工作室前缀 - let no_studio = RE_BRACKET.replace(&clean, "").trim().to_string(); - - // 移除模特称呼前缀 - let mut no_prefix = no_studio; - for prefix in MODEL_PREFIXES { - if no_prefix.starts_with(prefix) { - no_prefix = no_prefix[prefix.len()..].trim().to_string(); - break; - } - } - - // 移除编号前缀 - let no_prefix = RE_NUMBER_PREFIX.replace(&no_prefix, "").trim().to_string(); - - // 用分隔符拆分 - let parts: Vec<&str> = RE_SPLIT.splitn(&no_prefix, 2).collect(); - if parts.len() >= 2 { - let name = parts[0].trim(); - let name = RE_PHOTOS.replace(name, "").trim().to_string(); - if !name.is_empty() && name.len() >= 2 { - return Some(name); - } - } - - // 没有分隔符 - let name = RE_PHOTOS.replace(&no_prefix, "").trim().to_string(); - if name.len() >= 2 { - Some(name) - } else { - None - } - } - - /// 清洗人名 - pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) { - let mut aliases = Vec::new(); - let mut name = raw_name.to_string(); - - // 移除前缀 - for prefix in MODEL_PREFIXES { - if name.starts_with(prefix) { - aliases.push(("prefix".to_string(), prefix.to_string())); - name = name[prefix.len()..].trim().to_string(); - break; - } - } - - // 移除括号内容 - let mut cleaned = String::new(); - let mut last_end = 0; - for cap in RE_PAREN.captures_iter(&name) { - let m = cap.get(0).unwrap(); - let content = cap.get(1).map_or("", |c| c.as_str()); - cleaned.push_str(&name[last_end..m.start()]); - last_end = m.end(); - if !content.is_empty() { - if RE_CHINESE.is_match(content) { - aliases.push(("nickname".to_string(), content.to_string())); - } else { - aliases.push(("english".to_string(), content.to_string())); - } - } - } - cleaned.push_str(&name[last_end..]); - let name = cleaned.trim().to_string(); - - // 清理末尾标点 - let name = RE_TRAILING.replace(&name, "").trim().to_string(); - - // 提取英文名别名 - if let Some(caps) = RE_EN_SUFFIX.captures(&name) { - let chinese = caps[1].to_string(); - let english = caps[2].trim().to_string(); - if !english.is_empty() { - aliases.push(("chinese".to_string(), chinese)); - aliases.push(("english".to_string(), english)); - } - } - - let name = if name.is_empty() { raw_name.to_string() } else { name }; - (name, aliases) - } - - /// 从标题提取标签 - pub fn extract_tags(title: &str, person_name: Option<&str>) -> Vec<(String, String)> { - let mut clean = clean_title_suffix(title); - - // 移除人名 - if let Some(name) = person_name { - clean = clean.replace(name, " "); - for part in name.split_whitespace() { - clean = clean.replace(part, " "); - } - } - - // 移除工作室前缀 - clean = RE_BRACKET.replace(&clean, "").trim().to_string(); - - let mut tags = Vec::new(); - let mut seen = HashSet::new(); - - for cap in RE_WORD.find_iter(&clean) { - let word = cap.as_str().trim(); - if word.len() < 2 { continue; } - let word_lower = word.to_lowercase(); - if STOP_SET.contains(word_lower.as_str()) { continue; } - if word.chars().all(|c| c.is_ascii_digit() || c == '.') { continue; } - if !seen.insert(word_lower.clone()) { continue; } - - let tag_type = if RE_CHINESE.is_match(word) { "chinese" } else { "english" }; - tags.push((word.to_lowercase(), tag_type.to_string())); - } - - tags - } - - /// 判断是否为排除名 - pub fn is_excluded(name: &str) -> bool { - EXCLUDED_LOWER.contains(&name.to_lowercase()) - } -} - -// ============================================================ -// 数据模型 -// ============================================================ - -#[derive(Debug, Default)] -struct MigrationStats { - persons_imported: i64, - persons_skipped: i64, - persons_created: i64, - aliases_imported: i64, - albums_processed: i64, - albums_missing: i64, - tags_imported: i64, - person_albums_created: i64, -} - -#[derive(Debug, Clone)] -struct OldAlbum { - id: String, - title: String, - image_count: i64, - person_name_original: Option, - person_name_cleaned: Option, - tags: Option, - all_words: Option, -} - -#[derive(Debug, Clone)] -struct OldPerson { - id: i64, - name: String, - display_name: Option, - total_albums: i64, - total_images: i64, - original_names: Option, -} - -// ============================================================ -// 迁移器 -// ============================================================ - -struct Migrator { - old_pool: SqlitePool, - new_pool: SqlitePool, - stats: MigrationStats, -} - -impl Migrator { - async fn new(old_db_path: &str, new_db_path: &str) -> Result { - if !Path::new(old_db_path).exists() { - return Err(format!("老数据库不存在: {}", old_db_path)); - } - - // 创建新数据库目录 - if let Some(parent) = Path::new(new_db_path).parent() { - if !parent.as_os_str().is_empty() && !parent.exists() { - std::fs::create_dir_all(parent).map_err(|e| format!("创建目录失败: {}", e))?; - } - } - - println!("[连接] 老数据库: {}", old_db_path); - let old_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect(&format!("sqlite:{}", old_db_path)) - .await - .map_err(|e| format!("连接老数据库失败: {}", e))?; - - println!("[连接] 新数据库: {}", new_db_path); - let new_pool = SqlitePoolOptions::new() - .max_connections(5) - .connect(&format!("sqlite:{}", new_db_path)) - .await - .map_err(|e| format!("连接新数据库失败: {}", e))?; - - // SQLite 性能优化 - for p in [&new_pool] { - sqlx::query("PRAGMA journal_mode=WAL;").execute(p).await.ok(); - sqlx::query("PRAGMA synchronous=NORMAL;").execute(p).await.ok(); - sqlx::query("PRAGMA foreign_keys=ON;").execute(p).await.ok(); - } - - Ok(Self { old_pool, new_pool, stats: MigrationStats::default() }) - } - - async fn init_new_schema(&self) -> Result<(), String> { - println!("[Schema] 初始化新数据库表结构..."); - - let schema_sql = include_str!("../migrations/001_initial.sql"); - for stmt in schema_sql.split(';') { - let stmt = stmt.trim(); - if !stmt.is_empty() { - sqlx::query(stmt).execute(&self.new_pool).await - .map_err(|e| format!("执行 Schema 失败: {}", e))?; - } - } - - let schema_sql2 = include_str!("../migrations/002_metadata.sql"); - for stmt in schema_sql2.split(';') { - let stmt = stmt.trim(); - if !stmt.is_empty() { - sqlx::query(stmt).execute(&self.new_pool).await - .map_err(|e| format!("执行 Schema2 失败: {}", e))?; - } - } - - println!("[Schema] 表结构初始化完成"); - Ok(()) - } - - async fn read_old_albums(&self) -> Result, String> { - println!("[读取] 从老数据库读取相册..."); - let rows = sqlx::query( - "SELECT id, title, image_count, person_name_original, person_name_cleaned, tags, all_words - FROM albums ORDER BY image_count DESC" - ).fetch_all(&self.old_pool).await.map_err(|e| format!("读取相册失败: {}", e))?; - - let albums: Vec = rows.iter().map(|r| OldAlbum { - id: r.get("id"), - title: r.get("title"), - image_count: r.get::, _>("image_count").unwrap_or(0), - person_name_original: r.get("person_name_original"), - person_name_cleaned: r.get("person_name_cleaned"), - tags: r.get("tags"), - all_words: r.get("all_words"), - }).collect(); - - println!("[读取] 共读取 {} 个相册", albums.len()); - Ok(albums) - } - - async fn read_old_persons(&self) -> Result, String> { - println!("[读取] 从老数据库读取人物..."); - let rows = sqlx::query( - "SELECT id, name, display_name, total_albums, total_images, original_names - FROM persons ORDER BY total_albums DESC" - ).fetch_all(&self.old_pool).await.map_err(|e| format!("读取人物失败: {}", e))?; - - let persons: Vec = rows.iter().map(|r| OldPerson { - id: r.get("id"), - name: r.get("name"), - display_name: r.get("display_name"), - total_albums: r.get::, _>("total_albums").unwrap_or(0), - total_images: r.get::, _>("total_images").unwrap_or(0), - original_names: r.get("original_names"), - }).collect(); - - println!("[读取] 共读取 {} 个人物", persons.len()); - Ok(persons) - } - - async fn read_old_aliases(&self) -> Result, String> { - let rows = sqlx::query( - "SELECT person_id, alias_type, alias_name FROM person_aliases" - ).fetch_all(&self.old_pool).await.unwrap_or_default(); - - let aliases: Vec<(i64, String, String)> = rows.iter().map(|r| ( - r.get("person_id"), r.get("alias_type"), r.get("alias_name") - )).collect(); - - println!("[读取] 共读取 {} 个别名", aliases.len()); - Ok(aliases) - } - - async fn read_old_person_albums(&self) -> Result, String> { - let rows = sqlx::query( - "SELECT person_id, album_id FROM person_albums" - ).fetch_all(&self.old_pool).await.unwrap_or_default(); - - let links: Vec<(i64, String)> = rows.iter().map(|r| ( - r.get("person_id"), r.get("album_id") - )).collect(); - - println!("[读取] 共读取 {} 个人物-相册关联", links.len()); - Ok(links) - } - - async fn migrate(&mut self, reparse: bool) -> Result<(), String> { - self.init_new_schema().await?; - - // 读取老数据 - let old_albums = self.read_old_albums().await?; - let old_persons = self.read_old_persons().await?; - let old_aliases = self.read_old_aliases().await?; - let old_links = self.read_old_person_albums().await?; - - let mut old_to_new_person: HashMap = HashMap::new(); - - // Step 1: 迁移人物 - println!("\n[迁移] 步骤 1/5: 迁移人物..."); - for p in &old_persons { - if title_parser::is_excluded(&p.name) { - self.stats.persons_skipped += 1; - continue; - } - - sqlx::query( - "INSERT OR IGNORE INTO persons (name, display_name, total_albums, total_images, original_names) - VALUES (?, ?, ?, ?, ?)" - ).bind(&p.name).bind(p.display_name.as_deref().unwrap_or(&p.name)) - .bind(p.total_albums).bind(p.total_images).bind(p.original_names.as_deref()) - .execute(&self.new_pool).await.ok(); - - let new_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?") - .bind(&p.name).fetch_one(&self.new_pool).await.unwrap_or(0); - - if new_id > 0 { - old_to_new_person.insert(p.id, new_id); - self.stats.persons_imported += 1; - } - } - println!("[迁移] 导入 {} 人，跳过 {} 人", self.stats.persons_imported, self.stats.persons_skipped); - - // Step 2: 迁移别名 - println!("\n[迁移] 步骤 2/5: 迁移别名..."); - for (old_pid, alias_type, alias_name) in &old_aliases { - if let Some(&new_pid) = old_to_new_person.get(old_pid) { - sqlx::query( - "INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)" - ).bind(new_pid).bind(alias_type).bind(alias_name) - .execute(&self.new_pool).await.ok(); - self.stats.aliases_imported += 1; - } - } - println!("[迁移] 导入 {} 个别名", self.stats.aliases_imported); - - // Step 3: 迁移相册 + 解析title - println!( - "\n[迁移] 步骤 3/5: 迁移相册并解析title ({})...", - if reparse { "重新解析" } else { "保留原解析" } - ); - - let mut person_map: HashMap = HashMap::new(); - - for (i, album) in old_albums.iter().enumerate() { - if (i + 1) % 200 == 0 { - println!(" 处理进度: {}/{}", i + 1, old_albums.len()); - } - - let (person_orig, person_cleaned, tags, all_words) = - if reparse || album.person_name_cleaned.as_deref().unwrap_or("").is_empty() { - // 重新解析 - let person_name = title_parser::extract_person_name(&album.title); - let (cleaned, _aliases) = match &person_name { - Some(n) => { - let (c, a) = title_parser::clean_person_name(n); - if title_parser::is_excluded(&c) { - (None, a) - } else { - (Some(c), a) - } - } - None => (None, Vec::new()), - }; - - let tags_raw = title_parser::extract_tags(&album.title, person_name.as_deref()); - let tag_names: Vec = tags_raw.iter().map(|(t, _)| t.clone()).collect(); - let all_words: Vec<[String; 2]> = tags_raw.iter() - .map(|(t, tp)| [t.clone(), tp.clone()]).collect(); - - ( - person_name, - cleaned, - serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string()), - serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()), - ) - } else { - ( - album.person_name_original.clone(), - album.person_name_cleaned.clone(), - album.tags.clone().unwrap_or_else(|| "[]".to_string()), - album.all_words.clone().unwrap_or_else(|| "[]".to_string()), - ) - }; - - // 写入相册 - sqlx::query( - "INSERT OR REPLACE INTO albums - (id, title, image_count, person_name_original, person_name_cleaned, - tags, all_words, raw_title, processed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))" - ).bind(&album.id).bind(&album.title).bind(album.image_count) - .bind(person_orig.as_deref()).bind(person_cleaned.as_deref()) - .bind(&tags).bind(&all_words).bind(&album.title) - .execute(&self.new_pool).await.ok(); - - // 写入标签索引 - if let Ok(tag_list) = serde_json::from_str::>(&tags) { - if !tag_list.is_empty() { - for tag in &tag_list { - sqlx::query( - "INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES (?, ?)" - ).bind(&album.id).bind(tag).execute(&self.new_pool).await.ok(); - } - self.stats.tags_imported += 1; - } - } - - // 关联人物 - if let Some(pc) = &person_cleaned { - if !pc.is_empty() && !title_parser::is_excluded(pc) { - let pid = if let Some(&id) = person_map.get(pc) { - id - } else { - // 查找或创建 - let existing: Option = sqlx::query_scalar( - "SELECT id FROM persons WHERE name = ?" - ).bind(pc).fetch_optional(&self.new_pool).await.unwrap_or(None); - - let id = if let Some(eid) = existing { - eid - } else { - sqlx::query("INSERT INTO persons (name, display_name) VALUES (?, ?)") - .bind(pc).bind(pc).execute(&self.new_pool).await.ok(); - self.stats.persons_created += 1; - sqlx::query_scalar::<_, i64>("SELECT last_insert_rowid()") - .fetch_one(&self.new_pool).await.unwrap_or(0) - }; - person_map.insert(pc.clone(), id); - id - }; - - if pid > 0 { - sqlx::query( - "INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)" - ).bind(pid).bind(&album.id).execute(&self.new_pool).await.ok(); - } - } - } - - self.stats.albums_processed += 1; - } - - println!( - "[迁移] 处理 {} 个相册，{} 个标签导入，{} 个人物从title创建", - self.stats.albums_processed, self.stats.tags_imported, self.stats.persons_created - ); - - // Step 4: 迁移人物-相册关联 - println!("\n[迁移] 步骤 4/5: 迁移人物-相册关联..."); - for (old_pid, album_id) in &old_links { - if let Some(&new_pid) = old_to_new_person.get(old_pid) { - sqlx::query( - "INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)" - ).bind(new_pid).bind(album_id).execute(&self.new_pool).await.ok(); - self.stats.person_albums_created += 1; - } - } - println!("[迁移] 导入 {} 个关联", self.stats.person_albums_created); - - // Step 5: 更新人物统计 - println!("\n[迁移] 步骤 5/5: 更新人物统计..."); - let person_ids: Vec = sqlx::query_scalar("SELECT id FROM persons") - .fetch_all(&self.new_pool).await.unwrap_or_default(); - - for pid in &person_ids { - sqlx::query( - "UPDATE persons SET - total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?), - total_images = ( - SELECT COALESCE(SUM(a.image_count), 0) - FROM albums a JOIN person_albums pa ON a.id = pa.album_id - WHERE pa.person_id = ? - ), - updated_at = datetime('now') - WHERE id = ?" - ).bind(pid).bind(pid).bind(pid).execute(&self.new_pool).await.ok(); - } - println!("[迁移] 更新 {} 个人物统计", person_ids.len()); - - Ok(()) - } - - async fn print_stats(&self) { - let stats = self.get_new_stats().await; - println!("\n 新数据库统计:"); - println!(" 相册数: {}", stats.total_albums); - println!(" 图片数: {}", stats.total_images); - println!(" 人物数: {}", stats.total_persons); - println!(" 别名数: {}", stats.total_aliases); - println!(" 标签数: {}", stats.unique_tags); - println!(" 有标签相册: {}", stats.albums_with_tags); - println!(" 有人物相册: {}", stats.albums_with_person); - } - - async fn get_new_stats(&self) -> NewDbStats { - NewDbStats { - total_albums: sqlx::query_scalar("SELECT COUNT(*) FROM albums") - .fetch_one(&self.new_pool).await.unwrap_or(0), - total_images: sqlx::query_scalar("SELECT COALESCE(SUM(image_count), 0) FROM albums") - .fetch_one(&self.new_pool).await.unwrap_or(0), - total_persons: sqlx::query_scalar("SELECT COUNT(*) FROM persons") - .fetch_one(&self.new_pool).await.unwrap_or(0), - total_aliases: sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases") - .fetch_one(&self.new_pool).await.unwrap_or(0), - unique_tags: sqlx::query_scalar("SELECT COUNT(DISTINCT tag) FROM album_tags") - .fetch_one(&self.new_pool).await.unwrap_or(0), - albums_with_tags: sqlx::query_scalar( - "SELECT COUNT(*) FROM albums WHERE tags IS NOT NULL AND tags != '[]'" - ).fetch_one(&self.new_pool).await.unwrap_or(0), - albums_with_person: sqlx::query_scalar( - "SELECT COUNT(*) FROM albums WHERE person_name_cleaned IS NOT NULL AND person_name_cleaned != ''" - ).fetch_one(&self.new_pool).await.unwrap_or(0), - } - } -} - -#[derive(Debug)] -struct NewDbStats { - total_albums: i64, - total_images: i64, - total_persons: i64, - total_aliases: i64, - unique_tags: i64, - albums_with_tags: i64, - albums_with_person: i64, -} - -// ============================================================ -// 搜索验证器 -// ============================================================ - -struct SearchVerifier { - pool: SqlitePool, -} - -impl SearchVerifier { - async fn new(db_path: &str) -> Result { - let pool = SqlitePoolOptions::new() - .max_connections(2) - .connect(&format!("sqlite:{}", db_path)) - .await - .map_err(|e| format!("连接数据库失败: {}", e))?; - Ok(Self { pool }) - } - - async fn verify_all(&self) { - println!("\n{}", "=".repeat(60)); - println!(" 🔍 搜索功能验证 (Rust 实现)"); - println!("{}", "=".repeat(60)); - - // 1. 热门人物 - println!("\n[验证1] 热门人物 (Top 10):"); - let top_persons: Vec<(String, i64, i64)> = sqlx::query( - "SELECT name, total_albums, total_images FROM persons ORDER BY total_albums DESC LIMIT 10" - ).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("name"), r.get("total_albums"), r.get("total_images"))).collect(); - - for (i, (name, albums, images)) in top_persons.iter().enumerate() { - println!(" {:2d}. {:<25} {:4} 相册, {:5} 张图片", i + 1, name, albums, images); - } - - // 2. 热门标签 - println!("\n[验证2] 热门标签 (Top 20):"); - let top_tags: Vec<(String, i64, i64)> = sqlx::query( - "SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt - FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT 20" - ).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("tag"), r.get::("cnt"), r.get::("album_cnt"))).collect(); - - for (i, (tag, cnt, album_cnt)) in top_tags.iter().enumerate() { - println!(" {:2d}. {:<20} 出现 {:4} 次, {:4} 个相册", i + 1, tag, cnt, album_cnt); - } - - // 3. 人物搜索 - if let Some((test_person, _, _)) = top_persons.first() { - println!("\n[验证3] 人物搜索: \"{}\"", test_person); - let like = format!("%{}%", test_person); - let results: Vec<(String, i64)> = sqlx::query( - "SELECT DISTINCT p.name, p.total_albums FROM persons p - LEFT JOIN person_aliases pa ON pa.person_id = p.id - WHERE p.name LIKE ? OR pa.alias_name LIKE ? - ORDER BY p.total_albums DESC LIMIT 5" - ).bind(&like).bind(&like).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("name"), r.get("total_albums"))).collect(); - - for (name, albums) in &results { - println!(" - {} ({} 相册)", name, albums); - } - - // 4. 人物相册 - println!("\n[验证4] 人物相册: \"{}\" (前5个)", test_person); - let person_albums: Vec<(i64, String)> = sqlx::query( - "SELECT a.image_count, a.title FROM albums a - JOIN person_albums pa ON pa.album_id = a.id - JOIN persons p ON p.id = pa.person_id - WHERE p.name = ? ORDER BY a.image_count DESC LIMIT 5" - ).bind(test_person).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("image_count"), r.get("title"))).collect(); - - for (ic, title) in &person_albums { - let short: String = title.chars().take(60).collect(); - println!(" - [{:3}张] {}", ic, short); - } - } - - // 5. 标签搜索 - if let Some((test_tag, _, _)) = top_tags.first() { - println!("\n[验证5] 标签搜索: \"{}\" (前5个)", test_tag); - let tag_albums: Vec<(i64, String)> = sqlx::query( - "SELECT a.image_count, a.title FROM albums a - JOIN album_tags t ON t.album_id = a.id - WHERE t.tag = ? ORDER BY a.image_count DESC LIMIT 5" - ).bind(test_tag).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("image_count"), r.get("title"))).collect(); - - for (ic, title) in &tag_albums { - let short: String = title.chars().take(60).collect(); - println!(" - [{:3}张] {}", ic, short); - } - } - - // 6. 标题搜索 - println!("\n[验证6] 标题搜索: \"性感\" (前5个)"); - let like = "%性感%".to_string(); - let title_albums: Vec<(i64, String)> = sqlx::query( - "SELECT image_count, title FROM albums WHERE title LIKE ? ORDER BY image_count DESC LIMIT 5" - ).bind(&like).fetch_all(&self.pool).await.unwrap_or_default() - .iter().map(|r| (r.get("image_count"), r.get("title"))).collect(); - - for (ic, title) in &title_albums { - let short: String = title.chars().take(60).collect(); - println!(" - [{:3}张] {}", ic, short); - } - - println!("\n{}", "=".repeat(60)); - println!(" ✅ 搜索功能验证完成！"); - println!("{}", "=".repeat(60)); - } -} - -// ============================================================ -// CLI 入口 -// ============================================================ - -#[derive(Debug, clap::Parser)] -#[command(name = "migrate", about = "数据库迁移工具 (Rust 实现)")] -struct Cli { - /// 老数据库路径 - #[arg(long, default_value = "../yolov8_face_recognition/persons.db")] - old_db: String, - - /// 新数据库路径 - #[arg(long, default_value = "./data/imager.db")] - new_db: String, - - /// 重新解析所有 title - #[arg(long)] - reparse: bool, - - /// 迁移后验证搜索功能 - #[arg(long)] - verify: bool, - - /// 仅验证搜索功能 (不执行迁移) - #[arg(long)] - verify_only: bool, -} - -#[tokio::main] -async fn main() -> Result<(), String> { - // 检查是否有 clap - let args: Vec = std::env::args().collect(); - - let mut old_db = "../yolov8_face_recognition/persons.db".to_string(); - let mut new_db = "./data/imager.db".to_string(); - let mut reparse = false; - let mut verify = false; - let mut verify_only = false; - - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--old-db" => { i += 1; if i < args.len() { old_db = args[i].clone(); } } - "--new-db" => { i += 1; if i < args.len() { new_db = args[i].clone(); } } - "--reparse" => reparse = true, - "--verify" => verify = true, - "--verify-only" => verify_only = true, - "--help" | "-h" => { - println!("数据库迁移工具 (Rust 实现)"); - println!(); - println!("用法: cargo run --bin migrate -- [选项]"); - println!(); - println!("选项:"); - println!(" --old-db <路径> 老数据库路径 (默认: ../yolov8_face_recognition/persons.db)"); - println!(" --new-db <路径> 新数据库路径 (默认: ./data/imager.db)"); - println!(" --reparse 重新解析所有 title"); - println!(" --verify 迁移后验证搜索功能"); - println!(" --verify-only 仅验证搜索功能 (不执行迁移)"); - println!(" --help, -h 显示帮助"); - return Ok(()); - } - _ => {} - } - i += 1; - } - - let start = Instant::now(); - - // 仅验证模式 - if verify_only { - if !Path::new(&new_db).exists() { - return Err(format!("数据库不存在: {}", new_db)); - } - let verifier = SearchVerifier::new(&new_db).await?; - verifier.verify_all().await; - return Ok(()); - } - - println!("{}", "=".repeat(60)); - println!(" 🚀 数据库迁移工具 (Rust 原生实现)"); - println!("{}", "=".repeat(60)); - - // 执行迁移 - let mut migrator = Migrator::new(&old_db, &new_db).await?; - migrator.migrate(reparse).await?; - - println!("\n{}", "=".repeat(60)); - println!(" ✅ 迁移完成！用时: {:.2?}", start.elapsed()); - println!("{}", "=".repeat(60)); - migrator.print_stats().await; - - // 验证搜索 - if verify { - let verifier = SearchVerifier::new(&new_db).await?; - verifier.verify_all().await; - } - - Ok(()) -} diff --git a/src/config.rs b/src/config.rs index 809ff69..28e6997 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,12 +48,18 @@ pub struct OldDbConfig { pub path: String, #[serde(default = "default_auto_migrate")] pub auto_migrate: bool, + #[serde(default = "default_auto_parse")] + pub auto_parse: bool, } fn default_auto_migrate() -> bool { false } +fn default_auto_parse() -> bool { + true +} + impl Config { pub fn load() -> Result { let config = config::Config::builder() diff --git a/src/db/mod.rs b/src/db/mod.rs index 937d9e3..b269bef 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -8,8 +8,8 @@ pub mod metadata; pub mod migration; pub use connection::Database; -pub use album::*; -pub use image::*; -pub use favorite::*; -pub use metadata::*; +#[allow(unused)] pub use album::*; +#[allow(unused)] pub use image::*; +#[allow(unused)] pub use favorite::*; +#[allow(unused)] pub use metadata::*; pub use migration::MigrationStats; \ No newline at end of file diff --git a/src/engine/crawler.rs b/src/engine/crawler.rs index ee15a8d..343e941 100644 --- a/src/engine/crawler.rs +++ b/src/engine/crawler.rs @@ -77,7 +77,7 @@ impl Crawler { let resp = self.client.get(&url).send().await?; let data: serde_json::Value = resp.json().await?; - let albums: Vec = if let Some(arr) = data.as_array() { + let albums: Vec = if let Some(_arr) = data.as_array() { serde_json::from_value(data)? } else if let Some(arr) = data.get("albums").and_then(|v: &serde_json::Value| v.as_array()) { serde_json::from_value(serde_json::Value::Array(arr.clone()))? diff --git a/src/engine/mod.rs b/src/engine/mod.rs index b1714a9..7d18a2e 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -7,9 +7,9 @@ pub mod recommend; pub mod pipeline; pub mod crawler; -pub use title_parser::*; -pub use person_merger::*; -pub use tag_extractor::*; +#[allow(unused)] pub use title_parser::*; +#[allow(unused)] pub use person_merger::*; +#[allow(unused)] pub use tag_extractor::*; pub use recommend::*; -pub use pipeline::*; -pub use crawler::*; \ No newline at end of file +#[allow(unused)] pub use pipeline::*; +#[allow(unused)] pub use crawler::*; \ No newline at end of file diff --git a/src/engine/person_merger.rs b/src/engine/person_merger.rs index 16e8cb8..73d38bb 100644 --- a/src/engine/person_merger.rs +++ b/src/engine/person_merger.rs @@ -298,7 +298,7 @@ pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats let title = album.title.as_deref().unwrap_or(""); let current = album.person_name_original.as_deref().unwrap_or(""); - if let Some((text, person, source)) = match_title(title, patterns) { + if let Some((_text, person, source)) = match_title(title, patterns) { *stats.by_source.entry(source).or_insert(0) += 1; *stats.name_frequency.entry(person.clone()).or_insert(0) += 1; diff --git a/src/main.rs b/src/main.rs index a26aca6..4201a26 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,27 +47,29 @@ async fn main() -> std::io::Result<()> { } }; - // 自动迁移老数据库 - if let Some(ref old_db_cfg) = config.old_db { - if old_db_cfg.auto_migrate { - info!("[迁移] 检测到 old_db 配置，自动迁移: {}", old_db_cfg.path); - match db.migrate_from_old_db(&old_db_cfg.path).await { - Ok(stats) => info!("[迁移] {}", db.migration_summary(&stats)), - Err(e) => error!("[迁移] 失败: {}", e), - } - } - } - let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone())); - // 后台导入本地相册 + // 后台导入本地相册 + 自动解析title let config_clone = config.clone(); let db_clone = Arc::clone(&db); + let auto_parse = true; tokio::spawn(async move { info!("[后台任务] 开始扫描图片目录并导入新相册..."); - match services::import::import_existing_data(&config_clone, db_clone, None).await { - Ok(_) => info!("[后台任务] 相册导入完成"), - Err(e) => error!("[后台任务] 相册导入失败: {}", e), + let import_ok = match services::import::import_existing_data(&config_clone, Arc::clone(&db_clone), None).await { + Ok(_) => { + info!("[后台任务] 相册导入完成"); + true + } + Err(e) => { + error!("[后台任务] 相册导入失败: {}", e); + false + } + }; + + if import_ok && auto_parse { + info!("[后台任务] 开始解析相册 title（提取人名 + 标签）..."); + let (persons, tags) = engine::pipeline::process_all_albums(&db_clone).await; + info!("[后台任务] Title 解析完成：{} 个人名，{} 个标签", persons, tags); } }); diff --git a/src/routes/migration.rs b/src/routes/migration.rs index 695a68a..50f48f4 100644 --- a/src/routes/migration.rs +++ b/src/routes/migration.rs @@ -1,5 +1,5 @@ use crate::db::Database; -use crate::db::MigrationStats; +#[allow(unused)] use crate::db::MigrationStats; use actix_web::{HttpResponse, Responder, web}; use std::collections::HashMap; use std::sync::Arc; diff --git a/src/routes/recommend.rs b/src/routes/recommend.rs index d494362..0ec5c5c 100644 --- a/src/routes/recommend.rs +++ b/src/routes/recommend.rs @@ -1,6 +1,6 @@ use crate::db::Database; use crate::engine::RecommendEngine; -use crate::models::*; +#[allow(unused)] use crate::models::*; use actix_web::{HttpResponse, Responder, web}; use std::collections::HashMap; use std::sync::Arc; diff --git a/src/services/mod.rs b/src/services/mod.rs index e1d73b4..0da4431 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -2,4 +2,4 @@ pub mod import; -pub use import::*; \ No newline at end of file +#[allow(unused)] pub use import::*; \ No newline at end of file