update
This commit is contained in:
872
src/bin/migrate.rs
Normal file
872
src/bin/migrate.rs
Normal file
@@ -0,0 +1,872 @@
|
||||
//! 数据库迁移 CLI 工具
|
||||
//!
|
||||
//! 功能:
|
||||
//! 1. 从老数据库 (persons.db) 读取数据
|
||||
//! 2. 对 title 进行解析 (人名提取、标签提取)
|
||||
//! 3. 生成新的数据库
|
||||
//! 4. 验证搜索等功能
|
||||
//!
|
||||
//! 用法:
|
||||
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db
|
||||
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db --reparse
|
||||
//! cargo run --bin migrate -- --verify --new-db ./data/imager.db
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
// ============================================================
|
||||
// 标题解析模块 (Rust 原生实现)
|
||||
// ============================================================
|
||||
|
||||
mod title_parser {
|
||||
use regex::Regex;
|
||||
use std::collections::HashSet;
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
/// 模特称呼前缀
|
||||
const MODEL_PREFIXES: &[&str] = &[
|
||||
"美女模特", "性感女神", "新人模特", "新人嫩模", "新人妹子",
|
||||
"气质女神", "气质美女", "性感美女", "宅男女神", "丽质女神",
|
||||
"丽质美女", "红粉佳人", "美乳女神", "清纯女神", "性感嫩模",
|
||||
"性感御姐", "美女", "模特", "女神", "御姐", "嫩模",
|
||||
"新晋女神", "新晋美模", "日本", "日本人气",
|
||||
];
|
||||
|
||||
/// 停用词
|
||||
const STOP_WORDS: &[&str] = &[
|
||||
"photos", "photo", "phot", "images", "image", "video", "videos",
|
||||
"picture", "pictures", "pic", "gif", "gifs", "mp4", "webp",
|
||||
"cosplay", "cos", "set", "part", "collection", "series",
|
||||
"special", "vol", "no", "no.", "ver", "version", "edition",
|
||||
"new", "xxx", "preview", "teaser", "sample",
|
||||
"写真", "套图", "旅拍", "系列", "诱惑", "性感", "美女",
|
||||
"模特", "女神", "写真集", "首套", "版", "主题", "风格",
|
||||
"the", "a", "an", "in", "on", "at", "of", "to", "for",
|
||||
"with", "by", "and", "or", "but", "from", "is", "are",
|
||||
"this", "that", "these", "those", "it", "its",
|
||||
"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
|
||||
"一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
|
||||
"没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "些",
|
||||
];
|
||||
|
||||
/// 工作室排除名
|
||||
const EXCLUDED_NAMES: &[&str] = &[
|
||||
"xiuren", "秀人", "秀人网", "youmi", "尤蜜", "尤蜜荟",
|
||||
"xingyan", "星颜", "魅妍社", "mistar",
|
||||
"mygirl", "美媛馆", "imiss", "爱蜜社", "huayang", "花漾",
|
||||
"bololi", "波萝社", "feilin", "嗲囡囡", "qingdouke", "青豆客",
|
||||
"toutiao", "头条女神", "xiaoyu", "语画界", "youmei", "尤美",
|
||||
"ugirls", "尤果", "尤果网", "mfstar", "模范学院",
|
||||
"tgod", "推女神", "dgkirl", "御女郎", "candy", "糖果画报",
|
||||
"wanibooks", "bluecake", "pure media", "puremedia",
|
||||
];
|
||||
|
||||
lazy_static! {
|
||||
static ref RE_PHOTOS: Regex = Regex::new(
|
||||
r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$"
|
||||
).unwrap();
|
||||
static ref RE_COSPLAY: Regex = Regex::new(
|
||||
r"(?i)^(.+?)\s+cosplay\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
|
||||
).unwrap();
|
||||
static ref RE_COSPLAY_BY: Regex = Regex::new(
|
||||
r"(?i)^(.+?)\s+cosplay\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
|
||||
).unwrap();
|
||||
static ref RE_BRACKET: Regex = Regex::new(r"^[\s]*[\[【][^\]]*[\]】][\s]*").unwrap();
|
||||
static ref RE_NUMBER_PREFIX: Regex = Regex::new(
|
||||
r"(?i)^(?:No\.?\s*\d+|VOL\.?\s*\d+)\s+"
|
||||
).unwrap();
|
||||
static ref RE_SPLIT: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap();
|
||||
static ref RE_PAREN: Regex = Regex::new(r"[((]([^))]*)[))]").unwrap();
|
||||
static ref RE_TRAILING: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
|
||||
static ref RE_EN_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
|
||||
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
||||
static ref RE_WORD: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
|
||||
|
||||
static ref STOP_SET: HashSet<&'static str> = {
|
||||
let mut s = HashSet::new();
|
||||
for w in STOP_WORDS { s.insert(*w); }
|
||||
s
|
||||
};
|
||||
|
||||
static ref EXCLUDED_LOWER: HashSet<String> = {
|
||||
let mut s = HashSet::new();
|
||||
for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); }
|
||||
s
|
||||
};
|
||||
}
|
||||
|
||||
/// 清理标题末尾
|
||||
pub fn clean_title_suffix(title: &str) -> String {
|
||||
RE_PHOTOS.replace(title, "").trim().to_string()
|
||||
}
|
||||
|
||||
/// 从标题提取人名
|
||||
pub fn extract_person_name(title: &str) -> Option<String> {
|
||||
let clean = clean_title_suffix(title);
|
||||
if clean.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// cosplay 格式
|
||||
if let Some(caps) = RE_COSPLAY_BY.captures(&clean) {
|
||||
return Some(caps[2].trim().to_string());
|
||||
}
|
||||
if let Some(caps) = RE_COSPLAY.captures(&clean) {
|
||||
return Some(caps[1].trim().to_string());
|
||||
}
|
||||
|
||||
// 移除工作室前缀
|
||||
let no_studio = RE_BRACKET.replace(&clean, "").trim().to_string();
|
||||
|
||||
// 移除模特称呼前缀
|
||||
let mut no_prefix = no_studio;
|
||||
for prefix in MODEL_PREFIXES {
|
||||
if no_prefix.starts_with(prefix) {
|
||||
no_prefix = no_prefix[prefix.len()..].trim().to_string();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 移除编号前缀
|
||||
let no_prefix = RE_NUMBER_PREFIX.replace(&no_prefix, "").trim().to_string();
|
||||
|
||||
// 用分隔符拆分
|
||||
let parts: Vec<&str> = RE_SPLIT.splitn(&no_prefix, 2).collect();
|
||||
if parts.len() >= 2 {
|
||||
let name = parts[0].trim();
|
||||
let name = RE_PHOTOS.replace(name, "").trim().to_string();
|
||||
if !name.is_empty() && name.len() >= 2 {
|
||||
return Some(name);
|
||||
}
|
||||
}
|
||||
|
||||
// 没有分隔符
|
||||
let name = RE_PHOTOS.replace(&no_prefix, "").trim().to_string();
|
||||
if name.len() >= 2 {
|
||||
Some(name)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 清洗人名
|
||||
pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
|
||||
let mut aliases = Vec::new();
|
||||
let mut name = raw_name.to_string();
|
||||
|
||||
// 移除前缀
|
||||
for prefix in MODEL_PREFIXES {
|
||||
if name.starts_with(prefix) {
|
||||
aliases.push(("prefix".to_string(), prefix.to_string()));
|
||||
name = name[prefix.len()..].trim().to_string();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 移除括号内容
|
||||
let mut cleaned = String::new();
|
||||
let mut last_end = 0;
|
||||
for cap in RE_PAREN.captures_iter(&name) {
|
||||
let m = cap.get(0).unwrap();
|
||||
let content = cap.get(1).map_or("", |c| c.as_str());
|
||||
cleaned.push_str(&name[last_end..m.start()]);
|
||||
last_end = m.end();
|
||||
if !content.is_empty() {
|
||||
if RE_CHINESE.is_match(content) {
|
||||
aliases.push(("nickname".to_string(), content.to_string()));
|
||||
} else {
|
||||
aliases.push(("english".to_string(), content.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
cleaned.push_str(&name[last_end..]);
|
||||
let name = cleaned.trim().to_string();
|
||||
|
||||
// 清理末尾标点
|
||||
let name = RE_TRAILING.replace(&name, "").trim().to_string();
|
||||
|
||||
// 提取英文名别名
|
||||
if let Some(caps) = RE_EN_SUFFIX.captures(&name) {
|
||||
let chinese = caps[1].to_string();
|
||||
let english = caps[2].trim().to_string();
|
||||
if !english.is_empty() {
|
||||
aliases.push(("chinese".to_string(), chinese));
|
||||
aliases.push(("english".to_string(), english));
|
||||
}
|
||||
}
|
||||
|
||||
let name = if name.is_empty() { raw_name.to_string() } else { name };
|
||||
(name, aliases)
|
||||
}
|
||||
|
||||
/// 从标题提取标签
|
||||
pub fn extract_tags(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
|
||||
let mut clean = clean_title_suffix(title);
|
||||
|
||||
// 移除人名
|
||||
if let Some(name) = person_name {
|
||||
clean = clean.replace(name, " ");
|
||||
for part in name.split_whitespace() {
|
||||
clean = clean.replace(part, " ");
|
||||
}
|
||||
}
|
||||
|
||||
// 移除工作室前缀
|
||||
clean = RE_BRACKET.replace(&clean, "").trim().to_string();
|
||||
|
||||
let mut tags = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
|
||||
for cap in RE_WORD.find_iter(&clean) {
|
||||
let word = cap.as_str().trim();
|
||||
if word.len() < 2 { continue; }
|
||||
let word_lower = word.to_lowercase();
|
||||
if STOP_SET.contains(word_lower.as_str()) { continue; }
|
||||
if word.chars().all(|c| c.is_ascii_digit() || c == '.') { continue; }
|
||||
if !seen.insert(word_lower.clone()) { continue; }
|
||||
|
||||
let tag_type = if RE_CHINESE.is_match(word) { "chinese" } else { "english" };
|
||||
tags.push((word.to_lowercase(), tag_type.to_string()));
|
||||
}
|
||||
|
||||
tags
|
||||
}
|
||||
|
||||
/// 判断是否为排除名
|
||||
pub fn is_excluded(name: &str) -> bool {
|
||||
EXCLUDED_LOWER.contains(&name.to_lowercase())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 数据模型
|
||||
// ============================================================
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct MigrationStats {
|
||||
persons_imported: i64,
|
||||
persons_skipped: i64,
|
||||
persons_created: i64,
|
||||
aliases_imported: i64,
|
||||
albums_processed: i64,
|
||||
albums_missing: i64,
|
||||
tags_imported: i64,
|
||||
person_albums_created: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct OldAlbum {
|
||||
id: String,
|
||||
title: String,
|
||||
image_count: i64,
|
||||
person_name_original: Option<String>,
|
||||
person_name_cleaned: Option<String>,
|
||||
tags: Option<String>,
|
||||
all_words: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct OldPerson {
|
||||
id: i64,
|
||||
name: String,
|
||||
display_name: Option<String>,
|
||||
total_albums: i64,
|
||||
total_images: i64,
|
||||
original_names: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 迁移器
|
||||
// ============================================================
|
||||
|
||||
struct Migrator {
|
||||
old_pool: SqlitePool,
|
||||
new_pool: SqlitePool,
|
||||
stats: MigrationStats,
|
||||
}
|
||||
|
||||
impl Migrator {
|
||||
async fn new(old_db_path: &str, new_db_path: &str) -> Result<Self, String> {
|
||||
if !Path::new(old_db_path).exists() {
|
||||
return Err(format!("老数据库不存在: {}", old_db_path));
|
||||
}
|
||||
|
||||
// 创建新数据库目录
|
||||
if let Some(parent) = Path::new(new_db_path).parent() {
|
||||
if !parent.as_os_str().is_empty() && !parent.exists() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("创建目录失败: {}", e))?;
|
||||
}
|
||||
}
|
||||
|
||||
println!("[连接] 老数据库: {}", old_db_path);
|
||||
let old_pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&format!("sqlite:{}", old_db_path))
|
||||
.await
|
||||
.map_err(|e| format!("连接老数据库失败: {}", e))?;
|
||||
|
||||
println!("[连接] 新数据库: {}", new_db_path);
|
||||
let new_pool = SqlitePoolOptions::new()
|
||||
.max_connections(5)
|
||||
.connect(&format!("sqlite:{}", new_db_path))
|
||||
.await
|
||||
.map_err(|e| format!("连接新数据库失败: {}", e))?;
|
||||
|
||||
// SQLite 性能优化
|
||||
for p in [&new_pool] {
|
||||
sqlx::query("PRAGMA journal_mode=WAL;").execute(p).await.ok();
|
||||
sqlx::query("PRAGMA synchronous=NORMAL;").execute(p).await.ok();
|
||||
sqlx::query("PRAGMA foreign_keys=ON;").execute(p).await.ok();
|
||||
}
|
||||
|
||||
Ok(Self { old_pool, new_pool, stats: MigrationStats::default() })
|
||||
}
|
||||
|
||||
async fn init_new_schema(&self) -> Result<(), String> {
|
||||
println!("[Schema] 初始化新数据库表结构...");
|
||||
|
||||
let schema_sql = include_str!("../migrations/001_initial.sql");
|
||||
for stmt in schema_sql.split(';') {
|
||||
let stmt = stmt.trim();
|
||||
if !stmt.is_empty() {
|
||||
sqlx::query(stmt).execute(&self.new_pool).await
|
||||
.map_err(|e| format!("执行 Schema 失败: {}", e))?;
|
||||
}
|
||||
}
|
||||
|
||||
let schema_sql2 = include_str!("../migrations/002_metadata.sql");
|
||||
for stmt in schema_sql2.split(';') {
|
||||
let stmt = stmt.trim();
|
||||
if !stmt.is_empty() {
|
||||
sqlx::query(stmt).execute(&self.new_pool).await
|
||||
.map_err(|e| format!("执行 Schema2 失败: {}", e))?;
|
||||
}
|
||||
}
|
||||
|
||||
println!("[Schema] 表结构初始化完成");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_old_albums(&self) -> Result<Vec<OldAlbum>, String> {
|
||||
println!("[读取] 从老数据库读取相册...");
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, title, image_count, person_name_original, person_name_cleaned, tags, all_words
|
||||
FROM albums ORDER BY image_count DESC"
|
||||
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取相册失败: {}", e))?;
|
||||
|
||||
let albums: Vec<OldAlbum> = rows.iter().map(|r| OldAlbum {
|
||||
id: r.get("id"),
|
||||
title: r.get("title"),
|
||||
image_count: r.get::<Option<i64>, _>("image_count").unwrap_or(0),
|
||||
person_name_original: r.get("person_name_original"),
|
||||
person_name_cleaned: r.get("person_name_cleaned"),
|
||||
tags: r.get("tags"),
|
||||
all_words: r.get("all_words"),
|
||||
}).collect();
|
||||
|
||||
println!("[读取] 共读取 {} 个相册", albums.len());
|
||||
Ok(albums)
|
||||
}
|
||||
|
||||
async fn read_old_persons(&self) -> Result<Vec<OldPerson>, String> {
|
||||
println!("[读取] 从老数据库读取人物...");
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, name, display_name, total_albums, total_images, original_names
|
||||
FROM persons ORDER BY total_albums DESC"
|
||||
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取人物失败: {}", e))?;
|
||||
|
||||
let persons: Vec<OldPerson> = rows.iter().map(|r| OldPerson {
|
||||
id: r.get("id"),
|
||||
name: r.get("name"),
|
||||
display_name: r.get("display_name"),
|
||||
total_albums: r.get::<Option<i64>, _>("total_albums").unwrap_or(0),
|
||||
total_images: r.get::<Option<i64>, _>("total_images").unwrap_or(0),
|
||||
original_names: r.get("original_names"),
|
||||
}).collect();
|
||||
|
||||
println!("[读取] 共读取 {} 个人物", persons.len());
|
||||
Ok(persons)
|
||||
}
|
||||
|
||||
async fn read_old_aliases(&self) -> Result<Vec<(i64, String, String)>, String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT person_id, alias_type, alias_name FROM person_aliases"
|
||||
).fetch_all(&self.old_pool).await.unwrap_or_default();
|
||||
|
||||
let aliases: Vec<(i64, String, String)> = rows.iter().map(|r| (
|
||||
r.get("person_id"), r.get("alias_type"), r.get("alias_name")
|
||||
)).collect();
|
||||
|
||||
println!("[读取] 共读取 {} 个别名", aliases.len());
|
||||
Ok(aliases)
|
||||
}
|
||||
|
||||
async fn read_old_person_albums(&self) -> Result<Vec<(i64, String)>, String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT person_id, album_id FROM person_albums"
|
||||
).fetch_all(&self.old_pool).await.unwrap_or_default();
|
||||
|
||||
let links: Vec<(i64, String)> = rows.iter().map(|r| (
|
||||
r.get("person_id"), r.get("album_id")
|
||||
)).collect();
|
||||
|
||||
println!("[读取] 共读取 {} 个人物-相册关联", links.len());
|
||||
Ok(links)
|
||||
}
|
||||
|
||||
async fn migrate(&mut self, reparse: bool) -> Result<(), String> {
|
||||
self.init_new_schema().await?;
|
||||
|
||||
// 读取老数据
|
||||
let old_albums = self.read_old_albums().await?;
|
||||
let old_persons = self.read_old_persons().await?;
|
||||
let old_aliases = self.read_old_aliases().await?;
|
||||
let old_links = self.read_old_person_albums().await?;
|
||||
|
||||
let mut old_to_new_person: HashMap<i64, i64> = HashMap::new();
|
||||
|
||||
// Step 1: 迁移人物
|
||||
println!("\n[迁移] 步骤 1/5: 迁移人物...");
|
||||
for p in &old_persons {
|
||||
if title_parser::is_excluded(&p.name) {
|
||||
self.stats.persons_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"INSERT OR IGNORE INTO persons (name, display_name, total_albums, total_images, original_names)
|
||||
VALUES (?, ?, ?, ?, ?)"
|
||||
).bind(&p.name).bind(p.display_name.as_deref().unwrap_or(&p.name))
|
||||
.bind(p.total_albums).bind(p.total_images).bind(p.original_names.as_deref())
|
||||
.execute(&self.new_pool).await.ok();
|
||||
|
||||
let new_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?")
|
||||
.bind(&p.name).fetch_one(&self.new_pool).await.unwrap_or(0);
|
||||
|
||||
if new_id > 0 {
|
||||
old_to_new_person.insert(p.id, new_id);
|
||||
self.stats.persons_imported += 1;
|
||||
}
|
||||
}
|
||||
println!("[迁移] 导入 {} 人,跳过 {} 人", self.stats.persons_imported, self.stats.persons_skipped);
|
||||
|
||||
// Step 2: 迁移别名
|
||||
println!("\n[迁移] 步骤 2/5: 迁移别名...");
|
||||
for (old_pid, alias_type, alias_name) in &old_aliases {
|
||||
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
|
||||
sqlx::query(
|
||||
"INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)"
|
||||
).bind(new_pid).bind(alias_type).bind(alias_name)
|
||||
.execute(&self.new_pool).await.ok();
|
||||
self.stats.aliases_imported += 1;
|
||||
}
|
||||
}
|
||||
println!("[迁移] 导入 {} 个别名", self.stats.aliases_imported);
|
||||
|
||||
// Step 3: 迁移相册 + 解析title
|
||||
println!(
|
||||
"\n[迁移] 步骤 3/5: 迁移相册并解析title ({})...",
|
||||
if reparse { "重新解析" } else { "保留原解析" }
|
||||
);
|
||||
|
||||
let mut person_map: HashMap<String, i64> = HashMap::new();
|
||||
|
||||
for (i, album) in old_albums.iter().enumerate() {
|
||||
if (i + 1) % 200 == 0 {
|
||||
println!(" 处理进度: {}/{}", i + 1, old_albums.len());
|
||||
}
|
||||
|
||||
let (person_orig, person_cleaned, tags, all_words) =
|
||||
if reparse || album.person_name_cleaned.as_deref().unwrap_or("").is_empty() {
|
||||
// 重新解析
|
||||
let person_name = title_parser::extract_person_name(&album.title);
|
||||
let (cleaned, _aliases) = match &person_name {
|
||||
Some(n) => {
|
||||
let (c, a) = title_parser::clean_person_name(n);
|
||||
if title_parser::is_excluded(&c) {
|
||||
(None, a)
|
||||
} else {
|
||||
(Some(c), a)
|
||||
}
|
||||
}
|
||||
None => (None, Vec::new()),
|
||||
};
|
||||
|
||||
let tags_raw = title_parser::extract_tags(&album.title, person_name.as_deref());
|
||||
let tag_names: Vec<String> = tags_raw.iter().map(|(t, _)| t.clone()).collect();
|
||||
let all_words: Vec<[String; 2]> = tags_raw.iter()
|
||||
.map(|(t, tp)| [t.clone(), tp.clone()]).collect();
|
||||
|
||||
(
|
||||
person_name,
|
||||
cleaned,
|
||||
serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string()),
|
||||
serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
album.person_name_original.clone(),
|
||||
album.person_name_cleaned.clone(),
|
||||
album.tags.clone().unwrap_or_else(|| "[]".to_string()),
|
||||
album.all_words.clone().unwrap_or_else(|| "[]".to_string()),
|
||||
)
|
||||
};
|
||||
|
||||
// 写入相册
|
||||
sqlx::query(
|
||||
"INSERT OR REPLACE INTO albums
|
||||
(id, title, image_count, person_name_original, person_name_cleaned,
|
||||
tags, all_words, raw_title, processed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))"
|
||||
).bind(&album.id).bind(&album.title).bind(album.image_count)
|
||||
.bind(person_orig.as_deref()).bind(person_cleaned.as_deref())
|
||||
.bind(&tags).bind(&all_words).bind(&album.title)
|
||||
.execute(&self.new_pool).await.ok();
|
||||
|
||||
// 写入标签索引
|
||||
if let Ok(tag_list) = serde_json::from_str::<Vec<String>>(&tags) {
|
||||
if !tag_list.is_empty() {
|
||||
for tag in &tag_list {
|
||||
sqlx::query(
|
||||
"INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES (?, ?)"
|
||||
).bind(&album.id).bind(tag).execute(&self.new_pool).await.ok();
|
||||
}
|
||||
self.stats.tags_imported += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// 关联人物
|
||||
if let Some(pc) = &person_cleaned {
|
||||
if !pc.is_empty() && !title_parser::is_excluded(pc) {
|
||||
let pid = if let Some(&id) = person_map.get(pc) {
|
||||
id
|
||||
} else {
|
||||
// 查找或创建
|
||||
let existing: Option<i64> = sqlx::query_scalar(
|
||||
"SELECT id FROM persons WHERE name = ?"
|
||||
).bind(pc).fetch_optional(&self.new_pool).await.unwrap_or(None);
|
||||
|
||||
let id = if let Some(eid) = existing {
|
||||
eid
|
||||
} else {
|
||||
sqlx::query("INSERT INTO persons (name, display_name) VALUES (?, ?)")
|
||||
.bind(pc).bind(pc).execute(&self.new_pool).await.ok();
|
||||
self.stats.persons_created += 1;
|
||||
sqlx::query_scalar::<_, i64>("SELECT last_insert_rowid()")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0)
|
||||
};
|
||||
person_map.insert(pc.clone(), id);
|
||||
id
|
||||
};
|
||||
|
||||
if pid > 0 {
|
||||
sqlx::query(
|
||||
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
|
||||
).bind(pid).bind(&album.id).execute(&self.new_pool).await.ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.stats.albums_processed += 1;
|
||||
}
|
||||
|
||||
println!(
|
||||
"[迁移] 处理 {} 个相册,{} 个标签导入,{} 个人物从title创建",
|
||||
self.stats.albums_processed, self.stats.tags_imported, self.stats.persons_created
|
||||
);
|
||||
|
||||
// Step 4: 迁移人物-相册关联
|
||||
println!("\n[迁移] 步骤 4/5: 迁移人物-相册关联...");
|
||||
for (old_pid, album_id) in &old_links {
|
||||
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
|
||||
sqlx::query(
|
||||
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
|
||||
).bind(new_pid).bind(album_id).execute(&self.new_pool).await.ok();
|
||||
self.stats.person_albums_created += 1;
|
||||
}
|
||||
}
|
||||
println!("[迁移] 导入 {} 个关联", self.stats.person_albums_created);
|
||||
|
||||
// Step 5: 更新人物统计
|
||||
println!("\n[迁移] 步骤 5/5: 更新人物统计...");
|
||||
let person_ids: Vec<i64> = sqlx::query_scalar("SELECT id FROM persons")
|
||||
.fetch_all(&self.new_pool).await.unwrap_or_default();
|
||||
|
||||
for pid in &person_ids {
|
||||
sqlx::query(
|
||||
"UPDATE persons SET
|
||||
total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?),
|
||||
total_images = (
|
||||
SELECT COALESCE(SUM(a.image_count), 0)
|
||||
FROM albums a JOIN person_albums pa ON a.id = pa.album_id
|
||||
WHERE pa.person_id = ?
|
||||
),
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ?"
|
||||
).bind(pid).bind(pid).bind(pid).execute(&self.new_pool).await.ok();
|
||||
}
|
||||
println!("[迁移] 更新 {} 个人物统计", person_ids.len());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn print_stats(&self) {
|
||||
let stats = self.get_new_stats().await;
|
||||
println!("\n 新数据库统计:");
|
||||
println!(" 相册数: {}", stats.total_albums);
|
||||
println!(" 图片数: {}", stats.total_images);
|
||||
println!(" 人物数: {}", stats.total_persons);
|
||||
println!(" 别名数: {}", stats.total_aliases);
|
||||
println!(" 标签数: {}", stats.unique_tags);
|
||||
println!(" 有标签相册: {}", stats.albums_with_tags);
|
||||
println!(" 有人物相册: {}", stats.albums_with_person);
|
||||
}
|
||||
|
||||
async fn get_new_stats(&self) -> NewDbStats {
|
||||
NewDbStats {
|
||||
total_albums: sqlx::query_scalar("SELECT COUNT(*) FROM albums")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
total_images: sqlx::query_scalar("SELECT COALESCE(SUM(image_count), 0) FROM albums")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
total_persons: sqlx::query_scalar("SELECT COUNT(*) FROM persons")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
total_aliases: sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
unique_tags: sqlx::query_scalar("SELECT COUNT(DISTINCT tag) FROM album_tags")
|
||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
albums_with_tags: sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM albums WHERE tags IS NOT NULL AND tags != '[]'"
|
||||
).fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
albums_with_person: sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM albums WHERE person_name_cleaned IS NOT NULL AND person_name_cleaned != ''"
|
||||
).fetch_one(&self.new_pool).await.unwrap_or(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct NewDbStats {
|
||||
total_albums: i64,
|
||||
total_images: i64,
|
||||
total_persons: i64,
|
||||
total_aliases: i64,
|
||||
unique_tags: i64,
|
||||
albums_with_tags: i64,
|
||||
albums_with_person: i64,
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 搜索验证器
|
||||
// ============================================================
|
||||
|
||||
struct SearchVerifier {
|
||||
pool: SqlitePool,
|
||||
}
|
||||
|
||||
impl SearchVerifier {
|
||||
async fn new(db_path: &str) -> Result<Self, String> {
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect(&format!("sqlite:{}", db_path))
|
||||
.await
|
||||
.map_err(|e| format!("连接数据库失败: {}", e))?;
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
async fn verify_all(&self) {
|
||||
println!("\n{}", "=".repeat(60));
|
||||
println!(" 🔍 搜索功能验证 (Rust 实现)");
|
||||
println!("{}", "=".repeat(60));
|
||||
|
||||
// 1. 热门人物
|
||||
println!("\n[验证1] 热门人物 (Top 10):");
|
||||
let top_persons: Vec<(String, i64, i64)> = sqlx::query(
|
||||
"SELECT name, total_albums, total_images FROM persons ORDER BY total_albums DESC LIMIT 10"
|
||||
).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("name"), r.get("total_albums"), r.get("total_images"))).collect();
|
||||
|
||||
for (i, (name, albums, images)) in top_persons.iter().enumerate() {
|
||||
println!(" {:2d}. {:<25} {:4} 相册, {:5} 张图片", i + 1, name, albums, images);
|
||||
}
|
||||
|
||||
// 2. 热门标签
|
||||
println!("\n[验证2] 热门标签 (Top 20):");
|
||||
let top_tags: Vec<(String, i64, i64)> = sqlx::query(
|
||||
"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
|
||||
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT 20"
|
||||
).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("tag"), r.get::<i64, _>("cnt"), r.get::<i64, _>("album_cnt"))).collect();
|
||||
|
||||
for (i, (tag, cnt, album_cnt)) in top_tags.iter().enumerate() {
|
||||
println!(" {:2d}. {:<20} 出现 {:4} 次, {:4} 个相册", i + 1, tag, cnt, album_cnt);
|
||||
}
|
||||
|
||||
// 3. 人物搜索
|
||||
if let Some((test_person, _, _)) = top_persons.first() {
|
||||
println!("\n[验证3] 人物搜索: \"{}\"", test_person);
|
||||
let like = format!("%{}%", test_person);
|
||||
let results: Vec<(String, i64)> = sqlx::query(
|
||||
"SELECT DISTINCT p.name, p.total_albums FROM persons p
|
||||
LEFT JOIN person_aliases pa ON pa.person_id = p.id
|
||||
WHERE p.name LIKE ? OR pa.alias_name LIKE ?
|
||||
ORDER BY p.total_albums DESC LIMIT 5"
|
||||
).bind(&like).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("name"), r.get("total_albums"))).collect();
|
||||
|
||||
for (name, albums) in &results {
|
||||
println!(" - {} ({} 相册)", name, albums);
|
||||
}
|
||||
|
||||
// 4. 人物相册
|
||||
println!("\n[验证4] 人物相册: \"{}\" (前5个)", test_person);
|
||||
let person_albums: Vec<(i64, String)> = sqlx::query(
|
||||
"SELECT a.image_count, a.title FROM albums a
|
||||
JOIN person_albums pa ON pa.album_id = a.id
|
||||
JOIN persons p ON p.id = pa.person_id
|
||||
WHERE p.name = ? ORDER BY a.image_count DESC LIMIT 5"
|
||||
).bind(test_person).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
||||
|
||||
for (ic, title) in &person_albums {
|
||||
let short: String = title.chars().take(60).collect();
|
||||
println!(" - [{:3}张] {}", ic, short);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 标签搜索
|
||||
if let Some((test_tag, _, _)) = top_tags.first() {
|
||||
println!("\n[验证5] 标签搜索: \"{}\" (前5个)", test_tag);
|
||||
let tag_albums: Vec<(i64, String)> = sqlx::query(
|
||||
"SELECT a.image_count, a.title FROM albums a
|
||||
JOIN album_tags t ON t.album_id = a.id
|
||||
WHERE t.tag = ? ORDER BY a.image_count DESC LIMIT 5"
|
||||
).bind(test_tag).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
||||
|
||||
for (ic, title) in &tag_albums {
|
||||
let short: String = title.chars().take(60).collect();
|
||||
println!(" - [{:3}张] {}", ic, short);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 标题搜索
|
||||
println!("\n[验证6] 标题搜索: \"性感\" (前5个)");
|
||||
let like = "%性感%".to_string();
|
||||
let title_albums: Vec<(i64, String)> = sqlx::query(
|
||||
"SELECT image_count, title FROM albums WHERE title LIKE ? ORDER BY image_count DESC LIMIT 5"
|
||||
).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
|
||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
||||
|
||||
for (ic, title) in &title_albums {
|
||||
let short: String = title.chars().take(60).collect();
|
||||
println!(" - [{:3}张] {}", ic, short);
|
||||
}
|
||||
|
||||
println!("\n{}", "=".repeat(60));
|
||||
println!(" ✅ 搜索功能验证完成!");
|
||||
println!("{}", "=".repeat(60));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CLI 入口
|
||||
// ============================================================
|
||||
|
||||
#[derive(Debug, clap::Parser)]
|
||||
#[command(name = "migrate", about = "数据库迁移工具 (Rust 实现)")]
|
||||
struct Cli {
|
||||
/// 老数据库路径
|
||||
#[arg(long, default_value = "../yolov8_face_recognition/persons.db")]
|
||||
old_db: String,
|
||||
|
||||
/// 新数据库路径
|
||||
#[arg(long, default_value = "./data/imager.db")]
|
||||
new_db: String,
|
||||
|
||||
/// 重新解析所有 title
|
||||
#[arg(long)]
|
||||
reparse: bool,
|
||||
|
||||
/// 迁移后验证搜索功能
|
||||
#[arg(long)]
|
||||
verify: bool,
|
||||
|
||||
/// 仅验证搜索功能 (不执行迁移)
|
||||
#[arg(long)]
|
||||
verify_only: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), String> {
|
||||
// 检查是否有 clap
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
let mut old_db = "../yolov8_face_recognition/persons.db".to_string();
|
||||
let mut new_db = "./data/imager.db".to_string();
|
||||
let mut reparse = false;
|
||||
let mut verify = false;
|
||||
let mut verify_only = false;
|
||||
|
||||
let mut i = 1;
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
"--old-db" => { i += 1; if i < args.len() { old_db = args[i].clone(); } }
|
||||
"--new-db" => { i += 1; if i < args.len() { new_db = args[i].clone(); } }
|
||||
"--reparse" => reparse = true,
|
||||
"--verify" => verify = true,
|
||||
"--verify-only" => verify_only = true,
|
||||
"--help" | "-h" => {
|
||||
println!("数据库迁移工具 (Rust 实现)");
|
||||
println!();
|
||||
println!("用法: cargo run --bin migrate -- [选项]");
|
||||
println!();
|
||||
println!("选项:");
|
||||
println!(" --old-db <路径> 老数据库路径 (默认: ../yolov8_face_recognition/persons.db)");
|
||||
println!(" --new-db <路径> 新数据库路径 (默认: ./data/imager.db)");
|
||||
println!(" --reparse 重新解析所有 title");
|
||||
println!(" --verify 迁移后验证搜索功能");
|
||||
println!(" --verify-only 仅验证搜索功能 (不执行迁移)");
|
||||
println!(" --help, -h 显示帮助");
|
||||
return Ok(());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// 仅验证模式
|
||||
if verify_only {
|
||||
if !Path::new(&new_db).exists() {
|
||||
return Err(format!("数据库不存在: {}", new_db));
|
||||
}
|
||||
let verifier = SearchVerifier::new(&new_db).await?;
|
||||
verifier.verify_all().await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("{}", "=".repeat(60));
|
||||
println!(" 🚀 数据库迁移工具 (Rust 原生实现)");
|
||||
println!("{}", "=".repeat(60));
|
||||
|
||||
// 执行迁移
|
||||
let mut migrator = Migrator::new(&old_db, &new_db).await?;
|
||||
migrator.migrate(reparse).await?;
|
||||
|
||||
println!("\n{}", "=".repeat(60));
|
||||
println!(" ✅ 迁移完成!用时: {:.2?}", start.elapsed());
|
||||
println!("{}", "=".repeat(60));
|
||||
migrator.print_stats().await;
|
||||
|
||||
// 验证搜索
|
||||
if verify {
|
||||
let verifier = SearchVerifier::new(&new_db).await?;
|
||||
verifier.verify_all().await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user