update
This commit is contained in:
@@ -1,872 +0,0 @@
|
|||||||
//! 数据库迁移 CLI 工具
|
|
||||||
//!
|
|
||||||
//! 功能:
|
|
||||||
//! 1. 从老数据库 (persons.db) 读取数据
|
|
||||||
//! 2. 对 title 进行解析 (人名提取、标签提取)
|
|
||||||
//! 3. 生成新的数据库
|
|
||||||
//! 4. 验证搜索等功能
|
|
||||||
//!
|
|
||||||
//! 用法:
|
|
||||||
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db
|
|
||||||
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db --reparse
|
|
||||||
//! cargo run --bin migrate -- --verify --new-db ./data/imager.db
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row};
|
|
||||||
use std::collections::{HashMap, HashSet};
|
|
||||||
use std::path::Path;
|
|
||||||
use std::time::Instant;
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// 标题解析模块 (Rust 原生实现)
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
mod title_parser {
|
|
||||||
use regex::Regex;
|
|
||||||
use std::collections::HashSet;
|
|
||||||
use lazy_static::lazy_static;
|
|
||||||
|
|
||||||
/// 模特称呼前缀
|
|
||||||
const MODEL_PREFIXES: &[&str] = &[
|
|
||||||
"美女模特", "性感女神", "新人模特", "新人嫩模", "新人妹子",
|
|
||||||
"气质女神", "气质美女", "性感美女", "宅男女神", "丽质女神",
|
|
||||||
"丽质美女", "红粉佳人", "美乳女神", "清纯女神", "性感嫩模",
|
|
||||||
"性感御姐", "美女", "模特", "女神", "御姐", "嫩模",
|
|
||||||
"新晋女神", "新晋美模", "日本", "日本人气",
|
|
||||||
];
|
|
||||||
|
|
||||||
/// 停用词
|
|
||||||
const STOP_WORDS: &[&str] = &[
|
|
||||||
"photos", "photo", "phot", "images", "image", "video", "videos",
|
|
||||||
"picture", "pictures", "pic", "gif", "gifs", "mp4", "webp",
|
|
||||||
"cosplay", "cos", "set", "part", "collection", "series",
|
|
||||||
"special", "vol", "no", "no.", "ver", "version", "edition",
|
|
||||||
"new", "xxx", "preview", "teaser", "sample",
|
|
||||||
"写真", "套图", "旅拍", "系列", "诱惑", "性感", "美女",
|
|
||||||
"模特", "女神", "写真集", "首套", "版", "主题", "风格",
|
|
||||||
"the", "a", "an", "in", "on", "at", "of", "to", "for",
|
|
||||||
"with", "by", "and", "or", "but", "from", "is", "are",
|
|
||||||
"this", "that", "these", "those", "it", "its",
|
|
||||||
"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
|
|
||||||
"一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
|
|
||||||
"没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "些",
|
|
||||||
];
|
|
||||||
|
|
||||||
/// 工作室排除名
|
|
||||||
const EXCLUDED_NAMES: &[&str] = &[
|
|
||||||
"xiuren", "秀人", "秀人网", "youmi", "尤蜜", "尤蜜荟",
|
|
||||||
"xingyan", "星颜", "魅妍社", "mistar",
|
|
||||||
"mygirl", "美媛馆", "imiss", "爱蜜社", "huayang", "花漾",
|
|
||||||
"bololi", "波萝社", "feilin", "嗲囡囡", "qingdouke", "青豆客",
|
|
||||||
"toutiao", "头条女神", "xiaoyu", "语画界", "youmei", "尤美",
|
|
||||||
"ugirls", "尤果", "尤果网", "mfstar", "模范学院",
|
|
||||||
"tgod", "推女神", "dgkirl", "御女郎", "candy", "糖果画报",
|
|
||||||
"wanibooks", "bluecake", "pure media", "puremedia",
|
|
||||||
];
|
|
||||||
|
|
||||||
lazy_static! {
|
|
||||||
static ref RE_PHOTOS: Regex = Regex::new(
|
|
||||||
r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$"
|
|
||||||
).unwrap();
|
|
||||||
static ref RE_COSPLAY: Regex = Regex::new(
|
|
||||||
r"(?i)^(.+?)\s+cosplay\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
|
|
||||||
).unwrap();
|
|
||||||
static ref RE_COSPLAY_BY: Regex = Regex::new(
|
|
||||||
r"(?i)^(.+?)\s+cosplay\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
|
|
||||||
).unwrap();
|
|
||||||
static ref RE_BRACKET: Regex = Regex::new(r"^[\s]*[\[【][^\]]*[\]】][\s]*").unwrap();
|
|
||||||
static ref RE_NUMBER_PREFIX: Regex = Regex::new(
|
|
||||||
r"(?i)^(?:No\.?\s*\d+|VOL\.?\s*\d+)\s+"
|
|
||||||
).unwrap();
|
|
||||||
static ref RE_SPLIT: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap();
|
|
||||||
static ref RE_PAREN: Regex = Regex::new(r"[((]([^))]*)[))]").unwrap();
|
|
||||||
static ref RE_TRAILING: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
|
|
||||||
static ref RE_EN_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
|
|
||||||
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
|
|
||||||
static ref RE_WORD: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
|
|
||||||
|
|
||||||
static ref STOP_SET: HashSet<&'static str> = {
|
|
||||||
let mut s = HashSet::new();
|
|
||||||
for w in STOP_WORDS { s.insert(*w); }
|
|
||||||
s
|
|
||||||
};
|
|
||||||
|
|
||||||
static ref EXCLUDED_LOWER: HashSet<String> = {
|
|
||||||
let mut s = HashSet::new();
|
|
||||||
for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); }
|
|
||||||
s
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 清理标题末尾
|
|
||||||
pub fn clean_title_suffix(title: &str) -> String {
|
|
||||||
RE_PHOTOS.replace(title, "").trim().to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 从标题提取人名
|
|
||||||
pub fn extract_person_name(title: &str) -> Option<String> {
|
|
||||||
let clean = clean_title_suffix(title);
|
|
||||||
if clean.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
// cosplay 格式
|
|
||||||
if let Some(caps) = RE_COSPLAY_BY.captures(&clean) {
|
|
||||||
return Some(caps[2].trim().to_string());
|
|
||||||
}
|
|
||||||
if let Some(caps) = RE_COSPLAY.captures(&clean) {
|
|
||||||
return Some(caps[1].trim().to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 移除工作室前缀
|
|
||||||
let no_studio = RE_BRACKET.replace(&clean, "").trim().to_string();
|
|
||||||
|
|
||||||
// 移除模特称呼前缀
|
|
||||||
let mut no_prefix = no_studio;
|
|
||||||
for prefix in MODEL_PREFIXES {
|
|
||||||
if no_prefix.starts_with(prefix) {
|
|
||||||
no_prefix = no_prefix[prefix.len()..].trim().to_string();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 移除编号前缀
|
|
||||||
let no_prefix = RE_NUMBER_PREFIX.replace(&no_prefix, "").trim().to_string();
|
|
||||||
|
|
||||||
// 用分隔符拆分
|
|
||||||
let parts: Vec<&str> = RE_SPLIT.splitn(&no_prefix, 2).collect();
|
|
||||||
if parts.len() >= 2 {
|
|
||||||
let name = parts[0].trim();
|
|
||||||
let name = RE_PHOTOS.replace(name, "").trim().to_string();
|
|
||||||
if !name.is_empty() && name.len() >= 2 {
|
|
||||||
return Some(name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 没有分隔符
|
|
||||||
let name = RE_PHOTOS.replace(&no_prefix, "").trim().to_string();
|
|
||||||
if name.len() >= 2 {
|
|
||||||
Some(name)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 清洗人名
|
|
||||||
pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
|
|
||||||
let mut aliases = Vec::new();
|
|
||||||
let mut name = raw_name.to_string();
|
|
||||||
|
|
||||||
// 移除前缀
|
|
||||||
for prefix in MODEL_PREFIXES {
|
|
||||||
if name.starts_with(prefix) {
|
|
||||||
aliases.push(("prefix".to_string(), prefix.to_string()));
|
|
||||||
name = name[prefix.len()..].trim().to_string();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 移除括号内容
|
|
||||||
let mut cleaned = String::new();
|
|
||||||
let mut last_end = 0;
|
|
||||||
for cap in RE_PAREN.captures_iter(&name) {
|
|
||||||
let m = cap.get(0).unwrap();
|
|
||||||
let content = cap.get(1).map_or("", |c| c.as_str());
|
|
||||||
cleaned.push_str(&name[last_end..m.start()]);
|
|
||||||
last_end = m.end();
|
|
||||||
if !content.is_empty() {
|
|
||||||
if RE_CHINESE.is_match(content) {
|
|
||||||
aliases.push(("nickname".to_string(), content.to_string()));
|
|
||||||
} else {
|
|
||||||
aliases.push(("english".to_string(), content.to_string()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cleaned.push_str(&name[last_end..]);
|
|
||||||
let name = cleaned.trim().to_string();
|
|
||||||
|
|
||||||
// 清理末尾标点
|
|
||||||
let name = RE_TRAILING.replace(&name, "").trim().to_string();
|
|
||||||
|
|
||||||
// 提取英文名别名
|
|
||||||
if let Some(caps) = RE_EN_SUFFIX.captures(&name) {
|
|
||||||
let chinese = caps[1].to_string();
|
|
||||||
let english = caps[2].trim().to_string();
|
|
||||||
if !english.is_empty() {
|
|
||||||
aliases.push(("chinese".to_string(), chinese));
|
|
||||||
aliases.push(("english".to_string(), english));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let name = if name.is_empty() { raw_name.to_string() } else { name };
|
|
||||||
(name, aliases)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 从标题提取标签
|
|
||||||
pub fn extract_tags(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
|
|
||||||
let mut clean = clean_title_suffix(title);
|
|
||||||
|
|
||||||
// 移除人名
|
|
||||||
if let Some(name) = person_name {
|
|
||||||
clean = clean.replace(name, " ");
|
|
||||||
for part in name.split_whitespace() {
|
|
||||||
clean = clean.replace(part, " ");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 移除工作室前缀
|
|
||||||
clean = RE_BRACKET.replace(&clean, "").trim().to_string();
|
|
||||||
|
|
||||||
let mut tags = Vec::new();
|
|
||||||
let mut seen = HashSet::new();
|
|
||||||
|
|
||||||
for cap in RE_WORD.find_iter(&clean) {
|
|
||||||
let word = cap.as_str().trim();
|
|
||||||
if word.len() < 2 { continue; }
|
|
||||||
let word_lower = word.to_lowercase();
|
|
||||||
if STOP_SET.contains(word_lower.as_str()) { continue; }
|
|
||||||
if word.chars().all(|c| c.is_ascii_digit() || c == '.') { continue; }
|
|
||||||
if !seen.insert(word_lower.clone()) { continue; }
|
|
||||||
|
|
||||||
let tag_type = if RE_CHINESE.is_match(word) { "chinese" } else { "english" };
|
|
||||||
tags.push((word.to_lowercase(), tag_type.to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
tags
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 判断是否为排除名
|
|
||||||
pub fn is_excluded(name: &str) -> bool {
|
|
||||||
EXCLUDED_LOWER.contains(&name.to_lowercase())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// 数据模型
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
struct MigrationStats {
|
|
||||||
persons_imported: i64,
|
|
||||||
persons_skipped: i64,
|
|
||||||
persons_created: i64,
|
|
||||||
aliases_imported: i64,
|
|
||||||
albums_processed: i64,
|
|
||||||
albums_missing: i64,
|
|
||||||
tags_imported: i64,
|
|
||||||
person_albums_created: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
struct OldAlbum {
|
|
||||||
id: String,
|
|
||||||
title: String,
|
|
||||||
image_count: i64,
|
|
||||||
person_name_original: Option<String>,
|
|
||||||
person_name_cleaned: Option<String>,
|
|
||||||
tags: Option<String>,
|
|
||||||
all_words: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
struct OldPerson {
|
|
||||||
id: i64,
|
|
||||||
name: String,
|
|
||||||
display_name: Option<String>,
|
|
||||||
total_albums: i64,
|
|
||||||
total_images: i64,
|
|
||||||
original_names: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// 迁移器
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
struct Migrator {
|
|
||||||
old_pool: SqlitePool,
|
|
||||||
new_pool: SqlitePool,
|
|
||||||
stats: MigrationStats,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Migrator {
|
|
||||||
async fn new(old_db_path: &str, new_db_path: &str) -> Result<Self, String> {
|
|
||||||
if !Path::new(old_db_path).exists() {
|
|
||||||
return Err(format!("老数据库不存在: {}", old_db_path));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 创建新数据库目录
|
|
||||||
if let Some(parent) = Path::new(new_db_path).parent() {
|
|
||||||
if !parent.as_os_str().is_empty() && !parent.exists() {
|
|
||||||
std::fs::create_dir_all(parent).map_err(|e| format!("创建目录失败: {}", e))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("[连接] 老数据库: {}", old_db_path);
|
|
||||||
let old_pool = SqlitePoolOptions::new()
|
|
||||||
.max_connections(1)
|
|
||||||
.connect(&format!("sqlite:{}", old_db_path))
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("连接老数据库失败: {}", e))?;
|
|
||||||
|
|
||||||
println!("[连接] 新数据库: {}", new_db_path);
|
|
||||||
let new_pool = SqlitePoolOptions::new()
|
|
||||||
.max_connections(5)
|
|
||||||
.connect(&format!("sqlite:{}", new_db_path))
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("连接新数据库失败: {}", e))?;
|
|
||||||
|
|
||||||
// SQLite 性能优化
|
|
||||||
for p in [&new_pool] {
|
|
||||||
sqlx::query("PRAGMA journal_mode=WAL;").execute(p).await.ok();
|
|
||||||
sqlx::query("PRAGMA synchronous=NORMAL;").execute(p).await.ok();
|
|
||||||
sqlx::query("PRAGMA foreign_keys=ON;").execute(p).await.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Self { old_pool, new_pool, stats: MigrationStats::default() })
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn init_new_schema(&self) -> Result<(), String> {
|
|
||||||
println!("[Schema] 初始化新数据库表结构...");
|
|
||||||
|
|
||||||
let schema_sql = include_str!("../migrations/001_initial.sql");
|
|
||||||
for stmt in schema_sql.split(';') {
|
|
||||||
let stmt = stmt.trim();
|
|
||||||
if !stmt.is_empty() {
|
|
||||||
sqlx::query(stmt).execute(&self.new_pool).await
|
|
||||||
.map_err(|e| format!("执行 Schema 失败: {}", e))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let schema_sql2 = include_str!("../migrations/002_metadata.sql");
|
|
||||||
for stmt in schema_sql2.split(';') {
|
|
||||||
let stmt = stmt.trim();
|
|
||||||
if !stmt.is_empty() {
|
|
||||||
sqlx::query(stmt).execute(&self.new_pool).await
|
|
||||||
.map_err(|e| format!("执行 Schema2 失败: {}", e))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("[Schema] 表结构初始化完成");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_old_albums(&self) -> Result<Vec<OldAlbum>, String> {
|
|
||||||
println!("[读取] 从老数据库读取相册...");
|
|
||||||
let rows = sqlx::query(
|
|
||||||
"SELECT id, title, image_count, person_name_original, person_name_cleaned, tags, all_words
|
|
||||||
FROM albums ORDER BY image_count DESC"
|
|
||||||
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取相册失败: {}", e))?;
|
|
||||||
|
|
||||||
let albums: Vec<OldAlbum> = rows.iter().map(|r| OldAlbum {
|
|
||||||
id: r.get("id"),
|
|
||||||
title: r.get("title"),
|
|
||||||
image_count: r.get::<Option<i64>, _>("image_count").unwrap_or(0),
|
|
||||||
person_name_original: r.get("person_name_original"),
|
|
||||||
person_name_cleaned: r.get("person_name_cleaned"),
|
|
||||||
tags: r.get("tags"),
|
|
||||||
all_words: r.get("all_words"),
|
|
||||||
}).collect();
|
|
||||||
|
|
||||||
println!("[读取] 共读取 {} 个相册", albums.len());
|
|
||||||
Ok(albums)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_old_persons(&self) -> Result<Vec<OldPerson>, String> {
|
|
||||||
println!("[读取] 从老数据库读取人物...");
|
|
||||||
let rows = sqlx::query(
|
|
||||||
"SELECT id, name, display_name, total_albums, total_images, original_names
|
|
||||||
FROM persons ORDER BY total_albums DESC"
|
|
||||||
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取人物失败: {}", e))?;
|
|
||||||
|
|
||||||
let persons: Vec<OldPerson> = rows.iter().map(|r| OldPerson {
|
|
||||||
id: r.get("id"),
|
|
||||||
name: r.get("name"),
|
|
||||||
display_name: r.get("display_name"),
|
|
||||||
total_albums: r.get::<Option<i64>, _>("total_albums").unwrap_or(0),
|
|
||||||
total_images: r.get::<Option<i64>, _>("total_images").unwrap_or(0),
|
|
||||||
original_names: r.get("original_names"),
|
|
||||||
}).collect();
|
|
||||||
|
|
||||||
println!("[读取] 共读取 {} 个人物", persons.len());
|
|
||||||
Ok(persons)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_old_aliases(&self) -> Result<Vec<(i64, String, String)>, String> {
|
|
||||||
let rows = sqlx::query(
|
|
||||||
"SELECT person_id, alias_type, alias_name FROM person_aliases"
|
|
||||||
).fetch_all(&self.old_pool).await.unwrap_or_default();
|
|
||||||
|
|
||||||
let aliases: Vec<(i64, String, String)> = rows.iter().map(|r| (
|
|
||||||
r.get("person_id"), r.get("alias_type"), r.get("alias_name")
|
|
||||||
)).collect();
|
|
||||||
|
|
||||||
println!("[读取] 共读取 {} 个别名", aliases.len());
|
|
||||||
Ok(aliases)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_old_person_albums(&self) -> Result<Vec<(i64, String)>, String> {
|
|
||||||
let rows = sqlx::query(
|
|
||||||
"SELECT person_id, album_id FROM person_albums"
|
|
||||||
).fetch_all(&self.old_pool).await.unwrap_or_default();
|
|
||||||
|
|
||||||
let links: Vec<(i64, String)> = rows.iter().map(|r| (
|
|
||||||
r.get("person_id"), r.get("album_id")
|
|
||||||
)).collect();
|
|
||||||
|
|
||||||
println!("[读取] 共读取 {} 个人物-相册关联", links.len());
|
|
||||||
Ok(links)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn migrate(&mut self, reparse: bool) -> Result<(), String> {
|
|
||||||
self.init_new_schema().await?;
|
|
||||||
|
|
||||||
// 读取老数据
|
|
||||||
let old_albums = self.read_old_albums().await?;
|
|
||||||
let old_persons = self.read_old_persons().await?;
|
|
||||||
let old_aliases = self.read_old_aliases().await?;
|
|
||||||
let old_links = self.read_old_person_albums().await?;
|
|
||||||
|
|
||||||
let mut old_to_new_person: HashMap<i64, i64> = HashMap::new();
|
|
||||||
|
|
||||||
// Step 1: 迁移人物
|
|
||||||
println!("\n[迁移] 步骤 1/5: 迁移人物...");
|
|
||||||
for p in &old_persons {
|
|
||||||
if title_parser::is_excluded(&p.name) {
|
|
||||||
self.stats.persons_skipped += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR IGNORE INTO persons (name, display_name, total_albums, total_images, original_names)
|
|
||||||
VALUES (?, ?, ?, ?, ?)"
|
|
||||||
).bind(&p.name).bind(p.display_name.as_deref().unwrap_or(&p.name))
|
|
||||||
.bind(p.total_albums).bind(p.total_images).bind(p.original_names.as_deref())
|
|
||||||
.execute(&self.new_pool).await.ok();
|
|
||||||
|
|
||||||
let new_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?")
|
|
||||||
.bind(&p.name).fetch_one(&self.new_pool).await.unwrap_or(0);
|
|
||||||
|
|
||||||
if new_id > 0 {
|
|
||||||
old_to_new_person.insert(p.id, new_id);
|
|
||||||
self.stats.persons_imported += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
println!("[迁移] 导入 {} 人,跳过 {} 人", self.stats.persons_imported, self.stats.persons_skipped);
|
|
||||||
|
|
||||||
// Step 2: 迁移别名
|
|
||||||
println!("\n[迁移] 步骤 2/5: 迁移别名...");
|
|
||||||
for (old_pid, alias_type, alias_name) in &old_aliases {
|
|
||||||
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)"
|
|
||||||
).bind(new_pid).bind(alias_type).bind(alias_name)
|
|
||||||
.execute(&self.new_pool).await.ok();
|
|
||||||
self.stats.aliases_imported += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
println!("[迁移] 导入 {} 个别名", self.stats.aliases_imported);
|
|
||||||
|
|
||||||
// Step 3: 迁移相册 + 解析title
|
|
||||||
println!(
|
|
||||||
"\n[迁移] 步骤 3/5: 迁移相册并解析title ({})...",
|
|
||||||
if reparse { "重新解析" } else { "保留原解析" }
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut person_map: HashMap<String, i64> = HashMap::new();
|
|
||||||
|
|
||||||
for (i, album) in old_albums.iter().enumerate() {
|
|
||||||
if (i + 1) % 200 == 0 {
|
|
||||||
println!(" 处理进度: {}/{}", i + 1, old_albums.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
let (person_orig, person_cleaned, tags, all_words) =
|
|
||||||
if reparse || album.person_name_cleaned.as_deref().unwrap_or("").is_empty() {
|
|
||||||
// 重新解析
|
|
||||||
let person_name = title_parser::extract_person_name(&album.title);
|
|
||||||
let (cleaned, _aliases) = match &person_name {
|
|
||||||
Some(n) => {
|
|
||||||
let (c, a) = title_parser::clean_person_name(n);
|
|
||||||
if title_parser::is_excluded(&c) {
|
|
||||||
(None, a)
|
|
||||||
} else {
|
|
||||||
(Some(c), a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => (None, Vec::new()),
|
|
||||||
};
|
|
||||||
|
|
||||||
let tags_raw = title_parser::extract_tags(&album.title, person_name.as_deref());
|
|
||||||
let tag_names: Vec<String> = tags_raw.iter().map(|(t, _)| t.clone()).collect();
|
|
||||||
let all_words: Vec<[String; 2]> = tags_raw.iter()
|
|
||||||
.map(|(t, tp)| [t.clone(), tp.clone()]).collect();
|
|
||||||
|
|
||||||
(
|
|
||||||
person_name,
|
|
||||||
cleaned,
|
|
||||||
serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string()),
|
|
||||||
serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
(
|
|
||||||
album.person_name_original.clone(),
|
|
||||||
album.person_name_cleaned.clone(),
|
|
||||||
album.tags.clone().unwrap_or_else(|| "[]".to_string()),
|
|
||||||
album.all_words.clone().unwrap_or_else(|| "[]".to_string()),
|
|
||||||
)
|
|
||||||
};
|
|
||||||
|
|
||||||
// 写入相册
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR REPLACE INTO albums
|
|
||||||
(id, title, image_count, person_name_original, person_name_cleaned,
|
|
||||||
tags, all_words, raw_title, processed_at)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))"
|
|
||||||
).bind(&album.id).bind(&album.title).bind(album.image_count)
|
|
||||||
.bind(person_orig.as_deref()).bind(person_cleaned.as_deref())
|
|
||||||
.bind(&tags).bind(&all_words).bind(&album.title)
|
|
||||||
.execute(&self.new_pool).await.ok();
|
|
||||||
|
|
||||||
// 写入标签索引
|
|
||||||
if let Ok(tag_list) = serde_json::from_str::<Vec<String>>(&tags) {
|
|
||||||
if !tag_list.is_empty() {
|
|
||||||
for tag in &tag_list {
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES (?, ?)"
|
|
||||||
).bind(&album.id).bind(tag).execute(&self.new_pool).await.ok();
|
|
||||||
}
|
|
||||||
self.stats.tags_imported += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 关联人物
|
|
||||||
if let Some(pc) = &person_cleaned {
|
|
||||||
if !pc.is_empty() && !title_parser::is_excluded(pc) {
|
|
||||||
let pid = if let Some(&id) = person_map.get(pc) {
|
|
||||||
id
|
|
||||||
} else {
|
|
||||||
// 查找或创建
|
|
||||||
let existing: Option<i64> = sqlx::query_scalar(
|
|
||||||
"SELECT id FROM persons WHERE name = ?"
|
|
||||||
).bind(pc).fetch_optional(&self.new_pool).await.unwrap_or(None);
|
|
||||||
|
|
||||||
let id = if let Some(eid) = existing {
|
|
||||||
eid
|
|
||||||
} else {
|
|
||||||
sqlx::query("INSERT INTO persons (name, display_name) VALUES (?, ?)")
|
|
||||||
.bind(pc).bind(pc).execute(&self.new_pool).await.ok();
|
|
||||||
self.stats.persons_created += 1;
|
|
||||||
sqlx::query_scalar::<_, i64>("SELECT last_insert_rowid()")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0)
|
|
||||||
};
|
|
||||||
person_map.insert(pc.clone(), id);
|
|
||||||
id
|
|
||||||
};
|
|
||||||
|
|
||||||
if pid > 0 {
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
|
|
||||||
).bind(pid).bind(&album.id).execute(&self.new_pool).await.ok();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.stats.albums_processed += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"[迁移] 处理 {} 个相册,{} 个标签导入,{} 个人物从title创建",
|
|
||||||
self.stats.albums_processed, self.stats.tags_imported, self.stats.persons_created
|
|
||||||
);
|
|
||||||
|
|
||||||
// Step 4: 迁移人物-相册关联
|
|
||||||
println!("\n[迁移] 步骤 4/5: 迁移人物-相册关联...");
|
|
||||||
for (old_pid, album_id) in &old_links {
|
|
||||||
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
|
|
||||||
).bind(new_pid).bind(album_id).execute(&self.new_pool).await.ok();
|
|
||||||
self.stats.person_albums_created += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
println!("[迁移] 导入 {} 个关联", self.stats.person_albums_created);
|
|
||||||
|
|
||||||
// Step 5: 更新人物统计
|
|
||||||
println!("\n[迁移] 步骤 5/5: 更新人物统计...");
|
|
||||||
let person_ids: Vec<i64> = sqlx::query_scalar("SELECT id FROM persons")
|
|
||||||
.fetch_all(&self.new_pool).await.unwrap_or_default();
|
|
||||||
|
|
||||||
for pid in &person_ids {
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE persons SET
|
|
||||||
total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?),
|
|
||||||
total_images = (
|
|
||||||
SELECT COALESCE(SUM(a.image_count), 0)
|
|
||||||
FROM albums a JOIN person_albums pa ON a.id = pa.album_id
|
|
||||||
WHERE pa.person_id = ?
|
|
||||||
),
|
|
||||||
updated_at = datetime('now')
|
|
||||||
WHERE id = ?"
|
|
||||||
).bind(pid).bind(pid).bind(pid).execute(&self.new_pool).await.ok();
|
|
||||||
}
|
|
||||||
println!("[迁移] 更新 {} 个人物统计", person_ids.len());
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn print_stats(&self) {
|
|
||||||
let stats = self.get_new_stats().await;
|
|
||||||
println!("\n 新数据库统计:");
|
|
||||||
println!(" 相册数: {}", stats.total_albums);
|
|
||||||
println!(" 图片数: {}", stats.total_images);
|
|
||||||
println!(" 人物数: {}", stats.total_persons);
|
|
||||||
println!(" 别名数: {}", stats.total_aliases);
|
|
||||||
println!(" 标签数: {}", stats.unique_tags);
|
|
||||||
println!(" 有标签相册: {}", stats.albums_with_tags);
|
|
||||||
println!(" 有人物相册: {}", stats.albums_with_person);
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_new_stats(&self) -> NewDbStats {
|
|
||||||
NewDbStats {
|
|
||||||
total_albums: sqlx::query_scalar("SELECT COUNT(*) FROM albums")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
total_images: sqlx::query_scalar("SELECT COALESCE(SUM(image_count), 0) FROM albums")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
total_persons: sqlx::query_scalar("SELECT COUNT(*) FROM persons")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
total_aliases: sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
unique_tags: sqlx::query_scalar("SELECT COUNT(DISTINCT tag) FROM album_tags")
|
|
||||||
.fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
albums_with_tags: sqlx::query_scalar(
|
|
||||||
"SELECT COUNT(*) FROM albums WHERE tags IS NOT NULL AND tags != '[]'"
|
|
||||||
).fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
albums_with_person: sqlx::query_scalar(
|
|
||||||
"SELECT COUNT(*) FROM albums WHERE person_name_cleaned IS NOT NULL AND person_name_cleaned != ''"
|
|
||||||
).fetch_one(&self.new_pool).await.unwrap_or(0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct NewDbStats {
|
|
||||||
total_albums: i64,
|
|
||||||
total_images: i64,
|
|
||||||
total_persons: i64,
|
|
||||||
total_aliases: i64,
|
|
||||||
unique_tags: i64,
|
|
||||||
albums_with_tags: i64,
|
|
||||||
albums_with_person: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// 搜索验证器
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
struct SearchVerifier {
|
|
||||||
pool: SqlitePool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SearchVerifier {
|
|
||||||
async fn new(db_path: &str) -> Result<Self, String> {
|
|
||||||
let pool = SqlitePoolOptions::new()
|
|
||||||
.max_connections(2)
|
|
||||||
.connect(&format!("sqlite:{}", db_path))
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("连接数据库失败: {}", e))?;
|
|
||||||
Ok(Self { pool })
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn verify_all(&self) {
|
|
||||||
println!("\n{}", "=".repeat(60));
|
|
||||||
println!(" 🔍 搜索功能验证 (Rust 实现)");
|
|
||||||
println!("{}", "=".repeat(60));
|
|
||||||
|
|
||||||
// 1. 热门人物
|
|
||||||
println!("\n[验证1] 热门人物 (Top 10):");
|
|
||||||
let top_persons: Vec<(String, i64, i64)> = sqlx::query(
|
|
||||||
"SELECT name, total_albums, total_images FROM persons ORDER BY total_albums DESC LIMIT 10"
|
|
||||||
).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("name"), r.get("total_albums"), r.get("total_images"))).collect();
|
|
||||||
|
|
||||||
for (i, (name, albums, images)) in top_persons.iter().enumerate() {
|
|
||||||
println!(" {:2d}. {:<25} {:4} 相册, {:5} 张图片", i + 1, name, albums, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 热门标签
|
|
||||||
println!("\n[验证2] 热门标签 (Top 20):");
|
|
||||||
let top_tags: Vec<(String, i64, i64)> = sqlx::query(
|
|
||||||
"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
|
|
||||||
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT 20"
|
|
||||||
).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("tag"), r.get::<i64, _>("cnt"), r.get::<i64, _>("album_cnt"))).collect();
|
|
||||||
|
|
||||||
for (i, (tag, cnt, album_cnt)) in top_tags.iter().enumerate() {
|
|
||||||
println!(" {:2d}. {:<20} 出现 {:4} 次, {:4} 个相册", i + 1, tag, cnt, album_cnt);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 人物搜索
|
|
||||||
if let Some((test_person, _, _)) = top_persons.first() {
|
|
||||||
println!("\n[验证3] 人物搜索: \"{}\"", test_person);
|
|
||||||
let like = format!("%{}%", test_person);
|
|
||||||
let results: Vec<(String, i64)> = sqlx::query(
|
|
||||||
"SELECT DISTINCT p.name, p.total_albums FROM persons p
|
|
||||||
LEFT JOIN person_aliases pa ON pa.person_id = p.id
|
|
||||||
WHERE p.name LIKE ? OR pa.alias_name LIKE ?
|
|
||||||
ORDER BY p.total_albums DESC LIMIT 5"
|
|
||||||
).bind(&like).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("name"), r.get("total_albums"))).collect();
|
|
||||||
|
|
||||||
for (name, albums) in &results {
|
|
||||||
println!(" - {} ({} 相册)", name, albums);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. 人物相册
|
|
||||||
println!("\n[验证4] 人物相册: \"{}\" (前5个)", test_person);
|
|
||||||
let person_albums: Vec<(i64, String)> = sqlx::query(
|
|
||||||
"SELECT a.image_count, a.title FROM albums a
|
|
||||||
JOIN person_albums pa ON pa.album_id = a.id
|
|
||||||
JOIN persons p ON p.id = pa.person_id
|
|
||||||
WHERE p.name = ? ORDER BY a.image_count DESC LIMIT 5"
|
|
||||||
).bind(test_person).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
|
||||||
|
|
||||||
for (ic, title) in &person_albums {
|
|
||||||
let short: String = title.chars().take(60).collect();
|
|
||||||
println!(" - [{:3}张] {}", ic, short);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5. 标签搜索
|
|
||||||
if let Some((test_tag, _, _)) = top_tags.first() {
|
|
||||||
println!("\n[验证5] 标签搜索: \"{}\" (前5个)", test_tag);
|
|
||||||
let tag_albums: Vec<(i64, String)> = sqlx::query(
|
|
||||||
"SELECT a.image_count, a.title FROM albums a
|
|
||||||
JOIN album_tags t ON t.album_id = a.id
|
|
||||||
WHERE t.tag = ? ORDER BY a.image_count DESC LIMIT 5"
|
|
||||||
).bind(test_tag).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
|
||||||
|
|
||||||
for (ic, title) in &tag_albums {
|
|
||||||
let short: String = title.chars().take(60).collect();
|
|
||||||
println!(" - [{:3}张] {}", ic, short);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. 标题搜索
|
|
||||||
println!("\n[验证6] 标题搜索: \"性感\" (前5个)");
|
|
||||||
let like = "%性感%".to_string();
|
|
||||||
let title_albums: Vec<(i64, String)> = sqlx::query(
|
|
||||||
"SELECT image_count, title FROM albums WHERE title LIKE ? ORDER BY image_count DESC LIMIT 5"
|
|
||||||
).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
|
|
||||||
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
|
|
||||||
|
|
||||||
for (ic, title) in &title_albums {
|
|
||||||
let short: String = title.chars().take(60).collect();
|
|
||||||
println!(" - [{:3}张] {}", ic, short);
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("\n{}", "=".repeat(60));
|
|
||||||
println!(" ✅ 搜索功能验证完成!");
|
|
||||||
println!("{}", "=".repeat(60));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================
|
|
||||||
// CLI 入口
|
|
||||||
// ============================================================
|
|
||||||
|
|
||||||
#[derive(Debug, clap::Parser)]
|
|
||||||
#[command(name = "migrate", about = "数据库迁移工具 (Rust 实现)")]
|
|
||||||
struct Cli {
|
|
||||||
/// 老数据库路径
|
|
||||||
#[arg(long, default_value = "../yolov8_face_recognition/persons.db")]
|
|
||||||
old_db: String,
|
|
||||||
|
|
||||||
/// 新数据库路径
|
|
||||||
#[arg(long, default_value = "./data/imager.db")]
|
|
||||||
new_db: String,
|
|
||||||
|
|
||||||
/// 重新解析所有 title
|
|
||||||
#[arg(long)]
|
|
||||||
reparse: bool,
|
|
||||||
|
|
||||||
/// 迁移后验证搜索功能
|
|
||||||
#[arg(long)]
|
|
||||||
verify: bool,
|
|
||||||
|
|
||||||
/// 仅验证搜索功能 (不执行迁移)
|
|
||||||
#[arg(long)]
|
|
||||||
verify_only: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::main]
|
|
||||||
async fn main() -> Result<(), String> {
|
|
||||||
// 检查是否有 clap
|
|
||||||
let args: Vec<String> = std::env::args().collect();
|
|
||||||
|
|
||||||
let mut old_db = "../yolov8_face_recognition/persons.db".to_string();
|
|
||||||
let mut new_db = "./data/imager.db".to_string();
|
|
||||||
let mut reparse = false;
|
|
||||||
let mut verify = false;
|
|
||||||
let mut verify_only = false;
|
|
||||||
|
|
||||||
let mut i = 1;
|
|
||||||
while i < args.len() {
|
|
||||||
match args[i].as_str() {
|
|
||||||
"--old-db" => { i += 1; if i < args.len() { old_db = args[i].clone(); } }
|
|
||||||
"--new-db" => { i += 1; if i < args.len() { new_db = args[i].clone(); } }
|
|
||||||
"--reparse" => reparse = true,
|
|
||||||
"--verify" => verify = true,
|
|
||||||
"--verify-only" => verify_only = true,
|
|
||||||
"--help" | "-h" => {
|
|
||||||
println!("数据库迁移工具 (Rust 实现)");
|
|
||||||
println!();
|
|
||||||
println!("用法: cargo run --bin migrate -- [选项]");
|
|
||||||
println!();
|
|
||||||
println!("选项:");
|
|
||||||
println!(" --old-db <路径> 老数据库路径 (默认: ../yolov8_face_recognition/persons.db)");
|
|
||||||
println!(" --new-db <路径> 新数据库路径 (默认: ./data/imager.db)");
|
|
||||||
println!(" --reparse 重新解析所有 title");
|
|
||||||
println!(" --verify 迁移后验证搜索功能");
|
|
||||||
println!(" --verify-only 仅验证搜索功能 (不执行迁移)");
|
|
||||||
println!(" --help, -h 显示帮助");
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
let start = Instant::now();
|
|
||||||
|
|
||||||
// 仅验证模式
|
|
||||||
if verify_only {
|
|
||||||
if !Path::new(&new_db).exists() {
|
|
||||||
return Err(format!("数据库不存在: {}", new_db));
|
|
||||||
}
|
|
||||||
let verifier = SearchVerifier::new(&new_db).await?;
|
|
||||||
verifier.verify_all().await;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("{}", "=".repeat(60));
|
|
||||||
println!(" 🚀 数据库迁移工具 (Rust 原生实现)");
|
|
||||||
println!("{}", "=".repeat(60));
|
|
||||||
|
|
||||||
// 执行迁移
|
|
||||||
let mut migrator = Migrator::new(&old_db, &new_db).await?;
|
|
||||||
migrator.migrate(reparse).await?;
|
|
||||||
|
|
||||||
println!("\n{}", "=".repeat(60));
|
|
||||||
println!(" ✅ 迁移完成!用时: {:.2?}", start.elapsed());
|
|
||||||
println!("{}", "=".repeat(60));
|
|
||||||
migrator.print_stats().await;
|
|
||||||
|
|
||||||
// 验证搜索
|
|
||||||
if verify {
|
|
||||||
let verifier = SearchVerifier::new(&new_db).await?;
|
|
||||||
verifier.verify_all().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@@ -48,12 +48,18 @@ pub struct OldDbConfig {
|
|||||||
pub path: String,
|
pub path: String,
|
||||||
#[serde(default = "default_auto_migrate")]
|
#[serde(default = "default_auto_migrate")]
|
||||||
pub auto_migrate: bool,
|
pub auto_migrate: bool,
|
||||||
|
#[serde(default = "default_auto_parse")]
|
||||||
|
pub auto_parse: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_auto_migrate() -> bool {
|
fn default_auto_migrate() -> bool {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_auto_parse() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
pub fn load() -> Result<Self, config::ConfigError> {
|
pub fn load() -> Result<Self, config::ConfigError> {
|
||||||
let config = config::Config::builder()
|
let config = config::Config::builder()
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ pub mod metadata;
|
|||||||
pub mod migration;
|
pub mod migration;
|
||||||
|
|
||||||
pub use connection::Database;
|
pub use connection::Database;
|
||||||
pub use album::*;
|
#[allow(unused)] pub use album::*;
|
||||||
pub use image::*;
|
#[allow(unused)] pub use image::*;
|
||||||
pub use favorite::*;
|
#[allow(unused)] pub use favorite::*;
|
||||||
pub use metadata::*;
|
#[allow(unused)] pub use metadata::*;
|
||||||
pub use migration::MigrationStats;
|
pub use migration::MigrationStats;
|
||||||
@@ -77,7 +77,7 @@ impl Crawler {
|
|||||||
let resp = self.client.get(&url).send().await?;
|
let resp = self.client.get(&url).send().await?;
|
||||||
let data: serde_json::Value = resp.json().await?;
|
let data: serde_json::Value = resp.json().await?;
|
||||||
|
|
||||||
let albums: Vec<ApiAlbum> = if let Some(arr) = data.as_array() {
|
let albums: Vec<ApiAlbum> = if let Some(_arr) = data.as_array() {
|
||||||
serde_json::from_value(data)?
|
serde_json::from_value(data)?
|
||||||
} else if let Some(arr) = data.get("albums").and_then(|v: &serde_json::Value| v.as_array()) {
|
} else if let Some(arr) = data.get("albums").and_then(|v: &serde_json::Value| v.as_array()) {
|
||||||
serde_json::from_value(serde_json::Value::Array(arr.clone()))?
|
serde_json::from_value(serde_json::Value::Array(arr.clone()))?
|
||||||
|
|||||||
@@ -7,9 +7,9 @@ pub mod recommend;
|
|||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
pub mod crawler;
|
pub mod crawler;
|
||||||
|
|
||||||
pub use title_parser::*;
|
#[allow(unused)] pub use title_parser::*;
|
||||||
pub use person_merger::*;
|
#[allow(unused)] pub use person_merger::*;
|
||||||
pub use tag_extractor::*;
|
#[allow(unused)] pub use tag_extractor::*;
|
||||||
pub use recommend::*;
|
pub use recommend::*;
|
||||||
pub use pipeline::*;
|
#[allow(unused)] pub use pipeline::*;
|
||||||
pub use crawler::*;
|
#[allow(unused)] pub use crawler::*;
|
||||||
@@ -298,7 +298,7 @@ pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats
|
|||||||
let title = album.title.as_deref().unwrap_or("");
|
let title = album.title.as_deref().unwrap_or("");
|
||||||
let current = album.person_name_original.as_deref().unwrap_or("");
|
let current = album.person_name_original.as_deref().unwrap_or("");
|
||||||
|
|
||||||
if let Some((text, person, source)) = match_title(title, patterns) {
|
if let Some((_text, person, source)) = match_title(title, patterns) {
|
||||||
*stats.by_source.entry(source).or_insert(0) += 1;
|
*stats.by_source.entry(source).or_insert(0) += 1;
|
||||||
*stats.name_frequency.entry(person.clone()).or_insert(0) += 1;
|
*stats.name_frequency.entry(person.clone()).or_insert(0) += 1;
|
||||||
|
|
||||||
|
|||||||
32
src/main.rs
32
src/main.rs
@@ -47,27 +47,29 @@ async fn main() -> std::io::Result<()> {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// 自动迁移老数据库
|
|
||||||
if let Some(ref old_db_cfg) = config.old_db {
|
|
||||||
if old_db_cfg.auto_migrate {
|
|
||||||
info!("[迁移] 检测到 old_db 配置,自动迁移: {}", old_db_cfg.path);
|
|
||||||
match db.migrate_from_old_db(&old_db_cfg.path).await {
|
|
||||||
Ok(stats) => info!("[迁移] {}", db.migration_summary(&stats)),
|
|
||||||
Err(e) => error!("[迁移] 失败: {}", e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
|
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
|
||||||
|
|
||||||
// 后台导入本地相册
|
// 后台导入本地相册 + 自动解析title
|
||||||
let config_clone = config.clone();
|
let config_clone = config.clone();
|
||||||
let db_clone = Arc::clone(&db);
|
let db_clone = Arc::clone(&db);
|
||||||
|
let auto_parse = true;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
info!("[后台任务] 开始扫描图片目录并导入新相册...");
|
info!("[后台任务] 开始扫描图片目录并导入新相册...");
|
||||||
match services::import::import_existing_data(&config_clone, db_clone, None).await {
|
let import_ok = match services::import::import_existing_data(&config_clone, Arc::clone(&db_clone), None).await {
|
||||||
Ok(_) => info!("[后台任务] 相册导入完成"),
|
Ok(_) => {
|
||||||
Err(e) => error!("[后台任务] 相册导入失败: {}", e),
|
info!("[后台任务] 相册导入完成");
|
||||||
|
true
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("[后台任务] 相册导入失败: {}", e);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if import_ok && auto_parse {
|
||||||
|
info!("[后台任务] 开始解析相册 title(提取人名 + 标签)...");
|
||||||
|
let (persons, tags) = engine::pipeline::process_all_albums(&db_clone).await;
|
||||||
|
info!("[后台任务] Title 解析完成:{} 个人名,{} 个标签", persons, tags);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use crate::db::Database;
|
use crate::db::Database;
|
||||||
use crate::db::MigrationStats;
|
#[allow(unused)] use crate::db::MigrationStats;
|
||||||
use actix_web::{HttpResponse, Responder, web};
|
use actix_web::{HttpResponse, Responder, web};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use crate::db::Database;
|
use crate::db::Database;
|
||||||
use crate::engine::RecommendEngine;
|
use crate::engine::RecommendEngine;
|
||||||
use crate::models::*;
|
#[allow(unused)] use crate::models::*;
|
||||||
use actix_web::{HttpResponse, Responder, web};
|
use actix_web::{HttpResponse, Responder, web};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|||||||
@@ -2,4 +2,4 @@
|
|||||||
|
|
||||||
pub mod import;
|
pub mod import;
|
||||||
|
|
||||||
pub use import::*;
|
#[allow(unused)] pub use import::*;
|
||||||
Reference in New Issue
Block a user