Compare commits

..

60 Commits

Author SHA1 Message Date
lychang
ec52439622 update code level 2026-07-21 04:59:33 +08:00
lychang
7c9c80fe39 update code level 2026-07-21 04:46:01 +08:00
lychang
f6c7baee5e update code level 2026-07-21 04:21:57 +08:00
lychang
d86cce34bc update code level 2026-07-21 04:00:11 +08:00
lychang
4321cfa62f update code level 2026-07-21 03:50:02 +08:00
lychang
d64b85bc9a update code level 2026-07-21 03:33:35 +08:00
lychang
f1dba9f24d update code level 2026-07-21 03:10:27 +08:00
lychang
49c0a0d307 update code level 2026-07-21 02:53:47 +08:00
lychang
4a80e3272f update code level 2026-07-21 02:48:52 +08:00
lychang
992ee32e9b update code level 2026-07-21 02:45:14 +08:00
lychang
12880dd480 update code level 2026-07-21 02:40:41 +08:00
lychang
cff2eb2886 update code level 2026-07-21 02:25:36 +08:00
lychang
005688b58f update code level 2026-07-21 02:17:46 +08:00
lychang
f572c7370d update code level 2026-07-21 02:11:53 +08:00
lychang
9c3ce4ec0d update code level 2026-07-21 02:01:52 +08:00
lychang
c8778c0ed1 update code level 2026-07-21 01:46:57 +08:00
lychang
284e610140 update code level 2026-07-21 01:44:42 +08:00
lychang
f68a8f880d update code level 2026-07-21 01:43:16 +08:00
lychang
90c493d41b update code level 2026-07-21 01:25:27 +08:00
lychang
9d81c5077c update code level 2026-07-21 01:05:06 +08:00
lychang
a85b4c869e update code level 2026-07-21 00:51:55 +08:00
lychang
81bbc67432 update code level 2026-07-21 00:43:40 +08:00
lychang
0a5dd1c920 update code level 2026-07-21 00:00:27 +08:00
lychang
ee1092c5a9 update code level 2026-07-20 23:50:07 +08:00
lychang
56dd82fb63 update code level 2026-07-20 23:46:39 +08:00
lychang
84409207e2 update code level 2026-07-20 23:12:20 +08:00
lychang
923f9bd7cf update code level 2026-07-20 14:24:04 +08:00
lychang
faa5b416da update code level 2026-07-20 14:08:16 +08:00
lychang
c6639778a4 update code level 2026-07-20 07:43:29 +08:00
lychang
a33ef65bd7 update code level 2026-07-20 07:35:26 +08:00
lychang
5e6b27f26d update code level 2026-07-20 07:03:08 +08:00
lychang
76cf668c08 Merge remote-tracking branch 'origin/master'
# Conflicts:
#	.gitignore
#	Cargo.lock
2026-07-20 06:57:18 +08:00
lychang
9879aab614 update code level 2026-07-20 06:56:54 +08:00
lychang
94af7695ff update code level 2026-07-20 06:56:08 +08:00
lychang
390b548935 update 2026-07-20 05:23:31 +08:00
lychang
a6bfe7eda2 update 2026-07-20 04:46:02 +08:00
lychang
00af6e8528 update 2026-07-20 04:07:40 +08:00
lychang
9093f08441 update 2026-07-20 03:47:34 +08:00
lychang
d186aee171 update 2026-07-20 03:39:31 +08:00
lychang
61176cf2a3 update 2026-07-20 02:49:12 +08:00
lychang
8cf4bfe3d0 update 2026-07-20 02:47:40 +08:00
lychang
d958655c1e update 2026-07-20 02:19:54 +08:00
lychang
c96680fae8 update 2026-07-20 02:17:11 +08:00
lychang
be581977f3 update 2026-07-20 02:15:39 +08:00
lychang
1700585a36 update 2026-07-20 01:49:49 +08:00
lychang
97c2c208ba update 2026-07-20 01:43:33 +08:00
lychang
8d2f3361f8 update 2026-07-20 01:36:31 +08:00
lychang
5b803cc178 update 2026-07-20 01:33:44 +08:00
lychang
6b2f19ef47 update 2026-07-20 01:01:59 +08:00
lychang
da412c1c76 update 2026-07-20 00:50:45 +08:00
lychang
b1f0c2f36a update 2026-07-20 00:23:00 +08:00
lychang
36969a4fa0 update 2026-07-19 23:56:28 +08:00
lychang
34bcd67682 update 2026-07-19 19:25:49 +08:00
lychang
6ae3e3d1de update 2026-07-19 19:18:13 +08:00
lychang
847a80f470 update 2026-07-19 19:01:17 +08:00
lychang
d42f2c358e update 2026-07-19 18:57:47 +08:00
lychang
34fdb18919 update 2026-07-19 18:57:30 +08:00
lychang
879598585b update
Some checks failed
Rust CI / Build Release (push) Has been cancelled
Rust CI / Cargo Check (push) Has been cancelled
2026-07-19 17:54:42 +08:00
lychang
cb09f2caac update
Some checks failed
Rust CI / Build Release (push) Has been cancelled
Rust CI / Cargo Check (push) Has been cancelled
2026-07-19 17:44:08 +08:00
lychang
093924ea99 update
Some checks failed
Rust CI / Build Release (push) Has been cancelled
Rust CI / Cargo Check (push) Has been cancelled
2026-07-19 16:34:04 +08:00
55 changed files with 9476 additions and 3412 deletions

View File

@@ -1,74 +0,0 @@
name: Rust CI
on:
push:
branches: [main, master, develop]
pull_request:
branches: [main, master]
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: -D warnings
jobs:
check:
name: Cargo Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
components: clippy
- name: Cache cargo dependencies
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-
- name: Check compilation
run: cargo check --all-targets
- name: Clippy lint
run: cargo clippy --all-targets -- -D warnings
build:
name: Build Release
needs: check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- name: Cache cargo dependencies
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-
- name: Build release
run: cargo build --release
- name: Upload binary
uses: actions/upload-artifact@v4
with:
name: imager-${{ runner.os }}
path: target/release/imager
if-no-files-found: error

6
.gitignore vendored
View File

@@ -1,3 +1,5 @@
/target
/albums/
.idea
/albums
/.idea
/data
/scripts

319
Cargo.lock generated
View File

@@ -214,6 +214,12 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "ahash"
version = "0.7.8"
@@ -279,13 +285,13 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.89"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.0",
]
[[package]]
@@ -389,9 +395,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.1"
version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder"
@@ -426,6 +432,15 @@ dependencies = [
"shlex",
]
[[package]]
name = "cedarwood"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90"
dependencies = [
"smallvec",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -572,6 +587,25 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.13"
@@ -606,6 +640,12 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "dary_heap"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
[[package]]
name = "der"
version = "0.7.10"
@@ -745,9 +785,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]]
name = "fastrand"
version = "2.4.1"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
[[package]]
name = "fdeflate"
@@ -877,6 +917,15 @@ dependencies = [
"slab",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -970,6 +1019,17 @@ dependencies = [
"allocator-api2",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -1265,9 +1325,10 @@ dependencies = [
"chrono",
"config",
"image",
"jieba-rs",
"lazy_static",
"mimalloc",
"rand 0.8.7",
"rayon",
"regex",
"reqwest",
"serde",
@@ -1286,6 +1347,39 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870"
[[package]]
name = "include-flate"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f173716febb1ad596c16ea5637b5f1790ea32de8e627493ff82bc73b0876ce"
dependencies = [
"include-flate-codegen",
"include-flate-compress",
]
[[package]]
name = "include-flate-codegen"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a7875b62a72ad3f3203cdd8950d4cf9947db036030b974b8b37ceae90c8d8c0"
dependencies = [
"include-flate-compress",
"proc-macro-error3",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "include-flate-compress"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44fbb9c5ccb9a5b67b4afa2974c27e5507ea1bf6d22828cef418e4dfaeca51dd"
dependencies = [
"libflate",
"zstd",
]
[[package]]
name = "indexmap"
version = "2.14.0"
@@ -1308,6 +1402,30 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jieba-macros"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192"
dependencies = [
"phf_codegen",
]
[[package]]
name = "jieba-rs"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a"
dependencies = [
"cedarwood",
"fxhash",
"include-flate",
"jieba-macros",
"lazy_static",
"phf",
"regex",
]
[[package]]
name = "jobserver"
version = "0.1.35"
@@ -1367,21 +1485,36 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libflate"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
dependencies = [
"adler32",
"crc32fast",
"dary_heap",
"libflate_lz77",
"no_std_io2",
]
[[package]]
name = "libflate_lz77"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd"
dependencies = [
"hashbrown 0.16.1",
"no_std_io2",
"rle-decode-fast",
]
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
dependencies = [
"cc",
]
[[package]]
name = "libredox"
version = "0.1.18"
@@ -1471,15 +1604,6 @@ version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -1524,6 +1648,15 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "no_std_io2"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003"
dependencies = [
"memchr",
]
[[package]]
name = "nom"
version = "7.1.3"
@@ -1702,6 +1835,44 @@ dependencies = [
"pest",
]
[[package]]
name = "phf"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_codegen"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand 0.8.7",
]
[[package]]
name = "phf_shared"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -1779,19 +1950,41 @@ dependencies = [
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
name = "proc-macro-error-attr3"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
checksum = "34e4dd828515431dd6c4a030d26f7eaed7dd4778226e9d2bb968d65ca4ec3d4d"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "proc-macro-error3"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ee475e440453418ff1335189eddf7101ba502cd818ab7ae04209bc83aa925aa"
dependencies = [
"proc-macro-error-attr3",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.46"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
@@ -1855,6 +2048,26 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rayon"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -1963,6 +2176,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rle-decode-fast"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
[[package]]
name = "ron"
version = "0.7.1"
@@ -2093,9 +2312,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "serde"
version = "1.0.228"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
"serde_derive",
@@ -2103,22 +2322,22 @@ dependencies = [
[[package]]
name = "serde_core"
version = "1.0.228"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.0",
]
[[package]]
@@ -2168,6 +2387,12 @@ dependencies = [
"digest 0.11.3",
]
[[package]]
name = "sha1_smol"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
[[package]]
name = "sha2"
version = "0.10.9"
@@ -2220,6 +2445,12 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
[[package]]
name = "siphasher"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
[[package]]
name = "slab"
version = "0.4.12"
@@ -2519,6 +2750,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
@@ -2897,6 +3139,7 @@ dependencies = [
"getrandom 0.4.3",
"js-sys",
"serde_core",
"sha1_smol",
"wasm-bindgen",
]

View File

@@ -11,7 +11,7 @@ serde_json = "1"
tokio = { version = "1", features = ["full"] }
sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio", "migrate"] }
chrono = { version = "0.4", features = ["serde"] }
uuid = { version = "1.0", features = ["v4", "serde"] }
uuid = { version = "1.0", features = ["v4", "v5", "serde"] }
image = { version = "0.24", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] }
config = "0.13"
@@ -19,6 +19,7 @@ rand = "0.8"
tracing = "0.1"
tracing-subscriber = "0.3"
thiserror = "1.0"
mimalloc = "0.1.48"
regex = "1"
lazy_static = "1.4"
jieba-rs = "0.7"
rayon = "1"

View File

@@ -15,10 +15,12 @@
<i class="fas fa-arrow-left"></i> 返回首页
</a>
<h1 class="album-title" id="album-title">加载中...</h1>
<div class="album-info" id="album-info">
<i class="fas fa-images"></i> 正在加载...
<div class="album-header-row">
<div class="album-info" id="album-info">
正在加载...
</div>
<div class="album-meta-tags" id="album-meta-tags"></div>
</div>
<div class="album-meta-tags" id="album-meta-tags"></div>
</div>
<div class="images-grid" id="images-container">
@@ -38,13 +40,6 @@
<span class="preview-info" id="lightbox-album">来自: 相册名称</span>
</div>
<div class="preview-actions">
<button class="preview-nav preview-prev" onclick="prevImage()" id="lightbox-prev">
<i class="fas fa-chevron-left"></i>
</button>
<span class="preview-counter" id="lightbox-counter">1 / 1</span>
<button class="preview-nav preview-next" onclick="nextImage()" id="lightbox-next">
<i class="fas fa-chevron-right"></i>
</button>
<button class="preview-close" onclick="closeLightbox()">
<i class="fas fa-times"></i>
</button>
@@ -70,7 +65,19 @@
0 KB
</span>
</div>
<div class="preview-center-controls">
<button class="preview-nav preview-prev" onclick="prevImage()" id="lightbox-prev">
<i class="fas fa-chevron-left"></i>
</button>
<span class="preview-counter" id="lightbox-counter">1 / 1</span>
<button class="preview-nav preview-next" onclick="nextImage()" id="lightbox-next">
<i class="fas fa-chevron-right"></i>
</button>
</div>
<div class="preview-controls">
<button class="preview-action preview-action-close" onclick="closeLightbox()">
<i class="fas fa-times"></i> 关闭
</button>
<button class="preview-action" onclick="addToFavoritesFromLightbox()" id="lightbox-favorite-btn">
<i class="fas fa-heart"></i> 加入收藏
</button>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -8,39 +8,152 @@
<link rel="stylesheet" href="/assets/css/style.css">
</head>
<body>
<div class="container">
<div class="header">
<h1><i class="fas fa-images"></i> <span>流云图库</span></h1>
<p>发现精彩瞬间,浏览精美图片</p>
</div>
<div class="app-layout">
<div class="search-container">
<div class="search-box">
<label for="search-input"></label><input type="text" id="search-input" placeholder="搜索图片标题...">
<button onclick="searchAlbums()"><i class="fas fa-search"></i></button>
<!-- ============ 左侧边栏(可折叠) ============ -->
<aside class="sidebar" id="sidebar">
<!-- Logo 区域 -->
<div class="sidebar-brand">
<div class="brand-icon"><i class="fas fa-images"></i></div>
<div class="brand-text">流云图库</div>
</div>
<div class="action-buttons">
<button class="random-button" onclick="openRandomAlbum()">
<i class="fas fa-random"></i> 随机相册
</button>
<button class="favorites-button" onclick="showFavorites()">
<i class="fas fa-heart"></i> 收藏夹
<!-- 导航菜单 -->
<nav class="sidebar-nav">
<div class="nav-item active" data-view="home" onclick="switchView('home')">
<i class="fas fa-home"></i>
<span>首页</span>
</div>
<div class="nav-item" data-view="favorites" onclick="showFavorites()">
<i class="fas fa-heart"></i>
<span>收藏夹</span>
</div>
<div class="nav-item" onclick="openRandomAlbum()">
<i class="fas fa-random"></i>
<span>随机相册</span>
</div>
</nav>
<!-- 分隔线 -->
<div class="sidebar-divider"></div>
<!-- 热门人物 -->
<div class="sidebar-section-group">
<div class="sidebar-section-title collapsible" data-section="people" onclick="toggleSidebarSection('people')">
<i class="fas fa-users"></i>
<span>热门人物</span>
<i class="fas fa-chevron-down section-arrow"></i>
</div>
<div class="sidebar-section-content" id="sidebar-people-content">
<div class="sidebar-people" id="sidebar-people">
<!-- 动态填充 -->
</div>
</div>
</div>
<!-- 热门标签 -->
<div class="sidebar-section-group">
<div class="sidebar-section-title collapsible" data-section="tags" onclick="toggleSidebarSection('tags')">
<i class="fas fa-tags"></i>
<span>热门标签</span>
<i class="fas fa-chevron-down section-arrow"></i>
</div>
<div class="sidebar-section-content" id="sidebar-tags-content">
<div class="sidebar-tags" id="sidebar-tags">
<!-- 动态填充 -->
</div>
</div>
</div>
<!-- 底部折叠按钮 -->
<div class="sidebar-footer">
<button class="collapse-btn" onclick="toggleSidebar()" id="collapse-btn" title="收起侧边栏">
<i class="fas fa-chevron-left"></i>
</button>
</div>
</div>
</aside>
<div class="stats" id="stats">
正在加载相册数据...
</div>
<!-- ============ 主内容区 ============ -->
<main class="main-content">
<div class="albums-grid" id="albums-container">
<div class="loading">
<i class="fas fa-spinner fa-spin"></i><br>加载中...
<!-- 顶部工具栏 -->
<header class="topbar">
<!-- 移动端菜单按钮 -->
<button class="mobile-menu-btn" onclick="toggleMobileSidebar()">
<i class="fas fa-bars"></i>
</button>
<!-- 搜索框 -->
<div class="search-bar">
<i class="fas fa-search search-icon"></i>
<label for="search-input"></label>
<input type="text" id="search-input" placeholder="搜索相册标题、人物、标签..." onkeydown="if(event.key==='Enter') searchAlbums()">
<button class="search-clear" id="search-clear" onclick="clearSearch()" style="display:none;">
<i class="fas fa-times"></i>
</button>
</div>
<!-- 右侧快捷按钮 -->
<div class="topbar-actions">
<button class="action-btn" onclick="openRandomAlbum()" title="随机相册">
<i class="fas fa-random"></i>
</button>
<button class="action-btn" onclick="showFavorites()" title="收藏夹">
<i class="fas fa-heart"></i>
</button>
</div>
</header>
<!-- 筛选状态条 -->
<div class="filter-bar" id="filter-bar" style="display:none;">
<span class="filter-indicator">
<i class="fas fa-filter"></i>
当前筛选:
</span>
<span class="filter-pill" id="current-filter-pill"></span>
<button class="filter-remove" onclick="clearFilter()">
<i class="fas fa-times"></i>
</button>
</div>
</div>
<!-- 内容区域 -->
<div class="content-area">
<!-- 统计条 -->
<div class="stats-bar" id="stats-bar">
<div class="stat-item">
<span class="stat-icon"><i class="fas fa-folder-open"></i></span>
<span class="stat-value" id="stat-albums">0</span>
<span class="stat-label">个相册</span>
</div>
<div class="stat-divider"></div>
<div class="stat-item">
<span class="stat-icon"><i class="fas fa-image"></i></span>
<span class="stat-value" id="stat-images">0</span>
<span class="stat-label">张图片</span>
</div>
<div class="stat-divider"></div>
<div class="stat-item">
<span class="stat-icon"><i class="fas fa-user"></i></span>
<span class="stat-value" id="stat-people">0</span>
<span class="stat-label">位人物</span>
</div>
</div>
<!-- 相册网格 -->
<div class="albums-grid" id="albums-container">
<div class="loading">
<i class="fas fa-spinner fa-spin"></i><br>加载中...
</div>
</div>
</div>
</main>
</div>
<script src="/assets/js/app.js?v=4"></script>
<!-- 移动端遮罩 -->
<div class="mobile-overlay" id="mobile-overlay" onclick="toggleMobileSidebar()"></div>
<script src="/assets/js/app.js?v=6"></script>
</body>
</html>

View File

@@ -235,7 +235,7 @@ async function loadAlbum(albumId) {
// 显示骨架屏
showImageSkeleton();
const response = await fetch(`/api/album/${albumId}`);
const response = await fetch(`${API_BASE}/album/${albumId}`);
if (!response.ok) {
if (response.status === 404) {
throw new Error('相册不存在');
@@ -245,6 +245,9 @@ async function loadAlbum(albumId) {
const albumData = await response.json();
currentAlbum = albumData.album;
// tags/persons 在 albumData 顶层(AlbumWithImages),存到 currentAlbum 方便渲染
currentAlbum.tags = albumData.tags || [];
currentAlbum.persons = albumData.persons || [];
images = albumData.images;
updateAlbumInfo();
@@ -262,7 +265,7 @@ async function loadAlbum(albumId) {
function updateAlbumInfo() {
document.getElementById('album-title').textContent = currentAlbum.title;
let infoHtml = `<i class="fas fa-images"></i> 共 ${images.length} 张图片`;
let infoHtml = `${images.length} 张图片`;
document.getElementById('album-info').innerHTML = infoHtml;
// 显示相册元数据(人名 + 标签)
@@ -271,48 +274,80 @@ function updateAlbumInfo() {
document.title = `${currentAlbum.title} - 图片展示服务`;
}
// 更新相册元数据(人名 + 标签)点击可搜索
// 标签分类映射(与后端 tag_extractor.rs 保持一致)
const TAG_CATEGORIES = {
style: ['日系', '韩系', '森系', '复古', '甜美', '性感', '清新', '冷艳',
'优雅', '知性', '可爱', '俏皮', '温柔', '妩媚', '野性', '清纯',
'时尚', '潮流', '街头', '学院', '文艺', '古典', '民族', '异域',
'japanese', 'korean', 'retro', 'vintage', 'cute', 'sexy',
'elegant', 'fresh', 'sweet', 'wild', 'pure', 'fashion'],
outfit: ['比基尼', '泳装', '泳衣', '旗袍', '制服', '丝袜', '内衣', '文胸',
'包臀裙', '紧身裙', '超短裙', '连衣裙', '半身裙', '短裙', '长裙',
'牛仔裤', '热裤', '短裤', '休闲裤', '西装裤',
'T恤', '衬衫', '毛衣', '针织衫', '外套', '风衣', '大衣', '西装',
'吊带', '抹胸', '露背', '透视', '蕾丝', '镂空', '开叉',
'高跟鞋', '运动鞋', '凉鞋', '拖鞋', '靴子',
'bikini', 'swimsuit', 'stockings', 'lingerie', 'dress',
'skirt', 'jeans', 'shirt', 'sweater', 'coat', 'heels'],
scene: ['室内', '户外', '街拍', '浴室', '浴缸', '床上', '床照', '沙发',
'厨房', '阳台', '窗边', '楼梯', '走廊', '电梯', '车内', '驾驶座',
'海边', '沙滩', '泳池', '温泉', '森林', '草地', '花园', '公园',
'街道', '巷弄', '建筑', '天台', '屋顶', '咖啡馆', '餐厅', '酒店',
'indoor', 'outdoor', 'street', 'beach', 'pool', 'bathroom',
'bedroom', 'kitchen', 'balcony', 'car', 'cafe', 'hotel'],
body: ['美腿', '美胸', '翘臀', '玉足', '美背', '香肩', '蛮腰', '细腰',
'长腿', '酥胸', '丰臀', '白皙', '肌肤', '纹身',
'legs', 'breasts', 'ass', 'feet', 'back', 'shoulders', 'waist',
'skin', 'tattoo'],
mood: ['清纯', '诱惑', '唯美', '清新', '冷艳', '妩媚', '娇羞', '俏皮',
'温柔', '野性', '冷傲', '热情', '慵懒', '迷离', '甜美', '知性',
'pure', 'seductive', 'beautiful', 'fresh', 'cool', 'charming',
'shy', 'cute', 'gentle', 'wild', 'lazy', 'sweet'],
topic: ['写真', '私房', '街拍', '旅拍', '婚纱照', '艺术照', '生活照',
'自拍', '跟拍', '约拍', '商拍',
'photoshoot', 'portrait', 'selfie', 'travel', 'wedding'],
};
// 获取标签的分类
function getTagCategory(tag) {
const tagLower = tag.toLowerCase();
for (const [category, tags] of Object.entries(TAG_CATEGORIES)) {
if (tags.some(t => t.toLowerCase() === tagLower)) {
return category;
}
}
return 'default';
}
// 更新相册元数据(人名 + 标签)简洁版:只用颜色区分,不写分类名
function updateAlbumMeta() {
const metaContainer = document.getElementById('album-meta-tags');
if (!metaContainer) return;
let chips = [];
// 添加人名标签
if (currentAlbum.person_name_cleaned) {
chips.push({
text: currentAlbum.person_name_cleaned,
type: 'person',
query: currentAlbum.person_name_cleaned
// 添加人名标签(从 search.db 返回的 persons 数组)
if (currentAlbum.persons && Array.isArray(currentAlbum.persons)) {
currentAlbum.persons.forEach(name => {
chips.push({
text: name,
type: 'person',
category: 'person',
query: name
});
});
}
// 添加标签
if (currentAlbum.tags) {
try {
const tags = JSON.parse(currentAlbum.tags);
if (Array.isArray(tags)) {
tags.forEach(tag => {
chips.push({
text: tag,
type: 'tag',
query: tag
});
});
}
} catch (e) {
// 非JSON格式,当作逗号分隔
currentAlbum.tags.split(',').forEach(tag => {
const t = tag.trim();
if (t) {
chips.push({
text: t,
type: 'tag',
query: t
});
}
// 添加标签(从 search.db 返回的 tags 数组)
if (currentAlbum.tags && Array.isArray(currentAlbum.tags)) {
currentAlbum.tags.forEach(tag => {
chips.push({
text: tag,
type: 'tag',
category: getTagCategory(tag),
query: tag
});
}
});
}
if (chips.length === 0) {
@@ -323,7 +358,8 @@ function updateAlbumMeta() {
metaContainer.style.display = 'flex';
metaContainer.innerHTML = chips.map(chip => {
const icon = chip.type === 'person' ? 'fa-user' : 'fa-tag';
return `<span class="meta-chip meta-chip-${chip.type}" data-meta-type="${chip.type}" data-meta-query="${encodeURIComponent(chip.query)}">
const cssClass = chip.type === 'person' ? 'meta-chip-person' : `meta-chip-tag-${chip.category || 'default'}`;
return `<span class="meta-chip ${cssClass}" data-meta-type="${chip.type}" data-meta-query="${encodeURIComponent(chip.query)}" title="点击搜索: ${chip.text}">
<i class="fas ${icon}"></i> ${chip.text}
</span>`;
}).join('');
@@ -377,7 +413,7 @@ function displayImages() {
let dataSrc = '';
if (image.id) {
dataSrc = `/api/image/${image.id}`;
dataSrc = `${API_BASE}/image/${image.id}`;
imageUrl = '/assets/placeholder.svg'; // 初始使用占位图
} else if (image.original_url) {
dataSrc = image.original_url;
@@ -389,7 +425,7 @@ function displayImages() {
// 检查图片是否已被收藏
isImageFavorited(image.id || image.original_url);
return `
<div class="image-item ${orientation}" onclick="openLightbox(${index})">
<div class="image-item ${orientation}">
<img
src="${imageUrl}"
${image.id ? `data-image-id="${image.id}"` : ''}
@@ -397,6 +433,7 @@ function displayImages() {
class="lazy"
loading="lazy"
onerror="this.src='/assets/placeholder.svg'"
onclick="openLightbox(${index})"
>
<div class="image-caption">
${image.width ? `<small>${image.width}×${image.height}</small>` : ''}
@@ -490,7 +527,7 @@ function updateLightboxContent(newIndex) {
// 在lightbox中优先使用原始图片以获得更好的显示效果
let imageUrl = '/assets/placeholder.svg';
if (image.id) {
imageUrl = `/api/image/${image.id}`;
imageUrl = `${API_BASE}/image/${image.id}`;
} else if (image.original_url) {
imageUrl = image.original_url;
}

View File

@@ -1,6 +1,7 @@
// 全局变量
let allAlbums = []; // 存储所有相册数据
let displayedAlbums = []; // 当前显示的相册数据
let filteredAlbums = null; // 过滤后的相册数据(有筛选时使用,null=无筛选)
let searchTimeout = null;
let currentDisplayPage = 1; // 前端显示的分页
let albumsPerPage = 50; // 每次显示50个相册
@@ -10,6 +11,13 @@ let lazyLoadObserver = null;
let isAllAlbumsLoaded = false; // 标记是否已加载所有相册数据
let isInFavoritesPage = false; // 标记当前是否在收藏页
// 标签和人物筛选
let currentFilter = null; // { type: 'tag'|'person', value: string }
let collapsedSections = { people: false, tags: false };
// 侧边栏状态(持久化)
let sidebarCollapsed = localStorage.getItem('sidebarCollapsed') === 'true';
// API基础URL
const API_BASE = '/api';
@@ -102,6 +110,7 @@ document.addEventListener('DOMContentLoaded', function () {
initSearch();
initInfiniteScroll();
initLazyLoading(); // 初始化图片懒加载
initSidebarSections(); // 初始化侧边栏区块折叠状态
// 检查URL中是否有search参数
const urlParams = new URLSearchParams(window.location.search);
@@ -244,7 +253,9 @@ async function loadMoreAlbums() {
// 如果在收藏页,不加载更多数据
if (isInFavoritesPage) return;
if (isLoadingMore || !hasMoreAlbums || !isAllAlbumsLoaded) return;
if (isLoadingMore || !hasMoreAlbums) return;
// 全量模式下(无筛选)需等 isAllAlbumsLoaded;过滤模式下直接使用 filteredAlbums
if (!currentFilter && !isAllAlbumsLoaded) return;
isLoadingMore = true;
try {
@@ -358,7 +369,7 @@ function appendAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
dataSrc = `/api/image/${album.cover_image_id}`;
dataSrc = `${API_BASE}/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -437,6 +448,9 @@ async function loadAlbums(reset = false) {
// 预加载下一批图片
preloadNextBatchImages();
// 加载热门标签和人物
loadDiscoverData();
} catch (error) {
console.error('加载相册失败:', error);
showError('加载相册数据失败,请刷新页面重试');
@@ -445,11 +459,13 @@ async function loadAlbums(reset = false) {
// 显示当前页的相册数据
function displayCurrentPage() {
// 有筛选时用 filteredAlbums,否则用 allAlbums
const source = filteredAlbums || allAlbums;
const startIndex = (currentDisplayPage - 1) * albumsPerPage;
const endIndex = startIndex + albumsPerPage;
// 获取当前页要显示的相册
displayedAlbums = allAlbums.slice(startIndex, endIndex);
displayedAlbums = source.slice(startIndex, endIndex);
// 如果是第一页,直接显示;否则追加显示
if (currentDisplayPage === 1) {
@@ -459,7 +475,7 @@ function displayCurrentPage() {
}
// 检查是否还有更多数据可以显示
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < allAlbums.length;
hasMoreAlbums = (currentDisplayPage * albumsPerPage) < source.length;
}
// 搜索相册
@@ -468,14 +484,24 @@ async function debouncedSearch() {
const searchTerm = document.getElementById('search-input').value.toLowerCase().trim();
if (searchTerm === '') {
// 显示所有已加载的相册
// 搜索框为空 → 清除过滤状态,显示所有相册
currentFilter = null;
filteredAlbums = null;
displayAlbums(allAlbums);
updateStats();
return;
}
// 直接使用后端搜索API进行全站搜索
await performServerSearch(searchTerm);
const results = await performServerSearchWithResult(searchTerm);
if (results && results.length > 0) {
filteredAlbums = results;
currentDisplayPage = 1;
hasMoreAlbums = results.length > albumsPerPage;
displayAlbums(results);
updateStats(results);
initLazyLoading();
}
}
// 在当前已加载的数据集中搜索
@@ -511,11 +537,22 @@ function showSearchHint(searchTerm) {
// 执行服务器端搜索(全站搜索)
async function performServerSearch(searchTerm) {
try {
// 显示搜索中的骨架屏
showSkeleton(4);
const results = await performServerSearchWithResult(searchTerm);
if (results && results.length > 0) {
displayAlbums(results);
const oldStats = document.getElementById('stats');
if (oldStats) {
oldStats.innerHTML = `在全站找到 ${results.length} 个包含 "${searchTerm}" 的相册`;
}
updateStats(results);
initLazyLoading();
}
}
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=${albumsPerPage}`);
/// 执行服务器端搜索并返回结果数组(不直接渲染)
async function performServerSearchWithResult(searchTerm) {
try {
const response = await fetch(`${API_BASE}/search?q=${encodeURIComponent(searchTerm)}&page=1&limit=200`);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
@@ -523,20 +560,13 @@ async function performServerSearch(searchTerm) {
const serverResults = await response.json();
if (serverResults.length === 0) {
showError(`没有找到包含 "${searchTerm}" 的相册`);
return;
return [];
}
// 显示服务器搜索结果
displayAlbums(serverResults);
document.getElementById('stats').innerHTML = `在全站找到 ${serverResults.length} 个包含 "${searchTerm}" 的相册`;
// 初始化图片懒加载
initLazyLoading();
return serverResults;
} catch (error) {
console.error('全站搜索失败:', error);
showError('全站搜索失败,请重试');
return [];
}
}
@@ -555,7 +585,7 @@ function displayAlbums(albums) {
let dataSrc = '';
if (album.cover_image_id) {
dataSrc = `/api/image/${album.cover_image_id}`;
dataSrc = `${API_BASE}/image/${album.cover_image_id}`;
coverImageUrl = '/assets/placeholder.svg'; // 初始使用占位图
}
@@ -615,14 +645,32 @@ function updateStats(albums) {
albums = allAlbums;
}
const totalImages = albums.reduce((sum, album) => sum + album.image_count, 0);
document.getElementById('stats').innerHTML = `共发现 ${albums.length} 个相册,包含 ${totalImages} 张图片`;
// 更新统计条
const statAlbums = document.getElementById('stat-albums');
const statImages = document.getElementById('stat-images');
if (statAlbums) statAlbums.textContent = albums.length.toLocaleString();
if (statImages) statImages.textContent = totalImages.toLocaleString();
// 兼容旧版 stats
const oldStats = document.getElementById('stats');
if (oldStats) oldStats.innerHTML = `共发现 ${albums.length} 个相册,包含 ${totalImages} 张图片`;
// 获取人物数量(用 metadata stats API)
fetch(`${API_BASE}/metadata/stats`).then(r => r.ok && r.json()).then(data => {
if (data && data.total_persons !== undefined) {
const statPeople = document.getElementById('stat-people');
if (statPeople) statPeople.textContent = data.total_persons.toLocaleString();
}
}).catch(() => {});
}
// 显示错误信息
function showError(message) {
const container = document.getElementById('albums-container');
container.innerHTML = `<div class="no-results"><i class="fas fa-exclamation-triangle"></i><br>${message}</div>`;
document.getElementById('stats').innerHTML = '';
const oldStats = document.getElementById('stats');
if (oldStats) oldStats.innerHTML = '';
}
// 搜索相册函数
@@ -771,14 +819,20 @@ function openAlbum(albumId) {
// 随机打开相册
function openRandomAlbum() {
// 添加随机动画效果
const randomButton = document.querySelector('.random-button');
randomButton.style.transform = 'scale(0.95)';
randomButton.disabled = true;
// 添加随机动画效果 - 兼容侧边栏导航项
const navItems = document.querySelectorAll('.nav-item');
let randomButton = null;
navItems.forEach(item => {
if (item.textContent.includes('随机')) randomButton = item;
});
// 也试试顶部操作按钮
if (!randomButton) {
randomButton = document.querySelector('.action-btn[onclick*="openRandomAlbum"]');
}
// 显示加载状态
const originalText = randomButton.textContent;
randomButton.textContent = '随机加载中...';
if (randomButton) {
randomButton.style.transform = 'scale(0.95)';
}
// 调用新的随机相册API
fetch(`${API_BASE}/album/random`)
@@ -793,22 +847,22 @@ function openRandomAlbum() {
})
.then(randomAlbum => {
// 恢复按钮状态
setTimeout(() => {
randomButton.style.transform = '';
randomButton.disabled = false;
randomButton.textContent = originalText;
}, 200);
if (randomButton) {
setTimeout(() => {
randomButton.style.transform = '';
}, 200);
}
// 跳转到随机相册
window.location.href = `/album?id=${randomAlbum.id}`;
})
.catch(error => {
// 恢复按钮状态
setTimeout(() => {
randomButton.style.transform = '';
randomButton.disabled = false;
randomButton.textContent = originalText;
}, 200);
if (randomButton) {
setTimeout(() => {
randomButton.style.transform = '';
}, 200);
}
alert(error.message);
});
@@ -819,6 +873,11 @@ function showFavorites() {
// 设置当前在收藏页
isInFavoritesPage = true;
// 切换侧边栏选中状态
document.querySelectorAll('.nav-item').forEach(item => item.classList.remove('active'));
const favNav = document.querySelector('.nav-item[data-view="favorites"]');
if (favNav) favNav.classList.add('active');
// 从后端API获取收藏列表
fetch(`${API_BASE}/favorites`)
.then(response => {
@@ -891,8 +950,10 @@ function displayFavorites(favoriteImages) {
`}
`;
// 更新统计信息 - 显示收藏数量
document.getElementById('stats').innerHTML = `<div class="favorites-header">
// 更新统计信息 - 显示收藏数量(兼容新旧版布局)
const statsEl = document.getElementById('stats');
if (statsEl) {
statsEl.innerHTML = `<div class="favorites-header">
<div class="header-content">
<div class="header-left">
<div class="favorites-icon">
@@ -912,6 +973,7 @@ function displayFavorites(favoriteImages) {
</div>
</div>
</div>`;
}
// 初始化图片懒加载,确保收藏的图片被加载
initLazyLoading();
@@ -931,8 +993,12 @@ function formatDate(dateString) {
function backToHome() {
// 重置收藏页标记
isInFavoritesPage = false;
// 重新加载首页相册
loadAlbums(true);
// 切换导航高亮到首页
document.querySelectorAll('.nav-item').forEach(item => {
item.classList.toggle('active', item.dataset.view === 'home');
});
// 清除筛选
clearFilter();
}
// 全局变量存储收藏列表和当前预览索引
@@ -1199,6 +1265,406 @@ function openFavoriteImage(favoriteId) {
window.location.href = `/album?id=${favoriteId}`;
}
// ============================================================
// 标签和人物发现功能
// ============================================================
// 加载热门标签和人物数据
async function loadDiscoverData() {
try {
// 并行加载
const [tagsResp, peopleResp] = await Promise.all([
fetch(`${API_BASE}/tags/popular?limit=40`),
fetch(`${API_BASE}/recommend/popular-persons?limit=30`)
]);
if (tagsResp.ok) {
const tagsData = await tagsResp.json();
// 兼容两种格式:直接数组 或 { total, tags }
const tags = Array.isArray(tagsData) ? tagsData : (tagsData.tags || tagsData.items || []);
if (tags && tags.length > 0) {
renderTagCloud(tags);
}
}
if (peopleResp.ok) {
const people = await peopleResp.json();
if (people && people.length > 0) {
renderPeopleCloud(people);
}
}
} catch (error) {
console.warn('加载发现数据失败:', error);
}
}
// 渲染标签(侧边栏列表)
function renderTagCloud(tags) {
// 新版:侧边栏标签列表
const sidebarTags = document.getElementById('sidebar-tags');
if (sidebarTags && tags && tags.length) {
sidebarTags.innerHTML = tags.slice(0, 25).map(tag => {
const count = tag.album_cnt || tag.count || 1;
const name = tag.tag || tag.name || '';
const isActive = currentFilter && currentFilter.type === 'tag' && currentFilter.value === name;
return `<div class="sidebar-tag ${isActive ? 'active' : ''}" onclick="filterByTag('${name}')">
<span class="sidebar-tag-name">${name}</span>
<span class="sidebar-tag-count">${count}</span>
</div>`;
}).join('');
}
// 兼容旧版:标签云区域
const section = document.getElementById('tags-section');
const cloud = document.getElementById('tag-cloud');
if (section && cloud && tags && tags.length) {
section.style.display = 'block';
const maxCount = Math.max(...tags.map(t => t.album_cnt || t.count || 1));
const minCount = Math.min(...tags.map(t => t.album_cnt || t.count || 1));
cloud.innerHTML = tags.map(tag => {
const count = tag.album_cnt || tag.count || 1;
const name = tag.tag || tag.name || '';
const ratio = maxCount === minCount ? 0.5 : (count - minCount) / (maxCount - minCount);
const fontSize = 12 + ratio * 10;
const hue = 210 - ratio * 50;
const opacity = 0.6 + ratio * 0.4;
return `<span class="tag-chip"
style="font-size:${fontSize}px; color:hsl(${hue}, 70%, 45%); opacity:${opacity};"
onclick="filterByTag('${name}')"
title="${count} 个相册">
${name} <small>(${count})</small>
</span>`;
}).join('');
}
}
// 渲染热门人物(横向滚动条)
function renderPeopleCloud(people) {
// 侧边栏人物列表
const sidebarPeople = document.getElementById('sidebar-people');
if (sidebarPeople && people && people.length) {
sidebarPeople.innerHTML = people.slice(0, 25).map(p => {
const name = p.name || p.display_name || '';
const albumCount = p.total_albums || 0;
const isActive = currentFilter && currentFilter.type === 'person' && currentFilter.value === name;
return `<div class="sidebar-person ${isActive ? 'active' : ''}" onclick="filterByPerson('${name}')" title="${albumCount} 个相册">
<div class="sidebar-person-avatar">${name.charAt(0)}</div>
<div class="sidebar-person-info">
<div class="sidebar-person-name">${name}</div>
<div class="sidebar-person-count">${albumCount} 相册</div>
</div>
</div>`;
}).join('');
}
// 横向滚动条
const strip = document.getElementById('people-strip');
const scroll = document.getElementById('people-scroll');
if (strip && scroll && people && people.length) {
strip.style.display = 'flex';
scroll.innerHTML = people.slice(0, 20).map(p => {
const name = p.name || p.display_name || '';
const albumCount = p.total_albums || 0;
const isActive = currentFilter && currentFilter.type === 'person' && currentFilter.value === name;
return `<div class="person-chip-scroll ${isActive ? 'active' : ''}" onclick="filterByPerson('${name}')" title="${albumCount} 个相册">
<div class="person-chip-avatar">${name.charAt(0)}</div>
<div class="person-chip-info">
<div class="person-chip-name">${name}</div>
<div class="person-chip-count">${albumCount} 相册</div>
</div>
</div>`;
}).join('');
}
// 兼容旧版:人物卡片区域
const section = document.getElementById('people-section');
const cloud = document.getElementById('people-cloud');
if (section && cloud && people && people.length) {
section.style.display = 'block';
cloud.innerHTML = people.map(p => {
const name = p.name || p.display_name || '';
const albumCount = p.total_albums || 0;
const imageCount = p.total_images || 0;
return `<div class="person-chip" onclick="filterByPerson('${name}')" title="${albumCount} 个相册, ${imageCount} 张图片">
<div class="person-avatar">${name.charAt(0)}</div>
<div class="person-info">
<div class="person-name">${name}</div>
<div class="person-meta">${albumCount} 相册 · ${imageCount} 图</div>
</div>
</div>`;
}).join('');
}
}
// 侧边栏区块折叠/展开
function toggleSidebarSection(section) {
const group = document.querySelector(`.sidebar-section-group [data-section="${section}"]`)?.closest('.sidebar-section-group');
if (!group) return;
group.classList.toggle('collapsed');
// 保存状态到 localStorage
try {
const state = JSON.parse(localStorage.getItem('sidebarSections') || '{}');
state[section] = group.classList.contains('collapsed');
localStorage.setItem('sidebarSections', JSON.stringify(state));
} catch(e) {}
}
// 初始化侧边栏区块折叠状态
function initSidebarSections() {
try {
const state = JSON.parse(localStorage.getItem('sidebarSections') || '{}');
Object.keys(state).forEach(section => {
if (state[section]) {
const group = document.querySelector(`.sidebar-section-group [data-section="${section}"]`)?.closest('.sidebar-section-group');
if (group) group.classList.add('collapsed');
}
});
} catch(e) {}
}
// 按标签筛选
async function filterByTag(tag) {
currentFilter = { type: 'tag', value: tag };
showFilterStatus(`标签: ${tag}`);
renderTagCloud(await fetchTags().catch(() => []));
try {
showSkeleton(6);
const response = await fetch(`${API_BASE}/tags/search?tag=${encodeURIComponent(tag)}&limit=200`);
if (!response.ok) throw new Error('搜索失败');
const results = await response.json();
// 保存过滤结果到 filteredAlbums,重置分页
filteredAlbums = results;
currentDisplayPage = 1;
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
displayAlbums(results);
updateStats(results);
initLazyLoading();
} catch (error) {
console.error('标签筛选失败:', error);
showError('标签筛选失败');
}
}
// 获取标签(辅助函数)
async function fetchTags() {
const resp = await fetch(`${API_BASE}/tags/popular?limit=25`);
if (!resp.ok) return [];
return resp.json();
}
// 按人物筛选
async function filterByPerson(personName) {
currentFilter = { type: 'person', value: personName };
showFilterStatus(`人物: ${personName}`);
// 刷新侧边栏人物高亮
try {
const peopleResp = await fetch(`${API_BASE}/recommend/popular-persons?limit=30`);
if (peopleResp.ok) {
const peopleData = await peopleResp.json();
renderPeopleCloud(peopleData);
}
} catch(e) {}
try {
showSkeleton(6);
const response = await fetch(`${API_BASE}/people/search?q=${encodeURIComponent(personName)}`);
if (!response.ok) throw new Error('搜索失败');
const personData = await response.json();
const persons = personData.persons || personData;
if (persons && persons.length > 0) {
try {
const detailResp = await fetch(`${API_BASE}/people/${encodeURIComponent(personName)}`);
if (detailResp.ok) {
const detail = await detailResp.json();
if (detail && detail.albums) {
// 保存过滤结果到 filteredAlbums,重置分页
filteredAlbums = detail.albums;
currentDisplayPage = 1;
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
displayAlbums(detail.albums);
updateStats(detail.albums);
initLazyLoading();
return;
}
}
} catch (e) { /* 回退 */ }
}
// 回退到全站搜索
filteredAlbums = await performServerSearchWithResult(personName);
} catch (error) {
console.error('人物筛选失败:', error);
filteredAlbums = await performServerSearchWithResult(personName);
}
// 无论回退搜索是否找到结果,都要渲染页面(移除骨架屏)
if (filteredAlbums !== null) {
currentDisplayPage = 1;
hasMoreAlbums = filteredAlbums.length > albumsPerPage;
displayAlbums(filteredAlbums);
updateStats(filteredAlbums);
initLazyLoading();
}
}
// 显示筛选状态
function showFilterStatus(text) {
// 新版筛选条
const bar = document.getElementById('filter-bar');
const pill = document.getElementById('current-filter-pill');
if (bar && pill) {
pill.textContent = text;
bar.style.display = 'flex';
}
// 兼容旧版
const status = document.getElementById('filter-status');
const label = document.getElementById('current-filter');
if (status && label) {
label.textContent = text;
status.style.display = 'flex';
}
}
// 清除筛选
function clearFilter() {
currentFilter = null;
filteredAlbums = null; // 清除过滤结果缓存
currentDisplayPage = 1;
// 新版筛选条
const bar = document.getElementById('filter-bar');
if (bar) bar.style.display = 'none';
// 兼容旧版
const status = document.getElementById('filter-status');
if (status) status.style.display = 'none';
// 清空搜索框
const searchInput = document.getElementById('search-input');
if (searchInput) {
searchInput.value = '';
updateSearchClear();
}
// 重新渲染标签和人物(清除高亮)
loadDiscoverData();
// 重新加载所有相册
loadAlbums(true);
}
// ============ 侧边栏交互 ============
// 初始化侧边栏状态
function initSidebar() {
const sidebar = document.getElementById('sidebar');
if (sidebar && sidebarCollapsed) {
sidebar.classList.add('collapsed');
}
}
// 切换侧边栏折叠
function toggleSidebar() {
const sidebar = document.getElementById('sidebar');
if (!sidebar) return;
sidebarCollapsed = !sidebarCollapsed;
sidebar.classList.toggle('collapsed', sidebarCollapsed);
localStorage.setItem('sidebarCollapsed', sidebarCollapsed);
}
// 移动端切换侧边栏
function toggleMobileSidebar() {
const sidebar = document.getElementById('sidebar');
const overlay = document.getElementById('mobile-overlay');
if (!sidebar || !overlay) return;
sidebar.classList.toggle('mobile-open');
overlay.classList.toggle('active');
}
// 切换视图(首页/收藏夹)
function switchView(view) {
// 更新导航高亮
document.querySelectorAll('.nav-item').forEach(item => {
item.classList.toggle('active', item.dataset.view === view);
});
// 移动端关闭侧边栏
const sidebar = document.getElementById('sidebar');
const overlay = document.getElementById('mobile-overlay');
if (sidebar && sidebar.classList.contains('mobile-open')) {
sidebar.classList.remove('mobile-open');
overlay.classList.remove('active');
}
if (view === 'home') {
backToHome();
} else if (view === 'favorites') {
showFavorites();
}
}
// 搜索框清除按钮
function updateSearchClear() {
const input = document.getElementById('search-input');
const clear = document.getElementById('search-clear');
if (input && clear) {
clear.style.display = input.value.length > 0 ? 'flex' : 'none';
}
}
// 清除搜索
function clearSearch() {
const input = document.getElementById('search-input');
if (input) {
input.value = '';
updateSearchClear();
if (!currentFilter) {
loadAlbums(true);
}
}
}
// 折叠/展开区域(兼容旧版)
function toggleSection(name) {
collapsedSections[name] = !collapsedSections[name];
const cloudId = name === 'people' ? 'people-cloud' : 'tag-cloud';
const cloud = document.getElementById(cloudId);
const btn = document.querySelector(`#${name}-section .section-toggle i`);
if (cloud) {
if (collapsedSections[name]) {
cloud.style.display = 'none';
btn.className = 'fas fa-chevron-down';
} else {
cloud.style.display = '';
btn.className = 'fas fa-chevron-up';
}
}
}
// ============ 初始化 ============
document.addEventListener('DOMContentLoaded', function() {
initSidebar();
// 搜索框输入事件
const searchInput = document.getElementById('search-input');
if (searchInput) {
searchInput.addEventListener('input', updateSearchClear);
}
// 键盘快捷键:ESC 关闭筛选/搜索
document.addEventListener('keydown', function(e) {
if (e.key === 'Escape') {
if (currentFilter) clearFilter();
}
});
});
// 导出函数供全局使用
window.searchAlbums = searchAlbums;
window.openRandomAlbum = openRandomAlbum;
@@ -1210,3 +1676,11 @@ window.removeFavorite = removeFavorite;
window.previewFavorite = previewFavorite;
window.openFavoriteImage = openFavoriteImage;
window.removeFavoriteFromPreview = removeFavoriteFromPreview;
window.filterByTag = filterByTag;
window.filterByPerson = filterByPerson;
window.clearFilter = clearFilter;
window.toggleSection = toggleSection;
window.toggleSidebar = toggleSidebar;
window.toggleMobileSidebar = toggleMobileSidebar;
window.switchView = switchView;
window.clearSearch = clearSearch;

661
dict/tags.json Normal file
View File

@@ -0,0 +1,661 @@
{
"version": 4,
"description": "标签词典 v4 — 全量 keymap + stopwords(人物规则已融入 keymap)",
"metadata": {
"style": {
"label": "风格",
"color_bg": "#f0f9ff",
"color_fg": "#0284c7"
},
"outfit": {
"label": "服装",
"color_bg": "#fdf2f8",
"color_fg": "#db2777"
},
"scene": {
"label": "场景",
"color_bg": "#f0fdf4",
"color_fg": "#15803d"
},
"body": {
"label": "身体",
"color_bg": "#fef3c7",
"color_fg": "#b45309"
},
"mood": {
"label": "情绪",
"color_bg": "#faf5ff",
"color_fg": "#7c3aed"
},
"topic": {
"label": "主题",
"color_bg": "#f0f9ff",
"color_fg": "#0369a1"
},
"position": {
"label": "姿势",
"color_bg": "#fefce8",
"color_fg": "#a16207"
},
"location": {
"label": "地点",
"color_bg": "#f0fdfa",
"color_fg": "#0f766e"
},
"brand": {
"label": "品牌/工作室",
"color_bg": "#fff7ed",
"color_fg": "#c2410c"
},
"person": {
"label": "人物",
"color_bg": "#ecfdf5",
"color_fg": "#059669"
}
},
"keymap": {
"嫩模": "topic",
"108TV": "brand",
"108酱": "brand",
"Adidas": "brand",
"Agent Provocateur": "brand",
"Apple": "brand",
"Av": "brand",
"Beautyleg": "brand",
"BlueCake": "brand",
"BoLoli": "brand",
"CK": "brand",
"Calvin Klein": "brand",
"CANDY": "brand",
"CANDY糖果画报": "brand",
"Canon": "brand",
"Chanel": "brand",
"DKGirl": "brand",
"DKGirl御女郎": "brand",
"Dior": "brand",
"FEILIN": "brand",
"FEILIN嗲囡囡": "brand",
"Fantasy Factory": "brand",
"FeiLin": "brand",
"Feiling": "brand",
"Gucci": "brand",
"Hermes": "brand",
"HuaYang": "brand",
"IMISS": "brand",
"IMiss": "brand",
"IMiss爱蜜社": "brand",
"ISHOW": "brand",
"ISHOW爱秀": "brand",
"La Perla": "brand",
"LeiLei": "brand",
"Louis Vuitton": "brand",
"MFModel": "brand",
"MFStar": "brand",
"MFStar模特": "brand",
"MFStar模范学院": "brand",
"MiiTao": "brand",
"MiiTao蜜桃社": "brand",
"MiStar": "brand",
"Nike": "brand",
"OnlyFans": "brand",
"Prada": "brand",
"Puma": "brand",
"Pure Media": "brand",
"PureMedia": "brand",
"RuiSG": "brand",
"RuiSG瑞丝馆": "brand",
"SIW": "brand",
"SIW斯文传媒": "brand",
"SJA": "brand",
"SJA佳爷": "brand",
"SLADY": "brand",
"SLADY猎女神": "brand",
"SSA": "brand",
"SSA丝社": "brand",
"Shiseido": "brand",
"Sony": "brand",
"TASHA": "brand",
"TASHA系列": "brand",
"TGOD": "brand",
"TGOD推女神": "brand",
"Tristana": "brand",
"UXING": "brand",
"UXING优星馆": "brand",
"Ugirls": "brand",
"Ugirls爱尤物": "brand",
"V Girls": "brand",
"VGIRL": "brand",
"VGIRLS": "brand",
"VS": "brand",
"Victoria Secret": "brand",
"Victoria's Secret": "brand",
"Victorias Secret": "brand",
"WING": "brand",
"WingS": "brand",
"WingS影私荟": "brand",
"XIAOYU": "brand",
"XIAOYU语画界": "brand",
"XINGYAN": "brand",
"XINGYAN星颜社": "brand",
"XiuRen": "brand",
"XiuRen秀人网": "brand",
"YALAYI": "brand",
"YALAYI雅拉伊": "brand",
"YOUMI尤蜜荟": "brand",
"YouMi": "brand",
"YouMi尤蜜": "brand",
"YouWu": "brand",
"YouWu尤物馆": "brand",
"爱尤物": "brand",
"爱蜜社": "brand",
"波萝社": "brand",
"飞灵": "brand",
"绯月樱": "brand",
"花洋": "brand",
"花漾": "brand",
"美媛馆": "brand",
"魅妍社": "brand",
"蜜桃社": "brand",
"撸图": "brand",
"情欲萝莉": "brand",
"青豆客": "brand",
"瑞丝馆": "brand",
"头条女神": "brand",
"推女神": "brand",
"星颜社": "brand",
"秀人": "brand",
"秀人网": "brand",
"语画界": "brand",
"尤果网": "brand",
"尤果圈": "brand",
"尤蜜": "brand",
"尤蜜荟": "brand",
"尤物馆": "brand",
"旗袍": "outfit",
"比基尼": "outfit",
"泳装": "outfit",
"泳衣": "outfit",
"制服": "outfit",
"丝袜": "outfit",
"内衣": "outfit",
"文胸": "outfit",
"包臀裙": "outfit",
"紧身裙": "outfit",
"超短裙": "outfit",
"连衣裙": "outfit",
"半身裙": "outfit",
"短裙": "outfit",
"丁字裤": "outfit",
"长裙": "outfit",
"牛仔裤": "outfit",
"热裤": "outfit",
"短裤": "outfit",
"休闲裤": "outfit",
"T恤": "outfit",
"衬衫": "outfit",
"毛衣": "outfit",
"针织衫": "outfit",
"西装": "outfit",
"外套": "outfit",
"风衣": "outfit",
"大衣": "outfit",
"马甲": "outfit",
"卫衣": "outfit",
"吊带": "outfit",
"抹胸": "outfit",
"露背": "outfit",
"透视": "outfit",
"蕾丝": "outfit",
"镂空": "outfit",
"开衩": "outfit",
"一字肩": "outfit",
"露肩": "outfit",
"深V": "outfit",
"低胸": "outfit",
"高开衩": "outfit",
"学生服": "outfit",
"水手服": "outfit",
"护士服": "outfit",
"女仆装": "outfit",
"洛丽塔": "outfit",
"和服": "outfit",
"汉服": "outfit",
"浴衣": "outfit",
"校服": "outfit",
"白衬衫": "outfit",
"黑衬衫": "outfit",
"白裙": "outfit",
"黑裙": "outfit",
"红裙": "outfit",
"裹胸": "outfit",
"吊带裙": "outfit",
"修身": "outfit",
"长靴": "outfit",
"高跟鞋": "outfit",
"凉鞋": "outfit",
"运动鞋": "outfit",
"ol套装": "outfit",
"工作服": "outfit",
"空姐": "outfit",
"警服": "outfit",
"军装": "outfit",
"黑丝": "outfit",
"白丝": "outfit",
"肉丝": "outfit",
"网袜": "outfit",
"bikini": "outfit",
"swimsuit": "outfit",
"stockings": "outfit",
"lingerie": "outfit",
"dress": "outfit",
"skirt": "outfit",
"jeans": "outfit",
"shirt": "outfit",
"coat": "outfit",
"heels": "outfit",
"室内": "scene",
"户外": "scene",
"棚拍": "scene",
"外景": "scene",
"街景": "scene",
"夜景": "scene",
"海边": "scene",
"海滩": "scene",
"沙滩": "scene",
"泳池": "scene",
"浴室": "scene",
"浴缸": "scene",
"厨房": "scene",
"客厅": "scene",
"卧室": "scene",
"阳台": "scene",
"花园": "scene",
"庭院": "scene",
"天台": "scene",
"楼顶": "scene",
"屋顶": "scene",
"咖啡厅": "scene",
"酒店": "scene",
"民宿": "scene",
"教室": "scene",
"图书馆": "scene",
"办公室": "scene",
"楼梯": "scene",
"电梯": "scene",
"草地": "scene",
"森林": "scene",
"树林": "scene",
"竹林": "scene",
"花丛": "scene",
"花海": "scene",
"公园": "scene",
"操场": "scene",
"马路": "scene",
"公路": "scene",
"天桥": "scene",
"隧道": "scene",
"床上": "scene",
"indoor": "scene",
"outdoor": "scene",
"street": "scene",
"beach": "scene",
"pool": "scene",
"bathroom": "scene",
"bedroom": "scene",
"kitchen": "scene",
"car": "scene",
"cafe": "scene",
"hotel": "scene",
"美腿": "body",
"美胸": "body",
"翘臀": "body",
"玉足": "body",
"美背": "body",
"香肩": "body",
"酥胸": "body",
"纤腰": "body",
"蛮腰": "body",
"蜂腰": "body",
"小蛮腰": "body",
"事业线": "body",
"锁骨": "body",
"马甲线": "body",
"腹肌": "body",
"长腿": "body",
"美足": "body",
"白皙": "body",
"肌肤": "body",
"纹身": "body",
"legs": "body",
"breasts": "body",
"ass": "body",
"feet": "body",
"back": "body",
"shoulders": "body",
"waist": "body",
"skin": "body",
"tattoo": "body",
"诱惑": "mood",
"唯美": "mood",
"娇羞": "mood",
"冷傲": "mood",
"热情": "mood",
"慵懒": "mood",
"迷离": "mood",
"神秘": "mood",
"忧郁": "mood",
"伤感": "mood",
"快乐": "mood",
"开心": "mood",
"微笑": "mood",
"大笑": "mood",
"闭眼": "mood",
"凝视": "mood",
"魅惑": "mood",
"撩人": "mood",
"高冷": "mood",
"呆萌": "mood",
"天真": "mood",
"无辜": "mood",
"纯真": "mood",
"lazy": "mood",
"写真": "topic",
"cos": "topic",
"私房": "topic",
"街拍": "topic",
"旅拍": "topic",
"婚纱照": "topic",
"艺术照": "topic",
"生活照": "topic",
"自拍": "topic",
"cosplay": "topic",
"角色扮演": "topic",
"动漫": "topic",
"游戏": "topic",
"水着": "topic",
"竞泳": "topic",
"温泉": "topic",
"入浴": "topic",
"photoshoot": "topic",
"portrait": "topic",
"selfie": "topic",
"travel": "topic",
"wedding": "topic",
"Genshin": "topic",
"站立": "position",
"坐姿": "position",
"躺姿": "position",
"跪姿": "position",
"趴姿": "position",
"侧躺": "position",
"侧卧": "position",
"仰卧": "position",
"俯卧": "position",
"站姿": "position",
"站": "position",
"坐": "position",
"躺": "position",
"跪": "position",
"趴": "position",
"蹲": "position",
"回眸": "position",
"背影": "position",
"正面": "position",
"侧面": "position",
"standing": "position",
"sitting": "position",
"lying": "position",
"kneeling": "position",
"日本": "location",
"韩国": "location",
"中国": "location",
"台湾": "location",
"香港": "location",
"泰国": "location",
"越南": "location",
"东京": "location",
"首尔": "location",
"北京": "location",
"上海": "location",
"广州": "location",
"深圳": "location",
"成都": "location",
"重庆": "location",
"三亚": "location",
"厦门": "location",
"丽江": "location",
"大理": "location",
"曼谷": "location",
"芭提雅": "location",
"普吉": "location",
"济州": "location",
"巴厘岛": "location",
"马尔代夫": "location",
"沙巴": "location",
"塞班": "location",
"苏梅岛": "location",
"普吉岛": "location",
"冲绳": "location",
"京都": "location",
"大阪": "location",
"北海道": "location",
"阳朔": "location",
"japan": "location",
"korea": "location",
"china": "location",
"thailand": "location",
"tokyo": "location",
"seoul": "location",
"bangkok": "location",
"phuket": "location",
"bali": "location",
"日系": "style",
"韩系": "style",
"森系": "style",
"复古": "style",
"甜美": "style",
"性感": "style",
"清新": "style",
"冷艳": "style",
"优雅": "style",
"知性": "style",
"可爱": "style",
"俏皮": "style",
"温柔": "style",
"妩媚": "style",
"野性": "style",
"清纯": "style",
"时尚": "style",
"潮流": "style",
"街头": "style",
"学院": "style",
"文艺": "style",
"古典": "style",
"民族": "style",
"异域": "style",
"简约": "style",
"自然": "style",
"运动": "style",
"sexy": "style",
"wild": "style",
"retro": "style",
"korean": "style",
"fashion": "style",
"sweet": "style",
"pure": "style",
"fresh": "style",
"elegant": "style",
"cute": "style",
"vintage": "style",
"japanese": "style",
"杨晨晨": "person",
"水淼": "person",
"轩美宣": "person",
"王馨瑶": "person",
"雪儿": "person",
"顾雪儿": "person",
"曼曼": "person",
"佳佳": "person",
"小乔": "person",
"周于希": "person",
"王雨纯": "person",
"绮里嘉": "person",
"陆萱萱": "person",
"唐安琪": "person",
"鱼子酱": "person",
"安然": "person",
"林星阑": "person",
"筱慧": "person",
"蠢沫沫": "person",
"冯木木": "person",
"许诺": "person",
"言沫": "person",
"丽莎": "person",
"易阳": "person",
"陈良玲": "person",
"蓝夏": "person",
"凯竹": "person",
"郑颖姗": "person",
"刘飞儿": "person",
"赵小米": "person",
"于姬": "person",
"玉兔": "person",
"穆菲菲": "person",
"夏沫沫": "person",
"雪晴": "person",
"小夕": "person",
"谢芷馨": "person",
"顾奈奈": "person",
"唐琪儿": "person",
"艾小青": "person",
"李雅": "person",
"晓茜": "person",
"李宓儿": "person",
"林乐": "person",
"雪千寻": "person",
"艾静香": "person",
"阿薰": "person",
"李承美": "person",
"葛征": "person",
"沈佳熹": "person",
"陈思琪": "person",
"刘娅": "person",
"小瑶": "person",
"李梓": "person",
"程小烦": "person",
"马吉": "person",
"叶佳": "person",
"冯雪娇": "person",
"栗子": "person",
"丁筱南": "person",
"张思允": "person",
"小齿优喵": "person",
"KaYa Huang": "person",
"Miley": "person",
"Alice": "person",
"Zia": "person",
"Jooa": "person",
"Yome": "person",
"石川澪": "person",
"朱可儿": "person",
"苏雨": "person",
"熊吖": "person"
},
"stop_words": [
"re",
"photos",
"photo",
"pictures",
"picture",
"images",
"image",
"video",
"videos",
"cos",
"and",
"the",
"of",
"in",
"a",
"an",
"is",
"to",
"for",
"part",
"vol",
"collection",
"special",
"extra",
"bonus",
"new",
"old",
"selfie",
"official",
"original",
"set",
"gallery",
"album",
"page",
"issue",
"model",
"idol",
"star",
"actor",
"actress",
"singer",
"网",
"新人",
"图",
"套",
"合集",
"系列",
"作品",
"高清",
"原版",
"无水印",
"番号",
"编号",
"with",
"without",
"preview",
"patreon",
"no",
"site",
"web",
"online",
"digital",
"exclusive",
"premium",
"classic",
"photography",
"shoot",
"shot",
"xxx",
"com",
"www",
"https",
"http",
"Impact"
],
"person_prefixes": [
"写真",
"系列",
"美女模特",
"性感女神",
"清纯美女",
"气质美女",
"可爱女孩",
"甜美少女",
"时尚达人",
"女神",
"美女",
"模特",
"小姐姐",
"小仙女",
"御姐",
"萝莉",
"辣妹",
"潮人"
]
}

687
docs/api.md Normal file
View File

@@ -0,0 +1,687 @@
# Imager API 文档
Base URL: `http://<host>:<port>/api`
---
## 相册
### GET /api/albums
获取全量相册列表(从内存缓存)。
**参数**:无
**响应**:`AlbumInfo[]`
```json
[
{
"id": "uuid",
"title": "相册标题",
"document": "文件夹名",
"description": "描述",
"cover_image_id": "uuid",
"image_count": 100,
"created_at": "2024-01-01T00:00:00",
"tags": ["标签1", "标签2"],
"persons": ["人物1"]
}
]
```
---
### GET /api/search
组合搜索(关键词/标签/人物过滤)。
**参数**(query):
| 参数 | 类型 | 默认值 | 说明 |
|------|------|--------|------|
| q | string | - | 标题关键词模糊匹配 |
| tag | string | - | 标签精确匹配 |
| person | string | - | 人物精确或模糊匹配 |
| page | int | 1 | 页码 |
| limit | int | 50 | 每页条数(最大 200) |
**响应**:`AlbumInfo[]`
---
### GET /api/album/random
从缓存随机返回一个相册。
**参数**:无
**响应**:`AlbumInfo`
---
### GET /api/album/{id}
获取相册详情(含图片列表、标签、人物)。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | uuid | 相册 ID |
**响应**:`AlbumWithImages`
```json
{
"album": {
"id": "uuid",
"title": "标题",
"description": "描述",
"cover_image_id": "uuid",
"image_count": 100,
"created_at": "...",
"updated_at": "..."
},
"images": [
{
"id": "uuid",
"album_id": "uuid",
"filename": "1.jpg",
"original_url": "...",
"title": "图片标题",
"description": null,
"file_size": 123456,
"width": 1920,
"height": 1080,
"mime_type": "image/jpeg",
"created_at": "..."
}
],
"image_count": 100,
"tags": ["标签1", "标签2"],
"persons": ["人物1"]
}
```
---
### POST /api/albums
创建新相册。
**请求体**:
```json
{
"title": "相册标题",
"document": "文件夹名",
"description": "描述(可选)"
}
```
**响应**(201 Created):
```json
{
"id": "uuid"
}
```
---
## 图片
### GET /api/image/{id}
获取图片原始二进制数据。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | uuid | 图片 ID |
**响应**:二进制图片数据,Content-Type 按实际 mime 类型。带 ETag,Cache-Control: public, max-age=31536000。
---
## 收藏
### GET /api/favorites
获取当前用户的收藏列表。
**参数**:无(用户固定为 default_user)
**响应**:`FavoriteWithImage[]`
---
### GET /api/favorites/status/{id}
检查某图片的收藏状态。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | uuid | 图片 ID |
---
### POST /api/favorites
添加收藏。
**请求体**:
```json
{
"image_id": "uuid",
"album_id": "uuid"
}
```
**响应**(201 Created):
```json
{
"id": "uuid"
}
```
---
### DELETE /api/favorites/{id}
按收藏 ID 取消收藏。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | uuid | 收藏 ID |
---
### DELETE /api/favorites/image/{id}
按图片 ID 取消收藏。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | uuid | 图片 ID |
---
## 人物
### GET /api/people
从内存缓存获取所有人名列表。
**参数**:无
**响应**:
```json
{
"total": 100,
"persons": [
{
"id": 1,
"name": "人物名",
"display_name": "显示名",
"total_albums": 10,
"total_images": 50,
"aliases": ["别名1", "别名2"]
}
]
}
```
---
### GET /api/people/search
从内存缓存搜索人名。
**参数**(query):
| 参数 | 类型 | 说明 |
|------|------|------|
| q | string | 搜索关键词 |
**响应**:同上 `PersonListResponse`
---
### GET /api/people/{name}
获取人物详情及关联相册。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| name | string | 人物名称 |
**响应**:`PersonWithAlbums`
```json
{
"album_count": 10,
"person": {
"id": 1,
"name": "人物名",
"display_name": "显示名",
"total_albums": 10,
"total_images": 50,
"original_names": null,
"created_at": "",
"updated_at": ""
},
"albums": [/* AlbumInfo[] */]
}
```
---
### GET /api/people/{id}/aliases
查询人物别名。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| id | int | 人物 ID |
---
### POST /api/people/delete/{name}
删除人物(从 imager.db + search.db 中清除),并重新解析受影响的相册。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| name | string | 人物名(URL 编码) |
**响应**:
```json
{
"success": true,
"person": "人物名",
"albums_affected": 5,
"persons_updated": 3,
"tags_updated": 8,
"synced": 5,
"message": "人物 '人物名' 已删除,影响 5 个相册,重新解析完成..."
}
```
---
## 标签 / 元数据
### GET /api/metadata/stats
从内存缓存计算元数据统计。
**参数**:无
**响应**:
```json
{
"total_persons": 100,
"total_aliases": 50,
"total_albums": 10000,
"total_images": 50000,
"total_tags": 2000
}
```
---
### GET /api/tags/search
按标签搜索相册。
**参数**(query):
| 参数 | 类型 | 说明 |
|------|------|------|
| tag | string | **必填**,标签名 |
**响应**:`AlbumInfo[]`
---
### GET /api/tags/popular
获取热门标签。
**参数**(query):
| 参数 | 类型 | 默认值 | 说明 |
|------|------|--------|------|
| limit | int | 50 | 返回数量 |
**响应**:
```json
{
"total": 50,
"tags": [
{ "tag": "标签名", "count": 1000, "album_count": 1000 }
]
}
```
---
### POST /api/tags/delete/{name}
删除标签(从 dict/album_tags/tags 表中清除),并重新解析受影响的相册。
**参数**(path):
| 参数 | 类型 | 说明 |
|------|------|------|
| name | string | 标签名(URL 编码) |
**响应**:
```json
{
"success": true,
"tag": "标签名",
"albums_affected": 5,
"persons_updated": 3,
"tags_updated": 8,
"synced": 5,
"message": "标签 '标签名' 已删除,影响 5 个相册,重新解析完成..."
}
```
---
## 推荐
### GET /api/recommend/tags
标签自动补全推荐。
**参数**(query):
| 参数 | 类型 | 默认值 | 说明 |
|------|------|--------|------|
| q | string | - | 输入前缀 |
| limit | int | 10 | 返回数量 |
---
### GET /api/recommend/persons
人物自动补全推荐。
**参数**(query):
| 参数 | 类型 | 默认值 | 说明 |
|------|------|--------|------|
| q | string | - | 输入名称 |
| limit | int | 10 | 返回数量 |
---
### GET /api/recommend/related
关联推荐(标签共现 + 共现人物)。
**参数**(query):
| 参数 | 类型 | 说明 |
|------|------|------|
| context | string | **必填**,标签或人名 |
| limit | int | 默认 15 |
---
### GET /api/recommend/popular-persons
获取热门人物排行榜。
**参数**(query):
| 参数 | 类型 | 默认值 | 说明 |
|------|------|--------|------|
| limit | int | 20(最大 50) | 返回数量 |
**响应**:
```json
[
{
"name": "人物名",
"display_name": "显示名",
"total_albums": 10,
"total_images": 50,
"top_tags": ["标签1", "标签2"],
"aliases": ["别名1"]
}
]
```
---
## 管线
### POST /api/pipeline/run
手动触发完整管线(dict 优先匹配解析)。
**参数**:无
**响应**:
```json
{
"success": true,
"tags_cleaned": 0,
"short_tags_removed": 0,
"person_like_tags_removed": 0,
"dirty_albums_detected": 0,
"non_persons_deleted": 0,
"stop_word_persons_removed": 0,
"new_albums_found": 0,
"persons_updated": 0,
"tags_updated": 0,
"empty_albums_deleted": 0,
"dup_albums_deleted": 0,
"search_db_synced": 0,
"message": "管线执行完毕:..."
}
```
---
### POST /api/pipeline/reparse
重置所有相册为未处理状态,重新解析全部标签和人物并同步到 search.db。
**参数**:无
**响应**:
```json
{
"success": true,
"albums_parsed": 100,
"persons_updated": 80,
"tags_updated": 200,
"synced_to_search_db": 100,
"message": "重新解析完成:..."
}
```
---
### POST /api/pipeline/reparse-persons
重置所有相册,重新解析全部人物(清空 search.db 的人物关联后运行完整管线)。
**参数**:无
**响应**:
```json
{
"success": true,
"albums_parsed": 100,
"persons_updated": 80,
"non_persons_deleted": 2,
"stop_word_persons_removed": 1,
"synced_to_search_db": 100,
"message": "人物重新解析完成:..."
}
```
---
### POST /api/pipeline/reparse-tags
重置所有相册,重新解析全部标签(清空 search.db 的标签关联后运行完整管线)。
**参数**:无
**响应**:
```json
{
"success": true,
"albums_parsed": 100,
"tags_updated": 200,
"short_tags_removed": 5,
"synced_to_search_db": 100,
"message": "标签重新解析完成:..."
}
```
---
## 词典
### GET /api/dict/stats
查看标签词典状态。
**参数**:无
**响应**:
```json
{
"stats": "标签词典: ...",
"categories": [
{ "name": "style", "label": "风格", "count": 50 }
],
"total_tags": 2000,
"total_stop_words": 20,
"person_prefixes": ["prefix1"],
"excluded_person_names": ["name1"],
"included_person_names": ["name2"]
}
```
---
### POST /api/dict/reload
从 search.db 的 dict 表热重载词典到内存。
**参数**:无
**响应**:
```json
{
"success": true,
"message": "词典已从 search.db 重载: ..."
}
```
---
### POST /api/dict/update
批量更新 dict 表条目,并自动重载内存词典。
**请求体**(数组):
```json
[
{
"word": "写真",
"type": "tag",
"category": "topic",
"freq": null,
"merged_to": null
},
{
"word": "张三",
"type": "person",
"category": null,
"freq": null,
"merged_to": null
}
]
```
**type 可选值**:`tag`、`person`、`stop_word`、`person_prefix`、`excluded_person`
**响应**:
```json
{
"success": true,
"message": "更新 2 条字典条目,词典已从 search.db 重载: ...",
"updated": 2
}
```
---
## 缓存
### POST /api/cache/refresh
手动刷新所有内存缓存(AlbumsCache / TagCache / PersonCache)。
**参数**:无
**响应**:
```json
{
"success": true,
"message": "缓存已刷新"
}
```
---
## 迁移
### GET /api/migration/stats
查看当前数据库迁移状态。
**参数**:无
---
### POST /api/migration/run
执行从旧数据库的数据迁移。
**参数**(query):
| 参数 | 类型 | 说明 |
|------|------|------|
| path | string | **必填**,旧数据库文件路径 |
---
## 队列
### GET /api/queue/stats
获取任务队列状态。
**参数**:无
---
## 静态页面
| 方法 | 路径 | 说明 |
|------|------|------|
| GET | `/` | 首页(相册列表) |
| GET | `/album` | 相册详情页(需 ?id=xxx 参数) |
| GET | `/assets/*` | 静态资源文件 |
---
> 所有 API 响应均为 JSON 格式(仅 `/api/image/{id}` 返回二进制)。无认证鉴权。

View File

@@ -0,0 +1,226 @@
# 流云图库 — 元数据管线架构文档
> 版本:v2.0 · 更新:2026-07-19
---
## 总览
流云图库是一个 Rust 编写的图片管理应用,核心功能是从相册标题中自动提取**人物名称**和**分类标签**,并建立人物-相册关联关系。所有提取和关联逻辑由一套三阶段的**处理管线(Pipeline)** 驱动。
```
相册标题(如 "水淼Aqua cosplay Zani – Wuthering Waves 69 photos")
│
▼
┌─ Pipeline ──────────────────────────────────────┐
│ Phase 1 全量解析 提取人名 + 提取标签 │
│ Phase 2 人物合并 按中文核心合并重复人物 │
│ Phase 3 缓存刷新 重新加载内存缓存 │
└──────────────────────────────────────────────────┘
│
├──→ albums 表(person_name_original, tags 等)
├──→ persons 表(人物主表)
├──→ person_aliases 表(别名关联)
└──→ album_tags 表(标签关联)
```
---
## 启动流程与触发时机
应用启动后,管线任务按顺序串行执行三个阶段:
```
启动管线(单次,后台任务 A)
│
├── ① 扫描导入新相册(import)
├── ② 全量解析 + 人物合并(run_pipeline)
└── ③ 刷新 AlbumsCache
```
- **启动管线**:import → run_pipeline → refreshCache,三件事**顺序串行**,不并发
- **定时管线**:每 30 分钟触发一次,执行 run_pipeline → refreshCache(不重复 import)
- **API 触发**:POST /api/pipeline/run 可手动触发 run_pipeline
---
## Phase 1:全量解析
**入口函数:** `phase_parse_all(db)` → `(person_updated, tag_updated)`
对数据库中**每一个相册**独立执行以下流程:
```
for each album:
│
├── jieba 分词(词性标注)
│ ├──→ 提取推荐人名(词性 nr/nrfg/nrt/nz)
│ └──→ 提取推荐标签(词性 n/nz/ns/eng/x)
│
├── 确定最佳人名
│ ├── 有人物库匹配?→ 用库匹配结果
│ └── 否则 → 用 jieba 推荐
│
├── 写入 DB(强制覆盖,不比对旧值)
│
├── 提取标签
│ ├── 三步提取:分类词库 → jieba → 正则兜底
│ ├── unique_tags() 去重+排序+坏标签过滤
│ └── 写入 DB(强制覆盖,不比对旧值)
│
└── 下一个相册
```
### 人名提取策略
按优先级依次尝试:
1. **人物库精确匹配** — `person_merger::match_title()`,在已有 persons 表中查询标题子串
2. **jieba 词性标注** — `extract_person_from_tags()`,优先取 nr(人名)类词
3. **标题模式匹配** — `extract_person_name_from_title()`,尝试 cosplay 格式、分隔符切分、正则扫描
### 人名处理规则
- **禁止中英混杂**:人名主体为纯中文或纯英文(中文优先级更高)
- 中英文混合(如"水淼Aqua")由 `split_person_name_with_aliases` 拆分为:主名="水淼" + 别名="Aqua"
- 别名保存到 `person_aliases` 表,用于后续搜索匹配
### 标签分类体系
| 分类 | 标识 | 示例词 |
|------|------|--------|
| 风格 | style | 日系、韩系、复古、甜美、sexy |
| 服装 | outfit | 比基尼、旗袍、制服、丝袜 |
| 场景 | scene | 室内、户外、街拍、浴室、beach |
| 身体 | body | 美腿、翘臀、玉足、美背 |
| 情绪 | mood | 清纯、诱惑、唯美、冷艳 |
| 主题 | topic | 写真、私房、旅拍、cosplay |
| 名词 | noun | jieba 提取的普通名词 |
| 专名 | proper | jieba 提取的专有名词 |
| 地名 | location | jieba 提取的地名 |
| 其他 | other | 正则兜底提取的剩余关键词 |
### 标签过滤链(从外到内共 5 层)
```
1. is_punctuation_only() → 检测不含任何字/字母/数字,则跳过
2. 停用词表过滤 → photos、cosplay、美女、模特 等 50+ 词
3. 人名排除 → 排除标签提取时确定的人名
4. 词性排除 → 形容词(a)、动词(v)、名动词(vn)、机构(nt) 等不产出标签
5. unique_tags() 最终过滤 → 调用 is_bad_tag() 二次确认
```
### 强制覆盖写入
每次管线执行都是**完全重新解析**,提取到人名就写入覆盖、提取到标签就写入覆盖。不做"新旧好坏"比对:
- **人名**:jieba/fuzzy 提取到 → 直接写入 `albums.person_name_*`,创建/更新 `persons` 记录及别名关联
- **标签**:经过 5 层过滤链后仍有结果 → 直接写入 `albums.tags`,重建 `album_tags` 表
这样每次启动或定时触发时,标签和人名都会用最新的解析逻辑重新计算并强制覆盖,解析逻辑改进后老数据自动受益。
---
## Phase 2:人物合并
**入口函数:** `phase_merge_persons(db)` → `merged_count`
按**中文核心**自动合并重复人物:
```
1. 扫描所有 persons,提取其中文核心(开头的连续中文字符)
2. 将中文核心相同的 person 分为一组
3. 每组选相册数最多的作为目标(dst)
4. 其余合并到目标:
- 移动所有相册关联
- 移动所有别名
- 源人物原名也作为目标别名
- 删除源人物
5. 更新目标的统计计数
```
示例:`杨晨晨Yome` 和 `杨晨晨Sugar` 的中文核心都是 "杨晨晨" → 合并为一个人物
---
## 核心数据表结构
### albums 表(关键字段)
| 字段 | 类型 | 说明 |
|------|------|------|
| id | TEXT | UUID |
| title | TEXT | 相册标题,来源是文件夹名或 index.json |
| person_name_original | TEXT | 提取的原始人名(可能带英文后缀) |
| person_name_cleaned | TEXT | 清洗后的人名(仅中文主名) |
| tags | TEXT | JSON 数组,如 `["比基尼","室内","清纯"]` |
| all_words | TEXT | JSON 数组,如 `[["比基尼","outfit"],["室内","scene"]]` |
| processed_at | TEXT | 最后处理时间 |
### persons 表
| 字段 | 类型 | 说明 |
|------|------|------|
| id | INTEGER | 自增主键 |
| name | TEXT | 人物名(通常是中文主名) |
| display_name | TEXT | 显示名 |
| total_albums | INTEGER | 关联相册数 |
| total_images | INTEGER | 关联图片数 |
### person_aliases 表
| 字段 | 类型 | 说明 |
|------|------|------|
| id | INTEGER | 自增主键 |
| person_id | INTEGER | 关联 persons.id |
| alias_type | TEXT | 别名类型(english/nickname/chinese/original) |
| alias_name | TEXT | 别名文本 |
### album_tags 表
| 字段 | 类型 | 说明 |
|------|------|------|
| id | INTEGER | 自增主键 |
| album_id | TEXT | 关联 albums.id |
| tag | TEXT | 标签文本 |
---
## 定时任务
每 30 分钟触发一次,顺序执行:
```
1. run_pipeline() → Phase 1(全量解析)+ Phase 2(人物合并)
2. 刷新 AlbumsCache(内存缓存,为 API 提供快速查询)
```
---
## API 接口
| 方法 | 路径 | 说明 |
|------|------|------|
| POST | /api/pipeline/run | 手动触发一次完整管线 |
返回:
```json
{
"success": true,
"persons_updated": 12,
"tags_updated": 45,
"persons_merged": 3,
"message": "管线执行完毕:更新人名 12 个,更新标签 45 个,合并人物 3 人"
}
```
---
## 关键设计原则
1. **幂等性**:管线可反复执行,每次都是完整重新解析 + 强制覆盖写入
2. **渐进改进**:解析逻辑改进后,老数据在下次管线执行时自动受益
3. **强制覆盖**:提取到即写入,不做新旧比对,确保数据始终使用最新逻辑
4. **中文优先**:人名提取优先中文,中英文混合被拆分为主名+别名
5. **多层过滤**:标签经过 5 层过滤链,确保质量
6. **纯中文/纯英文**:人名主体不允许中英混杂

124
docs/pipeline-spec.md Normal file
View File

@@ -0,0 +1,124 @@
# 元数据管线 — 开发规格
> 2026-07-19
---
## 1. 需求概述
系统在启动和运行时,需要自动完成三个任务:
| # | 任务 | 说明 |
|---|------|------|
| 1 | **新相册解析** | 扫描磁盘导入的新相册 → 提取人名+标签 → 写入 DB |
| 2 | **老数据重解析** | 已有数据的相册用最新逻辑重新提取,强制覆盖 |
| 3 | **人物合并** | 按中文核心合并同一人的不同记录 |
这三件事通过**同一个管道**完成,管道在**启动时跑一次** + **定时每 30 分钟跑一次**。
---
## 2. 启动流程(时序)
当前问题:import 和 pipeline 是两个并发的 `tokio::spawn`,pipeline 固定等 3s,import 可能耗时更长,导致刚导入的相册没被解析。
**目标时序:**
```
服务启动
│
├── DB 初始化
│
├─── 后台任务 A:启动管线(单次)─────┐
│ ├── ① 扫描导入新相册 │
│ ├── ② Phase 1:全量解析(强制覆盖) │
│ ├── ③ Phase 2:人物合并 │
│ └── ④ 刷新内存缓存 │
│ │
├─── 后台任务 B:定时管线(每 30 分钟)──┤
│ ├── ① Phase 1:全量解析(强制覆盖) │
│ ├── ② Phase 2:人物合并 │
│ └── ③ 刷新内存缓存 │
│ │
└─── HTTP 服务启动 │
│
注:后台任务 A 和 B 互斥,不会同时执行
```
### 变化要点
- **启动管线 = import + parse + merge + refreshCache** 依次串行
- **定时管线 = parse + merge + refreshCache**(不重复 import)
- 管线内部顺序固定不可并发
---
## 3. 管线内部结构
```
run_pipeline(db, with_import: bool)
│
├── [with_import=true] 扫描导入新相册
│
├── Phase 1:全量解析
│ for each album:
│ ├── jieba 分词 → 候选人名 + 候选标签
│ ├── 人物库匹配 / jieba → 最佳人名
│ ├── 强制写入 albums.person_name_* + persons + aliases
│ ├── 标签三步提取(词库→jieba→正则)
│ ├── unique_tags(去重+排序+坏标签过滤)
│ └── 强制写入 albums.tags + album_tags
│
├── Phase 2:人物合并
│ ├── 提取所有人名的中文核心
│ ├── 分组 → 每组选最多相册的为目标
│ └── 合并相册关联 / 别名,删除源
│
└── Phase 3:刷新缓存
└── 重新加载 albums 到 AlbumsCache
```
---
## 4. 模块职责
| 模块 | 文件 | 职责 |
|------|------|------|
| Pipeline 引擎 | `engine/pipeline.rs` | `phase_parse_all()`、`phase_merge_persons()`、`run_pipeline()` |
| 标题解析 | `engine/title_parser.rs` | jieba 分词、人名提取、中英文拆分、别名生成 |
| 标签提取 | `engine/tag_extractor.rs` | 分类词库匹配、jieba 名词提取、标点/停用词/坏标签过滤 |
| 人物合并 | `engine/person_merger.rs` | 中文核心提取、分组分析、合并执行 |
| 导入 | `services/import.rs` | 扫描磁盘目录、创建相册和图片记录 |
| DB | `db/metadata.rs` | persons、aliases、album_tags 表的读写 |
| 缓存 | `cache/mod.rs` | AlbumsCache + TagCache + PersonCache |
---
## 5. 关键设计约束
1. **强制覆盖**:每次管线执行都是完整重解,直接覆盖 DB 数据
2. **幂等**:管线可反复执行,不影响正确性
3. **串行**:管线内部各阶段顺序执行,不并发
4. **多级过滤**:标签经过 5 层过滤才入库,确保质量
5. **中文优先**:人名优先中文,中英混杂拆为"主名+别名"
6. **纯中/纯英**:人名主体不允许中英混杂
---
## 6. 实现清单
### 6.1 Pipeline 引擎
- [x] `phase_parse_all()` — 全量解析
- [x] `phase_merge_persons()` — 人物合并
- [x] `phase_refresh_cache()` — 刷新缓存(在 main.rs 实现)
- [x] `run_pipeline(db)` — 整合解析+合并
### 6.2 main.rs 启动逻辑
- [x] 后台任务 A:顺序执行 import → run_pipeline → cache refresh
- [x] 后台任务 B:定时每 30 分钟 run_pipeline → cache refresh
### 6.3 API
- [x] POST /api/pipeline/run → 全量解析 + 人物合并

View File

@@ -0,0 +1,19 @@
-- 别名表增加唯一约束,防止同一个人名下同名别名重复
--
-- 注意:需要先清理重复数据,再创建索引。
-- 如果先 CREATE UNIQUE INDEX,已有重复行时会直接报错。
-- 1. 清理可能存在的重复别名(保留 id 最小的那条)
DELETE FROM person_aliases WHERE id NOT IN (
SELECT MIN(id) FROM person_aliases GROUP BY person_id, alias_name
);
-- 2. 创建唯一索引(此时已无重复数据,不会失败)
CREATE UNIQUE INDEX IF NOT EXISTS idx_aliases_unique ON person_aliases(person_id, alias_name);
-- 相册关联表上增加唯一索引(已有 PRIMARY KEY,添加显式唯一索引加速)
-- 同时增加每张专辑只能有一个 person 的约束
-- 先清理旧的重复关联(同一album_id关联了多个person_id的情况)
DELETE FROM person_albums WHERE rowid NOT IN (
SELECT MIN(rowid) FROM person_albums GROUP BY album_id, person_id
);

View File

@@ -0,0 +1,3 @@
-- 为 person_albums 添加 album_id 索引,优化 JOIN 查询和按相册查找的性能
-- person_id 已有主键索引 (person_id, album_id) 覆盖,无需重复创建
CREATE INDEX IF NOT EXISTS idx_pa_album ON person_albums(album_id);

View File

@@ -0,0 +1,6 @@
-- 清理不再使用的索引
-- 注意:不重建 albums 表,因为 4 张子表(images/favorites/album_tags/person_albums)
-- 通过 FOREIGN KEY 引用了 albums(id),DROP TABLE 即使关闭外键检查也会失败。
-- 未使用的列(person_name_original/all_words/tags)已由应用层忽略,不影响功能。
DROP INDEX IF EXISTS idx_albums_cleaned;

View File

@@ -1,872 +0,0 @@
//! 数据库迁移 CLI 工具
//!
//! 功能:
//! 1. 从老数据库 (persons.db) 读取数据
//! 2. 对 title 进行解析 (人名提取、标签提取)
//! 3. 生成新的数据库
//! 4. 验证搜索等功能
//!
//! 用法:
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db
//! cargo run --bin migrate -- --old-db ../yolov8_face_recognition/persons.db --new-db ./data/imager.db --reparse
//! cargo run --bin migrate -- --verify --new-db ./data/imager.db
use serde::{Deserialize, Serialize};
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row};
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::time::Instant;
// ============================================================
// 标题解析模块 (Rust 原生实现)
// ============================================================
mod title_parser {
use regex::Regex;
use std::collections::HashSet;
use lazy_static::lazy_static;
/// 模特称呼前缀
const MODEL_PREFIXES: &[&str] = &[
"美女模特", "性感女神", "新人模特", "新人嫩模", "新人妹子",
"气质女神", "气质美女", "性感美女", "宅男女神", "丽质女神",
"丽质美女", "红粉佳人", "美乳女神", "清纯女神", "性感嫩模",
"性感御姐", "美女", "模特", "女神", "御姐", "嫩模",
"新晋女神", "新晋美模", "日本", "日本人气",
];
/// 停用词
const STOP_WORDS: &[&str] = &[
"photos", "photo", "phot", "images", "image", "video", "videos",
"picture", "pictures", "pic", "gif", "gifs", "mp4", "webp",
"cosplay", "cos", "set", "part", "collection", "series",
"special", "vol", "no", "no.", "ver", "version", "edition",
"new", "xxx", "preview", "teaser", "sample",
"写真", "套图", "旅拍", "系列", "诱惑", "性感", "美女",
"模特", "女神", "写真集", "首套", "版", "主题", "风格",
"the", "a", "an", "in", "on", "at", "of", "to", "for",
"with", "by", "and", "or", "but", "from", "is", "are",
"this", "that", "these", "those", "it", "its",
"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
"一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
"没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", "些",
];
/// 工作室排除名
const EXCLUDED_NAMES: &[&str] = &[
"xiuren", "秀人", "秀人网", "youmi", "尤蜜", "尤蜜荟",
"xingyan", "星颜", "魅妍社", "mistar",
"mygirl", "美媛馆", "imiss", "爱蜜社", "huayang", "花漾",
"bololi", "波萝社", "feilin", "嗲囡囡", "qingdouke", "青豆客",
"toutiao", "头条女神", "xiaoyu", "语画界", "youmei", "尤美",
"ugirls", "尤果", "尤果网", "mfstar", "模范学院",
"tgod", "推女神", "dgkirl", "御女郎", "candy", "糖果画报",
"wanibooks", "bluecake", "pure media", "puremedia",
];
lazy_static! {
static ref RE_PHOTOS: Regex = Regex::new(
r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$"
).unwrap();
static ref RE_COSPLAY: Regex = Regex::new(
r"(?i)^(.+?)\s+cosplay\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
).unwrap();
static ref RE_COSPLAY_BY: Regex = Regex::new(
r"(?i)^(.+?)\s+cosplay\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
).unwrap();
static ref RE_BRACKET: Regex = Regex::new(r"^[\s]*[\[【][^\]]*[\]】][\s]*").unwrap();
static ref RE_NUMBER_PREFIX: Regex = Regex::new(
r"(?i)^(?:No\.?\s*\d+|VOL\.?\s*\d+)\s+"
).unwrap();
static ref RE_SPLIT: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap();
static ref RE_PAREN: Regex = Regex::new(r"[((]([^))]*)[))]").unwrap();
static ref RE_TRAILING: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
static ref RE_EN_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
static ref RE_WORD: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
static ref STOP_SET: HashSet<&'static str> = {
let mut s = HashSet::new();
for w in STOP_WORDS { s.insert(*w); }
s
};
static ref EXCLUDED_LOWER: HashSet<String> = {
let mut s = HashSet::new();
for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); }
s
};
}
/// 清理标题末尾
pub fn clean_title_suffix(title: &str) -> String {
RE_PHOTOS.replace(title, "").trim().to_string()
}
/// 从标题提取人名
pub fn extract_person_name(title: &str) -> Option<String> {
let clean = clean_title_suffix(title);
if clean.is_empty() {
return None;
}
// cosplay 格式
if let Some(caps) = RE_COSPLAY_BY.captures(&clean) {
return Some(caps[2].trim().to_string());
}
if let Some(caps) = RE_COSPLAY.captures(&clean) {
return Some(caps[1].trim().to_string());
}
// 移除工作室前缀
let no_studio = RE_BRACKET.replace(&clean, "").trim().to_string();
// 移除模特称呼前缀
let mut no_prefix = no_studio;
for prefix in MODEL_PREFIXES {
if no_prefix.starts_with(prefix) {
no_prefix = no_prefix[prefix.len()..].trim().to_string();
break;
}
}
// 移除编号前缀
let no_prefix = RE_NUMBER_PREFIX.replace(&no_prefix, "").trim().to_string();
// 用分隔符拆分
let parts: Vec<&str> = RE_SPLIT.splitn(&no_prefix, 2).collect();
if parts.len() >= 2 {
let name = parts[0].trim();
let name = RE_PHOTOS.replace(name, "").trim().to_string();
if !name.is_empty() && name.len() >= 2 {
return Some(name);
}
}
// 没有分隔符
let name = RE_PHOTOS.replace(&no_prefix, "").trim().to_string();
if name.len() >= 2 {
Some(name)
} else {
None
}
}
/// 清洗人名
pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
let mut aliases = Vec::new();
let mut name = raw_name.to_string();
// 移除前缀
for prefix in MODEL_PREFIXES {
if name.starts_with(prefix) {
aliases.push(("prefix".to_string(), prefix.to_string()));
name = name[prefix.len()..].trim().to_string();
break;
}
}
// 移除括号内容
let mut cleaned = String::new();
let mut last_end = 0;
for cap in RE_PAREN.captures_iter(&name) {
let m = cap.get(0).unwrap();
let content = cap.get(1).map_or("", |c| c.as_str());
cleaned.push_str(&name[last_end..m.start()]);
last_end = m.end();
if !content.is_empty() {
if RE_CHINESE.is_match(content) {
aliases.push(("nickname".to_string(), content.to_string()));
} else {
aliases.push(("english".to_string(), content.to_string()));
}
}
}
cleaned.push_str(&name[last_end..]);
let name = cleaned.trim().to_string();
// 清理末尾标点
let name = RE_TRAILING.replace(&name, "").trim().to_string();
// 提取英文名别名
if let Some(caps) = RE_EN_SUFFIX.captures(&name) {
let chinese = caps[1].to_string();
let english = caps[2].trim().to_string();
if !english.is_empty() {
aliases.push(("chinese".to_string(), chinese));
aliases.push(("english".to_string(), english));
}
}
let name = if name.is_empty() { raw_name.to_string() } else { name };
(name, aliases)
}
/// 从标题提取标签
pub fn extract_tags(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
let mut clean = clean_title_suffix(title);
// 移除人名
if let Some(name) = person_name {
clean = clean.replace(name, " ");
for part in name.split_whitespace() {
clean = clean.replace(part, " ");
}
}
// 移除工作室前缀
clean = RE_BRACKET.replace(&clean, "").trim().to_string();
let mut tags = Vec::new();
let mut seen = HashSet::new();
for cap in RE_WORD.find_iter(&clean) {
let word = cap.as_str().trim();
if word.len() < 2 { continue; }
let word_lower = word.to_lowercase();
if STOP_SET.contains(word_lower.as_str()) { continue; }
if word.chars().all(|c| c.is_ascii_digit() || c == '.') { continue; }
if !seen.insert(word_lower.clone()) { continue; }
let tag_type = if RE_CHINESE.is_match(word) { "chinese" } else { "english" };
tags.push((word.to_lowercase(), tag_type.to_string()));
}
tags
}
/// 判断是否为排除名
pub fn is_excluded(name: &str) -> bool {
EXCLUDED_LOWER.contains(&name.to_lowercase())
}
}
// ============================================================
// 数据模型
// ============================================================
#[derive(Debug, Default)]
struct MigrationStats {
persons_imported: i64,
persons_skipped: i64,
persons_created: i64,
aliases_imported: i64,
albums_processed: i64,
albums_missing: i64,
tags_imported: i64,
person_albums_created: i64,
}
#[derive(Debug, Clone)]
struct OldAlbum {
id: String,
title: String,
image_count: i64,
person_name_original: Option<String>,
person_name_cleaned: Option<String>,
tags: Option<String>,
all_words: Option<String>,
}
#[derive(Debug, Clone)]
struct OldPerson {
id: i64,
name: String,
display_name: Option<String>,
total_albums: i64,
total_images: i64,
original_names: Option<String>,
}
// ============================================================
// 迁移器
// ============================================================
struct Migrator {
old_pool: SqlitePool,
new_pool: SqlitePool,
stats: MigrationStats,
}
impl Migrator {
async fn new(old_db_path: &str, new_db_path: &str) -> Result<Self, String> {
if !Path::new(old_db_path).exists() {
return Err(format!("老数据库不存在: {}", old_db_path));
}
// 创建新数据库目录
if let Some(parent) = Path::new(new_db_path).parent() {
if !parent.as_os_str().is_empty() && !parent.exists() {
std::fs::create_dir_all(parent).map_err(|e| format!("创建目录失败: {}", e))?;
}
}
println!("[连接] 老数据库: {}", old_db_path);
let old_pool = SqlitePoolOptions::new()
.max_connections(1)
.connect(&format!("sqlite:{}", old_db_path))
.await
.map_err(|e| format!("连接老数据库失败: {}", e))?;
println!("[连接] 新数据库: {}", new_db_path);
let new_pool = SqlitePoolOptions::new()
.max_connections(5)
.connect(&format!("sqlite:{}", new_db_path))
.await
.map_err(|e| format!("连接新数据库失败: {}", e))?;
// SQLite 性能优化
for p in [&new_pool] {
sqlx::query("PRAGMA journal_mode=WAL;").execute(p).await.ok();
sqlx::query("PRAGMA synchronous=NORMAL;").execute(p).await.ok();
sqlx::query("PRAGMA foreign_keys=ON;").execute(p).await.ok();
}
Ok(Self { old_pool, new_pool, stats: MigrationStats::default() })
}
async fn init_new_schema(&self) -> Result<(), String> {
println!("[Schema] 初始化新数据库表结构...");
let schema_sql = include_str!("../migrations/001_initial.sql");
for stmt in schema_sql.split(';') {
let stmt = stmt.trim();
if !stmt.is_empty() {
sqlx::query(stmt).execute(&self.new_pool).await
.map_err(|e| format!("执行 Schema 失败: {}", e))?;
}
}
let schema_sql2 = include_str!("../migrations/002_metadata.sql");
for stmt in schema_sql2.split(';') {
let stmt = stmt.trim();
if !stmt.is_empty() {
sqlx::query(stmt).execute(&self.new_pool).await
.map_err(|e| format!("执行 Schema2 失败: {}", e))?;
}
}
println!("[Schema] 表结构初始化完成");
Ok(())
}
async fn read_old_albums(&self) -> Result<Vec<OldAlbum>, String> {
println!("[读取] 从老数据库读取相册...");
let rows = sqlx::query(
"SELECT id, title, image_count, person_name_original, person_name_cleaned, tags, all_words
FROM albums ORDER BY image_count DESC"
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取相册失败: {}", e))?;
let albums: Vec<OldAlbum> = rows.iter().map(|r| OldAlbum {
id: r.get("id"),
title: r.get("title"),
image_count: r.get::<Option<i64>, _>("image_count").unwrap_or(0),
person_name_original: r.get("person_name_original"),
person_name_cleaned: r.get("person_name_cleaned"),
tags: r.get("tags"),
all_words: r.get("all_words"),
}).collect();
println!("[读取] 共读取 {} 个相册", albums.len());
Ok(albums)
}
async fn read_old_persons(&self) -> Result<Vec<OldPerson>, String> {
println!("[读取] 从老数据库读取人物...");
let rows = sqlx::query(
"SELECT id, name, display_name, total_albums, total_images, original_names
FROM persons ORDER BY total_albums DESC"
).fetch_all(&self.old_pool).await.map_err(|e| format!("读取人物失败: {}", e))?;
let persons: Vec<OldPerson> = rows.iter().map(|r| OldPerson {
id: r.get("id"),
name: r.get("name"),
display_name: r.get("display_name"),
total_albums: r.get::<Option<i64>, _>("total_albums").unwrap_or(0),
total_images: r.get::<Option<i64>, _>("total_images").unwrap_or(0),
original_names: r.get("original_names"),
}).collect();
println!("[读取] 共读取 {} 个人物", persons.len());
Ok(persons)
}
async fn read_old_aliases(&self) -> Result<Vec<(i64, String, String)>, String> {
let rows = sqlx::query(
"SELECT person_id, alias_type, alias_name FROM person_aliases"
).fetch_all(&self.old_pool).await.unwrap_or_default();
let aliases: Vec<(i64, String, String)> = rows.iter().map(|r| (
r.get("person_id"), r.get("alias_type"), r.get("alias_name")
)).collect();
println!("[读取] 共读取 {} 个别名", aliases.len());
Ok(aliases)
}
async fn read_old_person_albums(&self) -> Result<Vec<(i64, String)>, String> {
let rows = sqlx::query(
"SELECT person_id, album_id FROM person_albums"
).fetch_all(&self.old_pool).await.unwrap_or_default();
let links: Vec<(i64, String)> = rows.iter().map(|r| (
r.get("person_id"), r.get("album_id")
)).collect();
println!("[读取] 共读取 {} 个人物-相册关联", links.len());
Ok(links)
}
async fn migrate(&mut self, reparse: bool) -> Result<(), String> {
self.init_new_schema().await?;
// 读取老数据
let old_albums = self.read_old_albums().await?;
let old_persons = self.read_old_persons().await?;
let old_aliases = self.read_old_aliases().await?;
let old_links = self.read_old_person_albums().await?;
let mut old_to_new_person: HashMap<i64, i64> = HashMap::new();
// Step 1: 迁移人物
println!("\n[迁移] 步骤 1/5: 迁移人物...");
for p in &old_persons {
if title_parser::is_excluded(&p.name) {
self.stats.persons_skipped += 1;
continue;
}
sqlx::query(
"INSERT OR IGNORE INTO persons (name, display_name, total_albums, total_images, original_names)
VALUES (?, ?, ?, ?, ?)"
).bind(&p.name).bind(p.display_name.as_deref().unwrap_or(&p.name))
.bind(p.total_albums).bind(p.total_images).bind(p.original_names.as_deref())
.execute(&self.new_pool).await.ok();
let new_id: i64 = sqlx::query_scalar("SELECT id FROM persons WHERE name = ?")
.bind(&p.name).fetch_one(&self.new_pool).await.unwrap_or(0);
if new_id > 0 {
old_to_new_person.insert(p.id, new_id);
self.stats.persons_imported += 1;
}
}
println!("[迁移] 导入 {} 人,跳过 {} 人", self.stats.persons_imported, self.stats.persons_skipped);
// Step 2: 迁移别名
println!("\n[迁移] 步骤 2/5: 迁移别名...");
for (old_pid, alias_type, alias_name) in &old_aliases {
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
sqlx::query(
"INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)"
).bind(new_pid).bind(alias_type).bind(alias_name)
.execute(&self.new_pool).await.ok();
self.stats.aliases_imported += 1;
}
}
println!("[迁移] 导入 {} 个别名", self.stats.aliases_imported);
// Step 3: 迁移相册 + 解析title
println!(
"\n[迁移] 步骤 3/5: 迁移相册并解析title ({})...",
if reparse { "重新解析" } else { "保留原解析" }
);
let mut person_map: HashMap<String, i64> = HashMap::new();
for (i, album) in old_albums.iter().enumerate() {
if (i + 1) % 200 == 0 {
println!(" 处理进度: {}/{}", i + 1, old_albums.len());
}
let (person_orig, person_cleaned, tags, all_words) =
if reparse || album.person_name_cleaned.as_deref().unwrap_or("").is_empty() {
// 重新解析
let person_name = title_parser::extract_person_name(&album.title);
let (cleaned, _aliases) = match &person_name {
Some(n) => {
let (c, a) = title_parser::clean_person_name(n);
if title_parser::is_excluded(&c) {
(None, a)
} else {
(Some(c), a)
}
}
None => (None, Vec::new()),
};
let tags_raw = title_parser::extract_tags(&album.title, person_name.as_deref());
let tag_names: Vec<String> = tags_raw.iter().map(|(t, _)| t.clone()).collect();
let all_words: Vec<[String; 2]> = tags_raw.iter()
.map(|(t, tp)| [t.clone(), tp.clone()]).collect();
(
person_name,
cleaned,
serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string()),
serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string()),
)
} else {
(
album.person_name_original.clone(),
album.person_name_cleaned.clone(),
album.tags.clone().unwrap_or_else(|| "[]".to_string()),
album.all_words.clone().unwrap_or_else(|| "[]".to_string()),
)
};
// 写入相册
sqlx::query(
"INSERT OR REPLACE INTO albums
(id, title, image_count, person_name_original, person_name_cleaned,
tags, all_words, raw_title, processed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))"
).bind(&album.id).bind(&album.title).bind(album.image_count)
.bind(person_orig.as_deref()).bind(person_cleaned.as_deref())
.bind(&tags).bind(&all_words).bind(&album.title)
.execute(&self.new_pool).await.ok();
// 写入标签索引
if let Ok(tag_list) = serde_json::from_str::<Vec<String>>(&tags) {
if !tag_list.is_empty() {
for tag in &tag_list {
sqlx::query(
"INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES (?, ?)"
).bind(&album.id).bind(tag).execute(&self.new_pool).await.ok();
}
self.stats.tags_imported += 1;
}
}
// 关联人物
if let Some(pc) = &person_cleaned {
if !pc.is_empty() && !title_parser::is_excluded(pc) {
let pid = if let Some(&id) = person_map.get(pc) {
id
} else {
// 查找或创建
let existing: Option<i64> = sqlx::query_scalar(
"SELECT id FROM persons WHERE name = ?"
).bind(pc).fetch_optional(&self.new_pool).await.unwrap_or(None);
let id = if let Some(eid) = existing {
eid
} else {
sqlx::query("INSERT INTO persons (name, display_name) VALUES (?, ?)")
.bind(pc).bind(pc).execute(&self.new_pool).await.ok();
self.stats.persons_created += 1;
sqlx::query_scalar::<_, i64>("SELECT last_insert_rowid()")
.fetch_one(&self.new_pool).await.unwrap_or(0)
};
person_map.insert(pc.clone(), id);
id
};
if pid > 0 {
sqlx::query(
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
).bind(pid).bind(&album.id).execute(&self.new_pool).await.ok();
}
}
}
self.stats.albums_processed += 1;
}
println!(
"[迁移] 处理 {} 个相册,{} 个标签导入,{} 个人物从title创建",
self.stats.albums_processed, self.stats.tags_imported, self.stats.persons_created
);
// Step 4: 迁移人物-相册关联
println!("\n[迁移] 步骤 4/5: 迁移人物-相册关联...");
for (old_pid, album_id) in &old_links {
if let Some(&new_pid) = old_to_new_person.get(old_pid) {
sqlx::query(
"INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)"
).bind(new_pid).bind(album_id).execute(&self.new_pool).await.ok();
self.stats.person_albums_created += 1;
}
}
println!("[迁移] 导入 {} 个关联", self.stats.person_albums_created);
// Step 5: 更新人物统计
println!("\n[迁移] 步骤 5/5: 更新人物统计...");
let person_ids: Vec<i64> = sqlx::query_scalar("SELECT id FROM persons")
.fetch_all(&self.new_pool).await.unwrap_or_default();
for pid in &person_ids {
sqlx::query(
"UPDATE persons SET
total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?),
total_images = (
SELECT COALESCE(SUM(a.image_count), 0)
FROM albums a JOIN person_albums pa ON a.id = pa.album_id
WHERE pa.person_id = ?
),
updated_at = datetime('now')
WHERE id = ?"
).bind(pid).bind(pid).bind(pid).execute(&self.new_pool).await.ok();
}
println!("[迁移] 更新 {} 个人物统计", person_ids.len());
Ok(())
}
async fn print_stats(&self) {
let stats = self.get_new_stats().await;
println!("\n 新数据库统计:");
println!(" 相册数: {}", stats.total_albums);
println!(" 图片数: {}", stats.total_images);
println!(" 人物数: {}", stats.total_persons);
println!(" 别名数: {}", stats.total_aliases);
println!(" 标签数: {}", stats.unique_tags);
println!(" 有标签相册: {}", stats.albums_with_tags);
println!(" 有人物相册: {}", stats.albums_with_person);
}
async fn get_new_stats(&self) -> NewDbStats {
NewDbStats {
total_albums: sqlx::query_scalar("SELECT COUNT(*) FROM albums")
.fetch_one(&self.new_pool).await.unwrap_or(0),
total_images: sqlx::query_scalar("SELECT COALESCE(SUM(image_count), 0) FROM albums")
.fetch_one(&self.new_pool).await.unwrap_or(0),
total_persons: sqlx::query_scalar("SELECT COUNT(*) FROM persons")
.fetch_one(&self.new_pool).await.unwrap_or(0),
total_aliases: sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases")
.fetch_one(&self.new_pool).await.unwrap_or(0),
unique_tags: sqlx::query_scalar("SELECT COUNT(DISTINCT tag) FROM album_tags")
.fetch_one(&self.new_pool).await.unwrap_or(0),
albums_with_tags: sqlx::query_scalar(
"SELECT COUNT(*) FROM albums WHERE tags IS NOT NULL AND tags != '[]'"
).fetch_one(&self.new_pool).await.unwrap_or(0),
albums_with_person: sqlx::query_scalar(
"SELECT COUNT(*) FROM albums WHERE person_name_cleaned IS NOT NULL AND person_name_cleaned != ''"
).fetch_one(&self.new_pool).await.unwrap_or(0),
}
}
}
#[derive(Debug)]
struct NewDbStats {
total_albums: i64,
total_images: i64,
total_persons: i64,
total_aliases: i64,
unique_tags: i64,
albums_with_tags: i64,
albums_with_person: i64,
}
// ============================================================
// 搜索验证器
// ============================================================
struct SearchVerifier {
pool: SqlitePool,
}
impl SearchVerifier {
async fn new(db_path: &str) -> Result<Self, String> {
let pool = SqlitePoolOptions::new()
.max_connections(2)
.connect(&format!("sqlite:{}", db_path))
.await
.map_err(|e| format!("连接数据库失败: {}", e))?;
Ok(Self { pool })
}
async fn verify_all(&self) {
println!("\n{}", "=".repeat(60));
println!(" 🔍 搜索功能验证 (Rust 实现)");
println!("{}", "=".repeat(60));
// 1. 热门人物
println!("\n[验证1] 热门人物 (Top 10):");
let top_persons: Vec<(String, i64, i64)> = sqlx::query(
"SELECT name, total_albums, total_images FROM persons ORDER BY total_albums DESC LIMIT 10"
).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("name"), r.get("total_albums"), r.get("total_images"))).collect();
for (i, (name, albums, images)) in top_persons.iter().enumerate() {
println!(" {:2d}. {:<25} {:4} 相册, {:5} 张图片", i + 1, name, albums, images);
}
// 2. 热门标签
println!("\n[验证2] 热门标签 (Top 20):");
let top_tags: Vec<(String, i64, i64)> = sqlx::query(
"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT 20"
).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("tag"), r.get::<i64, _>("cnt"), r.get::<i64, _>("album_cnt"))).collect();
for (i, (tag, cnt, album_cnt)) in top_tags.iter().enumerate() {
println!(" {:2d}. {:<20} 出现 {:4} 次, {:4} 个相册", i + 1, tag, cnt, album_cnt);
}
// 3. 人物搜索
if let Some((test_person, _, _)) = top_persons.first() {
println!("\n[验证3] 人物搜索: \"{}\"", test_person);
let like = format!("%{}%", test_person);
let results: Vec<(String, i64)> = sqlx::query(
"SELECT DISTINCT p.name, p.total_albums FROM persons p
LEFT JOIN person_aliases pa ON pa.person_id = p.id
WHERE p.name LIKE ? OR pa.alias_name LIKE ?
ORDER BY p.total_albums DESC LIMIT 5"
).bind(&like).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("name"), r.get("total_albums"))).collect();
for (name, albums) in &results {
println!(" - {} ({} 相册)", name, albums);
}
// 4. 人物相册
println!("\n[验证4] 人物相册: \"{}\" (前5个)", test_person);
let person_albums: Vec<(i64, String)> = sqlx::query(
"SELECT a.image_count, a.title FROM albums a
JOIN person_albums pa ON pa.album_id = a.id
JOIN persons p ON p.id = pa.person_id
WHERE p.name = ? ORDER BY a.image_count DESC LIMIT 5"
).bind(test_person).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
for (ic, title) in &person_albums {
let short: String = title.chars().take(60).collect();
println!(" - [{:3}张] {}", ic, short);
}
}
// 5. 标签搜索
if let Some((test_tag, _, _)) = top_tags.first() {
println!("\n[验证5] 标签搜索: \"{}\" (前5个)", test_tag);
let tag_albums: Vec<(i64, String)> = sqlx::query(
"SELECT a.image_count, a.title FROM albums a
JOIN album_tags t ON t.album_id = a.id
WHERE t.tag = ? ORDER BY a.image_count DESC LIMIT 5"
).bind(test_tag).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
for (ic, title) in &tag_albums {
let short: String = title.chars().take(60).collect();
println!(" - [{:3}张] {}", ic, short);
}
}
// 6. 标题搜索
println!("\n[验证6] 标题搜索: \"性感\" (前5个)");
let like = "%性感%".to_string();
let title_albums: Vec<(i64, String)> = sqlx::query(
"SELECT image_count, title FROM albums WHERE title LIKE ? ORDER BY image_count DESC LIMIT 5"
).bind(&like).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| (r.get("image_count"), r.get("title"))).collect();
for (ic, title) in &title_albums {
let short: String = title.chars().take(60).collect();
println!(" - [{:3}张] {}", ic, short);
}
println!("\n{}", "=".repeat(60));
println!(" ✅ 搜索功能验证完成!");
println!("{}", "=".repeat(60));
}
}
// ============================================================
// CLI 入口
// ============================================================
#[derive(Debug, clap::Parser)]
#[command(name = "migrate", about = "数据库迁移工具 (Rust 实现)")]
struct Cli {
/// 老数据库路径
#[arg(long, default_value = "../yolov8_face_recognition/persons.db")]
old_db: String,
/// 新数据库路径
#[arg(long, default_value = "./data/imager.db")]
new_db: String,
/// 重新解析所有 title
#[arg(long)]
reparse: bool,
/// 迁移后验证搜索功能
#[arg(long)]
verify: bool,
/// 仅验证搜索功能 (不执行迁移)
#[arg(long)]
verify_only: bool,
}
#[tokio::main]
async fn main() -> Result<(), String> {
// 检查是否有 clap
let args: Vec<String> = std::env::args().collect();
let mut old_db = "../yolov8_face_recognition/persons.db".to_string();
let mut new_db = "./data/imager.db".to_string();
let mut reparse = false;
let mut verify = false;
let mut verify_only = false;
let mut i = 1;
while i < args.len() {
match args[i].as_str() {
"--old-db" => { i += 1; if i < args.len() { old_db = args[i].clone(); } }
"--new-db" => { i += 1; if i < args.len() { new_db = args[i].clone(); } }
"--reparse" => reparse = true,
"--verify" => verify = true,
"--verify-only" => verify_only = true,
"--help" | "-h" => {
println!("数据库迁移工具 (Rust 实现)");
println!();
println!("用法: cargo run --bin migrate -- [选项]");
println!();
println!("选项:");
println!(" --old-db <路径> 老数据库路径 (默认: ../yolov8_face_recognition/persons.db)");
println!(" --new-db <路径> 新数据库路径 (默认: ./data/imager.db)");
println!(" --reparse 重新解析所有 title");
println!(" --verify 迁移后验证搜索功能");
println!(" --verify-only 仅验证搜索功能 (不执行迁移)");
println!(" --help, -h 显示帮助");
return Ok(());
}
_ => {}
}
i += 1;
}
let start = Instant::now();
// 仅验证模式
if verify_only {
if !Path::new(&new_db).exists() {
return Err(format!("数据库不存在: {}", new_db));
}
let verifier = SearchVerifier::new(&new_db).await?;
verifier.verify_all().await;
return Ok(());
}
println!("{}", "=".repeat(60));
println!(" 🚀 数据库迁移工具 (Rust 原生实现)");
println!("{}", "=".repeat(60));
// 执行迁移
let mut migrator = Migrator::new(&old_db, &new_db).await?;
migrator.migrate(reparse).await?;
println!("\n{}", "=".repeat(60));
println!(" ✅ 迁移完成!用时: {:.2?}", start.elapsed());
println!("{}", "=".repeat(60));
migrator.print_stats().await;
// 验证搜索
if verify {
let verifier = SearchVerifier::new(&new_db).await?;
verifier.verify_all().await;
}
Ok(())
}

193
src/cache/mod.rs vendored
View File

@@ -1,6 +1,9 @@
//! 缓存层 — 内存缓存管理
use crate::search_db::SearchDb;
use crate::db::Database;
use crate::models::*;
use sqlx::Row;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
@@ -88,14 +91,23 @@ impl AlbumsCache {
#[derive(Debug, Clone, Default)]
pub struct TagCache {
pub popular_tags: Vec<TagStat>,
/// tag → 包含该标签的相册列表(倒排索引,供按标签搜索用)
pub tag_albums: HashMap<String, Vec<AlbumInfo>>,
pub last_updated: chrono::DateTime<chrono::Utc>,
}
impl TagCache {
pub fn update(&mut self, tags: Vec<TagStat>) {
pub fn update(&mut self, tags: Vec<TagStat>, tag_albums: HashMap<String, Vec<AlbumInfo>>) {
self.popular_tags = tags;
self.tag_albums = tag_albums;
self.last_updated = chrono::Utc::now();
info!("标签缓存已更新,共 {} 个热门标签", self.popular_tags.len());
info!("标签缓存已更新,共 {} 个热门标签,{} 个标签倒排索引项",
self.popular_tags.len(), self.tag_albums.len());
}
/// 按标签搜索相册(从缓存中检索)
pub fn search_albums(&self, tag: &str) -> Vec<AlbumInfo> {
self.tag_albums.get(tag).cloned().unwrap_or_default()
}
}
@@ -103,30 +115,181 @@ impl TagCache {
#[derive(Debug, Clone, Default)]
pub struct PersonCache {
pub persons: Vec<PersonWithCounts>,
/// 搜索索引:关键词小写 → person_id 集合
pub search_index: HashMap<String, Vec<String>>,
/// person_id → person 投影(用于按 id 快速访问)
pub id_map: HashMap<String, PersonWithCounts>,
pub last_updated: chrono::DateTime<chrono::Utc>,
}
impl PersonCache {
pub fn update(&mut self, persons: Vec<PersonWithCounts>) {
self.search_index.clear();
self.id_map.clear();
for p in &persons {
// 构建搜索索引(姓名 + 所有别名)
let name_lower = p.name.to_lowercase();
for word in name_lower.split_whitespace() {
if word.len() >= 2 {
self.search_index.entry(word.to_string()).or_default().push(p.id.clone());
}
}
self.search_index.entry(name_lower.clone()).or_default().push(p.id.clone());
// 显示名
if let Some(ref dn) = p.display_name {
let dn_lower = dn.to_lowercase();
for word in dn_lower.split_whitespace() {
if word.len() >= 2 {
self.search_index.entry(word.to_string()).or_default().push(p.id.clone());
}
}
self.search_index.entry(dn_lower).or_default().push(p.id.clone());
}
// 别名
for alias in &p.aliases {
let alias_lower = alias.to_lowercase();
for word in alias_lower.split_whitespace() {
if word.len() >= 2 {
self.search_index.entry(word.to_string()).or_default().push(p.id.clone());
}
}
self.search_index.entry(alias_lower).or_default().push(p.id.clone());
}
self.id_map.insert(p.id.clone(), p.clone());
}
self.persons = persons;
self.last_updated = chrono::Utc::now();
info!("人物缓存已更新,共 {} 人", self.persons.len());
info!("人物缓存已更新,共 {} 人,{} 个搜索索引项",
self.persons.len(), self.search_index.len());
}
/// 从缓存搜索人物(按姓名/别名)
pub fn search(&self, query: &str) -> Vec<PersonWithCounts> {
let q = query.to_lowercase().trim().to_string();
if q.is_empty() || q.len() < 2 { return Vec::new(); }
let mut seen = std::collections::HashSet::new();
// 1. 完整匹配
if let Some(ids) = self.search_index.get(&q) {
for id in ids {
if seen.insert(id.clone()) {}
}
}
// 2. 单词匹配
for word in q.split_whitespace() {
if word.len() >= 2 {
if let Some(ids) = self.search_index.get(word) {
for id in ids {
seen.insert(id.clone());
}
}
}
}
// 3. 模糊匹配
if seen.is_empty() {
for p in &self.persons {
let full = format!("{} {} {}", p.name.to_lowercase(),
p.display_name.as_deref().unwrap_or(""),
p.aliases.join(" ").to_lowercase());
if full.contains(&q) {
seen.insert(p.id.clone());
}
}
}
seen.iter()
.filter_map(|id| self.id_map.get(id).cloned())
.collect()
}
}
/// 共享缓存
pub struct AppCache {
pub albums: Arc<RwLock<AlbumsCache>>,
pub tags: Arc<RwLock<TagCache>>,
pub persons: Arc<RwLock<PersonCache>>,
}
/// 刷新所有缓存(相册 + 标签 + 人物)
/// 相册从主库读取,标签/人物从 search.db 读取
pub async fn refresh_all_caches(
db: &Database,
search_db: &SearchDb,
albums_cache: &Arc<RwLock<AlbumsCache>>,
persons_cache: &Arc<RwLock<PersonCache>>,
tags_cache: &Arc<RwLock<TagCache>>,
) {
// 1. 从主库读取相册(tags 和 persons 从 search.db 关联)
if let Ok(albums) = db.get_all_albums().await {
// 从 search.db 批量获取 album → tags 映射
let tag_rows = sqlx::query(
r#"SELECT at.album_id, t.name FROM album_tags at JOIN tags t ON at.tag_id = t.id"#
).fetch_all(&search_db.pool).await.unwrap_or_default();
let mut album_tags: std::collections::HashMap<String, Vec<String>> = std::collections::HashMap::new();
for r in &tag_rows {
let aid: String = r.get("album_id");
let tag: String = r.get("name");
album_tags.entry(aid).or_default().push(tag);
}
impl AppCache {
pub fn new() -> Self {
Self {
albums: Arc::new(RwLock::new(AlbumsCache::new())),
tags: Arc::new(RwLock::new(TagCache::default())),
persons: Arc::new(RwLock::new(PersonCache::default())),
// 从 search.db 批量获取 album → persons 映射
let person_rows = sqlx::query(
r#"SELECT ap.album_id, p.name FROM album_persons ap JOIN persons p ON ap.person_id = p.id"#
).fetch_all(&search_db.pool).await.unwrap_or_default();
let mut album_persons: std::collections::HashMap<String, Vec<String>> = std::collections::HashMap::new();
for r in &person_rows {
let aid: String = r.get("album_id");
let name: String = r.get("name");
album_persons.entry(aid).or_default().push(name);
}
let enriched: Vec<AlbumInfo> = albums.into_iter().map(|mut a| {
let id_str = a.id.to_string();
a.tags = album_tags.remove(&id_str).unwrap_or_default();
a.persons = album_persons.remove(&id_str).unwrap_or_default();
a
}).collect();
// 2. 刷新标签(tag→albums 倒排索引直接从已富化的 album 列表构建,零 DB 查询)
let popular_tags = search_db.get_popular_tags(200).await;
let mut tag_albums: HashMap<String, Vec<AlbumInfo>> = HashMap::new();
for album in &enriched {
for tag in &album.tags {
tag_albums.entry(tag.clone()).or_default().push(album.clone());
}
}
let mut tc = tags_cache.write().await;
tc.update(popular_tags, tag_albums);
// 3. 刷新相册缓存(依赖 enriched 数据,必须在标签刷新之后执行)
let mut c = albums_cache.write().await;
c.update(enriched);
}
// 3. 刷新人物(从 search.db,修正 total_albums/total_images)
let mut persons = search_db.get_all_persons_with_aliases().await;
// 从 album_persons 重新计算实际相册数(修复缓存中 total_albums 可能为 0 的问题)
for p in &mut persons {
let actual: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM album_persons WHERE person_id = ?"
).bind(&p.id).fetch_one(&search_db.pool).await.unwrap_or(0);
if actual > 0 {
p.total_albums = actual;
}
}
let mut pc = persons_cache.write().await;
pc.update(persons);
info!("全部缓存已刷新");
}
/// 仅刷新人物缓存(管线执行后调用)
pub async fn refresh_persons_cache(search_db: &SearchDb, persons_cache: &Arc<RwLock<PersonCache>>) {
let mut persons = search_db.get_all_persons_with_aliases().await;
for p in &mut persons {
let actual: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM album_persons WHERE person_id = ?"
).bind(&p.id).fetch_one(&search_db.pool).await.unwrap_or(0);
if actual > 0 {
p.total_albums = actual;
}
}
let mut pc = persons_cache.write().await;
pc.update(persons);
}

View File

@@ -48,12 +48,18 @@ pub struct OldDbConfig {
pub path: String,
#[serde(default = "default_auto_migrate")]
pub auto_migrate: bool,
#[serde(default = "default_auto_parse")]
pub auto_parse: bool,
}
fn default_auto_migrate() -> bool {
false
}
fn default_auto_parse() -> bool {
true
}
impl Config {
pub fn load() -> Result<Self, config::ConfigError> {
let config = config::Config::builder()

View File

@@ -22,6 +22,102 @@ impl Database {
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}
}).collect())
}
/// 组合过滤搜索(懒加载):仅返回符合条件的相册,避免全量加载
///
/// 支持按 tag / person / 标题关键词 任意组合过滤(AND 关系)。
/// 参数传空字符串或 None 表示不过滤该维度。
/// 全参数为空时等价于 get_all_albums。
pub async fn search_albums_with_filters(
&self,
tag: Option<&str>,
person: Option<&str>,
title_q: Option<&str>,
limit: i64,
offset: i64,
) -> Result<Vec<AlbumInfo>, sqlx::Error> {
let has_tag = tag.map(|s| !s.is_empty()).unwrap_or(false);
let has_person = person.map(|s| !s.is_empty()).unwrap_or(false);
let has_q = title_q.map(|s| !s.is_empty()).unwrap_or(false);
if !has_tag && !has_person && !has_q {
// 无过滤条件 → 走缓存路径,这里回退到全量查询
return self.get_all_albums().await;
}
// 动态构建 SQL
let sql = {
let mut s = String::from(
"SELECT DISTINCT a.id, a.title, a.document, a.description, \
a.cover_image_id, a.image_count, a.created_at \
FROM albums a"
);
let mut joins = Vec::new();
let mut wheres = Vec::new();
if has_tag {
joins.push("JOIN album_tags t ON a.id = t.album_id");
wheres.push("t.tag = ?");
}
if has_person {
joins.push("JOIN person_albums pa ON a.id = pa.album_id");
joins.push("JOIN persons p ON pa.person_id = p.id");
wheres.push("(p.name = ? OR p.name LIKE ?)");
}
if has_q {
wheres.push("a.title LIKE ?");
}
for j in &joins {
s.push(' ');
s.push_str(j);
}
if !wheres.is_empty() {
s.push_str(" WHERE ");
s.push_str(&wheres.join(" AND "));
}
s.push_str(" ORDER BY a.created_at DESC");
if limit > 0 {
s.push_str(" LIMIT ? OFFSET ?");
}
s
};
let mut q = sqlx::query(&sql);
if has_tag {
q = q.bind(tag.unwrap());
}
if has_person {
let p = person.unwrap();
q = q.bind(p).bind(format!("%{}%", p));
}
if has_q {
q = q.bind(format!("%{}%", title_q.unwrap()));
}
if limit > 0 {
q = q.bind(limit).bind(offset);
}
let rows = q.fetch_all(&self.pool).await?;
Ok(rows.iter().map(|r| {
AlbumInfo {
id: Uuid::parse_str(&r.get::<String, _>("id")).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: r.get("document"),
description: r.get("description"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}
}).collect())
}
@@ -46,8 +142,6 @@ impl Database {
image_count: row.get("image_count"),
created_at: row.get("created_at"),
updated_at: row.get("updated_at"),
person_name_cleaned: row.get("person_name_cleaned"),
tags: row.get("tags"),
};
let img_rows = sqlx::query("SELECT * FROM images WHERE album_id = ?")
@@ -71,33 +165,58 @@ impl Database {
});
}
Ok(Some(AlbumWithImages { album, images: images.clone(), image_count: images.len() as i64 }))
Ok(Some(AlbumWithImages { album, images: images.clone(), image_count: images.len() as i64, tags: vec![], persons: vec![] }))
}
pub async fn get_all_albums_full(&self) -> Vec<AlbumMeta> {
let has_col = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM pragma_table_info('albums') WHERE name = 'person_name_original'"
).fetch_one(&self.pool).await.unwrap_or(0);
if has_col == 0 { return Vec::new(); }
let rows = sqlx::query(
r#"SELECT id, title, person_name_original, person_name_cleaned, tags, all_words, raw_title, processed_at, image_count
FROM albums ORDER BY created_at DESC"#
r#"SELECT id, title, cover_image_id, image_count
FROM albums"#
).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
id: r.get("id"), title: r.get("title"),
person_name_original: r.get("person_name_original"),
person_name_cleaned: r.get("person_name_cleaned"),
tags: r.get("tags"), all_words: r.get("all_words"),
raw_title: r.get("raw_title"), processed_at: r.get("processed_at"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
}).collect()
}
/// 按 ID 列表批量查询相册元数据
pub async fn get_albums_by_ids(&self, ids: &[String]) -> Vec<AlbumMeta> {
if ids.is_empty() { return Vec::new(); }
let placeholders: Vec<String> = (0..ids.len()).map(|_| "?".to_string()).collect();
let sql = format!(
"SELECT id, title, cover_image_id, image_count
FROM albums WHERE id IN ({})",
placeholders.join(", ")
);
let mut query = sqlx::query(&sql);
for id in ids {
query = query.bind(id);
}
let rows = query.fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| AlbumMeta {
id: r.get("id"), title: r.get("title"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
}).collect()
}
/// 获取所有未处理的相册 ID(processed_at IS NULL),用于管线收集新相册
pub async fn get_unprocessed_album_ids(&self) -> Vec<String> {
let rows = sqlx::query_scalar::<_, String>(
"SELECT id FROM albums WHERE processed_at IS NULL"
).fetch_all(&self.pool).await.unwrap_or_default();
rows
}
// ── 写入 ──
pub async fn create_album(&self, album: CreateAlbum) -> Result<Uuid, sqlx::Error> {
let _guard = self.write_lock.lock().await;
let id = Uuid::new_v4();
let now = chrono::Utc::now().to_rfc3339();
sqlx::query(
@@ -107,11 +226,81 @@ impl Database {
Ok(id)
}
/// 将所有相册重置为未处理状态(processed_at = NULL),下次管线会重新解析全部相册
pub async fn reset_all_processed_at(&self) -> usize {
sqlx::query("UPDATE albums SET processed_at = NULL")
.execute(&self.pool).await.unwrap_or_default()
.rows_affected() as usize
}
pub async fn delete_album(&self, album_id: Uuid) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM albums WHERE id = ?").bind(album_id.to_string()).execute(&self.pool).await?;
let _guard = self.write_lock.lock().await;
let aid = album_id.to_string();
// 先删除 person_albums(无 CASCADE)
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(&aid).execute(&self.pool).await?;
// 再删除 albums(images/favorites/album_tags 由 CASCADE 自动处理)
sqlx::query("DELETE FROM albums WHERE id = ?").bind(&aid).execute(&self.pool).await?;
Ok(())
}
/// 批量删除相册(级联清理所有关联数据)
pub async fn delete_albums_batch(&self, album_ids: &[Uuid]) -> usize {
if album_ids.is_empty() { return 0; }
let _guard = self.write_lock.lock().await;
for id in album_ids {
let aid = id.to_string();
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(&aid).execute(&self.pool).await.ok();
sqlx::query("DELETE FROM albums WHERE id = ?").bind(&aid).execute(&self.pool).await.ok();
}
album_ids.len()
}
/// 查询 image_count = 0 的空相册
pub async fn get_empty_albums(&self) -> Vec<AlbumInfo> {
let rows = sqlx::query(
r#"SELECT id, title, document, description, cover_image_id, image_count, created_at
FROM albums WHERE image_count IS NULL OR image_count = 0
ORDER BY created_at DESC"#
).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| {
let id_str = r.get::<String, _>("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"), document: r.get("document"),
description: r.get("description"), cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"), created_at: r.get("created_at"),
tags: vec![], persons: vec![],
}
}).collect()
}
/// 查找重名相册,返回结构:每个元素为 (keep_id, keep_document, [(dup_id, dup_document)])
pub async fn find_duplicate_albums(&self) -> Vec<(Uuid, String, Vec<(Uuid, String)>)> {
let rows = sqlx::query(
r#"SELECT id, title, document FROM albums WHERE title != '' ORDER BY created_at ASC"#
).fetch_all(&self.pool).await.unwrap_or_default();
// title → [(id, document)]
let mut title_map: std::collections::HashMap<String, Vec<(Uuid, String)>> = std::collections::HashMap::new();
for r in &rows {
let title: String = r.get("title");
let id_str: String = r.get("id");
let doc: String = r.get("document");
if let Ok(id) = Uuid::parse_str(&id_str) {
title_map.entry(title).or_default().push((id, doc));
}
}
title_map.into_iter()
.filter(|(_, items)| items.len() > 1)
.map(|(_, items)| {
let keep = items[0].clone(); // (keep_id, keep_document)
let dups = items[1..].to_vec();
(keep.0, keep.1, dups)
})
.collect()
}
pub async fn update_album_cover(&self, album_id: Uuid, image_id: Uuid) -> Result<(), sqlx::Error> {
sqlx::query("UPDATE albums SET cover_image_id = ? WHERE id = ?")
.bind(image_id.to_string()).bind(album_id.to_string()).execute(&self.pool).await?;

83
src/db/batch.rs Normal file
View File

@@ -0,0 +1,83 @@
//! 批量写入 — 在单个事务中写入多个相册的解析结果
//!
//! 配合 Phase 1a(并行解析)使用,所有脏相册在一个事务内写入,
//! 减少 round-trip 和 WAL fsync 次数。
use super::Database;
use crate::models::parse::ParseResult;
use sqlx::Transaction;
use sqlx::sqlite::Sqlite;
impl Database {
/// 批量写入多个相册的解析结果(Phase 1b)
///
/// 全部在一个事务中完成:
/// 1. 批量 upsert persons
/// 2. 批量 update 相册 person_name
/// 3. 批量 insert person_album 关联
/// 4. 批量 insert 别名
/// 5. 批量 update 标签
/// 6. 批量 update person_counts
pub async fn batch_write_parse_results(
&self,
tx: &mut Transaction<'_, Sqlite>,
results: &[ParseResult],
) -> Result<(), sqlx::Error> {
for result in results {
let album_id = &result.album_id;
// ── 标记处理时间(无论是否有人物,只要进了解析管线就标记)──
sqlx::query(
"UPDATE albums SET processed_at = datetime('now') WHERE id = ?"
)
.bind(album_id)
.execute(&mut **tx).await?;
// ── 先清空旧人物关联,再写新关联 ──
if !result.person_names.is_empty() {
sqlx::query("DELETE FROM person_albums WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;
for name in &result.person_names {
// ── insert or ignore 人物 ──
let pid = sqlx::query_scalar::<_, i64>(
"INSERT INTO persons (name, display_name) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET name = name RETURNING id"
)
.bind(name).bind(name)
.fetch_one(&mut **tx).await?;
// ── 写新关联 ──
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)")
.bind(pid).bind(album_id).execute(&mut **tx).await?;
// ── 更新该人物的计数 ──
sqlx::query(
"UPDATE persons SET total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?), \
total_images = COALESCE((SELECT SUM(image_count) FROM albums \
JOIN person_albums ON albums.id = person_albums.album_id \
WHERE person_albums.person_id = ?), 0) \
WHERE id = ?"
)
.bind(pid).bind(pid).bind(pid)
.execute(&mut **tx).await?;
}
} else {
// 无人名时也清空旧关联(避免残留)
sqlx::query("DELETE FROM person_albums WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;
}
// ── 写标签(tags/all_words 列已迁移到 search.db,只维护 album_tags 表)──
if !result.tags.is_empty() {
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id).execute(&mut **tx).await?;
for (t, _) in &result.tags {
sqlx::query("INSERT INTO album_tags (album_id, tag) VALUES (?, ?)")
.bind(album_id).bind(t).execute(&mut **tx).await?;
}
}
}
Ok(())
}
}

View File

@@ -1,9 +1,12 @@
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions};
use crate::config::Config;
use tokio::sync::Mutex;
pub struct Database {
pub pool: SqlitePool,
pub(crate) pool: SqlitePool,
pub config: Config,
/// 写互斥锁:单协程串行化所有写操作,消除 WAL 锁争用
pub(crate) write_lock: Mutex<()>,
}
impl Database {
@@ -22,30 +25,47 @@ impl Database {
}
let pool = SqlitePoolOptions::new()
.max_connections(10)
.min_connections(2)
.max_connections(2) // 写串行化后无需过多连接
.min_connections(1)
.acquire_timeout(std::time::Duration::from_secs(5))
.idle_timeout(std::time::Duration::from_secs(300))
.max_lifetime(std::time::Duration::from_secs(1800))
.connect(url)
.await?;
// SQLite 性能优化
// SQLite 高并发优化 PRAGMA
sqlx::query("PRAGMA journal_mode=WAL;").execute(&pool).await?;
sqlx::query("PRAGMA wal_autocheckpoint=4000;").execute(&pool).await?;
sqlx::query("PRAGMA synchronous=NORMAL;").execute(&pool).await?;
sqlx::query("PRAGMA cache_size=-64000;").execute(&pool).await?;
sqlx::query("PRAGMA busy_timeout=5000;").execute(&pool).await?; // 等待 5s 避免 SQLITE_BUSY
sqlx::query("PRAGMA wal_autocheckpoint=1000;").execute(&pool).await?; // 小 WAL → 短 checkpoint
sqlx::query("PRAGMA journal_size_limit=67108864;").execute(&pool).await?; // WAL 上限 64MB
sqlx::query("PRAGMA cache_size=-64000;").execute(&pool).await?; // 64KB 页缓存
sqlx::query("PRAGMA mmap_size=268435456;").execute(&pool).await?;
sqlx::query("PRAGMA temp_store=MEMORY;").execute(&pool).await?;
sqlx::query("PRAGMA foreign_keys=ON;").execute(&pool).await?;
sqlx::migrate!("./migrations").run(&pool).await?;
let db = Database { pool, config: config.clone() };
let db = Database { pool, config: config.clone(), write_lock: Mutex::new(()) };
db.ensure_image_count_field().await?;
Ok(db)
}
/// 执行 WAL checkpoint(TRUNCATE),将 WAL 完全合并回主文件并清理 -wal / -shm 文件
///
/// 应在服务优雅关闭时调用,确保数据库文件处于干净一致的状态,
/// 避免下次启动时 WAL 恢复带来的额外开销。
pub async fn wal_checkpoint(&self) {
let _guard = self.write_lock.lock().await;
// wal_checkpoint 返回一行 (busy, log, checkpointed),只关注结果忽略返回值
match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
.execute(&self.pool).await
{
Ok(_) => println!("[DB] WAL checkpoint(TRUNCATE) 完成,数据库文件已合并"),
Err(e) => eprintln!("[DB] WAL checkpoint 失败: {}", e),
}
}
/// 确保 image_count 字段存在
async fn ensure_image_count_field(&self) -> Result<(), sqlx::Error> {
let has_col = sqlx::query_scalar::<_, i64>(

View File

@@ -46,6 +46,7 @@ impl Database {
}
pub async fn add_favorite(&self, user_id: &str, fav: CreateFavorite) -> Result<Uuid, sqlx::Error> {
let _guard = self.write_lock.lock().await;
let id = Uuid::new_v4();
let now = chrono::Utc::now().to_rfc3339();
sqlx::query("INSERT INTO favorites (id, user_id, image_id, album_id, created_at) VALUES (?, ?, ?, ?, ?)")
@@ -55,11 +56,13 @@ impl Database {
}
pub async fn remove_favorite(&self, favorite_id: Uuid) -> Result<(), sqlx::Error> {
let _guard = self.write_lock.lock().await;
sqlx::query("DELETE FROM favorites WHERE id = ?").bind(favorite_id.to_string()).execute(&self.pool).await?;
Ok(())
}
pub async fn remove_favorite_by_image(&self, user_id: &str, image_id: Uuid) -> Result<(), sqlx::Error> {
let _guard = self.write_lock.lock().await;
sqlx::query("DELETE FROM favorites WHERE user_id = ? AND image_id = ?")
.bind(user_id).bind(image_id.to_string()).execute(&self.pool).await?;
Ok(())

View File

@@ -8,6 +8,7 @@ pub use crate::models::image::Image;
impl Database {
pub async fn add_images(&self, images: Vec<CreateImage>) -> Result<Vec<Uuid>, sqlx::Error> {
let _guard = self.write_lock.lock().await;
if images.is_empty() { return Ok(Vec::new()); }
let now = chrono::Utc::now().to_rfc3339();
let mut ids = Vec::new();

View File

@@ -1,6 +1,8 @@
use super::Database;
use crate::models::*;
use sqlx::Row;
use sqlx::{Row, Sqlite, Transaction};
use std::collections::HashMap;
use std::ops::DerefMut;
use uuid::Uuid;
impl Database {
@@ -43,6 +45,8 @@ impl Database {
title: r.get("title"), document: r.get("document"),
description: r.get("description"), cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"), created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}).collect()
}
@@ -64,45 +68,127 @@ impl Database {
WHERE p.name LIKE ? OR a.alias_name LIKE ? ORDER BY p.total_albums DESC"#
).bind(&pattern).bind(&pattern).fetch_all(&self.pool).await.unwrap_or_default();
let mut results = Vec::new();
for r in &rows {
let pids: Vec<i64> = rows.iter().map(|r| r.get::<i64, _>("id")).collect();
let aliases = self.batch_get_person_aliases(&pids).await;
rows.iter().map(|r| {
let pid = r.get::<i64, _>("id");
let aliases = self.get_person_aliases(pid).await;
results.push(PersonWithCounts {
id: pid, name: r.get("name"), display_name: r.get("display_name"),
PersonWithCounts {
id: pid.to_string(), name: r.get("name"), display_name: r.get("display_name"),
total_albums: r.get("total_albums"), total_images: r.get("total_images"),
aliases: aliases.into_iter().map(|a| a.alias_name).collect(),
});
}
results
aliases: aliases.get(&pid).cloned().unwrap_or_default(),
}
}).collect()
}
pub async fn get_all_persons_with_aliases(&self) -> Vec<PersonWithCounts> {
let rows = sqlx::query("SELECT id, name, display_name, total_albums, total_images FROM persons ORDER BY total_albums DESC")
.fetch_all(&self.pool).await.unwrap_or_default();
let mut results = Vec::new();
for r in &rows {
let pids: Vec<i64> = rows.iter().map(|r| r.get::<i64, _>("id")).collect();
let aliases = self.batch_get_person_aliases(&pids).await;
rows.iter().map(|r| {
let pid = r.get::<i64, _>("id");
let aliases = self.get_person_aliases(pid).await;
results.push(PersonWithCounts {
id: pid, name: r.get("name"), display_name: r.get("display_name"),
PersonWithCounts {
id: pid.to_string(), name: r.get("name"), display_name: r.get("display_name"),
total_albums: r.get("total_albums"), total_images: r.get("total_images"),
aliases: aliases.into_iter().map(|a| a.alias_name).collect(),
});
aliases: aliases.get(&pid).cloned().unwrap_or_default(),
}
}).collect()
}
/// 批量查询多个人物的别名(避免 N+1)
async fn batch_get_person_aliases(&self, pids: &[i64]) -> HashMap<i64, Vec<String>> {
if pids.is_empty() { return HashMap::new(); }
let placeholders: Vec<String> = pids.iter().map(|_| "?".to_string()).collect();
let sql = format!(
"SELECT person_id, alias_name FROM person_aliases WHERE person_id IN ({})",
placeholders.join(",")
);
let mut q = sqlx::query(&sql);
for pid in pids {
q = q.bind(pid);
}
results
let rows = q.fetch_all(&self.pool).await.unwrap_or_default();
let mut map: HashMap<i64, Vec<String>> = HashMap::new();
for r in &rows {
let pid: i64 = r.get("person_id");
let alias: String = r.get("alias_name");
map.entry(pid).or_default().push(alias);
}
map
}
// ── 标签查询 ──
pub async fn get_popular_tags(&self, limit: i64) -> Vec<TagStat> {
sqlx::query(
// 从 tag_dict 获取过滤词(停用词、品牌词、排除名)
// 注意:必须 clone 到本地变量,不能持 RwLockReadGuard 跨 await
let (stop_words, excluded, included) = {
let dict = crate::engine::tag_dict::get_tag_dict();
(
dict.stop_words.clone(),
dict.excluded_person_names.clone(),
dict.included_person_names.clone(),
)
};
// 标签自身小写的集合用于快速过滤
let filtered_tags: Vec<TagStat> = sqlx::query(
r#"SELECT tag, COUNT(*) as cnt, COUNT(DISTINCT album_id) as album_cnt
FROM album_tags GROUP BY tag ORDER BY cnt DESC LIMIT ?"#
).bind(limit).fetch_all(&self.pool).await.unwrap_or_default()
.iter().map(|r| TagStat {
tag: r.get("tag"), count: r.get("cnt"), album_count: r.get("album_cnt"),
}).collect()
).bind(limit * 3).fetch_all(&self.pool).await.unwrap_or_default()
.iter().filter_map(|r| {
let tag: String = r.get("tag");
let tag_lower = tag.to_lowercase();
// 过滤停用词
if stop_words.contains(&tag_lower) || stop_words.contains(&tag) {
return None;
}
// 过滤品牌/工作室名
if excluded.contains(&tag_lower) || excluded.contains(&tag) {
return None;
}
// 过滤包含人名(如果标签本身就是个人名,也不应该出现在标签中)
if included.contains(&tag_lower) || included.contains(&tag) {
return None;
}
Some(TagStat {
tag: r.get("tag"),
count: r.get("cnt"),
album_count: r.get("album_cnt"),
})
}).collect();
// 只保留 top limit 个
let mut result: Vec<TagStat> = filtered_tags;
result.truncate(limit as usize);
result
}
/// 批量获取所有 tag→albums 映射(供缓存使用)
pub async fn get_all_tag_albums(&self) -> HashMap<String, Vec<AlbumInfo>> {
let rows = sqlx::query(
r#"SELECT t.tag, a.id, a.title, a.document, a.description, a.cover_image_id, a.image_count, a.created_at
FROM album_tags t JOIN albums a ON t.album_id = a.id ORDER BY a.created_at DESC"#
).fetch_all(&self.pool).await.unwrap_or_default();
let mut map: HashMap<String, Vec<AlbumInfo>> = HashMap::new();
for r in &rows {
let tag: String = r.get("tag");
let album = AlbumInfo {
id: Uuid::parse_str(&r.get::<String, _>("id")).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"), document: r.get("document"),
description: r.get("description"), cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"), created_at: r.get("created_at"),
tags: vec![], persons: vec![],
};
map.entry(tag).or_default().push(album);
}
map
}
pub async fn get_tag_co_occurrences(&self, tag: &str, limit: i64) -> Vec<TagCoOccurrence> {
@@ -133,28 +219,38 @@ impl Database {
title: r.get("title"), document: r.get("document"),
description: r.get("description"), cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"), created_at: r.get("created_at"),
tags: vec![], persons: vec![],
}).collect()
}
// ── 统计 ──
pub async fn get_metadata_stats(&self) -> MetadataStats {
MetadataStats {
total_persons: sqlx::query_scalar("SELECT COUNT(*) FROM persons").fetch_one(&self.pool).await.unwrap_or(0),
total_aliases: sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases").fetch_one(&self.pool).await.unwrap_or(0),
total_albums_with_person: sqlx::query_scalar("SELECT COUNT(*) FROM albums WHERE person_name_original IS NOT NULL AND person_name_original != ''").fetch_one(&self.pool).await.unwrap_or(0),
total_albums_without_person: sqlx::query_scalar("SELECT COUNT(*) FROM albums WHERE person_name_original IS NULL OR person_name_original = ''").fetch_one(&self.pool).await.unwrap_or(0),
total_tags: sqlx::query_scalar("SELECT COUNT(*) FROM album_tags").fetch_one(&self.pool).await.unwrap_or(0),
}
}
// ── 写入 ──
pub async fn insert_person(&self, name: &str, display_name: Option<&str>) -> i64 {
if let Some(p) = self.get_person_by_name(name).await { return p.id; }
if let Some(p) = self.get_person_by_name_internal(name).await { return p; }
sqlx::query("INSERT OR IGNORE INTO persons (name, display_name) VALUES (?, ?)")
.bind(name).bind(display_name).execute(&self.pool).await.ok();
self.get_person_by_name(name).await.map(|p| p.id).unwrap_or(0)
self.get_person_by_name_internal(name).await.unwrap_or(0)
}
/// 内部方法:按名称查找人员,直接返回 i64 id(避免 String 转换开销)
async fn get_person_by_name_internal(&self, name: &str) -> Option<i64> {
sqlx::query_scalar::<_, i64>("SELECT id FROM persons WHERE name = ?")
.bind(name).fetch_optional(&self.pool).await.ok().flatten()
}
/// 与 insert_person 相同,但使用已有的事务
pub async fn insert_person_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, name: &str, display_name: Option<&str>) -> i64 {
if let Ok(Some(r)) = sqlx::query("SELECT id FROM persons WHERE name = ?")
.bind(name).fetch_optional(tx.deref_mut()).await {
return r.get::<i64, _>("id");
}
sqlx::query("INSERT OR IGNORE INTO persons (name, display_name) VALUES (?, ?)")
.bind(name).bind(display_name).execute(tx.deref_mut()).await.ok();
if let Ok(Some(r)) = sqlx::query("SELECT id FROM persons WHERE name = ?")
.bind(name).fetch_optional(tx.deref_mut()).await {
return r.get::<i64, _>("id");
}
0
}
pub async fn insert_person_alias(&self, person_id: i64, alias_type: &str, alias_name: &str) {
@@ -162,28 +258,185 @@ impl Database {
.bind(person_id).bind(alias_type).bind(alias_name).execute(&self.pool).await.ok();
}
pub async fn insert_person_alias_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, person_id: i64, alias_type: &str, alias_name: &str) {
sqlx::query("INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)")
.bind(person_id).bind(alias_type).bind(alias_name).execute(tx.deref_mut()).await.ok();
}
pub async fn insert_person_album(&self, person_id: i64, album_id: &str) {
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)")
.bind(person_id).bind(album_id).execute(&self.pool).await.ok();
}
pub async fn insert_person_album_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, person_id: i64, album_id: &str) {
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) VALUES (?, ?)")
.bind(person_id).bind(album_id).execute(tx.deref_mut()).await.ok();
}
pub async fn delete_person_albums(&self, person_id: i64) {
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
}
pub async fn update_album_person(&self, album_id: &str, person_name_original: &str, person_name_cleaned: &str) {
sqlx::query("UPDATE albums SET person_name_original = ?, person_name_cleaned = ?, processed_at = datetime('now') WHERE id = ?")
.bind(person_name_original).bind(person_name_cleaned).bind(album_id).execute(&self.pool).await.ok();
/// 删除某相册的所有人物关联,返回被影响的旧 person_id 列表
pub async fn delete_album_person_albums(&self, album_id: &str) -> Vec<i64> {
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
.bind(album_id).fetch_all(&self.pool).await.unwrap_or_default();
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok();
old_ids
}
pub async fn update_album_tags(&self, album_id: &str, tags_json: &str, all_words_json: &str) {
sqlx::query("UPDATE albums SET tags = ?, all_words = ?, processed_at = datetime('now') WHERE id = ?")
.bind(tags_json).bind(all_words_json).bind(album_id).execute(&self.pool).await.ok();
sqlx::query("DELETE FROM album_tags WHERE album_id = ?").bind(album_id).execute(&self.pool).await.ok();
/// 删除某相册的所有人物关联(事务版),返回被影响的旧 person_id 列表
pub async fn delete_album_person_albums_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str) -> Vec<i64> {
let old_ids: Vec<i64> = sqlx::query_scalar("SELECT person_id FROM person_albums WHERE album_id = ?")
.bind(album_id).fetch_all(tx.deref_mut()).await.unwrap_or_default();
sqlx::query("DELETE FROM person_albums WHERE album_id = ?").bind(album_id).execute(tx.deref_mut()).await.ok();
old_ids
}
/// 删除某人物下所有别名(写入新别名前清旧别名,配合 UNIQUE INDEX 防止重复)
pub async fn delete_person_aliases(&self, person_id: i64) {
sqlx::query("DELETE FROM person_aliases WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
}
pub async fn delete_person_aliases_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, person_id: i64) {
sqlx::query("DELETE FROM person_aliases WHERE person_id = ?").bind(person_id).execute(tx.deref_mut()).await.ok();
}
/// 彻底删除一个人物(包括别名、相册关联、自身)
pub async fn delete_person_completely(&self, person_id: i64) {
let _guard = self.write_lock.lock().await;
sqlx::query("DELETE FROM person_aliases WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(person_id).execute(&self.pool).await.ok();
sqlx::query("DELETE FROM persons WHERE id = ?").bind(person_id).execute(&self.pool).await.ok();
}
/// 把 src_id 的所有相册关联、别名移动到 dst_id,然后删除 src_id
pub async fn merge_person_into(&self, src_id: i64, dst_id: i64) {
let _guard = self.write_lock.lock().await;
// 整个合并过程是一个原子操作,用事务包裹避免逐条 autocommit
let mut tx = match self.pool.begin().await {
Ok(t) => t,
Err(_) => return,
};
// 1. 移动相册关联
sqlx::query("INSERT OR IGNORE INTO person_albums (person_id, album_id) SELECT ?, album_id FROM person_albums WHERE person_id = ?")
.bind(dst_id).bind(src_id).execute(tx.deref_mut()).await.ok();
// 2. 移动别名
sqlx::query("INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) SELECT ?, alias_type, alias_name FROM person_aliases WHERE person_id = ?")
.bind(dst_id).bind(src_id).execute(tx.deref_mut()).await.ok();
// 3. 把 src 的原名也作为别名加入 dst(使用事务连接,避免额外占用连接池)
if let Some(r) = sqlx::query("SELECT id, name, display_name FROM persons WHERE id = ?")
.bind(src_id).fetch_optional(tx.deref_mut()).await.ok().flatten()
{
let src_name: String = r.get("name");
let src_display: Option<String> = r.get("display_name");
self.insert_person_alias_with_tx(&mut tx, dst_id, "merged_original", &src_name).await;
if let Some(ref dn) = src_display {
self.insert_person_alias_with_tx(&mut tx, dst_id, "merged_display", dn).await;
}
}
// 4. 删除 src 的关联和别名
sqlx::query("DELETE FROM person_albums WHERE person_id = ?").bind(src_id).execute(tx.deref_mut()).await.ok();
sqlx::query("DELETE FROM person_aliases WHERE person_id = ?").bind(src_id).execute(tx.deref_mut()).await.ok();
// 5. 删除 src 人物
sqlx::query("DELETE FROM persons WHERE id = ?").bind(src_id).execute(tx.deref_mut()).await.ok();
// 6. 更新 dst 的统计数据
self.update_person_counts_with_tx(&mut tx, dst_id).await;
tx.commit().await.ok();
}
pub async fn get_person_by_id(&self, id: i64) -> Option<Person> {
let r = sqlx::query("SELECT * FROM persons WHERE id = ?").bind(id).fetch_optional(&self.pool).await.ok()??;
let db_id: i64 = r.get("id");
Some(Person {
id: db_id.to_string(), name: r.get("name"), display_name: r.get("display_name"),
total_albums: r.get("total_albums"), total_images: r.get("total_images"),
original_names: r.get("original_names"), created_at: r.get("created_at"), updated_at: r.get("updated_at"),
})
}
pub async fn update_album_person(&self, album_id: &str, _person_name_original: &str, _person_name_cleaned: &str) {
sqlx::query("UPDATE albums SET processed_at = datetime('now') WHERE id = ?")
.bind(album_id).execute(&self.pool).await.ok();
}
pub async fn update_album_person_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str, _person_name_original: &str, _person_name_cleaned: &str) {
sqlx::query("UPDATE albums SET processed_at = datetime('now') WHERE id = ?")
.bind(album_id).execute(tx.deref_mut()).await.ok();
}
/// 清空该人物所有关联相册的 person 字段(当该人物被确认为非人物后调用)
pub async fn update_album_person_by_person_id(&self, person_id: i64) {
sqlx::query(
"UPDATE albums SET processed_at = datetime('now') WHERE id IN (SELECT album_id FROM person_albums WHERE person_id = ?)"
).bind(person_id).execute(&self.pool).await.ok();
}
pub async fn update_album_tags(&self, album_id: &str, tags_json: &str, _all_words_json: &str) {
sqlx::query("UPDATE albums SET processed_at = datetime('now') WHERE id = ?")
.bind(album_id).execute(&self.pool).await.ok();
// 用一个事务批量替换 album_tags,避免逐条 autocommit 导致巨慢
if let Ok(tags) = serde_json::from_str::<Vec<String>>(tags_json) {
for tag in &tags {
sqlx::query("INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES (?, ?)")
.bind(album_id).bind(tag).execute(&self.pool).await.ok();
let mut tx = match self.pool.begin().await {
Ok(t) => t,
Err(_) => return,
};
// 单条 DELETE + 一条多值 INSERT
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id)
.execute(tx.deref_mut())
.await.ok();
// 构建多值 INSERT, 避免逐条执行
if !tags.is_empty() {
let placeholders: Vec<String> = tags.iter().map(|_| "(?, ?)".to_string()).collect();
let sql = format!(
"INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES {}",
placeholders.join(", ")
);
let mut q = sqlx::query(&sql);
for tag in &tags {
q = q.bind(album_id).bind(tag);
}
q.execute(tx.deref_mut()).await.ok();
}
tx.commit().await.ok();
}
}
/// 与 update_album_tags 相同,但使用已有的事务(不启动内部子事务)
pub async fn update_album_tags_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, album_id: &str, tags_json: &str, _all_words_json: &str) {
sqlx::query("UPDATE albums SET processed_at = datetime('now') WHERE id = ?")
.bind(album_id).execute(tx.deref_mut()).await.ok();
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id)
.execute(tx.deref_mut())
.await.ok();
if let Ok(tags) = serde_json::from_str::<Vec<String>>(tags_json) {
if !tags.is_empty() {
let placeholders: Vec<String> = tags.iter().map(|_| "(?, ?)".to_string()).collect();
let sql = format!(
"INSERT OR IGNORE INTO album_tags (album_id, tag) VALUES {}",
placeholders.join(", ")
);
let mut q = sqlx::query(&sql);
for tag in &tags {
q = q.bind(album_id).bind(tag);
}
q.execute(tx.deref_mut()).await.ok();
}
}
}
@@ -196,6 +449,14 @@ impl Database {
).bind(person_id).bind(person_id).bind(person_id).execute(&self.pool).await.ok();
}
pub async fn update_person_counts_with_tx(&self, tx: &mut Transaction<'_, Sqlite>, person_id: i64) {
sqlx::query(
r#"UPDATE persons SET total_albums = (SELECT COUNT(*) FROM person_albums WHERE person_id = ?),
total_images = (SELECT COALESCE(SUM(a.image_count), 0) FROM albums a JOIN person_albums pa ON a.id = pa.album_id WHERE pa.person_id = ?),
updated_at = datetime('now') WHERE id = ?"#
).bind(person_id).bind(person_id).bind(person_id).execute(tx.deref_mut()).await.ok();
}
/// 获取所有相册 ID 集合(用于迁移)
pub async fn get_all_album_ids(&self) -> Vec<String> {
sqlx::query_scalar::<_, String>("SELECT id FROM albums").fetch_all(&self.pool).await.unwrap_or_default()

View File

@@ -24,6 +24,9 @@ pub struct MigrationStats {
impl Database {
/// 从老数据库迁移元数据
pub async fn migrate_from_old_db(&self, old_db_path: &str) -> Result<MigrationStats, String> {
// 串行化写入:迁移过程中禁止其他写入操作
let _guard = self.write_lock.lock().await;
if !std::path::Path::new(old_db_path).exists() {
return Err(format!("老数据库不存在: {}", old_db_path));
}

View File

@@ -6,10 +6,11 @@ pub mod image;
pub mod favorite;
pub mod metadata;
pub mod migration;
pub mod batch;
pub use connection::Database;
pub use album::*;
pub use image::*;
pub use favorite::*;
pub use metadata::*;
#[allow(unused)] pub use album::*;
#[allow(unused)] pub use image::*;
#[allow(unused)] pub use favorite::*;
#[allow(unused)] pub use metadata::*;
pub use migration::MigrationStats;

View File

@@ -77,7 +77,7 @@ impl Crawler {
let resp = self.client.get(&url).send().await?;
let data: serde_json::Value = resp.json().await?;
let albums: Vec<ApiAlbum> = if let Some(arr) = data.as_array() {
let albums: Vec<ApiAlbum> = if let Some(_arr) = data.as_array() {
serde_json::from_value(data)?
} else if let Some(arr) = data.get("albums").and_then(|v: &serde_json::Value| v.as_array()) {
serde_json::from_value(serde_json::Value::Array(arr.clone()))?

View File

@@ -1,15 +1,15 @@
//! 核心引擎层 — 标题解析、人名匹配、标签提取、推荐计算
//! 核心引擎层 — 标题解析、标签提取、推荐计算
pub mod title_parser;
pub mod person_merger;
pub mod tag_dict;
pub mod tag_extractor;
pub mod recommend;
pub mod pipeline;
pub mod crawler;
pub use title_parser::*;
pub use person_merger::*;
pub use tag_extractor::*;
#[allow(unused)] pub use title_parser::*;
#[allow(unused)] pub use tag_dict::*;
#[allow(unused)] pub use tag_extractor::*;
pub use recommend::*;
pub use pipeline::*;
pub use crawler::*;
#[allow(unused)] pub use pipeline::*;
#[allow(unused)] pub use crawler::*;

View File

@@ -1,383 +0,0 @@
//! 人名匹配与合并
//! 对应 Python 的 match_person_from_title.py + merge_persons_by_core.py
//! 所有函数均为 async,因为底层 Database 操作是异步的
use crate::db::Database;
use crate::engine::title_parser::{extract_cn, is_excluded_name};
use std::collections::{HashMap, HashSet};
/// 匹配模式
#[derive(Debug, Clone)]
pub struct Pattern {
pub text: String,
pub person_name: String,
pub source: String, // "name" | "alias"
pub length: usize,
}
/// 匹配结果
#[derive(Debug, Clone)]
pub struct MatchResult {
pub matched: bool,
pub method: String,
pub person: Option<String>,
pub matched_text: Option<String>,
pub all_persons: Vec<String>,
pub multi_person: bool,
}
/// 深度匹配结果
#[derive(Debug, Clone)]
pub struct DeepMatchResult {
pub matched: bool,
pub method: String, // "jieba_extract" | "exact_name_match" | "alias_fuzzy_match"
pub person: Option<String>,
pub matched_text: Option<String>,
pub all_persons: Vec<String>,
pub multi_person: bool,
}
// ── Pass1: 收集模式 ──
/// 收集所有人名和别名,构建匹配模式列表
/// 按长度降序排列,确保长名优先匹配
pub async fn collect_person_patterns(db: &Database) -> Vec<Pattern> {
let mut patterns = Vec::new();
// 1. 收集所有人名
let rows = db.get_all_person_names().await;
for (name, _albums) in &rows {
if is_excluded_name(name) {
continue;
}
patterns.push(Pattern {
text: name.clone(),
person_name: name.clone(),
source: "name".to_string(),
length: name.len(),
});
}
// 2. 收集所有别名
let aliases = db.get_all_aliases().await;
for (alias_name, person_name) in &aliases {
if is_excluded_name(person_name) || is_excluded_name(alias_name) {
continue;
}
if alias_name != person_name {
patterns.push(Pattern {
text: alias_name.clone(),
person_name: person_name.clone(),
source: "alias".to_string(),
length: alias_name.len(),
});
}
}
// 3. 去重
let mut best_map: HashMap<String, Pattern> = HashMap::new();
for p in patterns {
let key = p.text.to_lowercase();
best_map.entry(key).or_insert(p);
}
let mut patterns: Vec<Pattern> = best_map.into_values().collect();
// 4. 按长度降序排列
patterns.sort_by(|a, b| b.length.cmp(&a.length));
patterns
}
/// Pass1: 简单子串匹配
pub fn match_title(title: &str, patterns: &[Pattern]) -> Option<(String, String, String)> {
if title.is_empty() {
return None;
}
for p in patterns {
if p.text.len() < 2 {
continue;
}
if title.contains(&p.text) {
return Some((p.text.clone(), p.person_name.clone(), p.source.clone()));
}
}
None
}
/// Pass2: 深度匹配(多重策略)
pub fn deep_match_title(
title: &str,
known_names: &HashSet<String>,
known_aliases: &HashSet<String>,
) -> DeepMatchResult {
let mut result = DeepMatchResult {
matched: false,
method: String::new(),
person: None,
matched_text: None,
all_persons: Vec::new(),
multi_person: false,
};
if title.is_empty() {
return result;
}
let title_clean = title.trim();
// 策略1: 用正则提取人名
if let Some(extracted) = crate::engine::title_parser::extract_person_name_from_title(title_clean) {
if known_names.contains(&extracted) {
result.matched = true;
result.method = "jieba_extract".to_string();
result.person = Some(extracted.clone());
result.matched_text = Some(extracted);
return result;
}
}
// 策略2: 标题中精确子串匹配已知人名(长名优先)
let mut found: Vec<String> = Vec::new();
let mut sorted_names: Vec<&String> = known_names.iter().collect();
sorted_names.sort_by(|a, b| b.len().cmp(&a.len()));
for name in &sorted_names {
if name.len() < 2 {
continue;
}
if title_clean.contains(name.as_str()) {
found.push(name.to_string());
}
}
if !found.is_empty() {
found.sort_by(|a, b| b.len().cmp(&a.len()));
let best = found[0].clone();
result.matched = true;
result.method = "exact_name_match".to_string();
result.person = Some(best.clone());
result.matched_text = Some(best);
result.all_persons = found.clone();
result.multi_person = found.len() > 1;
return result;
}
// 策略3: 别名模糊匹配
let title_lower = title_clean.to_lowercase();
let mut sorted_aliases: Vec<&String> = known_aliases.iter().collect();
sorted_aliases.sort_by(|a, b| b.len().cmp(&a.len()));
for alias in &sorted_aliases {
if alias.len() < 2 {
continue;
}
if title_lower.contains(alias.to_lowercase().as_str()) {
result.matched = true;
result.method = "alias_fuzzy_match".to_string();
result.person = Some(alias.to_string());
result.matched_text = Some(alias.to_string());
return result;
}
}
result
}
/// 收集所有已知人名和别名
pub async fn collect_known_names(db: &Database) -> (HashSet<String>, HashSet<String>) {
let mut known_names = HashSet::new();
let mut known_aliases = HashSet::new();
for (name, _) in db.get_all_person_names().await {
known_names.insert(name);
}
for (alias_name, _) in db.get_all_aliases().await {
known_aliases.insert(alias_name);
}
(known_names, known_aliases)
}
/// 检测含多个人名的相册
pub async fn detect_multi_person_titles(
db: &Database,
known_names: Option<&HashSet<String>>,
) -> Vec<(String, String, Vec<String>)> {
let names = match known_names {
Some(n) => n.clone(),
None => collect_known_names(db).await.0,
};
let albums = db.get_all_albums_full().await;
let mut multi_person = Vec::new();
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
if title.is_empty() {
continue;
}
let mut found: Vec<String> = Vec::new();
let mut sorted_names: Vec<&String> = names.iter().collect();
sorted_names.sort_by(|a, b| b.len().cmp(&a.len()));
for name in &sorted_names {
if name.len() < 2 {
continue;
}
if title.contains(name.as_str()) {
found.push(name.to_string());
}
}
if found.len() > 1 {
multi_person.push((
album.id.clone(),
title.to_string(),
found,
));
}
}
multi_person
}
// ── 中文核心合并分析 ──
/// 分析所有人名的中文核心分组
/// 返回: {core: [(id, name, albums, images), ...]}
pub async fn analyze_groups(db: &Database) -> HashMap<String, Vec<(i64, String, i64, i64)>> {
let persons = db.get_all_persons_with_counts().await;
let mut groups: HashMap<String, Vec<(i64, String, i64, i64)>> = HashMap::new();
for (pid, name, albums, images) in persons {
if let Some(cn) = extract_cn(&name) {
if cn.len() >= 2 {
groups.entry(cn).or_default().push((pid, name, albums, images));
}
}
}
for items in groups.values_mut() {
items.sort_by(|a, b| b.2.cmp(&a.2));
}
groups.retain(|_, v| v.len() >= 2);
groups
}
/// 统计匹配分析
#[derive(Debug, Clone)]
pub struct MatchStats {
pub total_albums: usize,
pub matched: usize,
pub unmatched: usize,
pub already_set: usize,
pub changed: usize,
pub by_source: HashMap<String, usize>,
pub name_frequency: HashMap<String, usize>,
}
/// 执行 Pass1 匹配分析
pub async fn analyze_matches(db: &Database, patterns: &[Pattern]) -> MatchStats {
let albums = db.get_all_albums_full().await;
let mut stats = MatchStats {
total_albums: albums.len(),
matched: 0,
unmatched: 0,
already_set: 0,
changed: 0,
by_source: HashMap::new(),
name_frequency: HashMap::new(),
};
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = album.person_name_original.as_deref().unwrap_or("");
if let Some((text, person, source)) = match_title(title, patterns) {
*stats.by_source.entry(source).or_insert(0) += 1;
*stats.name_frequency.entry(person.clone()).or_insert(0) += 1;
if current.is_empty() {
stats.matched += 1;
} else if current != person {
stats.changed += 1;
} else {
stats.already_set += 1;
}
} else {
stats.unmatched += 1;
}
}
stats
}
/// 深度匹配分析统计
#[derive(Debug, Clone)]
pub struct DeepMatchStats {
pub total_albums: usize,
pub pass1_matched: usize,
pub pass1_unmatched: usize,
pub pass2_jieba_match: usize,
pub pass2_exact_match: usize,
pub pass2_alias_match: usize,
pub pass2_still_unmatched: usize,
pub pass2_multi_person: usize,
}
/// 执行 Pass2 深度匹配分析
pub async fn analyze_deep_matches(db: &Database) -> DeepMatchStats {
let (known_names, known_aliases) = collect_known_names(db).await;
let albums = db.get_all_albums_full().await;
let mut stats = DeepMatchStats {
total_albums: albums.len(),
pass1_matched: 0,
pass1_unmatched: 0,
pass2_jieba_match: 0,
pass2_exact_match: 0,
pass2_alias_match: 0,
pass2_still_unmatched: 0,
pass2_multi_person: 0,
};
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
let current = album.person_name_original.as_deref().unwrap_or("");
if title.is_empty() {
continue;
}
if !current.is_empty() && known_names.contains(current) {
stats.pass1_matched += 1;
continue;
}
stats.pass1_unmatched += 1;
let result = deep_match_title(title, &known_names, &known_aliases);
if !result.matched {
stats.pass2_still_unmatched += 1;
continue;
}
match result.method.as_str() {
"jieba_extract" => stats.pass2_jieba_match += 1,
"exact_name_match" => stats.pass2_exact_match += 1,
"alias_fuzzy_match" => stats.pass2_alias_match += 1,
_ => {}
}
if result.multi_person {
stats.pass2_multi_person += 1;
}
}
stats
}

View File

@@ -1,81 +1,420 @@
//! 处理管线 — 批量处理相册元数据
//!
//! 核心架构:脏集追踪 + 增量解析,dict 优先匹配
//!
//! 流程:
//! Phase Z0 删除空相册
//! Phase Z1 去重同名相册
//! Phase ① 收集新相册(未处理)
//! Phase ② 合并人物清理(非人物/停用词人物)→ 1次人物扫描
//! Phase ③ dict 优先解析脏集相册 → 只扫脏集
//! Phase C 全量重计人物计数
//!
//! 脏集(DirtySet)贯穿各阶段,每个阶段发现的"需要重新解析"的相册都会加入脏集,
//! 最终只处理脏集里的相册,而非全库扫描,大幅提升效率。
use crate::db::Database;
use crate::engine::title_parser;
use crate::engine::tag_extractor;
use crate::engine::person_merger;
use std::collections::HashMap;
use crate::engine::tag_dict;
use crate::search_db::SearchDb;
use std::collections::HashSet;
use rayon::prelude::*;
use sqlx;
/// 处理所有相册的元数据:提取人名 + 标签
/// 两阶段:先用正则提取人名入库,再用匹配精炼,最后提取标签
pub async fn process_all_albums(db: &Database) -> (usize, usize) {
let albums = db.get_all_albums_full().await;
let mut person_count = 0;
let mut tag_count = 0;
// ════════════════════════════════════════════════
// 脏集 — 追踪需要重新解析的相册 ID
// ════════════════════════════════════════════════
// Phase 1: 先用正则从标题直接提取人名(冷启动,不需要已有 persons)
let patterns = person_merger::collect_person_patterns(db).await;
let use_patterns = !patterns.is_empty();
/// 脏集:记录哪些相册需要重新解析
#[derive(Debug, Clone)]
pub struct DirtySet {
ids: HashSet<String>,
}
for album in &albums {
let title = album.title.as_deref().unwrap_or("");
if title.is_empty() { continue; }
impl DirtySet {
pub fn new() -> Self { Self { ids: HashSet::new() } }
pub fn add(&mut self, id: &str) { self.ids.insert(id.to_string()); }
pub fn extend(&mut self, ids: impl IntoIterator<Item = String>) { self.ids.extend(ids); }
pub fn contains(&self, id: &str) -> bool { self.ids.contains(id) }
pub fn len(&self) -> usize { self.ids.len() }
pub fn is_empty(&self) -> bool { self.ids.is_empty() }
pub fn into_inner(self) -> HashSet<String> { self.ids }
}
// 提取人名
if album.person_name_original.is_none() || album.person_name_original.as_deref() == Some("") {
let person_name = if use_patterns {
// 已有人物库,用匹配
person_merger::match_title(title, &patterns).map(|(_, name, _)| name)
} else {
// 冷启动,直接用正则提取
title_parser::extract_person_name_from_title(title)
};
impl Default for DirtySet {
fn default() -> Self { Self::new() }
}
if let Some(person_name) = person_name {
let (cleaned, _aliases) = title_parser::clean_person_name(&person_name);
let final_name = if cleaned.is_empty() { person_name.clone() } else { cleaned };
db.update_album_person(&album.id, &person_name, &final_name).await;
// 创建人员
let pid = db.insert_person(&final_name, None).await;
db.insert_person_album(pid, &album.id).await;
db.update_person_counts(pid).await;
person_count += 1;
}
/// 判断一个标签是否"过短"——有效字符(汉字+字母)少于 2 个
fn is_short_tag(tag: &str) -> bool {
let mut valid_count = 0usize;
for ch in tag.chars() {
if ch.is_ascii_alphabetic() {
valid_count += 1;
} else if ('\u{4e00}'..='\u{9fff}').contains(&ch) {
valid_count += 1;
}
// 数字、符号等不算有效字符
}
valid_count < 2
}
// ════════════════════════════════════════════════
// Phase A3:收集新相册 + 半成品
// ════════════════════════════════════════════════
pub async fn phase_collect_new_albums(db: &Database) -> DirtySet {
let ids = db.get_unprocessed_album_ids().await;
let mut dirty = DirtySet::new();
for id in &ids { dirty.add(id); }
println!("[管线·PhaseA3] 收集相册:{} 个未处理", dirty.len());
dirty
}
// ════════════════════════════════════════════════
// Phase 1:增量解析 — 只处理脏集中的相册
// ┌─ Phase 1a:并行解析(CPU 密集,rayon par_iter)
// └─ Phase 1b:批量写库(IO 密集,一个事务)
pub async fn phase_parse_dirty(db: &Database, search_db: &SearchDb, dirty_set: &DirtySet) -> (usize, usize) {
if dirty_set.is_empty() {
println!("[管线·Phase3] dict 优先解析跳过:脏集为空");
return (0, 0);
}
// Phase 2: 提取标签(使用已填充的人名来排除)
// 重新获取 albums(因为 person_name_cleaned 已更新)
let albums_refreshed = db.get_all_albums_full().await;
let mut seen_albums: HashMap<String, String> = HashMap::new();
for album in &albums {
if let Some(pn) = &album.person_name_cleaned {
seen_albums.insert(album.id.clone(), pn.clone());
}
let dirty_ids: Vec<String> = dirty_set.clone().into_inner().into_iter().collect();
let albums = db.get_albums_by_ids(&dirty_ids).await;
// 加载 search.db 的 dict 条目(提供 merged_to 别名解析)
let dict_entries = search_db.load_dict_all().await.unwrap_or_default();
// ── Phase 3a:并行解析(CPU bound,rayon) ──
let dirty_albums: Vec<&crate::models::AlbumMeta> = albums.iter()
.filter(|a| dirty_set.contains(&a.id))
.collect();
let skipped = dirty_albums.iter().filter(|a| {
a.title.as_deref().map(|t| t.is_empty()).unwrap_or(true)
}).count();
let total = dirty_albums.len();
let parsed: Vec<crate::models::parse::ParseResult> = dirty_albums
.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
// 使用 dict 优先匹配解析
Some(title_parser::parse_album_title_with_dict(&album.id, title, &dict_entries))
})
.collect();
let person_count = parsed.iter().filter(|r| !r.person_names.is_empty()).count();
let tag_count = parsed.iter().filter(|r| !r.tags.is_empty()).count();
// ── Phase 3b:批量写入 imager.db ──
let _guard = db.write_lock.lock().await;
let mut tx = db.pool.begin().await.expect("begin tx");
db.batch_write_parse_results(&mut tx, &parsed).await.unwrap_or_else(|e| {
eprintln!("[管线·Phase3b] imager.db 批量写入失败: {}", e);
});
tx.commit().await.expect("commit tx");
// ── Phase 3c:直接写入 search.db(使用相同解析结果,无需重解析) ──
match search_db.batch_sync_parse_results(&parsed).await {
Ok(n) => println!("[管线·Phase3c] search.db 同步完成:{} 个相册", n),
Err(e) => eprintln!("[管线·Phase3c] search.db 同步失败: {}", e),
}
for album in &albums_refreshed {
let title = album.title.as_deref().unwrap_or("");
if title.is_empty() { continue; }
let existing_tags = album.tags.as_deref().unwrap_or("");
if existing_tags.is_empty() || existing_tags == "[]" {
let person = album.person_name_cleaned.as_deref()
.or_else(|| seen_albums.get(&album.id).map(|s| s.as_str()));
let tags = tag_extractor::extract_tags_from_title(title, person);
let unique = tag_extractor::unique_tags(tags);
if !unique.is_empty() {
let tag_names: Vec<String> = unique.iter().map(|(t, _)| t.clone()).collect();
let tags_json = serde_json::to_string(&tag_names).unwrap_or_else(|_| "[]".to_string());
let all_words: Vec<[String; 2]> = unique.iter().map(|(t, tp)| [t.clone(), tp.clone()]).collect();
let words_json = serde_json::to_string(&all_words).unwrap_or_else(|_| "[]".to_string());
db.update_album_tags(&album.id, &tags_json, &words_json).await;
tag_count += 1;
}
}
}
println!("[管线] 处理完成:{} 个人名,{} 个标签", person_count, tag_count);
println!("[管线·Phase3] dict 优先解析+批量写入完成: 脏集 {} 个, 有效 {} 个 (跳过 {} 无标题), 更新人名 {} 个, 更新标签 {} 个",
total, parsed.len(), skipped, person_count, tag_count);
(person_count, tag_count)
}
// ════════════════════════════════════════════════
// Phase C:全量重计 — 确保所有 person 的计数准确
// ════════════════════════════════════════════════
/// Phase C:全量重计所有人物相册和图片计数
///
/// 遍历所有 persons,用 person_albums 实际关联数更新 total_albums,
/// 用 JOIN albums 获取实际 image_count 更新 total_images。
/// 修复管线过程中因删除/迁移关联未更新旧 person 计数的遗留问题。
pub async fn phase_recount_all(db: &Database) -> usize {
let persons = db.get_all_persons_with_counts().await;
let mut fixed = 0usize;
for (pid, name, old_albums, _) in &persons {
let actual = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*) FROM person_albums WHERE person_id = ?"
).bind(pid).fetch_one(&db.pool).await.unwrap_or(0);
if actual != *old_albums {
db.update_person_counts(*pid).await;
fixed += 1;
}
}
if fixed > 0 {
println!("[管线·PhaseC] 全量重计完成:修正 {} 个人物的计数", fixed);
} else {
println!("[管线·PhaseC] 全量重计完成:所有计数正确");
}
fixed
}
// ════════════════════════════════════════════════
// Preprocess + Dirty Tracking
// ════════════════════════════════════════════════
/// 合并预处理结果
#[derive(Debug, Clone, Default)]
struct PreprocessResult {
tags_cleaned: usize,
short_tags_removed: usize,
person_like_tags_removed: usize,
new_albums_found: usize,
dirty: DirtySet,
}
/// 合并预处理:只做收集新相册,不再做标签清洗(已迁移到 search.db)
async fn phase_preprocess_combined(db: &Database) -> PreprocessResult {
let mut result = PreprocessResult::default();
let dirty = phase_collect_new_albums(db).await;
result.dirty = dirty;
result.new_albums_found = result.dirty.len();
println!("[管线·Phase①] 合并预处理完成(精简模式):新相册{},脏集合计{}",
result.new_albums_found, result.dirty.len());
result
}
/// 合并人物清理:一次人物遍历完成 2 件事
/// 1. 清理非人物
/// 2. 清理停用词重叠人物
async fn phase_clean_persons_combined(db: &Database) -> (DirtySet, usize, usize) {
let persons = db.get_all_persons_with_counts().await;
let mut dirty = DirtySet::new();
let mut non_persons_deleted = 0usize;
let mut stop_word_persons_removed = 0usize;
for (pid, name, _albums, _images) in &persons {
// 判断是否应该删除
let is_non_person = title_parser::is_excluded_name(name) || !title_parser::is_likely_person_name(name);
let is_stop_word = {
let excluded = tag_dict::get_tag_dict().is_excluded_person(name);
let cn_check = title_parser::extract_cn(name)
.map(|cn| tag_dict::get_tag_dict().is_excluded_person(&cn))
.unwrap_or(false);
excluded || cn_check
};
if !is_non_person && !is_stop_word { continue; }
let album_list = db.get_person_albums(*pid).await;
let album_ids: Vec<String> = album_list.into_iter().map(|a| a.id.to_string()).collect();
for aid in &album_ids { dirty.add(aid); }
if is_stop_word {
println!("[管线·Phase②] 排除人名命中:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
stop_word_persons_removed += 1;
} else {
println!("[管线·Phase②] 删除非人物:{} (id={}),释放 {} 个相册", name, pid, album_ids.len());
non_persons_deleted += 1;
}
db.update_album_person_by_person_id(*pid).await;
db.delete_person_completely(*pid).await;
}
if non_persons_deleted > 0 || stop_word_persons_removed > 0 {
println!("[管线·Phase②] 人物清理完成:删除非人物{},移除停用词人物{},脏相册{}",
non_persons_deleted, stop_word_persons_removed, dirty.len());
} else {
println!("[管线·Phase②] 人物清理完成:无异常");
}
(dirty, non_persons_deleted, stop_word_persons_removed)
}
// ════════════════════════════════════════════════
// Phase Z0:删除空相册(image_count = 0)
// ════════════════════════════════════════════════
/// 删除所有图片数为 0 的空相册
pub async fn phase_delete_empty_albums(db: &Database) -> usize {
let empty = db.get_empty_albums().await;
let count = empty.len();
if count == 0 {
println!("[管线·PhaseZ0] 无空相册需要删除");
return 0;
}
let ids: Vec<uuid::Uuid> = empty.iter().map(|a| a.id).collect();
let deleted = db.delete_albums_batch(&ids).await;
println!("[管线·PhaseZ0] 删除 {} 个空相册(无图片)", deleted);
deleted
}
// ════════════════════════════════════════════════
// Phase Z1:去重同名相册
// ════════════════════════════════════════════════
/// 查找同名的相册,只保留第一个(最早创建的),删除其余(包括文件夹和 search.db 数据)
pub async fn phase_dedup_albums(db: &Database, search_db: &SearchDb) -> usize {
let dups = db.find_duplicate_albums().await;
let images_dir = &db.config.storage.images_dir;
let mut total_deleted = 0usize;
for (keep_id, keep_document, to_delete) in &dups {
let dup_ids: Vec<uuid::Uuid> = to_delete.iter().map(|(id, _)| *id).collect();
// 删除数据库记录
db.delete_albums_batch(&dup_ids).await;
// 删除 search.db 数据
for (id, _) in to_delete {
search_db.delete_album(&id.to_string()).await.ok();
}
// 删除磁盘文件夹
for (_, doc) in to_delete {
let folder = images_dir.join(doc);
if folder.exists() {
if let Err(e) = std::fs::remove_dir_all(&folder) {
eprintln!("[管线·PhaseZ1] 删除文件夹失败 {}: {}", folder.display(), e);
} else {
println!("[管线·PhaseZ1] 删除文件夹 {}", folder.display());
}
}
}
total_deleted += to_delete.len();
for (id, doc) in to_delete {
println!("[管线·PhaseZ1] 删除重复相册 {} ({}),保留 {} ({})", id, doc, keep_id, keep_document);
}
}
if total_deleted == 0 {
println!("[管线·PhaseZ1] 无重复相册");
} else {
println!("[管线·PhaseZ1] 去重完成:删除 {} 个重复相册", total_deleted);
}
total_deleted
}
// ════════════════════════════════════════════════
// Pipeline 一键执行
// ════════════════════════════════════════════════
#[derive(Debug, Clone, Default)]
pub struct PipelineResult {
pub tags_cleaned: usize, // Phase ①
pub short_tags_removed: usize, // Phase ①
pub person_like_tags_removed: usize, // Phase ①
pub dirty_albums_detected: usize, // 总脏集
pub non_persons_deleted: usize, // Phase ②
pub stop_word_persons_removed: usize, // Phase ②
pub new_albums_found: usize, // Phase ①
pub persons_updated: usize, // Phase ③
pub tags_updated: usize, // Phase ③
pub empty_albums_deleted: usize, // Phase Z0
pub dup_albums_deleted: usize, // Phase Z1
}
/// 一键执行精简管线(dict 优先匹配)
///
/// Phase Z0 删除空相册
/// Phase Z1 去重同名相册
/// Phase ① 收集新相册
/// Phase ② 合并人物清理(非人物+停用词人物) → 1次人物扫描
/// Phase ③ dict 优先解析脏集 → 只扫脏集
/// Phase C 全量重计
pub async fn run_pipeline(db: &Database, search_db: &SearchDb) -> PipelineResult {
// Phase Z0:删除空相册
let empty_albums_deleted = phase_delete_empty_albums(db).await;
// Phase Z1:去重同名相册(含删除文件夹和 search.db 数据)
let dup_albums_deleted = phase_dedup_albums(db, search_db).await;
// Phase ①:收集新相册
let pre = phase_preprocess_combined(db).await;
let tags_cleaned = pre.tags_cleaned;
let short_tags_removed = pre.short_tags_removed;
let person_like_tags_removed = pre.person_like_tags_removed;
let new_albums_found = pre.new_albums_found;
// Phase ②:合并人物清理(1 次人物扫描)
let (dirty_persons, non_persons_deleted, stop_word_persons_removed) =
phase_clean_persons_combined(db).await;
// 合并脏集
let mut master_dirty = pre.dirty;
master_dirty.extend(dirty_persons.into_inner());
let dirty_albums_detected = master_dirty.len();
// Phase ③:dict 优先解析脏集
let (persons_updated, tags_updated) = phase_parse_dirty(db, search_db, &master_dirty).await;
// Phase C:全量重计(修正计数)
let _recounted = phase_recount_all(db).await;
PipelineResult {
tags_cleaned, short_tags_removed, person_like_tags_removed, dirty_albums_detected,
non_persons_deleted, stop_word_persons_removed, new_albums_found,
persons_updated, tags_updated,
empty_albums_deleted, dup_albums_deleted,
}
}
// ════════════════════════════════════════════════
// Phase S:同步管线结果到 search.db
// 只同步 tags/persons/aliases 到 search.db 的关联表,通过 album_id 关联
// 不同步 albums 表数据(albums 由 imager.db 管理)
// ════════════════════════════════════════════════
/// 全量同步:将所有相册的 tags/persons 解析结果写入 search.db 的关联表(单事务批量)
pub async fn sync_albums_to_search_db(db: &Database, search_db: &SearchDb) -> usize {
let albums = db.get_all_albums_full().await;
// 并行解析所有相册
let parsed: Vec<crate::models::parse::ParseResult> = albums.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
Some(title_parser::parse_album_title(&album.id, title))
})
.collect();
// 单事务批量写入 search.db
match search_db.batch_sync_parse_results(&parsed).await {
Ok(n) => {
println!("[Sync->search.db] 全量同步完成:{} 个相册", n);
n
}
Err(e) => {
eprintln!("[Sync->search.db] 批量同步失败: {}", e);
0
}
}
}
/// 增量同步:只处理脏集中的相册 ID(单事务批量)
pub async fn sync_dirty_albums(db: &Database, search_db: &SearchDb, dirty: &DirtySet) -> usize {
if dirty.is_empty() {
println!("[Sync->search.db·增量] 脏集为空,跳过");
return 0;
}
let dirty_ids: Vec<String> = dirty.clone().into_inner().into_iter().collect();
let albums = db.get_albums_by_ids(&dirty_ids).await;
// 并行解析脏集相册
let parsed: Vec<crate::models::parse::ParseResult> = albums.par_iter()
.filter_map(|album| {
let title = album.title.as_deref()?;
if title.is_empty() { return None; }
Some(title_parser::parse_album_title(&album.id, title))
})
.collect();
// 单事务批量写入 search.db
match search_db.batch_sync_parse_results(&parsed).await {
Ok(n) => {
println!("[Sync->search.db·增量] 脏集同步完成:{} 个相册", n);
n
}
Err(e) => {
eprintln!("[Sync->search.db·增量] 批量同步失败: {}", e);
0
}
}
}

View File

@@ -5,39 +5,35 @@
//! 2. 人物推荐:基于名称匹配 + 相册数排序
//! 3. 关联推荐:基于标签共现关系 + 人物共现关系
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use crate::engine::tag_dict;
use sqlx::Row;
/// 推荐引擎
pub struct RecommendEngine;
impl RecommendEngine {
/// 标签自动补全推荐
pub async fn recommend_tags(
db: &Database,
/// 标签自动补全推荐(从缓存的热门标签列表匹配)
pub fn recommend_tags(
popular_tags: &[TagStat],
query: &str,
limit: usize,
) -> RecommendResponse {
let limit = limit.min(20);
let q = query.trim().to_lowercase();
let tags = db.get_popular_tags(200).await;
let mut items: Vec<RecommendItem> = Vec::new();
for t in &tags {
for t in popular_tags {
let tag_lower = t.tag.to_lowercase();
let score = if q.is_empty() {
// 无查询时按热门度
t.count as f64
} else if tag_lower == q {
// 精确匹配
1000.0 + t.count as f64
} else if tag_lower.starts_with(&q) {
// 前缀匹配
500.0 + t.count as f64
} else if tag_lower.contains(&q) {
// 包含匹配
100.0 + t.count as f64
} else {
continue;
@@ -59,19 +55,18 @@ impl RecommendEngine {
RecommendResponse { items, total, category: "tags".to_string() }
}
/// 人物推荐
pub async fn recommend_persons(
db: &Database,
/// 人物推荐(从缓存的人物列表匹配)
pub fn recommend_persons(
persons: &[PersonWithCounts],
query: &str,
limit: usize,
) -> RecommendResponse {
let limit = limit.min(20);
let q = query.trim().to_lowercase();
let persons = db.get_all_persons_with_aliases().await;
let mut items: Vec<RecommendItem> = Vec::new();
for p in &persons {
for p in persons {
let name_lower = p.name.to_lowercase();
let score = if q.is_empty() {
p.total_albums as f64
@@ -82,7 +77,6 @@ impl RecommendEngine {
} else if name_lower.contains(&q) {
100.0 + p.total_albums as f64
} else {
// 检查别名
let alias_match = p.aliases.iter().any(|a| a.to_lowercase().contains(&q));
if alias_match { 200.0 + p.total_albums as f64 } else { continue }
};
@@ -111,14 +105,13 @@ impl RecommendEngine {
/// 关联推荐(共现标签 + 共现人物)
pub async fn recommend_related(
db: &Database,
context: &str, // 标签或人名
search_db: &SearchDb,
context: &str,
limit: usize,
) -> TagRecommendation {
let limit = limit.min(15);
// 获取共现标签
let co_tags = db.get_tag_co_occurrences(context, limit as i64).await;
let co_tags = search_db.get_tag_co_occurrences(context, limit as i64).await;
let related_tags: Vec<RecommendItem> = co_tags.iter().map(|c| {
RecommendItem {
value: c.co_tag.clone(),
@@ -129,20 +122,21 @@ impl RecommendEngine {
}
}).collect();
// 获取相关人物(通过标签)
let albums = db.search_albums_by_tag(context).await;
let albums = search_db.search_albums_by_tag(context).await;
let mut person_scores: std::collections::HashMap<String, (i64, f64)> = std::collections::HashMap::new();
let total = albums.len() as f64;
for album in &albums {
let meta = db.get_all_albums_full().await;
for m in &meta {
if m.id == album.id.to_string() {
if let Some(ref name) = m.person_name_cleaned {
let entry = person_scores.entry(name.clone()).or_insert((0, 0.0));
entry.0 += 1;
entry.1 = entry.0 as f64 / total;
}
}
let aid = album.id.to_string();
let person_rows = sqlx::query(
r#"SELECT p.name FROM persons p
JOIN album_persons ap ON p.id = ap.person_id
WHERE ap.album_id = ?"#
).bind(&aid).fetch_all(&search_db.pool).await.unwrap_or_default();
for row in &person_rows {
let name: String = row.get("name");
let entry = person_scores.entry(name).or_insert((0, 0.0));
entry.0 += 1;
entry.1 = entry.0 as f64 / total;
}
}
@@ -165,10 +159,32 @@ impl RecommendEngine {
}
}
/// 热门人物
pub async fn popular_persons(db: &Database, limit: usize) -> Vec<PopularPerson> {
let persons = db.get_all_persons_with_aliases().await;
let mut results: Vec<PopularPerson> = persons.iter().take(limit).map(|p| {
/// 热门人物(从缓存的人物列表过滤,top_tags 保留 search.db 查询)
pub async fn popular_persons(
search_db: &SearchDb,
persons_cache: &[PersonWithCounts],
limit: usize,
) -> Vec<PopularPerson> {
let dict = tag_dict::get_tag_dict();
let all_tag_words: std::collections::HashSet<String> = dict.tag_to_category.keys()
.map(|k| k.to_lowercase()).collect();
let filtered: Vec<_> = persons_cache.iter()
.filter(|p| {
if dict.is_excluded_person(&p.name) { return false; }
if !crate::engine::title_parser::is_likely_person_name(&p.name) {
return false;
}
let name_lower = p.name.to_lowercase();
if all_tag_words.contains(&name_lower) {
return false;
}
true
})
.collect();
let mut results: Vec<PopularPerson> = filtered.iter().take(limit).map(|p| {
PopularPerson {
name: p.name.clone(),
display_name: p.display_name.clone(),
@@ -179,23 +195,10 @@ impl RecommendEngine {
}
}).collect();
// 为热门人物获取 top tags
// (简化:从 person_albums 查标签)
for rp in &mut results {
if let Some(person) = db.get_person_by_name(&rp.name).await {
let albums = db.get_person_albums(person.id).await;
let mut tag_counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
for album in &albums {
let aid = album.id.to_string();
let rows = sqlx::query("SELECT tag FROM album_tags WHERE album_id = ?")
.bind(&aid).fetch_all(&db.pool).await.unwrap_or_default();
for row in &rows {
*tag_counts.entry(row.get::<String, _>("tag")).or_insert(0) += 1;
}
}
let mut tags: Vec<(String, i64)> = tag_counts.into_iter().collect();
tags.sort_by(|a, b| b.1.cmp(&a.1));
rp.top_tags = tags.into_iter().take(5).map(|(t, _)| t).collect();
if let Some(person) = search_db.get_person_by_name(&rp.name).await {
let top_tags = search_db.get_person_top_tags(&person.id, 5).await;
rp.top_tags = top_tags.into_iter().map(|(t, _)| t).collect();
}
}
results

629
src/engine/tag_dict.rs Normal file
View File

@@ -0,0 +1,629 @@
//! 标签词典 — 从外部 JSON 文件加载分类标签、停用词、人物规则
//!
//! 词典文件格式(dict/tags.json v4):
//! ```json
//! {
//! "version": 4,
//! "metadata": { "style": {"label":"风格","color_bg":"...","color_fg":"..."}, ... },
//! "keymap": { "日系": "style", "比基尼": "outfit", ... },
//! "stop_words": [...],
//! "person_rules": { "prefixes": [...], "excluded": [...], "include": [...] }
//! }
//! ```
use std::collections::{HashMap, HashSet};
use std::path::Path;
use serde::Deserialize;
use lazy_static::lazy_static;
use std::sync::RwLock;
// ── JSON 反序列化结构 (v4) ──
/// v4 分类元数据(label + 颜色)
#[derive(Debug, Clone, Deserialize)]
pub struct CategoryMeta {
#[serde(default)]
pub label: String,
#[serde(default = "default_color_bg")]
pub color_bg: String,
#[serde(default = "default_color_fg")]
pub color_fg: String,
}
fn default_color_bg() -> String { "#f1f5f9".to_string() }
fn default_color_fg() -> String { "#64748b".to_string() }
/// 词典文件的顶层结构 (v4)
#[derive(Debug, Clone, Deserialize)]
pub struct TagDictFile {
pub version: u32,
#[allow(unused)]
pub description: Option<String>,
/// v4: 分类元数据 { "style": {"label":"风格","color_bg":"...","color_fg":"..."}, ... }
#[serde(default)]
pub metadata: HashMap<String, CategoryMeta>,
/// v4: 扁平化匹配串→分类名 { "日系": "style", "比基尼": "outfit", ... }
#[serde(default)]
pub keymap: HashMap<String, String>,
/// 停用词列表
#[serde(default)]
pub stop_words: Vec<String>,
/// 人物称呼前缀(顶层字段,从 JSON 文件读取)
#[serde(default)]
pub person_prefixes: Vec<String>,
}
// ── 运行时词典结构 ──
/// 分类信息(运行时使用,兼容旧版 API)
#[derive(Debug, Clone)]
pub struct CategoryInfo {
pub name: String,
pub label: String,
pub color_bg: String,
pub color_fg: String,
pub words: Vec<String>,
}
/// 运行时标签词典
#[derive(Debug, Clone)]
pub struct TagDict {
/// 分类名称 → 分类详情(兼容旧版 API)
pub categories: HashMap<String, CategoryInfo>,
/// 标签 → 分类名称(兼容旧版 API,基于 keymap 构建)
pub tag_to_category: HashMap<String, String>,
/// 停用词集合
pub stop_words: HashSet<String>,
/// 人物称呼前缀(提取人名时直接 replace 移除)
pub person_prefixes: Vec<String>,
/// 排除的人物名(工作室/平台名,匹配上的人和包含这些词的人直接删除重解析)
pub excluded_person_names: HashSet<String>,
/// 强制包含的人物名(已知模特/演员名,只要出现就优先识别为人名)
pub included_person_names: HashSet<String>,
// ── v4 新字段 ──
/// 扁平化匹配串 → 分类名(v4 keymap)
pub keymap: HashMap<String, String>,
/// 分类元数据(v4 metadata)
pub metadata: HashMap<String, CategoryMeta>,
}
impl Default for TagDict {
fn default() -> Self {
let mut dict = TagDict {
categories: HashMap::new(),
tag_to_category: HashMap::new(),
stop_words: HashSet::new(),
person_prefixes: Vec::new(),
excluded_person_names: HashSet::new(),
included_person_names: HashSet::new(),
keymap: HashMap::new(),
metadata: HashMap::new(),
};
dict.fill_builtin();
dict
}
}
impl TagDict {
/// 从默认路径加载词典(`dict/tags.json`)
pub fn load() -> Self {
Self::load_from("dict/tags.json")
}
/// 从指定路径加载词典
pub fn load_from(path: impl AsRef<Path>) -> Self {
let path = path.as_ref();
if !path.exists() {
eprintln!("[词典] 词典文件不存在 {:?},使用内建回退", path);
return TagDict::default();
}
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(e) => {
eprintln!("[词典] 无法读取 {:?}: {},使用内建回退", path, e);
return TagDict::default();
}
};
let file: TagDictFile = match serde_json::from_str(&content) {
Ok(f) => f,
Err(e) => {
eprintln!("[词典] 解析 {:?} 失败: {},使用内建回退", path, e);
return TagDict::default();
}
};
// ── 人物前缀直接从顶层字段读取 ──
let person_prefixes = file.person_prefixes.clone();
let mut dict = TagDict {
categories: HashMap::new(),
tag_to_category: HashMap::new(),
stop_words: file.stop_words.into_iter().collect(),
person_prefixes,
excluded_person_names: HashSet::new(),
included_person_names: HashSet::new(),
keymap: HashMap::new(),
metadata: HashMap::new(),
};
// ── 加载 metadata(分类元数据) ──
dict.metadata = file.metadata.clone();
// ── 加载 keymap(扁平化匹配串→分类名) ──
dict.keymap = file.keymap.clone();
// ── 构建 tag_to_category(基于 keymap) ──
for (word, cat_name) in &file.keymap {
dict.tag_to_category.insert(word.to_lowercase(), cat_name.clone());
}
// ── 构建 categories(基于 metadata + keymap 提取 words) ──
// 从 keymap 中按分类收集所有词
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &file.keymap {
words_by_cat.entry(cat_name.clone())
.or_default()
.push(word.clone());
}
// 按 metadata 中的分类顺序构建
for (cat_name, meta) in &file.metadata {
let words = words_by_cat.remove(cat_name).unwrap_or_default();
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
// keymap 中有但 metadata 中没有的分类(兜底)
for (cat_name, words) in words_by_cat {
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label: cat_name.clone(),
color_bg: "#f1f5f9".to_string(),
color_fg: "#64748b".to_string(),
words,
});
}
// ── 从 keymap 构建排除人名和包含人名 ──
// person 类型的 key 加入 included_person_names
// 其他类型(style/outfit/scene/body/mood/topic/position/location/brand)加入 excluded_person_names
for (word, cat_name) in &file.keymap {
if cat_name == "person" {
dict.included_person_names.insert(word.to_lowercase());
} else {
dict.excluded_person_names.insert(word.to_lowercase());
}
}
if !path.as_os_str().is_empty() {
println!("[词典] v{} 已加载: {} 分类, {} 标签词(keymap), {} 停用词, {} 前缀, 排除+标签共 {} 词",
file.version, dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len());
}
dict
}
/// 热重载词典
pub fn reload(&mut self) {
*self = Self::load();
}
/// 获取所有分类名称(固定顺序:9 个标准分类优先)
pub fn get_category_order(&self) -> Vec<&str> {
let preferred = ["style", "outfit", "scene", "body", "mood", "topic", "position", "location", "brand"];
let mut result: Vec<&str> = preferred.iter()
.filter(|c| self.categories.contains_key(**c))
.map(|c| *c)
.collect();
for name in self.categories.keys() {
if !result.contains(&name.as_str()) {
result.push(name);
}
}
result
}
/// 获取分类信息
pub fn get_category(&self, name: &str) -> Option<&CategoryInfo> {
self.categories.get(name)
}
/// 获取分类的中文标签
pub fn get_category_label(&self, name: &str) -> String {
self.categories.get(name)
.map(|c| c.label.clone())
.unwrap_or_else(|| "其他".to_string())
}
/// 获取分类的颜色
pub fn get_category_color(&self, name: &str) -> (String, String) {
match self.categories.get(name) {
Some(c) => (c.color_bg.clone(), c.color_fg.clone()),
None => ("#f1f5f9".to_string(), "#64748b".to_string()),
}
}
/// 判断一个词属于哪个已知分类(基于 tag_to_category,由 keymap 构建)
pub fn classify(&self, word: &str) -> Option<&str> {
self.tag_to_category.get(&word.to_lowercase()).map(|s| s.as_str())
}
/// 在 keymap 中查找 word,返回分类名(与 classify 相同,v4 新增 API)
pub fn match_keymap(&self, word: &str) -> Option<&str> {
self.keymap.get(&word.to_lowercase()).map(|s| s.as_str())
}
/// 获取 v4 元数据
pub fn get_metadata(&self) -> &HashMap<String, CategoryMeta> {
&self.metadata
}
/// 判断是否是停用词
pub fn is_stop_word(&self, word: &str) -> bool {
self.stop_words.contains(&word.to_lowercase())
}
/// 获取所有标签→分类映射
pub fn get_tag_category_map(&self) -> &HashMap<String, String> {
&self.tag_to_category
}
/// 获取停用词集合
pub fn get_stop_words(&self) -> &HashSet<String> {
&self.stop_words
}
/// ── 人物规则 ──
/// 判断人名是否应被排除(精确匹配 或 被排除词包含)。
/// 注意:标签词典中的所有词汇已自动加入排除名单,因此标签词永不为人名。
pub fn is_excluded_person(&self, person_name: &str) -> bool {
let name_lower = person_name.to_lowercase();
// 精确匹配
if self.excluded_person_names.contains(person_name) {
return true;
}
if self.excluded_person_names.contains(&name_lower) {
return true;
}
// 包含匹配:排除词是 person_name 的子串
for excluded in &self.excluded_person_names {
if name_lower.contains(&excluded.to_lowercase()) {
return true;
}
}
false
}
/// 获取人物称呼前缀列表
pub fn get_person_prefixes(&self) -> &[String] {
&self.person_prefixes
}
/// 获取排除人名集合
pub fn get_excluded_persons(&self) -> &HashSet<String> {
&self.excluded_person_names
}
/// 判断人名是否在强制包含列表中(应优先识别为人名)
pub fn is_included_person(&self, name: &str) -> bool {
let name_lower = name.to_lowercase();
self.included_person_names.contains(name)
|| self.included_person_names.contains(&name_lower)
}
/// 在 include 列表中查找以 name 开头的最长匹配。
///
/// 例如 include 中有 "杨晨晨Yome", "杨晨晨Sugar",
/// 传入 "杨晨晨" 会返回 "杨晨晨Yome"(取最长的那条)。
/// 传入 "Yome" 也返回 "杨晨晨Yome"(取包含该词的最长的那条)。
pub fn find_longest_included_match(&self, name: &str) -> Option<String> {
let name_lower = name.to_lowercase();
self.included_person_names.iter()
.filter(|included| {
let inc_lower = included.to_lowercase();
// include 以 name 开头 ? 或 name 以 include 开头 ?
inc_lower.starts_with(&name_lower) || name_lower.starts_with(&inc_lower)
|| inc_lower.contains(&name_lower) // include 包含 name(中文核心→全名)
})
.max_by_key(|s| s.len())
.cloned()
}
/// 获取强制包含人名集合
pub fn get_included_persons(&self) -> &HashSet<String> {
&self.included_person_names
}
/// 内置回退
fn fill_builtin(&mut self) {
// ── 元数据(与 v4 JSON metadata 同构) ──
let builtin_metadata: Vec<( &str, &str, &str, &str)> = vec![
("style", "风格", "#f0f9ff", "#0284c7"),
("outfit", "服装", "#fdf2f8", "#db2777"),
("scene", "场景", "#f0fdf4", "#15803d"),
("body", "身体", "#fef3c7", "#b45309"),
("mood", "情绪", "#faf5ff", "#7c3aed"),
("topic", "主题", "#f0f9ff", "#0369a1"),
("position", "姿势", "#fefce8", "#a16207"),
("location", "地点", "#f0fdfa", "#0f766e"),
("brand", "品牌", "#fff7ed", "#c2410c"),
];
for (name, label, bg, fg) in &builtin_metadata {
self.metadata.insert(name.to_string(), CategoryMeta {
label: label.to_string(),
color_bg: bg.to_string(),
color_fg: fg.to_string(),
});
}
// ── keymap ──
let builtin_keymap: Vec<(&str, &str)> = vec![
// style
("日系", "style"), ("韩系", "style"), ("复古", "style"),
("甜美", "style"), ("性感", "style"), ("清新", "style"),
("冷艳", "style"), ("优雅", "style"), ("知性", "style"),
("可爱", "style"), ("俏皮", "style"), ("温柔", "style"),
("时尚", "style"), ("文艺", "style"), ("古典", "style"),
("elegant", "style"), ("fashion", "style"), ("japanese", "style"),
("korean", "style"), ("retro", "style"), ("sexy", "style"),
("vintage", "style"),
// outfit
("比基尼", "outfit"), ("泳装", "outfit"), ("旗袍", "outfit"),
("制服", "outfit"), ("丝袜", "outfit"), ("内衣", "outfit"),
("连衣裙", "outfit"), ("短裙", "outfit"), ("长裙", "outfit"),
("牛仔裤", "outfit"), ("热裤", "outfit"), ("吊带", "outfit"),
("抹胸", "outfit"), ("蕾丝", "outfit"), ("镂空", "outfit"),
("高跟鞋", "outfit"), ("T恤", "outfit"),
// scene
("室内", "scene"), ("户外", "scene"), ("浴室", "scene"),
("床上", "scene"), ("海边", "scene"), ("沙滩", "scene"),
("泳池", "scene"), ("花园", "scene"),
// body
("美腿", "body"), ("美胸", "body"), ("翘臀", "body"),
("玉足", "body"), ("美背", "body"), ("香肩", "body"),
("ass", "body"), ("back", "body"), ("breasts", "body"),
("feet", "body"), ("legs", "body"), ("shoulders", "body"),
("skin", "body"), ("tattoo", "body"), ("waist", "body"),
// mood
("清纯", "mood"), ("诱惑", "mood"), ("唯美", "mood"),
("妩媚", "mood"), ("beautiful", "mood"), ("charming", "mood"),
("cool", "mood"), ("cute", "mood"), ("fresh", "mood"),
("gentle", "mood"), ("lazy", "mood"), ("pure", "mood"),
("seductive", "mood"), ("shy", "mood"), ("sweet", "mood"),
("wild", "mood"),
// topic
("写真", "topic"), ("私房", "topic"), ("街拍", "topic"),
("旅拍", "topic"), ("婚纱照", "topic"),
("cosplay", "topic"), ("photoshoot", "topic"),
("portrait", "topic"), ("selfie", "topic"), ("travel", "topic"),
("wedding", "topic"),
// position
("站立", "position"), ("坐姿", "position"), ("躺姿", "position"),
("跪姿", "position"), ("侧卧", "position"),
("kneeling", "position"), ("lying", "position"),
("sitting", "position"), ("standing", "position"),
// location
("日本", "location"), ("韩国", "location"), ("中国", "location"),
("泰国", "location"), ("东京", "location"), ("首尔", "location"),
("北京", "location"), ("上海", "location"),
("japan", "location"), ("korea", "location"), ("china", "location"),
("bali", "location"), ("bangkok", "location"),
("phuket", "location"), ("seoul", "location"),
("thailand", "location"), ("tokyo", "location"),
// brand
("Victoria's Secret", "brand"), ("维密", "brand"),
("Calvin Klein", "brand"), ("CK", "brand"),
("Gucci", "brand"), ("Chanel", "brand"),
("Louis Vuitton", "brand"),
];
for (word, cat) in &builtin_keymap {
self.keymap.insert(word.to_string(), cat.to_string());
self.tag_to_category.insert(word.to_lowercase(), cat.to_string());
}
// ── categories(基于 metadata + keymap 派生) ──
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &self.keymap {
words_by_cat.entry(cat_name.clone())
.or_default()
.push(word.clone());
}
for (name, meta) in &self.metadata {
let words = words_by_cat.remove(name).unwrap_or_default();
self.categories.insert(name.clone(), CategoryInfo {
name: name.clone(),
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
// ── 停用词 ──
let stop_words: Vec<&str> = vec![
"photos", "photo", "pictures", "picture", "images", "image",
"video", "videos", "cosplay", "cos",
"and", "the", "of", "in", "a", "an", "is", "to", "for",
"part", "vol", "collection", "special", "extra", "bonus",
"new", "old", "selfie", "official", "original",
"set", "gallery", "album", "page", "issue",
"model", "idol", "star", "actor", "actress", "singer",
"秀人", "网", "图", "套", "合集", "系列", "作品",
"美女", "模特",
"高清", "原版", "无水印", "番号", "编号",
];
for w in &stop_words { self.stop_words.insert(w.to_string()); }
// ── 人物前缀 ──
let person_prefixes: Vec<&str> = vec![
"美女模特", "性感女神", "清纯美女", "气质美女", "可爱女孩",
"甜美少女", "时尚达人", "女神", "美女", "模特", "小姐姐",
"小仙女", "御姐", "萝莉", "辣妹", "潮人",
];
self.person_prefixes = person_prefixes.iter().map(|s| s.to_string()).collect();
// ── 排除人名(从 keymap 非 person 类型派生)──
for (word, cat) in &builtin_keymap {
if cat == &"person" {
self.included_person_names.insert(word.to_lowercase());
} else {
self.excluded_person_names.insert(word.to_lowercase());
}
}
// 额外内置排除(keymap 中未覆盖的工作室名)
let extra_excluded: Vec<&str> = vec![
"XiuRen", "xiuren", "秀人", "秀人网",
"MFStar", "MiiTao", "蜜桃社",
"YouMi", "尤蜜", "HuaYang", "花洋",
"FeiLin", "Feiling", "飞灵", "MFModel",
"LeiLei", "Tristana", "Av",
];
for n in &extra_excluded { self.excluded_person_names.insert(n.to_string()); }
// ── 包含人名(从 keymap person 类型已派生,再加几个默认) ──
let default_included: Vec<&str> = vec![
"杨晨晨", "水淼", "轩美宣", "王馨瑶",
"雪儿", "曼曼", "佳佳", "小乔",
];
for n in &default_included { self.included_person_names.insert(n.to_string()); }
}
}
// ── lazy_static 全局词典 ──
lazy_static! {
static ref TAG_DICT: RwLock<TagDict> = RwLock::new(TagDict::load());
}
/// 获取全局词典只读引用
pub fn get_tag_dict() -> std::sync::RwLockReadGuard<'static, TagDict> {
TAG_DICT.read().expect("tag_dict 锁被污染")
}
/// 热重载词典
pub fn reload_tag_dict() -> String {
let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染");
dict.reload();
format!(
"词典已重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)
}
/// 获取词典统计信息
pub fn get_tag_dict_stats() -> String {
let dict = get_tag_dict();
format!(
"标签词典: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名 | 文件: dict/tags.json",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)
}
// ════════════════════════════════════════════════
// 从 search.db 的 dict 表加载
// ════════════════════════════════════════════════
use crate::search_db::DictEntry;
impl TagDict {
/// 从 search.db 的 dict 表条目加载词典数据
///
/// dict 表条目类型(通过 type 字段区分):
/// - `tag` — word 是标签,category 是分类名 → 加入 keymap
/// - `person` — word 是人名 → 加入 included_person_names
/// - `stop_word` — word 是停用词
/// - `person_prefix` — word 是人物前缀
/// - `excluded_person` — word 是排除人名
///
/// 派生规则:
/// - 所有 tag 类型的 word 自动加入 excluded_person_names
/// - categories 结构基于 metadata(内建)和 keymap(从 tag 条目构建)
pub fn load_from_dict_entries(entries: &[DictEntry]) -> Self {
let mut dict = TagDict::default();
for entry in entries {
match entry.type_.as_str() {
"tag" => {
let cat = entry.category.as_deref().unwrap_or("other");
dict.keymap.insert(entry.word.clone(), cat.to_string());
dict.tag_to_category.insert(entry.word.to_lowercase(), cat.to_string());
dict.excluded_person_names.insert(entry.word.to_lowercase());
}
"person" => {
dict.included_person_names.insert(entry.word.to_lowercase());
}
"stop_word" => {
dict.stop_words.insert(entry.word.to_lowercase());
}
"person_prefix" => {
dict.person_prefixes.push(entry.word.clone());
}
"excluded_person" => {
dict.excluded_person_names.insert(entry.word.to_lowercase());
}
_ => {}
}
}
// ── 构建 categories(基于 metadata + keymap) ──
let mut words_by_cat: HashMap<String, Vec<String>> = HashMap::new();
for (word, cat_name) in &dict.keymap {
words_by_cat.entry(cat_name.clone()).or_default().push(word.clone());
}
for (name, meta) in &dict.metadata {
let words = words_by_cat.remove(name).unwrap_or_default();
dict.categories.insert(name.clone(), CategoryInfo {
name: name.clone(),
label: meta.label.clone(),
color_bg: meta.color_bg.clone(),
color_fg: meta.color_fg.clone(),
words,
});
}
for (cat_name, words) in words_by_cat {
dict.categories.insert(cat_name.clone(), CategoryInfo {
name: cat_name.clone(),
label: cat_name.clone(),
color_bg: "#f1f5f9".to_string(),
color_fg: "#64748b".to_string(),
words,
});
}
println!("[词典] 从 search.db dict 表加载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len());
dict
}
}
/// 从 search.db 的 dict 表重新加载标签词典
pub async fn reload_from_search_db(search_db: &crate::search_db::SearchDb) -> String {
let entries = match search_db.load_dict_all().await {
Ok(e) => e,
Err(e) => return format!("从 search.db 加载 dict 失败: {}", e),
};
let mut dict = TAG_DICT.write().expect("tag_dict 锁被污染");
*dict = TagDict::load_from_dict_entries(&entries);
format!(
"词典已从 search.db 重载: {} 分类, {} 标签(keymap), {} 停用词, {} 前缀, {} 排除人名, {} 包含人名",
dict.categories.len(), dict.keymap.len(),
dict.stop_words.len(), dict.person_prefixes.len(), dict.excluded_person_names.len(),
dict.included_person_names.len(),
)
}

View File

@@ -1,81 +1,373 @@
//! 标签提取器 — 从标题中提取标签词
//! 标签提取器 — 从标题中提取分类明确的标签
//!
//! 标签分类体系(外部词典 dict/tags.json v4 定义):
//! style outfit scene body mood topic position location brand person
//!
//! 核心设计:纯词典 keymap substring 匹配 + 正则兜底,无 jieba 依赖。
//! Person 类型由 title_parser 单独处理,此处不纳入标签。
use regex::Regex;
use lazy_static::lazy_static;
use std::collections::HashSet;
use crate::engine::tag_dict;
lazy_static! {
/// 匹配英文单词、数字、中文字符(用于正则兜底分词)
static ref WORD_RE: Regex = Regex::new(r"[a-zA-Z0-9\u{4e00}-\u{9fff}]+").unwrap();
static ref HAS_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
/// 去除头尾标点符号
static ref TAG_CORE_RE: Regex = Regex::new(
r#"^[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+|[\s.,;:!?'\"‘’“”\-–—()\[\]{}【】《》〈〉、,。;:?!…~·•]+$"#
).unwrap();
}
/// 停用词
const STOP_WORDS: &[&str] = &[
"photos", "photo", "pictures", "picture", "images", "image", "video", "videos",
"cosplay", "cos", "and", "the", "of", "in", "a", "an", "is", "to", "for",
"part", "vol", "collection", "special", "extra", "bonus",
"new", "old", "selfie", "official", "original", "preview", "patreon",
"秀人", "网", "写真", "图", "套", "合集", "系列", "作品",
];
/// 判断字符串是否全为标点符号
pub fn is_punctuation_only(s: &str) -> bool {
if s.is_empty() { return true; }
!s.chars().any(|c| {
c.is_ascii_alphanumeric()
|| ('\u{4e00}'..='\u{9fff}').contains(&c)
|| ('\u{3400}'..='\u{4dbf}').contains(&c)
|| ('\u{f900}'..='\u{faff}').contains(&c)
})
}
/// 从标题中提取标签
/// 清洗单个标签:去除头尾标点符号。
/// 清洗后如果只剩下标点或为空,则返回 None。
pub fn clean_tag(tag: &str) -> Option<String> {
let cleaned = TAG_CORE_RE.replace_all(tag.trim(), "").trim().to_string();
if cleaned.is_empty() || is_punctuation_only(&cleaned) || cleaned.len() < 2 {
None
} else {
Some(cleaned)
}
}
/// 获取分类优先级索引(从词典动态获取)。
fn category_order_index(category: &str) -> usize {
let dict = tag_dict::get_tag_dict();
let order = dict.get_category_order();
order.iter().position(|c| c == &category).unwrap_or(999)
}
/// 判断一个词属于哪个分类(在词典 keymap 中查找)。
/// 如果在 keymap 中找不到,返回 "other"。
pub fn classify_tag(word: &str) -> String {
let dict = tag_dict::get_tag_dict();
dict.classify(word)
.map(|c| c.to_string())
.unwrap_or_else(|| "other".to_string())
}
/// 获取标签的中文分类名。
pub fn get_category_label(category: &str) -> String {
tag_dict::get_tag_dict().get_category_label(category)
}
/// 获取标签分类的颜色 (bg, fg)。
pub fn get_category_color(category: &str) -> (String, String) {
tag_dict::get_tag_dict().get_category_color(category)
}
// ═══════════════════════════════════════════════════════
// 词典 keymap substring 匹配(最长匹配优先)
// ═══════════════════════════════════════════════════════
/// 从标题中通过词典 keymap substring 匹配提取分类标签,**最长匹配优先**。
///
/// 策略:
/// 1. 先从 `title_lower` 中移除 `person_name` 中的词(防止人名命中标签)
/// 2. keymap 中所有标签按长度降序排列(最长的先匹配)
/// 3. 匹配到某个标签后,将该标签从 `title_lower` 中**替换为空格**,
/// 防止其子串/短标签再次命中,实现最长匹配覆盖
/// 4. 跳过已被 `seen` 包含的词
/// 5. 跳过 `person` 类型的标签(由 title_parser 处理)
pub fn extract_dict_tags(
title: &str,
person_name: &[String],
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let mut tags = Vec::new();
let mut title_lower = title.to_lowercase();
// 移除人名(防止人名中的词命中标签)
for p in person_name {
title_lower = title_lower.replace(&p.to_lowercase(), " ");
}
let tag_category_map = dict.get_tag_category_map();
let mut sorted: Vec<(String, String)> = tag_category_map.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
// 长度降序排列:保证最长匹配优先
sorted.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
for (tag, category) in &sorted {
// 跳过 person 类型(由 title_parser 单独处理)
if category == "person" { continue; }
if seen.contains(tag) { continue; }
if !title_lower.contains(tag.as_str()) { continue; }
seen.insert(tag.clone());
tags.push((tag.clone(), category.clone()));
// 将匹配到的长标签从标题中移除(替换为空格),
// 避免其子串/短标签再次命中,实现最长匹配覆盖
title_lower = title_lower.replace(tag.as_str(), " ");
}
tags
}
// ═══════════════════════════════════════════════════════
// 正则兜底标签提取
// ═══════════════════════════════════════════════════════
/// 正则分词提取剩余关键词(兜底),全部归为 "other" 类。
///
/// 过滤规则:
/// - 长度 < 2、纯标点、纯数字 → 跳过
/// - 停用词 → 跳过
/// - 已被 `seen` 包含的词 → 跳过
/// - 人名 → 跳过
/// - 已在 keymap 中分类过的词 → 跳过(避免重复)
pub fn extract_regex_tags(
title: &str,
person_name: &[String],
stop_words: &HashSet<String>,
seen: &mut HashSet<String>,
) -> Vec<(String, String)> {
let mut tags = Vec::new();
let title_lower = title.to_lowercase();
for cap in WORD_RE.find_iter(&title_lower) {
let word = cap.as_str().trim();
let word_lower = word.to_lowercase();
if word.len() < 2 { continue; }
if is_punctuation_only(word) { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if stop_words.contains(&word_lower) { continue; }
if seen.contains(&word_lower) { continue; }
// 排除人名相关部分
if person_name.iter().any(|p| word_lower.contains(&p.to_lowercase())) {
continue;
}
// 已在词典分类中匹配过的跳过
if tag_dict::get_tag_dict().classify(&word_lower).is_some() {
continue;
}
seen.insert(word_lower);
tags.push((word.to_string(), "other".to_string()));
}
tags
}
// ═══════════════════════════════════════════════════════
// extract_tags_from_title(完整流程:词典匹配 → 正则兜底)
// ═══════════════════════════════════════════════════════
/// 从标题中提取分类标签(完整流程)。
///
/// 两步流程:
/// 1. **词典 keymap substring 匹配** — 在 `title_lower` 中搜索所有 keymap 中的 key,
/// 最长匹配优先,匹配后从标题中移除已匹配文本,避免短标签重复命中。
/// 跳过 person 类型标签。
/// 2. **正则兜底** — 用 WORD_RE 提取剩余单词,归为 "other" 类。
///
/// 过滤规则:
/// - 跳过停用词
/// - 跳过 person 类型(由 title_parser 处理)
/// - 跳过人名部分
///
/// # 参数
/// - `title` — 原始标题
/// - `person_name` — 已提取的人名(可选),用于从标题中排除,防止命中标签
pub fn extract_tags_from_title(
title: &str,
person_name: Option<&str>,
) -> Vec<(String, String)> {
let dict = tag_dict::get_tag_dict();
let stop_words = dict.get_stop_words();
// 构建人名排除列表:人名本身 + 按空白切分的各部分
let person_exclude: Vec<String> = person_name
.map(|n| {
let mut v = vec![n.to_lowercase()];
for part in n.split_whitespace() {
v.push(part.to_lowercase());
}
v
})
.unwrap_or_default();
let mut seen = HashSet::new();
let mut tags = Vec::new();
let mut title_lower = title.to_lowercase();
// 移除人名
if let Some(name) = person_name {
title_lower = title_lower.replace(&name.to_lowercase(), " ");
}
// Step 1: 词典 keymap substring 匹配(最长匹配优先)
tags.extend(extract_dict_tags(title, &person_exclude, &mut seen));
let stop_words: HashSet<&str> = STOP_WORDS.iter().copied().collect();
let person_name_lower = person_name.map(|n| n.to_lowercase());
for cap in WORD_RE.find_iter(&title_lower) {
let word = cap.as_str().trim();
if word.len() < 2 { continue; }
if word.chars().all(|c| c.is_ascii_digit()) { continue; }
if stop_words.contains(&word.to_lowercase().as_str()) { continue; }
if let Some(ref pn) = person_name_lower {
if word.to_lowercase().contains(pn.as_str()) { continue; }
}
let tag_type = if HAS_CHINESE.is_match(word) { "chinese" } else { "english" };
tags.push((word.to_string(), tag_type.to_string()));
}
// Step 2: 正则兜底(剩余未匹配的词,归 "other" 类)
tags.extend(extract_regex_tags(title, &person_exclude, stop_words, &mut seen));
tags
}
/// 标签去重并排序
/// 纯 keymap substring 匹配(无正则兜底),用于快速标签提取。
///
/// 只做词典匹配,不做正则兜底。适合在不需要"其他"类标签的场景下使用。
///
/// # 参数
/// - `title` — 原始标题
/// - `person_name` — 已提取的人名(可选),用于从标题中排除人名部分
pub fn extract_tags_with_keymap(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
let person_exclude: Vec<String> = person_name
.map(|n| {
let mut v = vec![n.to_lowercase()];
for part in n.split_whitespace() {
v.push(part.to_lowercase());
}
v
})
.unwrap_or_default();
let mut seen = HashSet::new();
extract_dict_tags(title, &person_exclude, &mut seen)
}
// ═══════════════════════════════════════════════════════
// 标签去重、排序、过滤
// ═══════════════════════════════════════════════════════
/// 标签去重并排序(分类标签优先,按词典中的 category_order 排序)。
///
/// 去重逻辑:
/// 1. 先按分类优先级排序(style < outfit < scene < ... < other)
/// 2. 逐个清洗标签(去头尾标点)
/// 3. 过滤坏标签(纯数字、纯标点、停用词等)
/// 4. 小写去重
pub fn unique_tags(tags: Vec<(String, String)>) -> Vec<(String, String)> {
let mut seen = HashSet::new();
let mut result = Vec::new();
for (tag, tag_type) in tags {
let key = tag.to_lowercase();
let mut sorted_tags = tags;
sorted_tags.sort_by(|a, b| {
let idx_a = category_order_index(&a.1);
let idx_b = category_order_index(&b.1);
idx_a.cmp(&idx_b)
});
for (tag, tag_type) in sorted_tags {
let cleaned_tag = match clean_tag(&tag) {
Some(t) => t,
None => continue,
};
if is_bad_tag(&cleaned_tag) { continue; }
let key = cleaned_tag.to_lowercase();
if seen.insert(key) {
result.push((tag, tag_type));
result.push((cleaned_tag, tag_type));
}
}
result
}
/// 判断一个标签是否"坏标签"(应被过滤掉)。
///
/// 规则:
/// - 空串或长度 < 2
/// - 纯数字
/// - 纯标点符号
/// - clean_tag 后变短(说明含不可见标点)
/// - 停用词
pub fn is_bad_tag(tag: &str) -> bool {
let tag = tag.trim();
if tag.is_empty() || tag.len() < 2 { return true; }
if tag.chars().all(|c| c.is_ascii_digit()) { return true; }
if is_punctuation_only(tag) { return true; }
if let Some(cleaned) = clean_tag(tag) {
if cleaned != tag { return true; }
}
let dict = tag_dict::get_tag_dict();
if dict.is_stop_word(tag.to_lowercase().as_str()) { return true; }
false
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_tags() {
// 杨晨晨Yome 是 person 类型,由 title_parser 处理,不应出现在标签中
// photos 是停用词,不应出现在标签中
let tags = extract_tags_from_title(
"XiuRen秀人网 – 杨晨晨Yome – Night Dress '81 photos'",
Some("杨晨晨Yome"),
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(names.contains(&"Night"));
assert!(names.contains(&"Dress"));
assert!(!names.contains(&"photos"));
assert!(!names.contains(&"photos"), "tags 不应包含停用词 'photos'");
assert!(!names.contains(&"杨晨晨"), "tags 不应包含 person 类型词 '杨晨晨'");
// 验证 keymap 中的词正确匹配
assert!(names.contains(&"dress"), "tags 应包含 keymap 中的 'dress'");
}
#[test]
fn test_longest_match_priority() {
// "连衣裙" (3字) 在内建词典 outfit 中。标题包含 "连衣裙",
// 匹配后无子串可再匹配。
let tags = extract_tags_from_title(
"粉色连衣裙 2025",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(names.contains(&"连衣裙"), "'连衣裙' 应在内建词典的结果中");
// "连衣裙" 被移除后,其子串不会再次匹配
assert_eq!(names.iter().filter(|t| **t == "连衣裙").count(), 1);
}
#[test]
fn test_classify_tag() {
// v4 JSON keymap 中 "日系":"style"
assert_eq!(classify_tag("日系"), "style");
assert_eq!(classify_tag("比基尼"), "outfit");
assert_eq!(classify_tag("浴室"), "scene");
assert_eq!(classify_tag("美腿"), "body");
assert_eq!(classify_tag("写真"), "topic");
assert_eq!(classify_tag("unknown"), "other");
}
#[test]
fn test_category_colors() {
let (bg, fg) = get_category_color("style");
assert_eq!(bg, "#f0f9ff");
assert_eq!(fg, "#0284c7");
}
#[test]
fn test_extract_tags_with_keymap() {
// 纯 keymap 匹配,不包含正则兜底的词
let tags = extract_tags_with_keymap(
"日系 比基尼 甜美 沙滩 2025",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(names.contains(&"日系"), "日系 应在 keymap 匹配结果中");
assert!(names.contains(&"比基尼"), "比基尼 应在 keymap 匹配结果中");
assert!(names.contains(&"沙滩"), "沙滩 应在 keymap 匹配结果中");
// "2025" 不在 keymap 中,不应出现(纯 keymap 无正则兜底)
assert!(!names.contains(&"2025"), "纯 keymap 匹配不应包含 '2025'");
}
#[test]
fn test_person_type_excluded_from_dict() {
// 杨晨晨 在 v4 keymap 中是 person 类型,不应被提取为标签
let tags = extract_tags_with_keymap(
"杨晨晨 日系 比基尼",
None,
);
let names: Vec<&str> = tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!names.contains(&"杨晨晨"), "person 类型的 '杨晨晨' 不应出现在标签中");
assert!(names.contains(&"日系"), "非 person 类型词应正常匹配");
}
}

View File

@@ -1,154 +1,387 @@
//! 标题解析、人名提取、标签提取
//! 对应 Python 的 tag_index.py + clean_person_names.py 的核心逻辑
//! 标题解析 — 统一最大匹配管线(v5)
//!
//! 核心思路:一次扫描同时识别 person 和 tags,两者互斥,不会混淆。
//!
//! 流程:
//! 1. 预处理:移除照片数后缀、[工作室]前缀、称呼前缀、编号后缀
//! 2. 切分:中文最大匹配 + 英文按空格切分
//! 3. 角色分类:每个片段有明确分类(person / style / outfit / ... / stop / other)
//! 4. 选人名:优先级 = keymap(person) > include 列表 > 启发式
//! 5. 其余分类即为标签
//!
//! 相比 v4 的改进:
//! - 中文连续文本也能正确切分(不再依赖空格)
//! - person 和 tag 在同一 pass 中确定,不会混淆
//! - 最长匹配优先,避免"性感"和"性感女神"都匹配的问题
//! - 英文只按空格切分,不逐字匹配,不移除英文介词等停用词
use regex::Regex;
use std::collections::HashSet;
use lazy_static::lazy_static;
use crate::engine::tag_dict;
// ── 常量 ──
/// 模特称呼前缀(移除用)
const MODEL_TITLE_PREFIX: &[&str] = &[
"美女模特", "性感女神", "清纯美女", "气质美女", "可爱女孩",
"甜美少女", "时尚达人", "女神", "美女", "模特", "小姐姐",
"小仙女", "御姐", "萝莉", "辣妹", "潮人",
];
/// 停用词(标签过滤用)
const STOP_WORDS: &[&str] = &[
"写真", "套图", "图片", "照片", "摄影", "拍摄", "风格",
"系列", "作品", "专辑", "主题", "人像", "艺术", "唯美",
"诱惑", "性感", "迷人", "魅力", "时尚", "潮流", "高清",
"原版", "无水印", "私房", "内衣", "比基尼", "泳装",
"旗袍", "制服", "丝袜", "美腿", "美胸", "翘臀",
"cosplay", "cos", "角色扮演", "动漫", "游戏",
"classic", "exclusive", "premium", "collection",
"photo", "photos", "video", "videos", "set", "vol",
"no", "番号", "编号", "期号",
];
/// 保留词性(仅用于 jieba 分词场景,这里做占位符)
const _KEEP_POS: &[&str] = &["n", "nr", "ns", "nz", "v", "vn", "a", "an", "eng", "x"];
/// 工作室/平台排除名
const EXCLUDED_NAMES: &[&str] = &[
"Pure Media", "XiuRen秀人网", "尤蜜", "魅妍社", "花漾",
"瑞丝馆", "撸图", "情欲萝莉", "蜜桃社", "tgod",
"MiStar", "BLUECAKE", "HuaYang", "YouMi", "秀人",
];
// ── 正则 ──
lazy_static! {
/// 编译后的正则
static ref RE_COSPLAY: Regex = Regex::new(
r"(?i)^(.+?)\s+cos(play)?\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
).unwrap();
static ref RE_COSPLAY_BY: Regex = Regex::new(
r"(?i)^(.+?)\s+cos(play)?\s+by\s+(.+?)(?:\s*[–—\-]\s*(.+))?$"
).unwrap();
static ref RE_BRACKET_PREFIX: Regex = Regex::new(r"^\[([^\]]+)\]\s*").unwrap();
static ref RE_NUMBER_SUFFIX: Regex = Regex::new(r"[\s#]*(?:no\.?\s*|vol\.?\s*|#)?\d+[\s]*$").unwrap();
static ref RE_PAREN_CONTENT: Regex = Regex::new(r"[((]([^))]*)[))]").unwrap();
static ref RE_SPLIT_DASH: Regex = Regex::new(r"\s*[–—\-]\s*").unwrap();
static ref RE_PHOTOS_SUFFIX: Regex = Regex::new(r"(?i)\s*\d+\s*photos?\s*(?:and\s*\d+\s*videos?)?\s*$").unwrap();
static ref RE_CHINESE: Regex = Regex::new(r"[\u{4e00}-\u{9fff}]").unwrap();
static ref RE_ENGLISH_SUFFIX: Regex = Regex::new(r"^([\u{4e00}-\u{9fff}]+)([A-Za-z].*)$").unwrap();
static ref RE_TRAILING_PUNCT: Regex = Regex::new(r"[\s.,;:,。;:\-–—]+$").unwrap();
/// 停用词集合
static ref STOP_WORDS_SET: HashSet<&'static str> = {
let mut s = HashSet::new();
for w in STOP_WORDS { s.insert(*w); }
s
};
/// 排除名集合(小写)
static ref EXCLUDED_LOWER: HashSet<String> = {
let mut s = HashSet::new();
for n in EXCLUDED_NAMES { s.insert(n.to_lowercase()); }
s.extend(["youmi", "huayang", "bluecake", "mistar", "xiuren", "秀人"].map(|s| s.to_string()));
s
};
static ref RE_PAREN_CONTENT: Regex = Regex::new(r"[((]([^))]*)[))]").unwrap();
/// 分隔符:空白、常见标点、中英文之间的边界
static ref RE_SEPARATOR: Regex = Regex::new(r"[\s\-–—_,.;:,。;:、/\\|·]+").unwrap();
}
// ── 人名提取 ──
// ── 预处理 ──
/// 从标题中提取模特/人物名称
/// 对应 Python tag_index.extract_person_name_from_title
pub fn extract_person_name_from_title(title: &str) -> Option<String> {
let title = title.trim();
if title.is_empty() {
return None;
}
/// 统一标题预处理:移除照片数后缀、[工作室] 前缀、称呼前缀、编号后缀
pub fn preprocess_title(title: &str) -> String {
let t = title.trim();
if t.is_empty() { return String::new(); }
// 清理末尾的 "N photos" 等
let title = RE_PHOTOS_SUFFIX.replace(title, "").trim().to_string();
let mut t = RE_PHOTOS_SUFFIX.replace(t, "").trim().to_string();
t = RE_BRACKET_PREFIX.replace(&t, "").trim().to_string();
// 1. cosplay 格式: "Name cosplay Character – Series"
if let Some(caps) = RE_COSPLAY_BY.captures(&title) {
return Some(caps[3].trim().to_string());
}
if let Some(caps) = RE_COSPLAY.captures(&title) {
return Some(caps[1].trim().to_string());
}
// 2. 中文摄影工作室格式
let title = RE_BRACKET_PREFIX.replace(&title, "").trim().to_string();
// 移除模特称呼前缀
let mut title = title;
for prefix in MODEL_TITLE_PREFIX {
if title.starts_with(prefix) {
title = title[prefix.len()..].trim().to_string();
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
if t.starts_with(prefix) {
t = t[prefix.len()..].trim().to_string();
break;
}
}
// 移除编号后缀
let title = RE_NUMBER_SUFFIX.replace(&title, "").trim().to_string();
// 用 `–—\-` 分割,取第一部分作为人名
let parts: Vec<&str> = RE_SPLIT_DASH.split(&title).collect();
let name = parts[0].trim().to_string();
if name.len() >= 2 {
Some(name)
} else {
None
}
t = RE_NUMBER_SUFFIX.replace(&t, "").trim().to_string();
t
}
// ── 切分片段:带分类 ──
/// 一个切分片段及其分类
#[derive(Debug, Clone)]
struct Segment {
text: String,
category: String,
}
// ── 切分核心算法 ──
/// 判断字符是否是中文字符
fn is_chinese_char(ch: char) -> bool {
('\u{4e00}'..='\u{9fff}').contains(&ch)
}
/// 对标题做切分,返回带分类的片段列表。
///
/// 算法分两路:
/// - 中文:从左到右最大匹配(逐字尝试,最长优先)
/// - 英文/数字:按空格/分隔符直接切分为完整单词,查词典分类
///
/// 匹配来源优先级:
/// keymap(person) > keymap(其他) > stop_words > include_person
///
/// 未匹配到的作为 "other" 片段保留。
fn max_match_segment(title: &str) -> Vec<Segment> {
let dict = tag_dict::get_tag_dict();
// 收集所有可匹配的词及分类
let mut all_words: Vec<(String, String)> = Vec::new();
// 1. keymap 中的所有词
for (word, cat) in dict.get_tag_category_map() {
all_words.push((word.clone(), cat.clone()));
}
// 2. include 列表中的人名 → "person_include"
for name in dict.get_included_persons() {
if !dict.get_tag_category_map().contains_key(name) {
all_words.push((name.clone(), "person_include".to_string()));
}
}
// 3. 停用词 → "stop"
for word in dict.get_stop_words() {
all_words.push((word.clone(), "stop".to_string()));
}
// 计算最大词长(中文最大匹配用)
let max_len = all_words.iter().map(|(w, _)| w.chars().count()).max().unwrap_or(8);
// 快速查找函数
let find_category = |word: &str| -> Option<String> {
let word_lower = word.to_lowercase();
for (w, cat) in &all_words {
if w.to_lowercase() == word_lower {
return Some(cat.clone());
}
}
None
};
let mut segments: Vec<Segment> = Vec::new();
let chars: Vec<char> = title.chars().collect();
let n = chars.len();
let mut i = 0;
while i < n {
let ch = chars[i];
// 跳过分隔符
if RE_SEPARATOR.is_match(&ch.to_string()) {
i += 1;
continue;
}
// ── 英文/数字:直接按分隔符切分整个单词,不逐字匹配 ──
if ch.is_ascii_alphanumeric() {
let start = i;
while i < n && chars[i].is_ascii_alphanumeric() {
i += 1;
}
let word: String = chars[start..i].iter().collect();
let cat = find_category(&word).unwrap_or_else(|| "other".to_string());
segments.push(Segment { text: word, category: cat });
continue;
}
// ── 中文:最大匹配 ──
if is_chinese_char(ch) {
let mut found = false;
let try_max = max_len.min(n - i);
'outer: for len in (1..=try_max).rev() {
let candidate: String = chars[i..i + len].iter().collect();
if let Some(cat) = find_category(&candidate) {
segments.push(Segment { text: candidate, category: cat });
i += len;
found = true;
break 'outer;
}
}
if !found {
// 未匹配:收集连续中文字符作为 other
let start = i;
while i < n && is_chinese_char(chars[i]) {
// 前瞻:如果下一个位置开始能匹配词典,就停在这里
if i > start {
let remaining = n - i;
let peek_max = max_len.min(remaining);
let mut matched = false;
for plen in (1..=peek_max).rev() {
let peek: String = chars[i..i + plen].iter().collect();
if find_category(&peek).is_some() {
matched = true;
break;
}
}
if matched { break; }
}
i += 1;
}
if i > start {
let other: String = chars[start..i].iter().collect();
segments.push(Segment { text: other, category: "other".to_string() });
}
}
continue;
}
// 其他字符跳过
i += 1;
}
segments
}
// ── 人名启发式判断 ──
/// 判断一个词是否"看起来像人名"
///
/// 注意:调用方应已排除 keymap 中的非 person 类型词、停用词等。
pub fn is_likely_person_name(name: &str) -> bool {
let name = name.trim();
if name.is_empty() { return false; }
let dict = tag_dict::get_tag_dict();
// 强制包含名单优先放行
if dict.is_included_person(name) {
return true;
}
// 字符数限制
let char_count = name.chars().count();
if char_count < 2 { return false; }
// keymap 中的非 person 类型词一定不是人名
if let Some(cat) = dict.classify(name) {
if cat != "person" { return false; }
}
// 排除已知非人名
if dict.is_excluded_person(name) { return false; }
let name_lower = name.to_lowercase();
// 纯数字排除
if name.chars().all(|c| c.is_ascii_digit()) { return false; }
// 含数字太多的排除
let digit_count = name.chars().filter(|c| c.is_ascii_digit()).count();
if digit_count * 2 > name.len() { return false; }
// 纯英文且过长排除
let has_cn = RE_CHINESE.is_match(name);
if !has_cn && name.len() > 15 { return false; }
// 以数字开头排除
if name.chars().next().map(|c| c.is_ascii_digit()).unwrap_or(false) { return false; }
// 含特殊字符过多排除
let special_count = name.chars().filter(|c| {
!c.is_alphanumeric() && !RE_CHINESE.is_match(&c.to_string())
}).count();
if special_count > 2 { return false; }
// 已知后缀排除
let known_suffix = ["photos", "series", "collection", "edition", "set", "vol", "album"];
for suffix in known_suffix {
if name_lower.ends_with(suffix) { return false; }
}
// 全大写且长度 > 6 排除(缩写/工作室名)
if name.chars().all(|c| c.is_ascii_uppercase()) && name.len() > 6 { return false; }
// 中文字符数判断:2-5 个中文字符更像人名
let cn_count: usize = name.chars().filter(|c| RE_CHINESE.is_match(&c.to_string())).count();
if has_cn && (cn_count < 2 || cn_count > 5) {
return false;
}
true
}
// ── 核心解析入口 ──
/// 统一相册标题解析入口(v5 最大匹配版本)
///
/// 流程:预处理 → 切分 → 选人 → 其余为标签
///
/// 返回 ParseResult 可直接用于批量写库。
pub fn parse_album_title(album_id: &str, title: &str) -> crate::models::parse::ParseResult {
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
let title = title.trim();
if title.is_empty() { return result; }
let cleaned = preprocess_title(title);
if cleaned.is_empty() { return result; }
// Step 1: 切分
let segments = max_match_segment(&cleaned);
// Step 2: 收集所有人名候选(按优先级排序,全部保留)
let mut person_candidates: Vec<(String, u8)> = Vec::new();
for seg in &segments {
match seg.category.as_str() {
"person" => {
person_candidates.push((seg.text.clone(), 0)); // 优先级 0 = 最高
}
"person_include" => {
person_candidates.push((seg.text.clone(), 1)); // 优先级 1
}
"other" => {
if is_likely_person_name(&seg.text) {
person_candidates.push((seg.text.clone(), 2)); // 优先级 2 = 启发式
}
}
_ => {}
}
}
// 按优先级排序
person_candidates.sort_by_key(|(_, p)| *p);
// 收集最终人名列表(去重)
let mut person_names: Vec<String> = Vec::new();
let mut seen_names: HashSet<String> = HashSet::new();
for (name, _) in &person_candidates {
let (primary, _) = split_person_name_with_aliases(name);
let final_name = if primary.is_empty() { name.clone() } else { primary };
if final_name.chars().count() >= 2 && seen_names.insert(final_name.to_lowercase()) {
person_names.push(final_name);
}
}
result.person_names = person_names;
// Step 3: 收集标签(跳过所有 person 片段)
// 构建所有已识别 person 片段的小写集合(用于过滤 other 类中与 person 重叠的片段)
let person_segment_lower: HashSet<String> = segments.iter()
.filter(|s| s.category == "person" || s.category == "person_include")
.map(|s| s.text.to_lowercase())
.collect();
let person_name_lower: HashSet<String> = result.person_names.iter()
.map(|n| n.to_lowercase())
.collect();
let mut tags: Vec<(String, String)> = Vec::new();
for seg in &segments {
let cat = &seg.category;
// 跳过停用词
if cat == "stop" { continue; }
// 跳过 person 相关片段
if cat == "person" || cat == "person_include" { continue; }
// 如果是 other 类,检查是否与任何人名重叠
if cat == "other" {
let seg_lower = seg.text.to_lowercase();
let overlaps_person = person_segment_lower.iter().any(|pl| {
seg_lower.contains(pl) || pl.contains(&seg_lower)
}) || person_name_lower.iter().any(|pl| {
seg_lower.contains(pl) || pl.contains(&seg_lower)
});
if overlaps_person { continue; }
// other 类也做个简单过滤:太短、纯数字等跳过
if seg.text.chars().count() < 2 { continue; }
if seg.text.chars().all(|c| c.is_ascii_digit()) { continue; }
// 太短的英文 other 也跳过(噪声)
if !RE_CHINESE.is_match(&seg.text) && seg.text.len() < 3 { continue; }
}
// 有效分类标签
let display_cat = if cat == "other" { "other".to_string() } else { cat.clone() };
tags.push((seg.text.clone(), display_cat));
}
// 去重(小写)
let mut seen = HashSet::new();
tags.retain(|(t, _)| seen.insert(t.to_lowercase()));
result.tags = tags;
result
}
// ── 人名清洗工具函数 ──
/// 提取中文核心(开头的连续中文字符)
/// 对应 Python merge_persons_by_core.extract_cn
pub fn extract_cn(name: &str) -> Option<String> {
let re = Regex::new(r"^([\u{4e00}-\u{9fff}]+)").unwrap();
re.captures(name).map(|c| c[1].to_string())
}
/// 判断后缀是否有效(不含中文=有效英文后缀)
/// 对应 Python merge_persons_by_core.is_valid_suffix
pub fn is_valid_suffix(suffix: &str) -> bool {
if suffix.is_empty() {
return true;
}
!RE_CHINESE.is_match(suffix)
}
/// 判断是否为工作室/平台名(需排除)
pub fn is_excluded_name(name: &str) -> bool {
EXCLUDED_LOWER.contains(&name.to_lowercase())
}
// ── 人名清洗 ──
/// 清洗人名:移除前缀、括号内容、末尾标点
/// 返回 (cleaned_name, aliases)
/// 对应 Python clean_person_names.clean_person_name
/// 清理人名:移除前缀、括号内容、末尾标点
pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
let mut aliases = Vec::new();
let mut name = raw_name.to_string();
// 1. 移除前缀
for prefix in MODEL_TITLE_PREFIX {
for prefix in tag_dict::get_tag_dict().get_person_prefixes() {
if name.starts_with(prefix) {
aliases.push(("prefix".to_string(), prefix.to_string()));
name = name[prefix.len()..].trim().to_string();
@@ -156,7 +389,6 @@ pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
}
}
// 2. 移除末尾括号内容
let mut cleaned = String::new();
let mut last_end = 0;
for cap in RE_PAREN_CONTENT.captures_iter(&name) {
@@ -164,7 +396,6 @@ pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
let content = cap.get(1).map_or("", |c| c.as_str());
cleaned.push_str(&name[last_end..m.start()]);
last_end = m.end();
if !content.is_empty() {
if RE_CHINESE.is_match(content) {
aliases.push(("nickname".to_string(), content.to_string()));
@@ -174,12 +405,10 @@ pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
}
}
cleaned.push_str(&name[last_end..]);
let name = cleaned.trim().to_string();
let name = RE_TRAILING_PUNCT.replace(&cleaned.trim(), "").trim().to_string();
// 3. 清理末尾标点
let name = RE_TRAILING_PUNCT.replace(&name, "").trim().to_string();
aliases.push(("original".to_string(), raw_name.trim().to_string()));
// 4. 从清洗后名字提取英文名别名
if let Some(caps) = RE_ENGLISH_SUFFIX.captures(&name) {
let chinese_part = caps[1].to_string();
let english_part = caps[2].to_string();
@@ -189,67 +418,48 @@ pub fn clean_person_name(raw_name: &str) -> (String, Vec<(String, String)>) {
}
}
if name.is_empty() {
(raw_name.to_string(), aliases)
} else {
(name, aliases)
}
if name.is_empty() { (raw_name.to_string(), aliases) } else { (name, aliases) }
}
/// 获取归并键(小写化+移除空格和标点)
pub fn get_person_key(name: &str) -> String {
name.to_lowercase()
.chars()
.filter(|c| !c.is_whitespace() && !c.is_ascii_punctuation())
.collect()
/// 拆分人名为「中文主名 + 别名列表」
pub fn split_person_name_with_aliases(raw_name: &str) -> (String, Vec<(String, String)>) {
let (cleaned, mut aliases) = clean_person_name(raw_name);
let mut primary_name = cleaned.clone();
if let Some(cn_part) = extract_cn(&cleaned) {
if !cn_part.is_empty() && cn_part != cleaned {
primary_name = cn_part.clone();
if let Some(rest) = cleaned.strip_prefix(&cn_part) {
let english_suffix = rest.trim().to_string();
if !english_suffix.is_empty() {
let already_has = aliases.iter().any(|(_, a)| a == &english_suffix);
if !already_has {
aliases.push(("english".to_string(), english_suffix));
}
}
}
}
}
if cleaned != primary_name {
let already_has = aliases.iter().any(|(_, a)| a == &cleaned);
if !already_has {
aliases.push(("cleaned".to_string(), cleaned));
}
}
(primary_name, aliases)
}
// ── 标签提取 ──
/// 判断是否为工作室/平台名(需排除)
pub fn is_excluded_name(name: &str) -> bool {
tag_dict::get_tag_dict().is_excluded_person(name)
}
/// 从标题中提取标签(排除人名部分)
/// 返回 Vec<(tag, pos)>
/// 对应 Python tag_index.extract_tags_from_title
#[allow(dead_code)]
pub fn extract_tags_from_title_legacy(title: &str, person_name: Option<&str>) -> Vec<(String, String)> {
let mut title = title.to_string();
// 清理末尾的 "N photos" 等
title = RE_PHOTOS_SUFFIX.replace(&title, "").to_string();
// 移除人名
if let Some(name) = person_name {
title = title.replace(name, " ");
for part in name.split_whitespace() {
title = title.replace(part, " ");
}
}
// 移除 [工作室] 前缀
title = RE_BRACKET_PREFIX.replace(&title, "").to_string();
// 按空白分割,过滤停用词和短词
let mut tags = Vec::new();
let mut seen = HashSet::new();
for word in title.split_whitespace() {
let word = word.trim().to_lowercase();
if word.len() < 2 {
continue;
}
if STOP_WORDS_SET.contains(word.as_str()) {
continue;
}
if word.chars().all(|c| c.is_ascii_digit() || c == '.') {
continue;
}
if seen.contains(&word) {
continue;
}
seen.insert(word.clone());
tags.push((word, "x".to_string()));
}
tags
/// 判断后缀是否有效
pub fn is_valid_suffix(suffix: &str) -> bool {
if suffix.is_empty() { return true; }
!RE_CHINESE.is_match(suffix)
}
/// 标签归一化
@@ -257,16 +467,188 @@ pub fn normalize_tag(tag: &str) -> String {
tag.trim().to_lowercase()
}
// ── dict 优先匹配 ──
/// 使用 dict 表优先匹配 + title_parser 回退的混合解析
///
/// 流程:
/// 1. dict 最长匹配提取 person 和 tags(带 merged_to 解析)
/// 2. 从标题移除已匹配的 dict 词
/// 3. 剩余文本走 title_parser 解析(tag_dict 回退)
/// 4. 合并:dict 结果优先,parser 补充
pub fn parse_album_title_with_dict(
album_id: &str,
title: &str,
dict_entries: &[crate::search_db::DictEntry],
) -> crate::models::parse::ParseResult {
let title = title.trim();
if title.is_empty() {
return crate::models::parse::ParseResult::new(album_id.to_string());
}
// Phase 1: dict 优先匹配
let (dict_person, dict_tags, remaining) = extract_with_dict(title, dict_entries);
// Phase 2: title_parser 回退(解析剩余文本)
let parse_result = if remaining.is_empty() {
crate::models::parse::ParseResult::new(album_id.to_string())
} else {
parse_album_title(album_id, &remaining)
};
// Phase 3: 合并结果 — dict 优先,parser 补充
let mut result = crate::models::parse::ParseResult::new(album_id.to_string());
// persons: dict 命中 + parser 结果合并去重
let mut seen_persons = HashSet::new();
for name in dict_person.iter().chain(parse_result.person_names.iter()) {
if seen_persons.insert(name.to_lowercase()) {
result.person_names.push(name.clone());
}
}
// tags: dict 标签优先,parser 标签补充去重
let mut seen = HashSet::new();
for (t, _) in &dict_tags {
seen.insert(t.to_lowercase());
result.tags.push((t.clone(), "other".to_string()));
}
for (t, c) in &parse_result.tags {
if seen.insert(t.to_lowercase()) {
result.tags.push((t.clone(), c.clone()));
}
}
result
}
/// dict 条目按词长降序匹配标题,返回 (persons, tags, 剩余文本)
fn extract_with_dict(
title: &str,
dict_entries: &[crate::search_db::DictEntry],
) -> (Vec<String>, Vec<(String, String)>, String) {
// 按词长降序排列
let mut sorted: Vec<&crate::search_db::DictEntry> = dict_entries.iter().collect();
sorted.sort_by(|a, b| b.word.len().cmp(&a.word.len()));
let title_lower = title.to_lowercase();
let mut matched_ranges: Vec<(usize, usize)> = Vec::new();
let mut persons: Vec<String> = Vec::new();
let mut seen_persons: HashSet<String> = HashSet::new();
let mut tags: Vec<(String, String)> = Vec::new();
let mut seen_tag_words = HashSet::new();
for entry in &sorted {
let word_lower = entry.word.to_lowercase();
// 在当前标题中查找该词(忽略大小写)
let mut search_start = 0;
while let Some(pos) = title_lower[search_start..].find(&word_lower) {
let abs_start = search_start + pos;
let abs_end = abs_start + entry.word.len();
// 检查是否已被更长的 dict 词覆盖
if matched_ranges.iter().any(|&(s, e)| s <= abs_start && abs_end <= e) {
break;
}
matched_ranges.push((abs_start, abs_end));
match entry.type_.as_str() {
"person" => {
let resolved = entry.merged_to.as_ref().unwrap_or(&entry.word).clone();
if seen_persons.insert(resolved.to_lowercase()) {
persons.push(resolved);
}
}
"tag" => {
if seen_tag_words.insert(entry.word.clone()) {
tags.push((entry.word.clone(), entry.category.clone().unwrap_or_default()));
}
}
_ => {}
}
break; // 每个词只匹配一次
}
}
// 构建剩余文本:去掉已匹配的部分
if matched_ranges.is_empty() {
return (persons, tags, title.to_string());
}
matched_ranges.sort_by_key(|&(s, _)| s);
let mut remaining = String::new();
let mut last = 0;
for &(s, e) in &matched_ranges {
if s > last {
remaining.push_str(&title[last..s]);
}
remaining.push(' ');
last = e;
}
if last < title.len() {
remaining.push_str(&title[last..]);
}
// 合并连续空格
let remaining = remaining.split_whitespace().collect::<Vec<_>>().join(" ");
(persons, tags, remaining)
}
// ── 兼容函数(供 pipeline / person_merger 调用) ──
/// 从标题中提取人物名列表(兼容函数)
pub fn extract_person_name_from_title(title: &str) -> Vec<String> {
parse_album_title("", title).person_names
}
/// 从标题中提取人物名和标签(兼容函数)
pub fn extract_person_and_tags(title: &str) -> (Vec<String>, Vec<(String, String)>) {
let result = parse_album_title("", title);
(result.person_names, result.tags)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_person_name() {
assert_eq!(extract_person_name_from_title("杨晨晨Yome 私房写真").unwrap(), "杨晨晨Yome");
assert_eq!(extract_person_name_from_title("王馨瑶yanni 比基尼").unwrap(), "王馨瑶yanni");
assert_eq!(extract_person_name_from_title("[XiuRen] 杨晨晨 性感").unwrap(), "杨晨晨");
assert_eq!(extract_person_name_from_title("美女模特 杨晨晨 私房").unwrap(), "杨晨晨");
fn test_parse_album_title_with_person() {
let r = parse_album_title("id1", "杨晨晨Yome 私房写真");
assert_eq!(r.person_names.first().map(|s| s.as_str()), Some("杨晨晨"));
assert!(!r.tags.is_empty());
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(tag_names.contains(&"私房"), "应包含 '私房' 标签");
assert!(tag_names.contains(&"写真"), "应包含 '写真' 标签");
}
#[test]
fn test_parse_album_title_no_person() {
let r = parse_album_title("id2", "浴室 可爱 性感 写真");
assert!(r.person_names.is_empty());
assert!(!r.tags.is_empty());
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(tag_names.contains(&"浴室"));
assert!(tag_names.contains(&"性感"));
assert!(tag_names.contains(&"写真"));
}
#[test]
fn test_parse_album_title_with_prefix() {
let r = parse_album_title("id3", "美女模特 杨晨晨 私房");
assert_eq!(r.person_names.first().map(|s| s.as_str()), Some("杨晨晨"));
}
#[test]
fn test_parse_continuous_chinese() {
// 连续中文:杨晨晨私房写真 — 应能切分出人名和标签
let r = parse_album_title("id4", "杨晨晨私房写真");
assert_eq!(r.person_names.first().map(|s| s.as_str()), Some("杨晨晨"));
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(tag_names.contains(&"私房"), "应包含 '私房'");
assert!(tag_names.contains(&"写真"), "应包含 '写真'");
}
#[test]
@@ -277,9 +659,35 @@ mod tests {
}
#[test]
fn test_clean_person_name() {
let (cleaned, aliases) = clean_person_name("美女模特 杨晨晨Yome");
assert_eq!(cleaned, "美女模特 杨晨晨Yome");
assert!(!aliases.is_empty());
fn test_split_person_name_with_aliases() {
let (name, aliases) = split_person_name_with_aliases("杨晨晨Yome");
assert_eq!(name, "杨晨晨");
let alias_names: Vec<&str> = aliases.iter().map(|(_, a)| a.as_str()).collect();
assert!(alias_names.contains(&"Yome"));
let (name2, _) = split_person_name_with_aliases("杨晨晨");
assert_eq!(name2, "杨晨晨");
let (name3, _) = split_person_name_with_aliases("Yome");
assert_eq!(name3, "Yome");
}
#[test]
fn test_person_not_in_tags() {
// 人名不应出现在标签中
let r = parse_album_title("id5", "杨晨晨 日系 比基尼");
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!tag_names.contains(&"杨晨晨"), "人名 '杨晨晨' 不应出现在标签中");
assert!(tag_names.contains(&"日系"));
assert!(tag_names.contains(&"比基尼"));
}
#[test]
fn test_stop_words_removed() {
// 停用词不应出现在标签或人名中
let r = parse_album_title("id6", "杨晨晨 photos collection");
let tag_names: Vec<&str> = r.tags.iter().map(|(t, _)| t.as_str()).collect();
assert!(!tag_names.contains(&"photos"), "停用词 'photos' 不应在标签中");
assert!(!tag_names.contains(&"collection"), "停用词 'collection' 不应在标签中");
}
}

View File

@@ -7,22 +7,30 @@ mod routes;
mod services;
mod image_processor;
mod queue;
mod search_db;
use crate::cache::AlbumsCache;
use crate::search_db::SearchDb;
use actix_files as afs;
use actix_web::{App, HttpResponse, HttpServer, middleware, web};
use std::env;
use std::fs;
use std::sync::Arc;
use tokio::time::{Duration, interval};
use tracing::{error, info};
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[actix_web::main]
async fn main() -> std::io::Result<()> {
tracing_subscriber::fmt::init();
let args: Vec<String> = env::args().collect();
let export_to_search = args.iter().any(|a| a == "--export-search");
let search_db_cli_path = args.iter().find(|a| a.starts_with("--search-db="))
.map(|s| s.strip_prefix("--search-db=").unwrap_or("search.db"))
.unwrap_or("search.db");
let clear_search = args.iter().any(|a| a == "--clear-search");
let wal_checkpoint = args.iter().any(|a| a == "--wal-checkpoint");
// 加载配置
let config = match config::Config::load() {
Ok(cfg) => cfg,
@@ -39,7 +47,11 @@ async fn main() -> std::io::Result<()> {
let db = match db::Database::new(&config).await {
Ok(db) => {
info!("数据库连接成功");
Arc::new(db)
let db = Arc::new(db);
// 启动时先合并上次未完成的 WAL,确保数据库干净
info!("正在合并数据库 WAL...");
db.wal_checkpoint().await;
db
}
Err(e) => {
error!("数据库连接失败: {}", e);
@@ -47,66 +59,123 @@ async fn main() -> std::io::Result<()> {
}
};
// 自动迁移老数据库
if let Some(ref old_db_cfg) = config.old_db {
if old_db_cfg.auto_migrate {
info!("[迁移] 检测到 old_db 配置,自动迁移: {}", old_db_cfg.path);
match db.migrate_from_old_db(&old_db_cfg.path).await {
Ok(stats) => info!("[迁移] {}", db.migration_summary(&stats)),
Err(e) => error!("[迁移] 失败: {}", e),
if export_to_search {
info!("[搜索导出] 开始将元数据导出到 search.db...");
match services::search_export::export_to_search_db(&db, search_db_cli_path, clear_search).await {
Ok(result) => {
info!("[搜索导出] 导出完成: 相册{}, 标签{}, 人物{}, 别名{}, 关联{}",
result.albums_processed, result.tags_created, result.persons_created,
result.aliases_created, result.album_tags_created + result.album_persons_created);
std::process::exit(0);
}
Err(e) => {
error!("[搜索导出] 导出失败: {}", e);
std::process::exit(1);
}
}
}
if wal_checkpoint {
info!("[WAL合并] 主数据库 WAL checkpoint 已在初始化时完成");
info!("[WAL合并] 正在合并 search.db WAL...");
let search_db = SearchDb::new("data/search.db").await
.expect("search.db 连接失败");
search_db.wal_checkpoint().await;
info!("[WAL合并] 两个数据库均已合并完成");
std::process::exit(0);
}
let image_processor = Arc::new(image_processor::ImageProcessor::new(config.storage.clone()));
// 后台导入本地相册
let config_clone = config.clone();
let db_clone = Arc::clone(&db);
tokio::spawn(async move {
info!("[后台任务] 开始扫描图片目录并导入新相册...");
match services::import::import_existing_data(&config_clone, db_clone, None).await {
Ok(_) => info!("[后台任务] 相册导入完成"),
Err(e) => error!("[后台任务] 相册导入失败: {}", e),
}
});
// ── 初始化 search.db(运行时搜索专用)──
let search_db_path = "data/search.db";
let search_db = Arc::new(
SearchDb::new(search_db_path).await
.expect("search.db 连接失败")
);
info!("search.db 连接成功");
search_db.wal_checkpoint().await;
// 相册内存缓存
// ── 从 search.db 的 dict 表加载标签词典 ──
match engine::tag_dict::reload_from_search_db(&search_db).await {
msg => info!("{}", msg),
}
// ── 内存缓存(必须在管线任务之前创建)──
let albums_cache = Arc::new(tokio::sync::RwLock::new(AlbumsCache::new()));
let persons_cache = Arc::new(tokio::sync::RwLock::new(cache::PersonCache::default()));
let tags_cache = Arc::new(tokio::sync::RwLock::new(cache::TagCache::default()));
// 初始加载缓存
let initial_cache = Arc::clone(&albums_cache);
let initial_db = Arc::clone(&db);
// ── 启动管线(单次):导入 → 增量解析 → 条件全量 sync → 缓存刷新 ──
// 不做全量 reparse/merge,只处理新发现的相册
let boot_config = config.clone();
let boot_db = Arc::clone(&db);
let boot_search_db = Arc::clone(&search_db);
let boot_albums_cache = Arc::clone(&albums_cache);
let boot_persons_cache = Arc::clone(&persons_cache);
let boot_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
match initial_db.get_all_albums().await {
Ok(albums) => {
let mut cache = initial_cache.write().await;
cache.update(albums);
println!("✅ 初始相册缓存加载完成");
}
Err(e) => {
eprintln!("❌ 初始缓存加载失败: {}", e);
}
// ① 扫描导入新相册
info!("[启动] ① 扫描导入新相册...");
match services::import::import_existing_data(&boot_config, Arc::clone(&boot_db), None).await {
Ok(_) => info!("[启动] 导入完成"),
Err(e) => error!("[启动] 导入失败: {}", e),
}
// ② 清理:删除空相册 + 去重
info!("[启动] ② 清理空相册和重复相册...");
let empty = engine::pipeline::phase_delete_empty_albums(&boot_db).await;
let dedup = engine::pipeline::phase_dedup_albums(&boot_db, &boot_search_db).await;
if empty > 0 || dedup > 0 {
info!("[启动] 清理完成:删除空相册 {},去重删除 {}", empty, dedup);
}
// ③ 查找新相册 / 半成品
info!("[启动] ③ 查找新相册...");
let dirty = engine::pipeline::phase_collect_new_albums(&boot_db).await;
let new_count = dirty.len();
if new_count > 0 {
info!("[启动] 发现 {} 个新相册,同步 tags/persons 到 search.db...", new_count);
let synced = engine::pipeline::sync_dirty_albums(&boot_db, &boot_search_db, &dirty).await;
info!("[启动] 增量同步完成:{} 个相册(仅 tags/persons,不同步 albums 表)", synced);
} else {
info!("[启动] 没有新相册");
}
// ④ 刷新所有内存缓存(从 search.db 读取)
info!("[启动] ④ 刷新所有内存缓存...");
cache::refresh_all_caches(&boot_db, &boot_search_db, &boot_albums_cache, &boot_persons_cache, &boot_tags_cache).await;
info!("[启动] 启动管线执行完毕(新相册={})", new_count);
});
// 定时刷新缓存(每30分钟)
let refresh_cache = Arc::clone(&albums_cache);
let refresh_db = Arc::clone(&db);
// ── 定时管线(每 30 分钟):导入 → 增量解析(有新相册时)→ 缓存刷新(始终执行)──
let timer_cache = Arc::clone(&albums_cache);
let timer_db = Arc::clone(&db);
let timer_search_db = Arc::clone(&search_db);
let timer_persons_cache = Arc::clone(&persons_cache);
let timer_tags_cache = Arc::clone(&tags_cache);
tokio::spawn(async move {
let mut interval = interval(Duration::from_secs(30 * 60));
loop {
interval.tick().await;
match refresh_db.get_all_albums().await {
Ok(albums) => {
let mut cache = refresh_cache.write().await;
cache.update(albums);
println!("🔄 相册缓存定时刷新完成");
}
Err(e) => {
eprintln!("❌ 缓存刷新失败: {}", e);
}
// ① 扫描导入新相册
info!("[定时] ① 扫描导入新相册...");
match services::import::import_existing_data(&timer_db.config, Arc::clone(&timer_db), None).await {
Ok(_) => {},
Err(e) => error!("[定时] 导入失败: {}", e),
}
// ② 查找新相册并增量解析
let new_albums = engine::pipeline::phase_collect_new_albums(&timer_db).await;
if !new_albums.is_empty() {
info!("[定时] 发现 {} 个新相册,增量解析(同时写入 imager.db + search.db)...", new_albums.len());
engine::pipeline::phase_parse_dirty(&timer_db, &timer_search_db, &new_albums).await;
}
// ③ 缓存刷新(始终执行)
info!("[定时] 刷新缓存...");
cache::refresh_all_caches(&timer_db, &timer_search_db, &timer_cache, &timer_persons_cache, &timer_tags_cache).await;
}
});
@@ -133,9 +202,11 @@ async fn main() -> std::io::Result<()> {
println!(" - GET /api/people/search?q=关键词 - 搜索人员");
println!(" - GET /api/people/{{name}} - 人员详情");
println!(" - GET /api/people/{{id}}/aliases - 人员别名");
println!(" - POST /api/people/delete/{{name}} - 删除人员并重解析受影响相册");
println!(" - GET /api/metadata/stats - 元数据统计");
println!(" - GET /api/tags/search?tag=标签 - 按标签搜索");
println!(" - GET /api/tags/popular?limit=50 - 热门标签");
println!(" - POST /api/tags/delete/{{name}} - 删除标签并重解析受影响相册");
println!("推荐接口:");
println!(" - GET /api/recommend/tags?q=xxx - 标签推荐");
println!(" - GET /api/recommend/persons?q=xxx - 人物推荐");
@@ -145,17 +216,33 @@ async fn main() -> std::io::Result<()> {
println!(" - GET /api/migration/stats - 迁移状态");
println!(" - POST /api/migration/run?path=persons.db - 执行迁移");
println!(" - GET /api/queue/stats - 队列状态");
println!("管线接口:");
println!(" - POST /api/pipeline/run - 执行管线(增量)");
println!(" - POST /api/pipeline/reparse - 重置并重新解析全部相册(标签+人物)");
println!(" - POST /api/pipeline/reparse-persons - 重置并重新解析全部人物");
println!(" - POST /api/pipeline/reparse-tags - 重置并重新解析全部标签");
println!("词典管理:");
println!(" - GET /api/dict/stats - 词典状态");
println!(" - POST /api/dict/reload - 热重载词典");
println!(" - POST /api/dict/update - 批量更新 dict 表(自动重载内存词典)");
println!("缓存管理:");
println!(" - POST /api/cache/refresh - 手动刷新内存缓存");
println!("按 Ctrl+C 停止服务");
fs::create_dir_all(&config.storage.images_dir).unwrap_or(());
HttpServer::new(move || {
// 在 move 闭包之前 clone 一份 db 用于关闭信号处理
let shutdown_db = Arc::clone(&db);
let server = HttpServer::new(move || {
App::new()
.wrap(middleware::Compress::default())
.wrap(middleware::Logger::default())
.app_data(web::Data::new(Arc::clone(&db)))
.app_data(web::Data::new(Arc::clone(&search_db)))
.app_data(web::Data::new(Arc::clone(&image_processor)))
.app_data(web::Data::new(Arc::clone(&albums_cache)))
.app_data(web::Data::new(Arc::clone(&persons_cache)))
.app_data(web::Data::new(Arc::clone(&tags_cache)))
.app_data(web::Data::new(Arc::clone(&task_queue)))
// 相册 API
.route("/api/albums", web::get().to(routes::get_albums))
@@ -175,10 +262,12 @@ async fn main() -> std::io::Result<()> {
.route("/api/people/search", web::get().to(routes::search_people))
.route("/api/people/{name}", web::get().to(routes::get_person_detail))
.route("/api/people/{id}/aliases", web::get().to(routes::get_person_aliases))
.route("/api/people/delete/{name}", web::post().to(routes::delete_person))
// 标签 API
.route("/api/metadata/stats", web::get().to(routes::get_metadata_stats))
.route("/api/tags/search", web::get().to(routes::search_by_tag))
.route("/api/tags/popular", web::get().to(routes::get_popular_tags))
.route("/api/tags/delete/{name}", web::post().to(routes::delete_tag))
// 推荐 API
.route("/api/recommend/tags", web::get().to(routes::recommend_tags))
.route("/api/recommend/persons", web::get().to(routes::recommend_persons))
@@ -189,8 +278,17 @@ async fn main() -> std::io::Result<()> {
.route("/api/migration/run", web::post().to(routes::migration_run))
// 管线 API
.route("/api/pipeline/run", web::post().to(routes::pipeline_run))
.route("/api/pipeline/reparse", web::post().to(routes::pipeline_reparse))
.route("/api/pipeline/reparse-persons", web::post().to(routes::pipeline_reparse_persons))
.route("/api/pipeline/reparse-tags", web::post().to(routes::pipeline_reparse_tags))
// 队列 API
.route("/api/queue/stats", web::get().to(routes::get_queue_stats))
// 词典 API
.route("/api/dict/stats", web::get().to(routes::dict_stats))
.route("/api/dict/reload", web::post().to(routes::dict_reload))
.route("/api/dict/update", web::post().to(routes::dict_update))
// 缓存管理 API
.route("/api/cache/refresh", web::post().to(routes::cache_refresh))
// 静态文件
.service(
afs::Files::new("/assets", "./assets")
@@ -217,6 +315,20 @@ async fn main() -> std::io::Result<()> {
}))
})
.bind(config.server_address())?
.run()
.await
.run();
// ── 信号监听:捕获 Ctrl+C / SIGINT,优雅关闭 ──
let server_handle = server.handle();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.expect("监听关闭信号失败");
println!("\n[关闭] 收到关闭信号,正在优雅停止服务器...");
// 停止接受新请求,等待正在处理中的请求完成(最多 10s)
server_handle.stop(true).await;
// 服务器已停止 → 合并 WAL
println!("[关闭] 服务已停止,正在合并数据库 WAL...");
shutdown_db.wal_checkpoint().await;
println!("[关闭] 数据库文件已合并,再见!");
});
server.await
}

View File

@@ -10,10 +10,6 @@ pub struct Album {
pub image_count: i64,
pub created_at: String,
pub updated_at: String,
#[serde(default)]
pub person_name_cleaned: Option<String>,
#[serde(default)]
pub tags: Option<String>, // JSON: [tag, ...]
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -25,6 +21,8 @@ pub struct AlbumInfo {
pub cover_image_id: Option<String>,
pub image_count: i64,
pub created_at: String,
pub tags: Vec<String>,
pub persons: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -32,6 +30,8 @@ pub struct AlbumWithImages {
pub album: Album,
pub images: Vec<super::image::Image>,
pub image_count: i64,
pub tags: Vec<String>,
pub persons: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -47,16 +47,11 @@ pub struct AlbumIndex {
pub images: Vec<String>,
}
/// 相册完整元数据(含扩展字段)
/// 相册完整元数据(精简版:仅保留 albums 表原始字段)
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AlbumMeta {
pub id: String,
pub title: Option<String>,
pub person_name_original: Option<String>,
pub person_name_cleaned: Option<String>,
pub tags: Option<String>,
pub all_words: Option<String>,
pub raw_title: Option<String>,
pub processed_at: Option<String>,
pub cover_image_id: Option<String>,
pub image_count: Option<i64>,
}

View File

@@ -4,6 +4,7 @@ pub mod person;
pub mod tag;
pub mod favorite;
pub mod recommend;
pub mod parse;
pub use album::*;
pub use image::*;

25
src/models/parse.rs Normal file
View File

@@ -0,0 +1,25 @@
//! 相册解析结果 — 单个相册标题的一次性解析产物
//! 用于串联并行解析(Phase 1a)和批量写库(Phase 1b)
use serde::{Serialize, Deserialize};
/// 一次 title 解析的完整结果
///
/// 在一个相册上通过一次分词,同时提取:
/// - 人名列表(支持多个)
/// - 标签列表(含分类)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParseResult {
/// 相册 UUID
pub album_id: String,
/// 最终识别的人名列表 | 空 = 未找到
pub person_names: Vec<String>,
/// 标签列表 (tag, category)
pub tags: Vec<(String, String)>,
}
impl ParseResult {
pub fn new(album_id: String) -> Self {
Self { album_id, person_names: Vec::new(), tags: Vec::new() }
}
}

View File

@@ -3,7 +3,7 @@ use super::album::AlbumInfo;
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Person {
pub id: i64,
pub id: String,
pub name: String,
pub display_name: Option<String>,
pub total_albums: i64,
@@ -16,7 +16,7 @@ pub struct Person {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct PersonAlias {
pub id: i64,
pub person_id: i64,
pub person_id: String,
pub alias_type: String,
pub alias_name: String,
pub created_at: String,
@@ -31,7 +31,7 @@ pub struct PersonWithAlbums {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct PersonWithCounts {
pub id: i64,
pub id: String,
pub name: String,
pub display_name: Option<String>,
pub total_albums: i64,

View File

@@ -20,8 +20,8 @@ pub struct TagListResponse {
pub struct MetadataStats {
pub total_persons: i64,
pub total_aliases: i64,
pub total_albums_with_person: i64,
pub total_albums_without_person: i64,
pub total_albums: i64,
pub total_images: i64,
pub total_tags: i64,
}

View File

@@ -1,5 +1,6 @@
use crate::cache::AlbumsCache;
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
@@ -7,38 +8,68 @@ use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
/// GET /api/albums — 从缓存返回全量相册(无过滤时)
pub async fn get_albums(cache: web::Data<Arc<RwLock<AlbumsCache>>>) -> impl Responder {
let c = cache.read().await;
HttpResponse::Ok().json(&c.albums)
}
/// GET /api/search — 组合搜索(支持 tag/person/q 过滤)
///
/// 参数:
/// - q: 关键词(标题模糊匹配)
/// - tag: 标签名(精确匹配)
/// - person: 人物名(精确或模糊匹配)
/// - page: 页码(默认 1)
/// - limit: 每页条数(默认 50,最大 200)
///
/// 规则:有任意过滤条件时 → 走 DB 懒加载(只查符合条件的相册)
/// 无过滤条件时 → 走缓存全量返回
pub async fn search_albums(
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<AlbumsCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
if q.len() > 100 { return HttpResponse::BadRequest().json(serde_json::json!({"error": "搜索查询过长"})); }
if q.chars().any(|c| c.is_control() || c == '\0') { return HttpResponse::BadRequest().json(serde_json::json!({"error": "无效字符"})); }
let tag = query.get("tag").map(|s| s.as_str()).unwrap_or("");
let person = query.get("person").map(|s| s.as_str()).unwrap_or("");
let page: usize = query.get("page").and_then(|s| s.parse().ok()).unwrap_or(1);
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(50).min(200);
let offset = (page - 1) * limit;
// 校验
let err = |msg: &str| HttpResponse::BadRequest().json(serde_json::json!({"error": msg}));
if q.len() > 100 { return err("搜索查询过长"); }
if q.chars().any(|c| c.is_control() || c == '\0') { return err("无效字符"); }
let has_filter = !q.is_empty() || !tag.is_empty() || !person.is_empty();
if has_filter {
// ── 有过滤条件:走 DB 懒加载 ──
let page: i64 = query.get("page").and_then(|s| s.parse().ok()).unwrap_or(1);
let limit: i64 = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(50).min(200);
let offset = (page - 1) * limit;
let tag_arg = if tag.is_empty() { None } else { Some(tag) };
let person_arg = if person.is_empty() { None } else { Some(person) };
let q_arg = if q.is_empty() { None } else { Some(q) };
match search_db.search_albums(tag_arg, person_arg, q_arg, limit, offset).await {
Ok(results) => HttpResponse::Ok().json(results),
Err(e) => HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{}", e)})),
}
} else {
// ── 无过滤条件:走缓存(现有行为) ──
let c = cache.read().await;
let page: usize = query.get("page").and_then(|s| s.parse().ok()).unwrap_or(1);
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(50).min(200);
let offset = (page - 1) * limit;
let c = cache.read().await;
let results = if q.is_empty() {
let total = c.albums.len();
let start = offset.min(total);
let end = (offset + limit).min(total);
c.albums[start..end].to_vec()
} else {
let mut results = c.search(q);
let total = results.len();
let start = offset.min(total);
let end = (offset + limit).min(total);
if start < end { results.drain(start..end).collect() } else { Vec::new() }
};
let results = c.albums[start..end].to_vec();
HttpResponse::Ok().json(results)
HttpResponse::Ok().json(results)
}
}
pub async fn get_random_album(cache: web::Data<Arc<RwLock<AlbumsCache>>>) -> impl Responder {
@@ -48,10 +79,30 @@ pub async fn get_random_album(cache: web::Data<Arc<RwLock<AlbumsCache>>>) -> imp
HttpResponse::Ok().json(&c.albums[idx])
}
pub async fn get_album_detail(db: web::Data<Arc<Database>>, path: web::Path<String>) -> impl Responder {
pub async fn get_album_detail(
db: web::Data<Arc<Database>>,
cache: web::Data<Arc<RwLock<AlbumsCache>>>,
path: web::Path<String>,
) -> impl Responder {
let id = match Uuid::parse_str(&path.into_inner()) { Ok(id) => id, Err(_) => return HttpResponse::BadRequest().json(serde_json::json!({"error": "无效ID"})) };
let id_str = id.to_string();
// 从内存缓存查 tags/persons
let (tags, persons) = {
let c = cache.read().await;
let found = c.albums.iter().find(|a| a.id == id);
match found {
Some(a) => (a.tags.clone(), a.persons.clone()),
None => (vec![], vec![]),
}
};
match db.get_album_with_images(id).await {
Ok(Some(album)) => HttpResponse::Ok().json(album),
Ok(Some(mut album)) => {
album.tags = tags;
album.persons = persons;
HttpResponse::Ok().json(album)
}
Ok(None) => HttpResponse::NotFound().json(serde_json::json!({"error": "相册不存在"})),
Err(e) => HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{}", e)})),
}

27
src/routes/cache.rs Normal file
View File

@@ -0,0 +1,27 @@
use crate::cache;
use crate::db::Database;
use crate::search_db::SearchDb;
use crate::cache::{AlbumsCache, TagCache, PersonCache};
use actix_web::{HttpResponse, Responder, web};
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::info;
/// POST /api/cache/refresh — 手动刷新所有内存缓存
pub async fn cache_refresh(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
albums_cache: web::Data<Arc<RwLock<AlbumsCache>>>,
persons_cache: web::Data<Arc<RwLock<PersonCache>>>,
tags_cache: web::Data<Arc<RwLock<TagCache>>>,
) -> impl Responder {
info!("[API] 手动触发缓存刷新...");
cache::refresh_all_caches(
db.get_ref(),
search_db.get_ref(),
albums_cache.get_ref(),
persons_cache.get_ref(),
tags_cache.get_ref(),
).await;
HttpResponse::Ok().json(serde_json::json!({"success": true, "message": "缓存已刷新"}))
}

83
src/routes/dict.rs Normal file
View File

@@ -0,0 +1,83 @@
//! 标签词典管理 API — 查看/热重载
use std::sync::Arc;
use actix_web::{HttpResponse, Responder, web};
use crate::engine::tag_dict;
use crate::search_db::SearchDb;
/// GET /api/dict/stats — 查看词典状态
pub async fn dict_stats() -> impl Responder {
let stats = tag_dict::get_tag_dict_stats();
let dict = tag_dict::get_tag_dict();
let mut categories: Vec<serde_json::Value> = Vec::new();
for name in dict.get_category_order() {
if let Some(cat) = dict.get_category(name) {
categories.push(serde_json::json!({
"name": name,
"label": cat.label,
"count": cat.words.len(),
}));
}
}
HttpResponse::Ok().json(serde_json::json!({
"stats": stats,
"categories": categories,
"total_tags": dict.get_tag_category_map().len(),
"total_stop_words": dict.get_stop_words().len(),
"person_prefixes": dict.get_person_prefixes(),
"excluded_person_names": dict.excluded_person_names.iter().collect::<Vec<_>>(),
"included_person_names": dict.included_person_names.iter().collect::<Vec<_>>(),
}))
}
/// POST /api/dict/reload — 热重载词典
pub async fn dict_reload(search_db: web::Data<Arc<SearchDb>>) -> impl Responder {
let msg = tag_dict::reload_from_search_db(search_db.get_ref()).await;
println!("[API] 词典热重载: {}", msg);
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": msg,
}))
}
/// dict 条目请求体
#[derive(serde::Deserialize)]
pub struct DictEntryInput {
pub word: String,
#[serde(rename = "type")]
pub type_: String,
pub category: Option<String>,
pub freq: Option<i64>,
pub merged_to: Option<String>,
}
/// POST /api/dict/update — 更新 dict 表(新增或覆盖条目)
pub async fn dict_update(
search_db: web::Data<Arc<SearchDb>>,
body: web::Json<Vec<DictEntryInput>>,
) -> impl Responder {
let entries = body.into_inner();
let mut inserted = 0usize;
for e in &entries {
match search_db.upsert_dict_entry(
&e.word, &e.type_, e.category.as_deref(), e.freq, e.merged_to.as_deref(),
).await {
Ok(_) => inserted += 1,
Err(err) => {
return HttpResponse::InternalServerError().json(serde_json::json!({
"success": false,
"message": format!("更新词条 '{}' 失败: {}", e.word, err),
}));
}
}
}
// 自动重载内存词典
let msg = tag_dict::reload_from_search_db(search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"message": format!("更新 {} 条字典条目,{}", inserted, msg),
"updated": inserted,
}))
}

View File

@@ -1,5 +1,5 @@
use crate::db::Database;
use crate::db::MigrationStats;
#[allow(unused)] use crate::db::MigrationStats;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
use std::sync::Arc;

View File

@@ -1,6 +1,7 @@
//! API 路由层 — 按领域拆分
pub mod album;
pub mod cache;
pub mod image;
pub mod favorite;
pub mod person;
@@ -9,8 +10,10 @@ pub mod recommend;
pub mod queue;
pub mod migration;
pub mod pipeline;
pub mod dict;
pub use album::*;
pub use cache::*;
pub use image::*;
pub use favorite::*;
pub use person::*;
@@ -19,3 +22,4 @@ pub use recommend::*;
pub use queue::*;
pub use migration::*;
pub use pipeline::*;
pub use dict::*;

View File

@@ -1,36 +1,190 @@
use crate::cache::{PersonCache, AlbumsCache, TagCache};
use crate::db::Database;
use crate::engine;
use crate::search_db::SearchDb;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use sqlx::Row;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
pub async fn get_people(db: web::Data<Arc<Database>>) -> impl Responder {
let persons = db.get_all_persons_with_aliases().await;
HttpResponse::Ok().json(PersonListResponse { total: persons.len(), persons })
/// GET /api/people — 从内存缓存获取所有人名列表
pub async fn get_people(cache: web::Data<Arc<RwLock<PersonCache>>>) -> impl Responder {
let c = cache.read().await;
HttpResponse::Ok().json(PersonListResponse {
total: c.persons.len(),
persons: c.persons.clone(),
})
}
pub async fn search_people(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
/// GET /api/people/search?q=关键词 — 从内存缓存搜索人名
pub async fn search_people(
cache: web::Data<Arc<RwLock<PersonCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
let c = cache.read().await;
if q.is_empty() {
let persons = db.get_all_persons_with_aliases().await;
return HttpResponse::Ok().json(PersonListResponse { total: persons.len(), persons });
return HttpResponse::Ok().json(PersonListResponse {
total: c.persons.len(),
persons: c.persons.clone(),
});
}
let persons = db.search_persons(q).await;
HttpResponse::Ok().json(PersonListResponse { total: persons.len(), persons })
let persons = c.search(q);
HttpResponse::Ok().json(PersonListResponse {
total: persons.len(),
persons,
})
}
pub async fn get_person_detail(db: web::Data<Arc<Database>>, path: web::Path<String>) -> impl Responder {
/// GET /api/people/{name} — 查找人物详情
/// 人物基本信息从内存缓存获取,相册列表从 imager.db 查询
pub async fn get_person_detail(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<PersonCache>>>,
path: web::Path<String>,
) -> impl Responder {
let name = path.into_inner();
match db.get_person_by_name(&name).await {
let c = cache.read().await;
let found = c.search(&name);
let p = found.into_iter().find(|p| p.name.to_lowercase() == name.to_lowercase());
match p {
Some(p) => {
let albums = db.get_person_albums(p.id).await;
HttpResponse::Ok().json(PersonWithAlbums { album_count: albums.len(), person: p, albums })
// 从 search.db 获取相册 ID 列表
let album_ids = search_db.get_person_album_ids(&p.id).await;
// 从 imager.db 查询相册详情
let albums = if album_ids.is_empty() {
Vec::new()
} else {
let placeholders: Vec<String> = (0..album_ids.len()).map(|_| "?".to_string()).collect();
let sql = format!(
"SELECT id, title, document, description, cover_image_id, image_count, created_at \
FROM albums WHERE id IN ({}) ORDER BY created_at DESC",
placeholders.join(", ")
);
let mut query = sqlx::query(&sql);
for id in &album_ids {
query = query.bind(id);
}
query.fetch_all(&db.pool).await.unwrap_or_default()
.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: r.get("document"),
description: r.get("description"),
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}
}).collect()
};
HttpResponse::Ok().json(PersonWithAlbums {
album_count: albums.len(),
person: Person {
id: p.id,
name: p.name,
display_name: p.display_name,
total_albums: p.total_albums,
total_images: p.total_images,
original_names: None,
created_at: String::new(),
updated_at: String::new(),
},
albums,
})
}
None => HttpResponse::NotFound().json(serde_json::json!({"error": format!("人员 '{}' 不存在", name)})),
}
}
pub async fn get_person_aliases(db: web::Data<Arc<Database>>, path: web::Path<i64>) -> impl Responder {
let aliases = db.get_person_aliases(path.into_inner()).await;
/// GET /api/people/{id}/aliases — 从 search.db 查别名
pub async fn get_person_aliases(search_db: web::Data<Arc<SearchDb>>, path: web::Path<String>) -> impl Responder {
let aliases = search_db.get_person_aliases(&path.into_inner()).await;
HttpResponse::Ok().json(aliases)
}
/// POST /api/people/delete/{name} — 删除 person(从 imager.db + search.db)并重解析受影响的相册
pub async fn delete_person(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
albums_cache: web::Data<Arc<RwLock<AlbumsCache>>>,
persons_cache: web::Data<Arc<RwLock<PersonCache>>>,
tags_cache: web::Data<Arc<RwLock<TagCache>>>,
path: web::Path<String>,
) -> impl Responder {
let person_name = path.into_inner().trim().to_string();
if person_name.is_empty() {
return HttpResponse::BadRequest().json(serde_json::json!({
"success": false, "message": "人名不能为空"
}));
}
// 1. 从 search.db 删除并获取受影响 album_ids
let album_ids = match search_db.delete_person_and_get_albums(&person_name).await {
Ok(ids) => ids,
Err(e) => return HttpResponse::InternalServerError().json(serde_json::json!({
"success": false,
"message": format!("从 search.db 删除人物 '{}' 失败: {}", person_name, e),
})),
};
// 2. 从 imager.db 删除(persons + person_aliases + person_albums)
// person.id 来自 imager.db,存储为 i64,模型层转为 String,这里转回 i64
if let Some(person) = db.get_person_by_name(&person_name).await {
if let Ok(pid) = person.id.parse::<i64>() {
db.delete_person_completely(pid).await;
}
}
// 3. 重置受影响相册的 processed_at(触发重新解析)
for aid in &album_ids {
sqlx::query("UPDATE albums SET processed_at = NULL WHERE id = ?")
.bind(aid)
.execute(&db.pool)
.await
.ok();
}
// 4. 重新解析受影响相册,同步到 search.db
if !album_ids.is_empty() {
let mut dirty = engine::pipeline::DirtySet::new();
for aid in &album_ids { dirty.add(aid); }
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &search_db, &dirty).await;
// 5. 重载内存词典
engine::tag_dict::reload_from_search_db(&search_db).await;
// 6. 刷新缓存
crate::cache::refresh_all_caches(&db, &search_db, &albums_cache, &persons_cache, &tags_cache).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"person": person_name,
"albums_affected": album_ids.len(),
"persons_updated": persons_updated,
"tags_updated": tags_updated,
"message": format!(
"人物 '{}' 已删除,影响 {} 个相册,重新解析完成(更新人物 {},更新标签 {})",
person_name, album_ids.len(), persons_updated, tags_updated,
)
}))
} else {
// 无受影响相册,只需刷新缓存
crate::cache::refresh_all_caches(&db, &search_db, &albums_cache, &persons_cache, &tags_cache).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"person": person_name,
"albums_affected": 0,
"message": format!("人物 '{}' 已删除,无关联相册", person_name),
}))
}
}

View File

@@ -1,15 +1,144 @@
use crate::db::Database;
use crate::engine::pipeline;
use crate::engine;
use crate::search_db::SearchDb;
use actix_web::{HttpResponse, Responder, web};
use std::sync::Arc;
/// POST /api/pipeline/run — 处理所有相册元数据(提取人名+标签)
pub async fn pipeline_run(db: web::Data<Arc<Database>>) -> impl Responder {
let (persons, tags) = pipeline::process_all_albums(db.get_ref()).await;
pub async fn pipeline_run(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
let result = engine::pipeline::run_pipeline(db.get_ref(), search_db.get_ref()).await;
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"persons_updated": persons,
"tags_updated": tags,
"message": format!("处理完成:{} 个人名,{} 个标签", persons, tags)
"tags_cleaned": result.tags_cleaned,
"short_tags_removed": result.short_tags_removed,
"person_like_tags_removed": result.person_like_tags_removed,
"dirty_albums_detected": result.dirty_albums_detected,
"non_persons_deleted": result.non_persons_deleted,
"stop_word_persons_removed": result.stop_word_persons_removed,
"new_albums_found": result.new_albums_found,
"persons_updated": result.persons_updated,
"tags_updated": result.tags_updated,
"empty_albums_deleted": result.empty_albums_deleted,
"dup_albums_deleted": result.dup_albums_deleted,
"search_db_synced": synced,
"message": format!(
"管线执行完毕:删除空相册{},去重{},清洗标点{}相册,移除短标签{},检测人名标签{},脏集{}相册,删除非人物{},清除停用词重叠人物{},发现新相册{},更新人名{},更新标签{},同步标签/人物到search.db{}",
result.empty_albums_deleted, result.dup_albums_deleted,
result.tags_cleaned, result.short_tags_removed, result.person_like_tags_removed, result.dirty_albums_detected,
result.non_persons_deleted, result.stop_word_persons_removed, result.new_albums_found,
result.persons_updated, result.tags_updated, synced
)
}))
}
/// POST /api/pipeline/reparse — 重置所有相册为未处理状态,重新解析全部标签和人物并同步到 search.db
pub async fn pipeline_reparse(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
// 1. 清除 search.db 中的旧关联(不影响 tags/persons 字典)
sqlx::query("DELETE FROM album_tags").execute(&search_db.pool).await.ok();
sqlx::query("DELETE FROM album_persons").execute(&search_db.pool).await.ok();
// 2. 重置所有相册为未处理
let reset_count = db.reset_all_processed_at().await;
tracing::info!("[reparse] 重置 {} 个相册的 processed_at", reset_count);
// 3. 重新运行完整管线+同步
let result = engine::pipeline::run_pipeline(db.get_ref(), search_db.get_ref()).await;
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"albums_parsed": result.new_albums_found,
"persons_updated": result.persons_updated,
"tags_updated": result.tags_updated,
"synced_to_search_db": synced,
"message": format!(
"重新解析完成:重置{}个相册,解析{}个,更新人物{}个,标签{}个,同步{}个到search.db",
reset_count, result.new_albums_found,
result.persons_updated, result.tags_updated, synced,
)
}))
}
/// POST /api/pipeline/reparse-persons — 重置所有相册,重新解析全部人物
///
/// 流程:
/// 清空 search.db 的 album_persons → 重置所有相册 processed_at →
/// 运行完整管线(人物相关逻辑全部执行)→ 全量同步到 search.db
pub async fn pipeline_reparse_persons(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
// 1. 清空 search.db 中的人物相关表(album_persons + persons + person_aliases)
sqlx::query("DELETE FROM album_persons").execute(&search_db.pool).await.ok();
sqlx::query("DELETE FROM person_aliases").execute(&search_db.pool).await.ok();
sqlx::query("DELETE FROM persons").execute(&search_db.pool).await.ok();
// 2. 重置所有相册为未处理(触发管线重新解析)
let reset_count = db.reset_all_processed_at().await;
tracing::info!("[reparse-persons] 重置 {} 个相册的 processed_at", reset_count);
// 3. 重新运行完整管线(Phase ② 人物清理 + Phase ③ 解析 + Phase ④ 人物合并 + Phase C 重计)
let result = engine::pipeline::run_pipeline(db.get_ref(), search_db.get_ref()).await;
// 4. 全量同步到 search.db(tags 和 persons 都会同步,但 album_persons 已清空确保了干净状态)
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"albums_parsed": result.new_albums_found,
"persons_updated": result.persons_updated,
"non_persons_deleted": result.non_persons_deleted,
"stop_word_persons_removed": result.stop_word_persons_removed,
"synced_to_search_db": synced,
"message": format!(
"人物重新解析完成:重置{}个相册,解析{}个,更新人物{}个,删除非人物{},清除重叠{},同步{}个到search.db",
reset_count, result.new_albums_found,
result.persons_updated,
result.non_persons_deleted, result.stop_word_persons_removed, synced,
)
}))
}
/// POST /api/pipeline/reparse-tags — 重置所有相册,重新解析全部标签
///
/// 流程:
/// 清空 search.db 的 album_tags → 重置所有相册 processed_at →
/// 运行完整管线(标签相关逻辑全部执行)→ 全量同步到 search.db
pub async fn pipeline_reparse_tags(
db: web::Data<Arc<Database>>,
search_db: web::Data<Arc<SearchDb>>,
) -> impl Responder {
// 1. 清空 search.db 中的标签相关表(album_tags + tags)
sqlx::query("DELETE FROM album_tags").execute(&search_db.pool).await.ok();
sqlx::query("DELETE FROM tags").execute(&search_db.pool).await.ok();
// 2. 重置所有相册为未处理(触发管线重新解析)
let reset_count = db.reset_all_processed_at().await;
tracing::info!("[reparse-tags] 重置 {} 个相册的 processed_at", reset_count);
// 3. 重新运行完整管线(Phase ③ 解析会重新提取全部标签)
let result = engine::pipeline::run_pipeline(db.get_ref(), search_db.get_ref()).await;
// 4. 全量同步到 search.db
let synced = engine::pipeline::sync_albums_to_search_db(db.get_ref(), search_db.get_ref()).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"albums_parsed": result.new_albums_found,
"tags_updated": result.tags_updated,
"short_tags_removed": result.short_tags_removed,
"synced_to_search_db": synced,
"message": format!(
"标签重新解析完成:重置{}个相册,解析{}个,更新标签{}个,移除短标签{},同步{}个到search.db",
reset_count, result.new_albums_found,
result.tags_updated, result.short_tags_removed, synced,
)
}))
}

View File

@@ -1,36 +1,54 @@
use crate::db::Database;
use crate::cache::{TagCache, PersonCache};
use crate::search_db::SearchDb;
use crate::engine::RecommendEngine;
use crate::models::*;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
/// 标签推荐(自动补全)
pub async fn recommend_tags(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
/// 标签推荐(自动补全)— 从 TagCache 匹配
pub async fn recommend_tags(
cache: web::Data<Arc<RwLock<TagCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10);
HttpResponse::Ok().json(RecommendEngine::recommend_tags(&db, q, limit).await)
let c = cache.read().await;
HttpResponse::Ok().json(RecommendEngine::recommend_tags(&c.popular_tags, q, limit))
}
/// 人物推荐(自动补全)
pub async fn recommend_persons(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
/// 人物推荐(自动补全)— 从 PersonCache 匹配
pub async fn recommend_persons(
cache: web::Data<Arc<RwLock<PersonCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let q = query.get("q").map(|s| s.as_str()).unwrap_or("");
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(10);
HttpResponse::Ok().json(RecommendEngine::recommend_persons(&db, q, limit).await)
let c = cache.read().await;
HttpResponse::Ok().json(RecommendEngine::recommend_persons(&c.persons, q, limit))
}
/// 关联推荐(标签共现 + 共现人物)
pub async fn recommend_related(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
pub async fn recommend_related(
search_db: web::Data<Arc<SearchDb>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let context = match query.get("context") {
Some(c) if !c.is_empty() => c,
_ => return HttpResponse::BadRequest().json(serde_json::json!({"error": "请提供context参数"})),
};
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(15);
HttpResponse::Ok().json(RecommendEngine::recommend_related(&db, context, limit).await)
HttpResponse::Ok().json(RecommendEngine::recommend_related(&search_db, context, limit).await)
}
/// 热门人物
pub async fn popular_persons(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
/// 热门人物 — 从 PersonCache 过滤,top_tags 保留 search.db 查询
pub async fn popular_persons(
search_db: web::Data<Arc<SearchDb>>,
cache: web::Data<Arc<RwLock<PersonCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(20).min(50);
HttpResponse::Ok().json(RecommendEngine::popular_persons(&db, limit).await)
let c = cache.read().await;
HttpResponse::Ok().json(RecommendEngine::popular_persons(&search_db, &c.persons, limit).await)
}

View File

@@ -1,24 +1,111 @@
use crate::db::Database;
use crate::cache::{TagCache, AlbumsCache, PersonCache};
use crate::engine;
use crate::models::*;
use crate::search_db::SearchDb;
use actix_web::{HttpResponse, Responder, web};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
pub async fn get_metadata_stats(db: web::Data<Arc<Database>>) -> impl Responder {
HttpResponse::Ok().json(db.get_metadata_stats().await)
/// GET /api/metadata/stats — 从内存缓存计算统计数据
pub async fn get_metadata_stats(
albums_cache: web::Data<Arc<RwLock<AlbumsCache>>>,
persons_cache: web::Data<Arc<RwLock<PersonCache>>>,
tags_cache: web::Data<Arc<RwLock<TagCache>>>,
) -> impl Responder {
let ac = albums_cache.read().await;
let pc = persons_cache.read().await;
let tc = tags_cache.read().await;
let stats = MetadataStats {
total_persons: pc.persons.len() as i64,
total_aliases: pc.persons.iter().map(|p| p.aliases.len() as i64).sum(),
total_albums: ac.albums.len() as i64,
total_images: ac.albums.iter().map(|a| a.image_count).sum(),
total_tags: tc.tag_albums.len() as i64,
};
HttpResponse::Ok().json(stats)
}
pub async fn search_by_tag(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
/// GET /api/tags/search?tag=xxx — 从缓存按标签搜索相册
pub async fn search_by_tag(
cache: web::Data<Arc<RwLock<TagCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let tag = match query.get("tag") {
Some(t) if !t.is_empty() => t,
_ => return HttpResponse::BadRequest().json(serde_json::json!({"error": "请提供tag参数"})),
};
let albums = db.search_albums_by_tag(tag).await;
let c = cache.read().await;
let albums = c.search_albums(tag);
HttpResponse::Ok().json(albums)
}
pub async fn get_popular_tags(db: web::Data<Arc<Database>>, query: web::Query<HashMap<String, String>>) -> impl Responder {
let limit: i64 = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(50);
let tags = db.get_popular_tags(limit).await;
/// GET /api/tags/popular?limit=50 — 从缓存获取热门标签
pub async fn get_popular_tags(
cache: web::Data<Arc<RwLock<TagCache>>>,
query: web::Query<HashMap<String, String>>,
) -> impl Responder {
let limit: usize = query.get("limit").and_then(|s| s.parse().ok()).unwrap_or(50);
let c = cache.read().await;
let tags: Vec<TagStat> = c.popular_tags.iter().take(limit).cloned().collect();
HttpResponse::Ok().json(TagListResponse { total: tags.len(), tags })
}
/// POST /api/tags/delete/{name} — 删除 tag(从 dict/album_tags/tags)并重解析受影响的相册
pub async fn delete_tag(
db: web::Data<Arc<crate::db::Database>>,
search_db: web::Data<Arc<SearchDb>>,
albums_cache: web::Data<Arc<RwLock<AlbumsCache>>>,
persons_cache: web::Data<Arc<RwLock<PersonCache>>>,
tags_cache: web::Data<Arc<RwLock<TagCache>>>,
path: web::Path<String>,
) -> impl Responder {
let tag_name = path.into_inner().trim().to_string();
if tag_name.is_empty() {
return HttpResponse::BadRequest().json(serde_json::json!({
"success": false, "message": "标签名不能为空"
}));
}
// 1. 删除 tag(dict/album_tags/tags),获取受影响 album_ids
let album_ids = match search_db.delete_tag_and_get_albums(&tag_name).await {
Ok(ids) => ids,
Err(e) => return HttpResponse::InternalServerError().json(serde_json::json!({
"success": false,
"message": format!("删除标签 '{}' 失败: {}", tag_name, e),
})),
};
// 2. 重置受影响相册的 processed_at,触发重新解析
for aid in &album_ids {
sqlx::query("UPDATE albums SET processed_at = NULL WHERE id = ?")
.bind(aid)
.execute(&db.pool)
.await
.ok();
}
// 3. 重新解析受影响相册(同时写入 imager.db + search.db)
let mut dirty = engine::pipeline::DirtySet::new();
for aid in &album_ids { dirty.add(aid); }
let (persons_updated, tags_updated) = engine::pipeline::phase_parse_dirty(&db, &search_db, &dirty).await;
// 4. 重载内存词典(dict 表已更新)
engine::tag_dict::reload_from_search_db(&search_db).await;
// 5. 刷新内存缓存
crate::cache::refresh_all_caches(&db, &search_db, &albums_cache, &persons_cache, &tags_cache).await;
HttpResponse::Ok().json(serde_json::json!({
"success": true,
"tag": tag_name,
"albums_affected": album_ids.len(),
"persons_updated": persons_updated,
"tags_updated": tags_updated,
"message": format!(
"标签 '{}' 已删除,影响 {} 个相册,重新解析完成(更新人物 {},更新标签 {})",
tag_name, album_ids.len(), persons_updated, tags_updated,
)
}))
}

557
src/search_db/mod.rs Normal file
View File

@@ -0,0 +1,557 @@
mod search;
pub use search::*;
use sqlx::{SqlitePool, sqlite::SqlitePoolOptions, Row};
use uuid::Uuid;
/// 从 name 计算确定性 tag UUID(无需查数据库即可得到 id)
pub fn tag_uuid(name: &str) -> String {
Uuid::new_v5(
&Uuid::NAMESPACE_URL,
format!("https://imager.dev/tag/{}", name).as_bytes(),
)
.to_string()
}
/// 从 name 计算确定性 person UUID(无需查数据库即可得到 id)
pub fn person_uuid(name: &str) -> String {
Uuid::new_v5(
&Uuid::NAMESPACE_URL,
format!("https://imager.dev/person/{}", name).as_bytes(),
)
.to_string()
}
#[derive(Debug, Clone)]
pub struct DictEntry {
pub word: String,
pub type_: String,
pub category: Option<String>,
pub freq: Option<i64>,
pub merged_to: Option<String>,
}
pub struct SearchDb {
pub pool: SqlitePool,
}
impl SearchDb {
pub async fn new(db_path: &str) -> Result<Self, sqlx::Error> {
// 确保父目录存在
let path = std::path::Path::new(db_path);
if let Some(parent) = path.parent() {
if !parent.exists() {
std::fs::create_dir_all(parent)?;
}
}
// 转换为绝对路径并创建文件(如果不存在)
let abs_path = std::fs::canonicalize(db_path)
.unwrap_or_else(|_| {
// canonicalize 要求文件存在,所以先创建再转换
std::fs::File::create(db_path).ok();
std::fs::canonicalize(db_path).unwrap_or_else(|_| path.to_path_buf())
});
// 使用 sqlite:/// 前缀 + 绝对路径,避免相对路径解析错误
let url = format!("sqlite:///{}", abs_path.to_string_lossy().replace('\\', "/"));
let pool = SqlitePoolOptions::new()
.max_connections(2)
.connect(&url)
.await?;
sqlx::query("PRAGMA journal_mode=WAL;").execute(&pool).await?;
sqlx::query("PRAGMA synchronous=NORMAL;").execute(&pool).await?;
sqlx::query("PRAGMA busy_timeout=5000;").execute(&pool).await?;
sqlx::query("PRAGMA wal_autocheckpoint=1000;").execute(&pool).await?;
sqlx::query("PRAGMA journal_size_limit=67108864;").execute(&pool).await?;
sqlx::query("PRAGMA cache_size=-256000;").execute(&pool).await?;
sqlx::query("PRAGMA temp_store=MEMORY;").execute(&pool).await?;
sqlx::query("PRAGMA mmap_size=268435456;").execute(&pool).await?;
Self::create_tables(&pool).await?;
Ok(Self { pool })
}
/// 在批量写入前临时调低 synchronous 级别提升性能
pub async fn set_sync_off(&self) {
let _ = sqlx::query("PRAGMA synchronous=OFF;").execute(&self.pool).await;
}
/// 批量写入后恢复 synchronous 级别
pub async fn set_sync_normal(&self) {
let _ = sqlx::query("PRAGMA synchronous=NORMAL;").execute(&self.pool).await;
}
/// 执行 WAL checkpoint(TRUNCATE),将 WAL 完全合并回主文件并清理 -wal / -shm 文件
///
/// 在启动时调用,确保 search.db 文件干净,避免下次启动时 WAL 恢复开销。
pub async fn wal_checkpoint(&self) {
match sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
.execute(&self.pool).await
{
Ok(_) => println!("[search.db] WAL checkpoint(TRUNCATE) 完成,数据库文件已合并"),
Err(e) => eprintln!("[search.db] WAL checkpoint 失败: {}", e),
}
}
async fn create_tables(pool: &SqlitePool) -> Result<(), sqlx::Error> {
// ── 迁移检测:如果 tags 表存在且 id 类型是 INTEGER(旧版),重建表 ──
let has_old_schema: bool = sqlx::query(
"SELECT type FROM pragma_table_info('tags') WHERE name = 'id'"
).fetch_optional(pool).await.ok()
.flatten()
.map(|r| {
let t: String = r.get("type");
// 旧版用 INTEGER,新版用 TEXT
t.contains("INTEGER")
})
.unwrap_or(false);
if has_old_schema {
println!("[search.db] 检测到旧版 INTEGER ID 模式,重建表为 TEXT UUID 模式...");
sqlx::query("DROP TABLE IF EXISTS album_tags").execute(pool).await?;
sqlx::query("DROP TABLE IF EXISTS album_persons").execute(pool).await?;
sqlx::query("DROP TABLE IF EXISTS person_aliases").execute(pool).await?;
sqlx::query("DROP TABLE IF EXISTS tags").execute(pool).await?;
sqlx::query("DROP TABLE IF EXISTS persons").execute(pool).await?;
println!("[search.db] 旧表已删除,即将重建");
}
// ── 标签表(TEXT UUID 主键) ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS tags (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
category TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
// ── 人物表(TEXT UUID 主键) ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS persons (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
display_name TEXT,
total_albums INTEGER DEFAULT 0,
total_images INTEGER DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
// ── 人物别名 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS person_aliases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
person_id TEXT NOT NULL,
alias_type TEXT NOT NULL,
alias_name TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
// ── 相册-标签关联 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS album_tags (
album_id TEXT NOT NULL,
tag_id TEXT NOT NULL,
PRIMARY KEY (album_id, tag_id),
FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
// ── 相册-人物关联 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS album_persons (
album_id TEXT NOT NULL,
person_id TEXT NOT NULL,
PRIMARY KEY (album_id, person_id),
FOREIGN KEY (person_id) REFERENCES persons(id) ON DELETE CASCADE
)
"#).execute(pool).await?;
// ── albums 表:搜索返回需要的相册基础信息 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS albums (
id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT '',
cover_image_id TEXT,
image_count INTEGER DEFAULT 0,
created_at TEXT
)
"#).execute(pool).await?;
// ── dict 表 ──
sqlx::query(r#"
CREATE TABLE IF NOT EXISTS dict (
word TEXT NOT NULL PRIMARY KEY,
type TEXT NOT NULL,
category TEXT,
freq INTEGER,
merged_to TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
"#).execute(pool).await?;
// ── 索引 ──
sqlx::query("CREATE INDEX IF NOT EXISTS idx_albums_title ON albums(title)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_tags_name ON tags(name)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_persons_name ON persons(name)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_album ON album_tags(album_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_tags_tag ON album_tags(tag_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_album ON album_persons(album_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_album_persons_person ON album_persons(person_id)").execute(pool).await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_person_aliases_name ON person_aliases(alias_name)").execute(pool).await?;
// 兼容旧表:尝试添加 merged_to 列(已有则忽略)
let _ = sqlx::query("ALTER TABLE dict ADD COLUMN merged_to TEXT").execute(pool).await;
Ok(())
}
// ── upsert 写入方法 ──
pub async fn upsert_album(&self, album_id: &str, title: &str,
cover_image_id: Option<&str>, image_count: i64,
created_at: Option<&str>) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT INTO albums (id, title, cover_image_id, image_count, created_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
title = excluded.title,
cover_image_id = excluded.cover_image_id,
image_count = excluded.image_count,
created_at = excluded.created_at
"#)
.bind(album_id).bind(title).bind(cover_image_id)
.bind(image_count).bind(created_at)
.execute(&self.pool).await?;
Ok(())
}
pub async fn delete_album(&self, album_id: &str) -> Result<(), sqlx::Error> {
let mut tx = self.pool.begin().await?;
sqlx::query("DELETE FROM album_tags WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?;
sqlx::query("DELETE FROM album_persons WHERE album_id = ?").bind(album_id).execute(&mut *tx).await?;
sqlx::query("DELETE FROM albums WHERE id = ?").bind(album_id).execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
/// 插入标签(使用确定性 UUID,无需 SELECT 回查)
pub async fn insert_tag(&self, name: &str, category: Option<&str>) -> Result<String, sqlx::Error> {
let id = tag_uuid(name);
sqlx::query(r#"INSERT OR IGNORE INTO tags (id, name, category) VALUES (?, ?, ?)"#)
.bind(&id).bind(name).bind(category)
.execute(&self.pool).await?;
Ok(id)
}
/// 插入人物(使用确定性 UUID,无需 SELECT 回查)
pub async fn insert_person(&self, name: &str, display_name: Option<&str>) -> Result<String, sqlx::Error> {
let id = person_uuid(name);
sqlx::query(r#"INSERT OR IGNORE INTO persons (id, name, display_name) VALUES (?, ?, ?)"#)
.bind(&id).bind(name).bind(display_name)
.execute(&self.pool).await?;
Ok(id)
}
pub async fn update_person_counts(&self, person_id: &str, total_albums: i64, total_images: i64) -> Result<(), sqlx::Error> {
sqlx::query("UPDATE persons SET total_albums = ?, total_images = ?, updated_at = datetime('now') WHERE id = ?")
.bind(total_albums).bind(total_images).bind(person_id)
.execute(&self.pool).await?;
Ok(())
}
pub async fn insert_person_alias(&self, person_id: &str, alias_type: &str, alias_name: &str) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)
"#)
.bind(person_id)
.bind(alias_type)
.bind(alias_name)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn insert_album_tag(&self, album_id: &str, tag_id: &str) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO album_tags (album_id, tag_id) VALUES (?, ?)
"#)
.bind(album_id)
.bind(tag_id)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn insert_album_person(&self, album_id: &str, person_id: &str) -> Result<(), sqlx::Error> {
sqlx::query(r#"
INSERT OR IGNORE INTO album_persons (album_id, person_id) VALUES (?, ?)
"#)
.bind(album_id)
.bind(person_id)
.execute(&self.pool)
.await?;
Ok(())
}
/// 清空某相册的所有关联标签
pub async fn clear_album_tags(&self, album_id: &str) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM album_tags WHERE album_id = ?")
.bind(album_id).execute(&self.pool).await?;
Ok(())
}
/// 清空某相册的所有关联人物
pub async fn clear_album_persons(&self, album_id: &str) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM album_persons WHERE album_id = ?")
.bind(album_id).execute(&self.pool).await?;
Ok(())
}
pub async fn load_dict(&self) -> Result<Vec<DictEntry>, sqlx::Error> {
let rows = sqlx::query("SELECT word, type, category, freq, merged_to FROM dict")
.fetch_all(&self.pool)
.await?;
let entries = rows.iter().map(|r| DictEntry {
word: r.get("word"),
type_: r.get("type"),
category: r.get("category"),
freq: r.get("freq"),
merged_to: r.get("merged_to"),
}).collect();
Ok(entries)
}
/// 加载 dict 表全部条目(不限类型),用于初始化 TagDict
pub async fn load_dict_all(&self) -> Result<Vec<DictEntry>, sqlx::Error> {
self.load_dict().await
}
/// 插入或更新一条 dict 条目(word 为 PK,相同 word 会覆盖)
pub async fn upsert_dict_entry(&self, word: &str, type_: &str, category: Option<&str>, freq: Option<i64>, merged_to: Option<&str>) -> Result<(), sqlx::Error> {
let rows = sqlx::query(
r#"SELECT COUNT(*) as cnt FROM dict WHERE word = ?"#
).bind(word).fetch_one(&self.pool).await?;
let exists: i64 = rows.get("cnt");
if exists > 0 {
sqlx::query(
"UPDATE dict SET type = ?, category = ?, freq = ?, merged_to = ? WHERE word = ?"
).bind(type_).bind(category).bind(freq).bind(merged_to).bind(word)
.execute(&self.pool).await?;
} else {
sqlx::query(
"INSERT INTO dict (word, type, category, freq, merged_to) VALUES (?, ?, ?, ?, ?)"
).bind(word).bind(type_).bind(category).bind(freq).bind(merged_to)
.execute(&self.pool).await?;
}
Ok(())
}
/// 删除 tag:从 dict/album_tags/tags 表中删除,返回受影响的 album_id 列表
pub async fn delete_tag_and_get_albums(&self, tag_name: &str) -> Result<Vec<String>, sqlx::Error> {
// 1. 计算确定性 tag UUID
let tag_id = tag_uuid(tag_name);
// 2. 查找关联 album_ids
let album_ids: Vec<String> = sqlx::query_scalar::<_, String>(
"SELECT DISTINCT album_id FROM album_tags WHERE tag_id = ?"
).bind(&tag_id).fetch_all(&self.pool).await.unwrap_or_default();
// 3. 删除 album_tags 关联
sqlx::query("DELETE FROM album_tags WHERE tag_id = ?")
.bind(&tag_id).execute(&self.pool).await?;
// 4. 删除 tags 表条目
sqlx::query("DELETE FROM tags WHERE id = ?").bind(&tag_id).execute(&self.pool).await?;
// 5. 删除 dict 表条目
sqlx::query("DELETE FROM dict WHERE word = ? AND type = 'tag'").bind(tag_name).execute(&self.pool).await?;
Ok(album_ids)
}
/// 删除 person:从 search.db 的 album_persons/persons/person_aliases 中删除,返回受影响的 album_id 列表
pub async fn delete_person_and_get_albums(&self, person_name: &str) -> Result<Vec<String>, sqlx::Error> {
// 1. 计算确定性 person UUID
let person_id = person_uuid(person_name);
// 2. 查找关联 album_ids
let album_ids: Vec<String> = sqlx::query_scalar::<_, String>(
"SELECT DISTINCT album_id FROM album_persons WHERE person_id = ?"
).bind(&person_id).fetch_all(&self.pool).await.unwrap_or_default();
// 3. 删除关联表
sqlx::query("DELETE FROM album_persons WHERE person_id = ?")
.bind(&person_id).execute(&self.pool).await?;
sqlx::query("DELETE FROM person_aliases WHERE person_id = ?")
.bind(&person_id).execute(&self.pool).await?;
sqlx::query("DELETE FROM persons WHERE id = ?")
.bind(&person_id).execute(&self.pool).await?;
// 4. 删除 dict 表中的对应条目(如果存在)
sqlx::query("DELETE FROM dict WHERE word = ? AND type = 'person'")
.bind(person_name).execute(&self.pool).await?;
Ok(album_ids)
}
/// 批量同步解析结果到 search.db(单事务 + 预计算 UUID + 批量操作)
///
/// 核心优化:
/// 1. 预加载全部 tags/persons 到 HashMap(仅 2 次 SELECT)
/// 2. 预计算所有 tag/person 的 UUID(零数据库查询得到 ID)
/// 3. 全量 `DELETE WHERE album_id IN (...)` 代替逐条 DELETE
/// 4. 逐条 INSERT 但无需 SELECT 查 ID(UUID 直接写入)
/// 5. 每事务一次 person counts 刷新
pub async fn batch_sync_parse_results(&self, results: &[crate::models::parse::ParseResult]) -> Result<usize, sqlx::Error> {
use std::collections::HashMap;
let mut tx = self.pool.begin().await?;
// 1. 预加载现有 tags/persons(仅 2 次 SELECT)
let existing_tags: HashMap<String, String> = sqlx::query_as::<_, (String, String)>(
"SELECT name, id FROM tags"
).fetch_all(&mut *tx).await?.into_iter().collect();
let existing_persons: HashMap<String, String> = sqlx::query_as::<_, (String, String)>(
"SELECT name, id FROM persons"
).fetch_all(&mut *tx).await?.into_iter().collect();
// 2. 收集所有相册 ID,用于批量 DELETE
let album_ids: Vec<&str> = results.iter().map(|r| r.album_id.as_str()).collect();
if !album_ids.is_empty() {
let ph: Vec<String> = album_ids.iter().map(|_| "?".to_string()).collect();
let sql = format!("DELETE FROM album_tags WHERE album_id IN ({})", ph.join(","));
let mut q = sqlx::query(&sql);
for id in &album_ids { q = q.bind(id); }
q.execute(&mut *tx).await?;
let ph2: Vec<String> = album_ids.iter().map(|_| "?".to_string()).collect();
let sql2 = format!("DELETE FROM album_persons WHERE album_id IN ({})", ph2.join(","));
let mut q2 = sqlx::query(&sql2);
for id in &album_ids { q2 = q2.bind(id); }
q2.execute(&mut *tx).await?;
}
// 3. 收集去重后的 tag 和 person 名称
let mut unique_tags: Vec<(String, Option<String>)> = Vec::new();
let mut seen_tags = std::collections::HashSet::new();
let mut unique_persons: Vec<String> = Vec::new();
let mut seen_persons = std::collections::HashSet::new();
for result in results {
for (tag_name, category) in &result.tags {
if seen_tags.insert(tag_name.clone()) {
unique_tags.push((tag_name.clone(), Some(category.clone())));
}
}
for person_name in &result.person_names {
if seen_persons.insert(person_name.clone()) {
unique_persons.push(person_name.clone());
}
}
}
// 4a. 写入标签:对不存在的标签 INSERT(使用预计算 UUID)
for (tag_name, category) in &unique_tags {
if !existing_tags.contains_key(tag_name) {
let id = tag_uuid(tag_name);
sqlx::query("INSERT OR IGNORE INTO tags (id, name, category) VALUES (?, ?, ?)")
.bind(&id).bind(tag_name).bind(category)
.execute(&mut *tx).await?;
}
}
// 4b. 写入人物:对不存在的人物 INSERT(使用预计算 UUID)
for person_name in &unique_persons {
if !existing_persons.contains_key(person_name) {
let id = person_uuid(person_name);
sqlx::query("INSERT OR IGNORE INTO persons (id, name) VALUES (?, ?)")
.bind(&id).bind(person_name)
.execute(&mut *tx).await?;
}
}
// 5. 写入批量关联数据(无 SELECT 查找,直接使用预计算的 UUID)
let mut tag_id_cache: HashMap<String, String> = existing_tags.clone();
let mut person_id_cache: HashMap<String, String> = existing_persons.clone();
for result in results {
let album_id = &result.album_id;
// ── album_tags ──
for (tag_name, _category) in &result.tags {
let tid = tag_id_cache.get(tag_name.as_str())
.cloned()
.unwrap_or_else(|| tag_uuid(tag_name));
// 缓存新写入的 tag(即使上一个缓存中已更新,这里确保缓存完整)
tag_id_cache.entry(tag_name.clone()).or_insert_with(|| tid.clone());
sqlx::query("INSERT OR IGNORE INTO album_tags (album_id, tag_id) VALUES (?, ?)")
.bind(album_id).bind(&tid).execute(&mut *tx).await?;
}
// ── album_persons ──
for person_name in &result.person_names {
let pid = person_id_cache.get(person_name.as_str())
.cloned()
.unwrap_or_else(|| person_uuid(person_name));
person_id_cache.entry(person_name.clone()).or_insert_with(|| pid.clone());
sqlx::query("INSERT OR IGNORE INTO album_persons (album_id, person_id) VALUES (?, ?)")
.bind(album_id).bind(&pid).execute(&mut *tx).await?;
// 别名写入
let (primary, aliases) = crate::engine::title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
sqlx::query("INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)")
.bind(&pid).bind(alias_type).bind(alias_name).execute(&mut *tx).await?;
}
}
if primary != person_name.as_str() && !primary.is_empty() {
sqlx::query("INSERT OR IGNORE INTO person_aliases (person_id, alias_type, alias_name) VALUES (?, ?, ?)")
.bind(&pid).bind("primary").bind(&primary).execute(&mut *tx).await?;
}
}
}
}
// 6. 更新 person counts(一次批量)
for (_name, pid) in &person_id_cache {
let count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM album_persons WHERE person_id = ?"
).bind(pid).fetch_one(&mut *tx).await.unwrap_or(0);
sqlx::query("UPDATE persons SET total_albums = ?, total_images = 0 WHERE id = ?")
.bind(count).bind(pid).execute(&mut *tx).await?;
}
tx.commit().await?;
let total = results.len();
println!("[batch_sync] 单事务同步完成:{} 个相册,{} 个标签(key),{} 个人物(key)",
total, tag_id_cache.len(), person_id_cache.len());
Ok(total)
}
pub async fn clear_all(&self) -> Result<(), sqlx::Error> {
let mut tx = self.pool.begin().await?;
sqlx::query("DELETE FROM album_tags").execute(&mut *tx).await?;
sqlx::query("DELETE FROM album_persons").execute(&mut *tx).await?;
sqlx::query("DELETE FROM person_aliases").execute(&mut *tx).await?;
sqlx::query("DELETE FROM tags").execute(&mut *tx).await?;
sqlx::query("DELETE FROM persons").execute(&mut *tx).await?;
sqlx::query("DELETE FROM albums").execute(&mut *tx).await?;
sqlx::query("DELETE FROM dict").execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
}

350
src/search_db/search.rs Normal file
View File

@@ -0,0 +1,350 @@
//! 搜索读取方法 — SearchDb 的只读接口
//!
//! 所有搜索相关的数据读取都从这里进行,完全不碰主库。
use sqlx::Row;
use std::collections::HashMap;
use uuid::Uuid;
use crate::models::*;
use super::SearchDb;
impl SearchDb {
// ═══════════════════════════════════════
// 相册搜索
// ═══════════════════════════════════════
/// 组合搜索:按 tag/person/标题关键词 任意组合过滤(AND)
pub async fn search_albums(
&self,
tag: Option<&str>,
person: Option<&str>,
title_q: Option<&str>,
limit: i64,
offset: i64,
) -> Result<Vec<AlbumInfo>, sqlx::Error> {
let has_tag = tag.map(|s| !s.is_empty()).unwrap_or(false);
let has_person = person.map(|s| !s.is_empty()).unwrap_or(false);
let has_q = title_q.map(|s| !s.is_empty()).unwrap_or(false);
if !has_tag && !has_person && !has_q {
return self.get_all_albums().await;
}
let mut sql = String::from(
"SELECT DISTINCT a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM albums a"
);
let mut conditions: Vec<String> = Vec::new();
if has_tag {
sql.push_str(" JOIN album_tags at ON a.id = at.album_id JOIN tags t ON at.tag_id = t.id");
conditions.push("t.name = ?".to_string());
}
if has_person {
sql.push_str(" JOIN album_persons ap ON a.id = ap.album_id JOIN persons p ON ap.person_id = p.id");
conditions.push("(p.name = ? OR LOWER(p.name) LIKE ?)".to_string());
}
if !conditions.is_empty() {
sql.push_str(" WHERE ");
sql.push_str(&conditions.join(" AND "));
}
if has_q {
if conditions.is_empty() {
sql.push_str(" WHERE ");
} else {
sql.push_str(" AND ");
}
sql.push_str("a.title LIKE ?");
}
sql.push_str(" ORDER BY a.created_at DESC LIMIT ? OFFSET ?");
let mut query = sqlx::query(&sql);
if has_tag { query = query.bind(tag.unwrap()); }
if has_person {
let p = person.unwrap();
query = query.bind(p).bind(format!("%{}%", p));
}
if has_q { query = query.bind(format!("%{}%", title_q.unwrap())); }
query = query.bind(limit).bind(offset);
let rows = query.fetch_all(&self.pool).await?;
Ok(rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}
}).collect())
}
/// 获取全量相册列表(缓存烘焙用,含 tags 和 persons)
pub async fn get_all_albums(&self) -> Result<Vec<AlbumInfo>, sqlx::Error> {
let rows = sqlx::query(
"SELECT id, title, cover_image_id, image_count, created_at FROM albums ORDER BY created_at DESC"
).fetch_all(&self.pool).await?;
// 批量获取所有 album-tag 关系
let tag_rows = sqlx::query(
r#"SELECT at.album_id, t.name FROM album_tags at JOIN tags t ON at.tag_id = t.id"#
).fetch_all(&self.pool).await?;
let mut album_tags: std::collections::HashMap<String, Vec<String>> = std::collections::HashMap::new();
for r in &tag_rows {
let aid: String = r.get("album_id");
let name: String = r.get("name");
album_tags.entry(aid).or_default().push(name);
}
// 批量获取所有 album-person 关系
let person_rows = sqlx::query(
r#"SELECT ap.album_id, p.name FROM album_persons ap JOIN persons p ON ap.person_id = p.id"#
).fetch_all(&self.pool).await?;
let mut album_persons: std::collections::HashMap<String, Vec<String>> = std::collections::HashMap::new();
for r in &person_rows {
let aid: String = r.get("album_id");
let name: String = r.get("name");
album_persons.entry(aid).or_default().push(name);
}
Ok(rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: album_tags.remove(&id_str).unwrap_or_default(),
persons: album_persons.remove(&id_str).unwrap_or_default(),
}
}).collect())
}
// ═══════════════════════════════════════
// 标签搜索
// ═══════════════════════════════════════
/// 获取热门标签(按使用次数排序)
pub async fn get_popular_tags(&self, limit: usize) -> Vec<TagStat> {
let limit = limit.min(500);
let rows = sqlx::query(
r#"SELECT t.name, COUNT(at.album_id) as cnt
FROM tags t
JOIN album_tags at ON t.id = at.tag_id
GROUP BY t.id, t.name
ORDER BY cnt DESC LIMIT ?"#
).bind(limit as i64).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| TagStat {
tag: r.get("name"),
count: r.get("cnt"),
album_count: r.get("cnt"),
}).collect()
}
/// 按标签搜索相册
pub async fn search_albums_by_tag(&self, tag: &str) -> Vec<AlbumInfo> {
let rows = sqlx::query(
r#"SELECT a.id, a.title, a.cover_image_id, a.image_count, a.created_at
FROM albums a
JOIN album_tags at ON a.id = at.album_id
JOIN tags t ON at.tag_id = t.id
WHERE t.name = ?
ORDER BY a.created_at DESC"#
).bind(tag).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| {
let id_str: String = r.get("id");
AlbumInfo {
id: Uuid::parse_str(&id_str).unwrap_or_else(|_| Uuid::nil()),
title: r.get("title"),
document: String::new(),
description: None,
cover_image_id: r.get("cover_image_id"),
image_count: r.get("image_count"),
created_at: r.get("created_at"),
tags: vec![],
persons: vec![],
}
}).collect()
}
/// 获取标签共现数据(关联推荐用)
pub async fn get_tag_co_occurrences(&self, tag: &str, limit: i64) -> Vec<TagCoOccurrence> {
let rows = sqlx::query(
r#"SELECT t2.name as co_tag, COUNT(*) as cnt
FROM album_tags at1
JOIN album_tags at2 ON at1.album_id = at2.album_id AND at1.tag_id != at2.tag_id
JOIN tags t1 ON at1.tag_id = t1.id
JOIN tags t2 ON at2.tag_id = t2.id
WHERE t1.name = ?
GROUP BY t2.name
ORDER BY cnt DESC LIMIT ?"#
).bind(tag).bind(limit).fetch_all(&self.pool).await.unwrap_or_default();
let total: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM album_tags at JOIN tags t ON at.tag_id = t.id WHERE t.name = ?"
).bind(tag).fetch_one(&self.pool).await.unwrap_or(1);
rows.iter().map(|r| TagCoOccurrence {
tag: tag.to_string(),
co_tag: r.get("co_tag"),
count: r.get("cnt"),
score: r.get::<i64, _>("cnt") as f64 / total as f64,
}).collect()
}
// ═══════════════════════════════════════
// 人物搜索
// ═══════════════════════════════════════
/// 获取人物别名列表
pub async fn get_person_aliases(&self, person_id: &str) -> Vec<PersonAlias> {
let rows = sqlx::query(
"SELECT id, person_id, alias_type, alias_name, created_at FROM person_aliases WHERE person_id = ?"
).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| PersonAlias {
id: r.get("id"),
person_id: r.get("person_id"),
alias_type: r.get("alias_type"),
alias_name: r.get("alias_name"),
created_at: r.get("created_at"),
}).collect()
}
/// 获取所有人物(含别名、排序)
pub async fn get_all_persons_with_aliases(&self) -> Vec<PersonWithCounts> {
let persons = sqlx::query(
"SELECT id, name, display_name, total_albums, total_images FROM persons ORDER BY total_albums DESC"
).fetch_all(&self.pool).await.unwrap_or_default();
let mut results = Vec::new();
for r in &persons {
let id: String = r.get("id");
let name: String = r.get("name");
let aliases: Vec<String> = sqlx::query_scalar(
"SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type"
).bind(&id).fetch_all(&self.pool).await.unwrap_or_default();
results.push(PersonWithCounts {
id,
name,
display_name: r.get("display_name"),
total_albums: r.get("total_albums"),
total_images: r.get("total_images"),
aliases,
});
}
results
}
/// 按名称查找人物
pub async fn get_person_by_name(&self, name: &str) -> Option<PersonWithCounts> {
let row = sqlx::query(
"SELECT id, name, display_name, total_albums, total_images FROM persons WHERE name = ?"
).bind(name).fetch_optional(&self.pool).await.ok()??;
let id: String = row.get("id");
let aliases: Vec<String> = sqlx::query_scalar(
"SELECT alias_name FROM person_aliases WHERE person_id = ? ORDER BY alias_type"
).bind(&id).fetch_all(&self.pool).await.unwrap_or_default();
Some(PersonWithCounts {
id,
name: row.get("name"),
display_name: row.get("display_name"),
total_albums: row.get("total_albums"),
total_images: row.get("total_images"),
aliases,
})
}
/// 获取人物的所有相册 ID(从 search.db 的 album_persons 关联表查询)
pub async fn get_person_album_ids(&self, person_id: &str) -> Vec<String> {
sqlx::query_scalar::<_, String>(
"SELECT album_id FROM album_persons WHERE person_id = ?"
).bind(person_id).fetch_all(&self.pool).await.unwrap_or_default()
}
/// 获取人物的 top 标签(用于详情页)
pub async fn get_person_top_tags(&self, person_id: &str, limit: usize) -> Vec<(String, i64)> {
let rows = sqlx::query(
r#"SELECT t.name, COUNT(*) as cnt
FROM tags t
JOIN album_tags at ON t.id = at.tag_id
JOIN album_persons ap ON at.album_id = ap.album_id
WHERE ap.person_id = ?
GROUP BY t.name
ORDER BY cnt DESC LIMIT ?"#
).bind(person_id).bind(limit as i64)
.fetch_all(&self.pool).await.unwrap_or_default();
rows.iter().map(|r| (r.get("name"), r.get("cnt"))).collect()
}
/// 搜索人物(按名称/别名模糊匹配,内存)
pub fn search_persons_inline(persons: &[PersonWithCounts], query: &str) -> Vec<PersonWithCounts> {
let q = query.to_lowercase().trim().to_string();
if q.is_empty() || q.len() < 2 { return Vec::new(); }
let mut seen = std::collections::HashSet::new();
let mut results = Vec::new();
for p in persons {
let name_lower = p.name.to_lowercase();
let alias_matches = p.aliases.iter().any(|a| a.to_lowercase().contains(&q));
let matched = name_lower == q
|| name_lower.starts_with(&q)
|| name_lower.contains(&q)
|| alias_matches;
if matched && seen.insert(p.id.clone()) {
results.push(p.clone());
}
}
results.sort_by(|a, b| b.total_albums.cmp(&a.total_albums));
results
}
// ═══════════════════════════════════════
// 元数据统计
// ═══════════════════════════════════════
/// 获取全局统计信息
pub async fn get_metadata_stats(&self) -> MetadataStats {
let total_persons: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM persons")
.fetch_one(&self.pool).await.unwrap_or(0);
let total_aliases: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM person_aliases")
.fetch_one(&self.pool).await.unwrap_or(0);
let total_albums: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM albums")
.fetch_one(&self.pool).await.unwrap_or(0);
let total_images: i64 = sqlx::query_scalar("SELECT COALESCE(SUM(image_count), 0) FROM albums")
.fetch_one(&self.pool).await.unwrap_or(0);
let total_tags: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM album_tags")
.fetch_one(&self.pool).await.unwrap_or(0);
MetadataStats {
total_persons,
total_aliases,
total_albums,
total_images,
total_tags,
}
}
}

View File

@@ -1,5 +1,7 @@
//! 服务层 — 导入、处理等业务逻辑
pub mod import;
pub mod search_export;
pub use import::*;
#[allow(unused)] pub use import::*;
#[allow(unused)] pub use search_export::*;

View File

@@ -0,0 +1,192 @@
use crate::db::Database;
use crate::engine::title_parser;
use crate::search_db::SearchDb;
use std::collections::HashMap;
pub struct SearchExportResult {
pub albums_processed: usize,
pub tags_created: usize,
pub persons_created: usize,
pub aliases_created: usize,
pub album_tags_created: usize,
pub album_persons_created: usize,
pub dict_matched_persons: usize,
pub dict_matched_tags: usize,
pub dict_resolved_aliases: usize,
}
impl Default for SearchExportResult {
fn default() -> Self {
Self {
albums_processed: 0,
tags_created: 0,
persons_created: 0,
aliases_created: 0,
album_tags_created: 0,
album_persons_created: 0,
dict_matched_persons: 0,
dict_matched_tags: 0,
dict_resolved_aliases: 0,
}
}
}
pub async fn export_to_search_db(
source_db: &Database,
search_db_path: &str,
clear_existing: bool,
) -> Result<SearchExportResult, String> {
let search_db = SearchDb::new(search_db_path).await.map_err(|e| format!("创建 search.db 失败: {}", e))?;
if clear_existing {
search_db.clear_all().await.map_err(|e| format!("清空 search.db 失败: {}", e))?;
println!("[搜索导出] 已清空 search.db 现有数据");
}
// ── 加载 dict 表 ──
let dict_entries = search_db
.load_dict()
.await
.map_err(|e| format!("加载 dict 表失败: {}", e))?;
let person_count = dict_entries.iter().filter(|e| e.type_ == "person").count();
let tag_count = dict_entries.iter().filter(|e| e.type_ == "tag").count();
let alias_count = dict_entries.iter().filter(|e| e.merged_to.is_some()).count();
println!(
"[搜索导出] dict 表加载完成: {} 条 (person={}, tag={}, 别名映射={})",
dict_entries.len(),
person_count,
tag_count,
alias_count,
);
let albums = source_db.get_all_albums().await.map_err(|e| format!("读取相册列表失败: {}", e))?;
println!("[搜索导出] 共读取 {} 个相册(来自 imager.db)", albums.len());
let mut tag_cache: HashMap<String, String> = HashMap::new();
let mut person_cache: HashMap<String, String> = HashMap::new();
let mut result = SearchExportResult::default();
for album in &albums {
let album_id = album.id.to_string();
let title = &album.title;
// ── 1. dict 优先解析(dict 优先匹配 → tag_dict 回退 → 合并) ──
let parse_result = title_parser::parse_album_title_with_dict(&album_id, title, &dict_entries);
// 统计 dict 命中
let dict_matched_person = parse_result.person_names.iter().any(|pn| {
dict_entries.iter().any(|e| {
e.type_ == "person"
&& (e.word.to_lowercase() == pn.to_lowercase()
|| e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == pn.to_lowercase()))
})
});
if dict_matched_person {
result.dict_matched_persons += 1;
}
// ── 2. 写入 search.db ──
for (tag_name, category) in &parse_result.tags {
// 检查是否是 dict 命中的 tag
let from_dict = dict_entries.iter().any(|e| {
e.type_ == "tag" && e.word.to_lowercase() == tag_name.to_lowercase()
});
if from_dict {
result.dict_matched_tags += 1;
}
// 使用确定性 UUID,无需查数据库得到 id
let tag_id = match tag_cache.get(tag_name) {
Some(id) => id.clone(),
None => {
let id = search_db
.insert_tag(tag_name, Some(category))
.await
.map_err(|e| format!("插入标签 {} 失败: {}", tag_name, e))?;
tag_cache.insert(tag_name.clone(), id.clone());
result.tags_created += 1;
id
}
};
search_db
.insert_album_tag(&album_id, &tag_id)
.await
.map_err(|e| format!("关联相册标签 {} → {} 失败: {}", album_id, tag_name, e))?;
result.album_tags_created += 1;
}
for person_name in &parse_result.person_names {
// 检查是否有 merged_to 别名解析
let resolved_from = dict_entries.iter().find(|e| {
e.type_ == "person"
&& e.merged_to.as_ref().map_or(false, |m| m.to_lowercase() == person_name.to_lowercase())
});
if resolved_from.is_some() {
result.dict_resolved_aliases += 1;
}
// 使用确定性 UUID,无需查数据库得到 id
let person_id = match person_cache.get(person_name) {
Some(id) => id.clone(),
None => {
let id = search_db
.insert_person(person_name, None)
.await
.map_err(|e| format!("插入人物 {} 失败: {}", person_name, e))?;
person_cache.insert(person_name.clone(), id.clone());
result.persons_created += 1;
id
}
};
search_db
.insert_album_person(&album_id, &person_id)
.await
.map_err(|e| format!("关联相册人物 {} → {} 失败: {}", album_id, person_name, e))?;
result.album_persons_created += 1;
// 同步别名
let (primary, aliases) = title_parser::split_person_name_with_aliases(person_name);
if !aliases.is_empty() {
for (alias_type, alias_name) in &aliases {
if alias_name != person_name {
search_db
.insert_person_alias(&person_id, alias_type, alias_name)
.await
.map_err(|e| format!("插入别名 {} → {} 失败: {}", person_name, alias_name, e))?;
result.aliases_created += 1;
}
}
if primary != person_name.as_str() && !primary.is_empty() {
search_db
.insert_person_alias(&person_id, "primary", &primary)
.await
.map_err(|e| format!("插入别名 {} → {} 失败: {}", person_name, primary, e))?;
result.aliases_created += 1;
}
}
}
result.albums_processed += 1;
if result.albums_processed % 100 == 0 {
println!("[搜索导出] 已处理 {} / {} 相册", result.albums_processed, albums.len());
}
}
println!("[搜索导出] 完成:");
println!(" - 相册: {}", result.albums_processed);
println!(" - 标签: {}", result.tags_created);
println!(" - 人物: {}", result.persons_created);
println!(" - 别名: {}", result.aliases_created);
println!(" - 相册标签关联: {}", result.album_tags_created);
println!(" - 相册人物关联: {}", result.album_persons_created);
println!(" - dict 命中 person: {}", result.dict_matched_persons);
println!(" - dict 命中 tag: {}", result.dict_matched_tags);
println!(" - dict 别名解析: {}", result.dict_resolved_aliases);
Ok(result)
}