更新数据截止日2025.12.24

This commit is contained in:
lychang
2025-12-26 08:20:45 +08:00
parent 7169b568dd
commit de0842fbe1
17 changed files with 3466 additions and 1977 deletions

View File

@@ -4,6 +4,8 @@ import os.path
from typing import List, Tuple
import logging
import os
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from requests.adapters import HTTPAdapter
@@ -209,48 +211,101 @@ class ChatSessionManager:
else:
return None
def download_chat_history(self, year: int, month: int, day: int, hour: int) -> bool:
def _download_file(self, file_info, files_dir, history, lock):
"""
下载指定时间的聊天历史
下载单个文件的线程函数
Args:
file_info: 文件信息
files_dir: 文件保存目录
history: 聊天历史字典
lock: 线程锁
"""
try:
path = file_info["path"].replace("s3://", "")
file_content = self.obj.get_file(path)
if path.endswith(".json"):
data = json.loads(file_content)
with lock:
history[data["msgid"]] = data
else:
# 保存非JSON文件
filename = path.split("/")[-1]
with open(os.path.join(files_dir, filename), "wb") as f:
f.write(file_content)
return path, True
except Exception as e:
return path, False
def download_chat_history(self, year: int, month: int, day: int, hour: int, max_workers: int = 5) -> bool:
"""
下载指定时间的聊天历史(多线程版本)
Args:
year: 年份
month: 月份
day: 日期
hour: 小时
max_workers: 最大线程数
Returns:
下载是否成功
"""
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d}")
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} (多线程模式)")
try:
file_list = self.obj.get_file_list(f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/")
print(f"共计{len(file_list)}个文件")
print(f"共计{len(file_list)}个文件,使用{max_workers}个线程下载")
history = {}
count = 1
for file_info in file_list:
path = file_info["path"].replace("s3://", "")
file_content = self.obj.get_file(path)
print(f"正在处理第{count}个文件: {path}")
count += 1
if path.endswith(".json"):
data = json.loads(file_content)
history[data["msgid"]] = data
else:
# 保存非JSON文件
filename = path.split("/")[-1]
with open(os.path.join(self.files_dir, filename), "wb") as f:
f.write(file_content)
failed_files = []
if file_list:
lock = threading.Lock()
# 使用线程池下载文件
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# 提交所有下载任务
future_to_file = {
executor.submit(self._download_file, file_info, self.files_dir, history, lock): file_info
for file_info in file_list
}
# 处理完成的任务
completed_count = 0
total_count = len(file_list)
for future in as_completed(future_to_file):
file_info = future_to_file[future]
completed_count += 1
try:
path, success = future.result()
if success:
print(f"[{completed_count}/{total_count}] 下载成功: {path}")
else:
print(f"[{completed_count}/{total_count}] 下载失败: {path}")
failed_files.append(path)
except Exception as e:
print(f"[{completed_count}/{total_count}] 处理异常: {file_info['path']} - {str(e)}")
failed_files.append(file_info['path'])
# 检查是否有失败的文件
if failed_files:
print(f"警告: {len(failed_files)}个文件下载失败")
print("失败文件列表:", failed_files)
# 保存聊天历史JSON文件
output_path = os.path.join(self.messages_dir, message_history_name)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(history, f, ensure_ascii=False, indent=2)
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} complete")
return True
print(f"成功下载: {len(file_list) - len(failed_files)}/{len(file_list)} 个文件")
# 如果有失败的文件,返回False
return len(failed_files) == 0
except Exception as e:
error_file = os.path.join(self.messages_dir, f"{message_history_name}.unfinished")
@@ -341,7 +396,7 @@ class ChatSessionManager:
# 如果文件不存在,先下载
if not os.path.exists(history_path):
self.download_chat_history(year, month, day, hour)
self.download_chat_history(year, month, day, hour,10)
with open(history_path, "r", encoding="utf-8") as f:
message_history = json.load(f)
@@ -677,4 +732,4 @@ class ChatSessionManager:
group_messages[group_id] = []
group_messages[group_id].append(i)
group_qa = self.analyze_qa_content(group_messages)
self.save_qa_to_sqlite(group_qa)
self.save_qa_to_sqlite(group_qa)