更新数据截止日2025.12.24
This commit is contained in:
@@ -4,6 +4,8 @@ import os.path
|
||||
from typing import List, Tuple
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
@@ -209,48 +211,101 @@ class ChatSessionManager:
|
||||
else:
|
||||
return None
|
||||
|
||||
def download_chat_history(self, year: int, month: int, day: int, hour: int) -> bool:
|
||||
def _download_file(self, file_info, files_dir, history, lock):
|
||||
"""
|
||||
下载指定时间的聊天历史
|
||||
下载单个文件的线程函数
|
||||
|
||||
Args:
|
||||
file_info: 文件信息
|
||||
files_dir: 文件保存目录
|
||||
history: 聊天历史字典
|
||||
lock: 线程锁
|
||||
"""
|
||||
try:
|
||||
path = file_info["path"].replace("s3://", "")
|
||||
file_content = self.obj.get_file(path)
|
||||
|
||||
if path.endswith(".json"):
|
||||
data = json.loads(file_content)
|
||||
with lock:
|
||||
history[data["msgid"]] = data
|
||||
else:
|
||||
# 保存非JSON文件
|
||||
filename = path.split("/")[-1]
|
||||
with open(os.path.join(files_dir, filename), "wb") as f:
|
||||
f.write(file_content)
|
||||
|
||||
return path, True
|
||||
except Exception as e:
|
||||
return path, False
|
||||
|
||||
def download_chat_history(self, year: int, month: int, day: int, hour: int, max_workers: int = 5) -> bool:
|
||||
"""
|
||||
下载指定时间的聊天历史(多线程版本)
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
day: 日期
|
||||
hour: 小时
|
||||
max_workers: 最大线程数
|
||||
|
||||
Returns:
|
||||
下载是否成功
|
||||
"""
|
||||
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
|
||||
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d}")
|
||||
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} (多线程模式)")
|
||||
|
||||
try:
|
||||
file_list = self.obj.get_file_list(f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/")
|
||||
print(f"共计{len(file_list)}个文件")
|
||||
print(f"共计{len(file_list)}个文件,使用{max_workers}个线程下载")
|
||||
history = {}
|
||||
count = 1
|
||||
for file_info in file_list:
|
||||
path = file_info["path"].replace("s3://", "")
|
||||
file_content = self.obj.get_file(path)
|
||||
print(f"正在处理第{count}个文件: {path}")
|
||||
count += 1
|
||||
if path.endswith(".json"):
|
||||
data = json.loads(file_content)
|
||||
history[data["msgid"]] = data
|
||||
else:
|
||||
# 保存非JSON文件
|
||||
filename = path.split("/")[-1]
|
||||
with open(os.path.join(self.files_dir, filename), "wb") as f:
|
||||
f.write(file_content)
|
||||
failed_files = []
|
||||
if file_list:
|
||||
lock = threading.Lock()
|
||||
|
||||
# 使用线程池下载文件
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
# 提交所有下载任务
|
||||
future_to_file = {
|
||||
executor.submit(self._download_file, file_info, self.files_dir, history, lock): file_info
|
||||
for file_info in file_list
|
||||
}
|
||||
|
||||
# 处理完成的任务
|
||||
completed_count = 0
|
||||
total_count = len(file_list)
|
||||
|
||||
for future in as_completed(future_to_file):
|
||||
file_info = future_to_file[future]
|
||||
completed_count += 1
|
||||
|
||||
try:
|
||||
path, success = future.result()
|
||||
if success:
|
||||
print(f"[{completed_count}/{total_count}] 下载成功: {path}")
|
||||
else:
|
||||
print(f"[{completed_count}/{total_count}] 下载失败: {path}")
|
||||
failed_files.append(path)
|
||||
except Exception as e:
|
||||
print(f"[{completed_count}/{total_count}] 处理异常: {file_info['path']} - {str(e)}")
|
||||
failed_files.append(file_info['path'])
|
||||
|
||||
# 检查是否有失败的文件
|
||||
if failed_files:
|
||||
print(f"警告: {len(failed_files)}个文件下载失败")
|
||||
print("失败文件列表:", failed_files)
|
||||
|
||||
# 保存聊天历史JSON文件
|
||||
output_path = os.path.join(self.messages_dir, message_history_name)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(history, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} complete")
|
||||
return True
|
||||
print(f"成功下载: {len(file_list) - len(failed_files)}/{len(file_list)} 个文件")
|
||||
|
||||
# 如果有失败的文件,返回False
|
||||
return len(failed_files) == 0
|
||||
|
||||
except Exception as e:
|
||||
error_file = os.path.join(self.messages_dir, f"{message_history_name}.unfinished")
|
||||
@@ -341,7 +396,7 @@ class ChatSessionManager:
|
||||
|
||||
# 如果文件不存在,先下载
|
||||
if not os.path.exists(history_path):
|
||||
self.download_chat_history(year, month, day, hour)
|
||||
self.download_chat_history(year, month, day, hour,10)
|
||||
|
||||
with open(history_path, "r", encoding="utf-8") as f:
|
||||
message_history = json.load(f)
|
||||
@@ -677,4 +732,4 @@ class ChatSessionManager:
|
||||
group_messages[group_id] = []
|
||||
group_messages[group_id].append(i)
|
||||
group_qa = self.analyze_qa_content(group_messages)
|
||||
self.save_qa_to_sqlite(group_qa)
|
||||
self.save_qa_to_sqlite(group_qa)
|
||||
Reference in New Issue
Block a user