Files
SmartChat/chat_session.py
lychang d10ec1f911 init
2025-11-01 02:40:17 +08:00

598 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import datetime
import json
import os.path
import sqlite3
from typing import List, Tuple
import pytz
from file import ObjectFileManager, LocalFileManager
TZ = pytz.timezone('Asia/Shanghai')
def convert_to_utc(time_str, time_format='%Y-%m-%d %H:%M:%S', source_timezone=None):
# 解析时间字符串为datetime对象
local_dt = datetime.datetime.strptime(time_str, time_format)
# 如果指定了源时区,则将本地时间转换为UTC时间
if source_timezone:
tz = pytz.timezone(source_timezone)
local_dt = tz.localize(local_dt)
utc_dt = local_dt.astimezone(pytz.UTC)
else:
# 如果没有指定源时区,则假设时间字符串已经是UTC时间
utc_dt = pytz.UTC.localize(local_dt)
return utc_dt
class ChatSessionManager:
"""聊天会话管理器类"""
def __init__(self, chat_history_dir: str = "../chat_history"):
"""
初始化聊天会话管理器
Args:
chat_history_dir: 聊天历史文件目录
"""
self.chat_history_dir = chat_history_dir
self.files_dir = os.path.join(chat_history_dir, "files")
self.messages_dir = os.path.join(chat_history_dir, "messages")
self.db_path = os.path.join("", "data.sqlite")
self.cache_path = None
# 初始化文件管理器
self.obj = ObjectFileManager()
self.lfm = LocalFileManager(self.files_dir)
self.users = []
self.groups = []
self.members = {}
self.messages = []
self.files = {}
# 初始化数据库连接
self.conn = sqlite3.connect(self.db_path)
self.cursor = self.conn.cursor()
# 确保目录存在
os.makedirs(self.files_dir, exist_ok=True)
os.makedirs(self.messages_dir, exist_ok=True)
def download_chat_history(self, year: int, month: int, day: int, hour: int) -> bool:
"""
下载指定时间的聊天历史
Args:
year: 年份
month: 月份
day: 日期
hour: 小时
Returns:
下载是否成功
"""
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d}")
try:
file_list = self.obj.get_file_list(f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/")
print(f"共计{len(file_list)}个文件")
history = {}
count = 1
for file_info in file_list:
path = file_info["path"].replace("s3://", "")
file_content = self.obj.get_file(path)
print(f"正在处理第{count}个文件: {path}")
count += 1
if path.endswith(".json"):
data = json.loads(file_content)
history[data["msgid"]] = data
else:
# 保存非JSON文件
filename = path.split("/")[-1]
with open(os.path.join(self.files_dir, filename), "wb") as f:
f.write(file_content)
# 保存聊天历史JSON文件
output_path = os.path.join(self.messages_dir, message_history_name)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(history, f, ensure_ascii=False, indent=2)
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} complete")
return True
except Exception as e:
error_file = os.path.join(self.messages_dir, f"{message_history_name}.unfinished")
with open(error_file, "w", encoding="utf-8") as f:
f.write(str(e))
print(
f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} failed: please check the {message_history_name}.unfinished")
return False
def _parse_message(self, data, is_mixed: bool = False, idx: int = 0):
msg_type = data["msgtype"]
if msg_type == "text":
content = data['text']["content"]
elif msg_type in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]:
file_data = data[msg_type]
suffixs = [f"_{idx}", ".png", ".jpg", ".gif", ".mp4", "_.png", "_.jpg", "_.gif", "_.mp4",
("_" + file_data["filename"]) if msg_type == "file" else ""]
file_path = ""
for suffix in suffixs:
temp_path = f"{data['msgid']}_mixed{suffix}" if is_mixed else f"{data['msgid']}_{msg_type}{suffix}"
if self.lfm.exists(temp_path):
file_path = temp_path
break
if file_path != "":
ext_mapping = {
"file": (file_data.get('fileext', ''), "filesize"),
"image": ("png", file_data.get("filesize", 0)),
"video": ("mp4", file_data.get("filesize", 0)),
"voice": ("wav", file_data.get("voice_size", 0)),
"emotion": ("gif" if file_data.get("type", 0) == 1 else "png", file_data.get("imagesize", 0)),
"meeting_voice_call": ("wav", self.lfm.get_size(file_path)),
}
file_info = {
"id": file_data.get("md5sum", self.lfm.get_md5(file_path)),
"size": ext_mapping[msg_type][1],
"extension": ext_mapping[msg_type][0],
"content": file_path,
"original": self.cache_path + file_path,
"type": msg_type
}
self.files[file_info["id"]] = file_info
content = file_info["id"]
else:
file_path = f"{data['msgid']}_{msg_type}_0"
file_info = {
"id": file_data.get("md5sum", file_path),
"size": file_data.get('filesize', 0),
"extension": file_data.get('fileext', "unknown"),
"content": file_path,
"original": self.cache_path + file_path,
"type": msg_type
}
self.files[file_info["id"]] = file_info
content = file_info["id"]
elif msg_type == "link":
data = data[msg_type]
content = f'{data["description"]}{data["title"]}({data["link_url"]})'
else:
print(f"未知消息类型: {msg_type}")
content = json.dumps(data, ensure_ascii=False)
return content
def _parse_mixed_message(self, param, idx: int):
tp = param["msgtype"]
content = self._parse_message(param, True, idx)
if tp in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]:
self.files[content]["content"] = f'{param["msgid"]}_mixed_{idx}'
idx += 1
return idx, content
def parse_chat_history(self, year: int, month: int, day: int, hour: int):
"""
解析聊天历史数据
Args:
year: 年份
month: 月份
day: 日期
hour: 小时
Returns:
(用户列表, 群组列表, 成员映射, 消息列表, 文件映射)
"""
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
history_path = os.path.join(self.messages_dir, message_history_name)
# 如果文件不存在,先下载
if not os.path.exists(history_path):
self.download_chat_history(year, month, day, hour)
with open(history_path, "r", encoding="utf-8") as f:
message_history = json.load(f)
try:
for message_data in message_history.values():
# 只处理发送的消息
if message_data.get("action") != "send":
continue
# 提取基本信息
from_user = message_data["from"]
to_users = message_data["tolist"]
room_id = message_data.get("roomid", "")
# 更新用户列表
self.users.extend([from_user] + to_users)
self.users = list(set(self.users))
# 确定接收者类型
to_type = 1 if room_id else 0
receiver = room_id if to_type == 1 else to_users[0]
# 处理群组信息
if room_id:
if room_id not in self.members:
self.members[room_id] = to_users + [from_user]
else:
self.members[room_id].extend(to_users + [from_user])
self.members[room_id] = list(set(self.members[room_id]))
if room_id not in self.groups:
self.groups.append(room_id)
# 构建消息对象
message = {
"id": message_data["msgid"],
"type": message_data["msgtype"],
"time": message_data["msgtime"],
"toType": to_type,
"from": from_user,
"to": receiver
}
# 根据消息类型处理内容
msg_type = message_data["msgtype"]
if msg_type == "mixed":
temp = []
idx = 0
for t in message_data["mixed"]["item"]:
idx, content = self._parse_mixed_message({"msgid": message_data["msgid"],
"msgtype": t["type"],
t["type"]: json.loads(t["content"])}, idx)
temp.append([t["type"], content])
message["content"] = json.dumps(temp, ensure_ascii=False)
else:
message["content"] = self._parse_message(message_data)
self.messages.append(message)
except Exception as e:
print(f"解析聊天记录失败: {e}")
raise e
def save_to_sqlite(self, sql: str, data: List[Tuple]) -> None:
"""
批量保存数据到SQLite数据库
Args:
sql: SQL语句
data: 数据列表
"""
if not data:
return
print(f"开始插入 {len(data)} 条记录")
self.cursor.executemany(sql, data)
self.conn.commit()
print(f"成功插入 {len(data)} 条记录")
def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int,
group_num: int) -> None:
"""
保存会话信息到SQLite数据库
Args:
date_time: 会话时间
message_num: 消息数量
file_num: 文件数量
user_num: 用户数量
group_num: 群组数量
"""
self.save_to_sqlite('''
INSERT OR IGNORE INTO "information"
(datetime, message, file, user, "group")
VALUES (?, ?, ?, ?, ?)
''', [(date_time, message_num, file_num, user_num, group_num)])
def save_all_to_sqlite(self) -> None:
"""
保存所有聊天数据到SQLite数据库
Args:
users: 用户列表
groups: 群组列表
members: 成员映射
messages: 消息列表
files: 文件映射
"""
# 保存文件信息
file_data = []
for file_info in self.files.values():
file_data.append((
file_info.get('id'),
file_info.get('type'),
file_info.get('content'),
file_info.get('original', ''),
file_info.get('extension', ''),
file_info.get('size', 0)
))
self.save_to_sqlite('''
INSERT OR IGNORE INTO file
(id, type, content, original, extension, size)
VALUES (?, ?, ?, ?, ?, ?)
''', file_data)
# 保存用户信息
user_data = [(user, user) for user in self.users]
self.save_to_sqlite('''
INSERT OR IGNORE INTO user
(id, name)
VALUES (?, ?)
''', user_data)
# 保存消息信息
message_data = []
for message in self.messages:
message_data.append((
message.get('id'),
message.get('type'),
message.get('time'),
message.get('toType'),
message.get('from'),
message.get('to'),
message.get('content')
))
self.save_to_sqlite('''
INSERT OR IGNORE INTO message
(id, type, time, toType, sender, receiver, content)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', message_data)
# 保存群组信息
group_data = [(group, group) for group in self.groups]
self.save_to_sqlite('''
INSERT OR IGNORE INTO "group"
(id, name)
VALUES (?, ?)
''', group_data)
# 保存成员信息
member_data = []
for group_id, group_users in self.members.items():
for user in group_users:
member_data.append((group_id, user))
self.save_to_sqlite('''
INSERT OR IGNORE INTO "member"
(groupId, userId)
VALUES (?, ?)
''', member_data)
def process_chat_session(self, year: int, month: int, day: int, hour: int) -> bool:
"""
处理完整的聊天会话流程
Args:
year: 年份
month: 月份
day: 日期
hour: 小时
Returns:
处理是否成功
"""
self.users = []
self.groups = []
self.members = {}
self.messages = []
self.files = {}
try:
self.cache_path = f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/"
# 解析聊天历史
self.parse_chat_history(year, month, day, hour)
# 保存到数据库
self.save_all_to_sqlite()
self.save_information_to_sqlite(f"{year}-{month:02d}-{day:02d}-{hour:02d}", len(self.messages),
len(self.files), len(self.users), len(self.groups))
print(f"成功处理 {year}-{month:02d}-{day:02d}-{hour:02d} 的聊天会话")
return True
except Exception as e:
print(f"处理聊天会话失败: {e}")
return False
def get_chat_history(self, start_time_str: str, end_time_str: str) -> list:
"""
获取指定时间的聊天历史记录
Args:
start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS"
end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS"
Returns:
聊天历史记录列表
"""
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
cursor = self.conn.cursor()
cursor.execute('''
SELECT message.time, message.content
FROM message
WHERE (message.type = 'text'
or message.type = 'mixed')
AND message.time >= ?
AND message.time <= ?
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
return res
def get_history(self, start_time_str: str, end_time_str: str) -> list:
"""
获取指定时间的聊天历史记录
Args:
start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS"
end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS"
Returns:
聊天历史记录列表
"""
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
cursor = self.conn.cursor()
cursor.execute('''
SELECT count(1) as count
FROM message
WHERE message.time >= ?
AND message.time <= ?
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
print(f"总消息数量: {res[0][0]}")
cursor.execute('''
SELECT 1 as count
FROM message
WHERE message.time >= ?
AND message.time <= ?
group by message.sender
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
print(f"发送者数量: {len(res)}")
cursor.execute('''
SELECT *
FROM message
WHERE message.type in ('text', 'mixed', 'chatrecord')
AND message.time >= ?
AND message.time <= ?
AND message.toType == 1
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
return res
def close(self) -> None:
"""关闭数据库连接"""
if self.conn:
self.conn.close()
def __del__(self) -> None:
"""析构函数,确保资源被正确释放"""
self.close()
if __name__ == '__main__':
chat_session = ChatSessionManager()
# for day in range(22, 30+1):
# for hour in range(0, 24):
# chat_session.process_chat_session(2025, 9, day, hour)
# for day in range(1, 29+1):
# for hour in range(0, 24):
# chat_session.process_chat_session(2025, 10, day, hour)
# start = "2025-10-15 00:00:00"
# end = "2025-10-30 23:59:59"
# data = chat_session.get_history(start, end)
# group_messages = {}
# for i in data:
# group_id = i[-1]
# if group_id not in group_messages:
# group_messages[group_id] = []
# group_messages[group_id].append((i[5], i[2], i[3]))
# group_qa = {}
# for group_id in group_messages:
# print(f"群ID: {group_id}")
# print(f"消息数量: {len(group_messages[group_id])}")
# content = ""
# if len(group_messages[group_id]) < 2:
# continue
# elif len(group_messages[group_id]) == 2:
# print(group_messages[group_id])
# else:
# for msg in sorted(group_messages[group_id], key=lambda i: i[1]):
# content += json.dumps({
# "user": msg[0],
# "time": msg[1],
# "content": msg[2]
# },ensure_ascii=False) + ",\n"
#
# template = f"""
# 以下是群中的消息记录
#
# ```json
# [
# {content}
# ]
# ```
# """ + """
# 找到其中的问题和回答,如果没有回答就只整理问题,问题是与产品填报系统,上市公司,证券政策相关的问题
# 问题可能不只由一句话构成,回答也可能不只由一句话构成
# 多个问题需要分开放置
# 输出格式
#
# ```json
# [
# {
# "question": [{
# "user": "用户1",
# "time": "时间戳",
# "content": "问题"
# }],
# "answer": [
# {
# "user": "用户1",
# "time": "时间戳",
# "content": "回答"
# }
# ]
# }
# ]
# ```
# """
# answer = agent.chat_deepseek(template)
# answer = answer.replace("```json", "").replace("```", "") if "```json" in answer else answer
# try:
# answer = json.loads(answer)
# except:
# print(answer)
# answer = []
# group_qa[group_id] = answer
# print(group_qa)
# with open(f"group_qa-{start.split(' ')[0]}-{end.split(' ')[0]}.json", "w", encoding="utf-8") as f:
# f.write(json.dumps(group_qa, ensure_ascii=False))
# push_message(f"群聊问答-{start.split(' ')[0]}-{end.split(' ')[0]}", "问题提取完成", "")
with open(f"group_qa.json", "r", encoding="utf-8") as f:
data = json.load(f)
cursor = chat_session.conn.cursor()
a = cursor.execute('''
select *
from message
''').fetchall()
mapping = {}
for i in a:
mapping[i[2] + "-" + i[5]] = i[0]
dar = []
for r in data:
qas = data.get(r)
for qa in qas:
Q = []
A = []
q = qa["question"]
for qq in q:
if qq["time"] + "-" + qq["user"] not in mapping:
print(qq)
else:
Q.append(mapping[qq["time"] + "-" + qq["user"]])
a = qa["answer"]
for aa in a:
if aa["time"] + "-" + aa["user"] not in mapping:
print(aa)
else:
A.append(mapping[aa["time"] + "-" + aa["user"]])
dar.append((r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False)))
a = cursor.executemany('''
insert into qa (groupId, question, answer)
values (?, ?, ?)
''', dar)
chat_session.conn.commit()