init
This commit is contained in:
597
chat_session.py
Normal file
597
chat_session.py
Normal file
@@ -0,0 +1,597 @@
|
||||
import datetime
|
||||
import json
|
||||
import os.path
|
||||
import sqlite3
|
||||
from typing import List, Tuple
|
||||
|
||||
import pytz
|
||||
|
||||
from file import ObjectFileManager, LocalFileManager
|
||||
|
||||
TZ = pytz.timezone('Asia/Shanghai')
|
||||
|
||||
|
||||
def convert_to_utc(time_str, time_format='%Y-%m-%d %H:%M:%S', source_timezone=None):
|
||||
# 解析时间字符串为datetime对象
|
||||
local_dt = datetime.datetime.strptime(time_str, time_format)
|
||||
|
||||
# 如果指定了源时区,则将本地时间转换为UTC时间
|
||||
if source_timezone:
|
||||
tz = pytz.timezone(source_timezone)
|
||||
local_dt = tz.localize(local_dt)
|
||||
utc_dt = local_dt.astimezone(pytz.UTC)
|
||||
else:
|
||||
# 如果没有指定源时区,则假设时间字符串已经是UTC时间
|
||||
utc_dt = pytz.UTC.localize(local_dt)
|
||||
|
||||
return utc_dt
|
||||
|
||||
|
||||
class ChatSessionManager:
|
||||
"""聊天会话管理器类"""
|
||||
|
||||
def __init__(self, chat_history_dir: str = "../chat_history"):
|
||||
"""
|
||||
初始化聊天会话管理器
|
||||
|
||||
Args:
|
||||
chat_history_dir: 聊天历史文件目录
|
||||
"""
|
||||
self.chat_history_dir = chat_history_dir
|
||||
self.files_dir = os.path.join(chat_history_dir, "files")
|
||||
self.messages_dir = os.path.join(chat_history_dir, "messages")
|
||||
self.db_path = os.path.join("", "data.sqlite")
|
||||
self.cache_path = None
|
||||
# 初始化文件管理器
|
||||
self.obj = ObjectFileManager()
|
||||
self.lfm = LocalFileManager(self.files_dir)
|
||||
self.users = []
|
||||
self.groups = []
|
||||
self.members = {}
|
||||
self.messages = []
|
||||
self.files = {}
|
||||
# 初始化数据库连接
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
# 确保目录存在
|
||||
os.makedirs(self.files_dir, exist_ok=True)
|
||||
os.makedirs(self.messages_dir, exist_ok=True)
|
||||
|
||||
def download_chat_history(self, year: int, month: int, day: int, hour: int) -> bool:
|
||||
"""
|
||||
下载指定时间的聊天历史
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
day: 日期
|
||||
hour: 小时
|
||||
|
||||
Returns:
|
||||
下载是否成功
|
||||
"""
|
||||
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
|
||||
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d}")
|
||||
|
||||
try:
|
||||
file_list = self.obj.get_file_list(f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/")
|
||||
print(f"共计{len(file_list)}个文件")
|
||||
history = {}
|
||||
count = 1
|
||||
for file_info in file_list:
|
||||
path = file_info["path"].replace("s3://", "")
|
||||
file_content = self.obj.get_file(path)
|
||||
print(f"正在处理第{count}个文件: {path}")
|
||||
count += 1
|
||||
if path.endswith(".json"):
|
||||
data = json.loads(file_content)
|
||||
history[data["msgid"]] = data
|
||||
else:
|
||||
# 保存非JSON文件
|
||||
filename = path.split("/")[-1]
|
||||
with open(os.path.join(self.files_dir, filename), "wb") as f:
|
||||
f.write(file_content)
|
||||
|
||||
# 保存聊天历史JSON文件
|
||||
output_path = os.path.join(self.messages_dir, message_history_name)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(history, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} complete")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
error_file = os.path.join(self.messages_dir, f"{message_history_name}.unfinished")
|
||||
with open(error_file, "w", encoding="utf-8") as f:
|
||||
f.write(str(e))
|
||||
print(
|
||||
f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} failed: please check the {message_history_name}.unfinished")
|
||||
return False
|
||||
|
||||
def _parse_message(self, data, is_mixed: bool = False, idx: int = 0):
|
||||
msg_type = data["msgtype"]
|
||||
if msg_type == "text":
|
||||
content = data['text']["content"]
|
||||
|
||||
elif msg_type in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]:
|
||||
file_data = data[msg_type]
|
||||
suffixs = [f"_{idx}", ".png", ".jpg", ".gif", ".mp4", "_.png", "_.jpg", "_.gif", "_.mp4",
|
||||
("_" + file_data["filename"]) if msg_type == "file" else ""]
|
||||
file_path = ""
|
||||
for suffix in suffixs:
|
||||
temp_path = f"{data['msgid']}_mixed{suffix}" if is_mixed else f"{data['msgid']}_{msg_type}{suffix}"
|
||||
if self.lfm.exists(temp_path):
|
||||
file_path = temp_path
|
||||
break
|
||||
if file_path != "":
|
||||
ext_mapping = {
|
||||
"file": (file_data.get('fileext', ''), "filesize"),
|
||||
"image": ("png", file_data.get("filesize", 0)),
|
||||
"video": ("mp4", file_data.get("filesize", 0)),
|
||||
"voice": ("wav", file_data.get("voice_size", 0)),
|
||||
"emotion": ("gif" if file_data.get("type", 0) == 1 else "png", file_data.get("imagesize", 0)),
|
||||
"meeting_voice_call": ("wav", self.lfm.get_size(file_path)),
|
||||
}
|
||||
file_info = {
|
||||
"id": file_data.get("md5sum", self.lfm.get_md5(file_path)),
|
||||
"size": ext_mapping[msg_type][1],
|
||||
"extension": ext_mapping[msg_type][0],
|
||||
"content": file_path,
|
||||
"original": self.cache_path + file_path,
|
||||
"type": msg_type
|
||||
}
|
||||
self.files[file_info["id"]] = file_info
|
||||
content = file_info["id"]
|
||||
else:
|
||||
file_path = f"{data['msgid']}_{msg_type}_0"
|
||||
file_info = {
|
||||
"id": file_data.get("md5sum", file_path),
|
||||
"size": file_data.get('filesize', 0),
|
||||
"extension": file_data.get('fileext', "unknown"),
|
||||
"content": file_path,
|
||||
"original": self.cache_path + file_path,
|
||||
"type": msg_type
|
||||
}
|
||||
self.files[file_info["id"]] = file_info
|
||||
content = file_info["id"]
|
||||
elif msg_type == "link":
|
||||
data = data[msg_type]
|
||||
content = f'{data["description"]}{data["title"]}({data["link_url"]})'
|
||||
else:
|
||||
print(f"未知消息类型: {msg_type}")
|
||||
content = json.dumps(data, ensure_ascii=False)
|
||||
return content
|
||||
|
||||
def _parse_mixed_message(self, param, idx: int):
|
||||
tp = param["msgtype"]
|
||||
content = self._parse_message(param, True, idx)
|
||||
|
||||
if tp in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]:
|
||||
self.files[content]["content"] = f'{param["msgid"]}_mixed_{idx}'
|
||||
idx += 1
|
||||
return idx, content
|
||||
|
||||
def parse_chat_history(self, year: int, month: int, day: int, hour: int):
|
||||
"""
|
||||
解析聊天历史数据
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
day: 日期
|
||||
hour: 小时
|
||||
|
||||
Returns:
|
||||
(用户列表, 群组列表, 成员映射, 消息列表, 文件映射)
|
||||
"""
|
||||
message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json"
|
||||
history_path = os.path.join(self.messages_dir, message_history_name)
|
||||
|
||||
# 如果文件不存在,先下载
|
||||
if not os.path.exists(history_path):
|
||||
self.download_chat_history(year, month, day, hour)
|
||||
|
||||
with open(history_path, "r", encoding="utf-8") as f:
|
||||
message_history = json.load(f)
|
||||
|
||||
try:
|
||||
|
||||
for message_data in message_history.values():
|
||||
# 只处理发送的消息
|
||||
if message_data.get("action") != "send":
|
||||
continue
|
||||
|
||||
# 提取基本信息
|
||||
from_user = message_data["from"]
|
||||
to_users = message_data["tolist"]
|
||||
room_id = message_data.get("roomid", "")
|
||||
|
||||
# 更新用户列表
|
||||
self.users.extend([from_user] + to_users)
|
||||
self.users = list(set(self.users))
|
||||
|
||||
# 确定接收者类型
|
||||
to_type = 1 if room_id else 0
|
||||
receiver = room_id if to_type == 1 else to_users[0]
|
||||
|
||||
# 处理群组信息
|
||||
if room_id:
|
||||
if room_id not in self.members:
|
||||
self.members[room_id] = to_users + [from_user]
|
||||
else:
|
||||
self.members[room_id].extend(to_users + [from_user])
|
||||
self.members[room_id] = list(set(self.members[room_id]))
|
||||
|
||||
if room_id not in self.groups:
|
||||
self.groups.append(room_id)
|
||||
|
||||
# 构建消息对象
|
||||
message = {
|
||||
"id": message_data["msgid"],
|
||||
"type": message_data["msgtype"],
|
||||
"time": message_data["msgtime"],
|
||||
"toType": to_type,
|
||||
"from": from_user,
|
||||
"to": receiver
|
||||
}
|
||||
|
||||
# 根据消息类型处理内容
|
||||
msg_type = message_data["msgtype"]
|
||||
if msg_type == "mixed":
|
||||
temp = []
|
||||
idx = 0
|
||||
for t in message_data["mixed"]["item"]:
|
||||
idx, content = self._parse_mixed_message({"msgid": message_data["msgid"],
|
||||
"msgtype": t["type"],
|
||||
t["type"]: json.loads(t["content"])}, idx)
|
||||
temp.append([t["type"], content])
|
||||
message["content"] = json.dumps(temp, ensure_ascii=False)
|
||||
else:
|
||||
message["content"] = self._parse_message(message_data)
|
||||
self.messages.append(message)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"解析聊天记录失败: {e}")
|
||||
raise e
|
||||
|
||||
def save_to_sqlite(self, sql: str, data: List[Tuple]) -> None:
|
||||
"""
|
||||
批量保存数据到SQLite数据库
|
||||
|
||||
Args:
|
||||
sql: SQL语句
|
||||
data: 数据列表
|
||||
"""
|
||||
if not data:
|
||||
return
|
||||
|
||||
print(f"开始插入 {len(data)} 条记录")
|
||||
self.cursor.executemany(sql, data)
|
||||
self.conn.commit()
|
||||
print(f"成功插入 {len(data)} 条记录")
|
||||
|
||||
def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int,
|
||||
group_num: int) -> None:
|
||||
"""
|
||||
保存会话信息到SQLite数据库
|
||||
|
||||
Args:
|
||||
date_time: 会话时间
|
||||
message_num: 消息数量
|
||||
file_num: 文件数量
|
||||
user_num: 用户数量
|
||||
group_num: 群组数量
|
||||
"""
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO "information"
|
||||
(datetime, message, file, user, "group")
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
''', [(date_time, message_num, file_num, user_num, group_num)])
|
||||
|
||||
def save_all_to_sqlite(self) -> None:
|
||||
"""
|
||||
保存所有聊天数据到SQLite数据库
|
||||
|
||||
Args:
|
||||
users: 用户列表
|
||||
groups: 群组列表
|
||||
members: 成员映射
|
||||
messages: 消息列表
|
||||
files: 文件映射
|
||||
"""
|
||||
# 保存文件信息
|
||||
file_data = []
|
||||
for file_info in self.files.values():
|
||||
file_data.append((
|
||||
file_info.get('id'),
|
||||
file_info.get('type'),
|
||||
file_info.get('content'),
|
||||
file_info.get('original', ''),
|
||||
file_info.get('extension', ''),
|
||||
file_info.get('size', 0)
|
||||
))
|
||||
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO file
|
||||
(id, type, content, original, extension, size)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
''', file_data)
|
||||
|
||||
# 保存用户信息
|
||||
user_data = [(user, user) for user in self.users]
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO user
|
||||
(id, name)
|
||||
VALUES (?, ?)
|
||||
''', user_data)
|
||||
|
||||
# 保存消息信息
|
||||
message_data = []
|
||||
for message in self.messages:
|
||||
message_data.append((
|
||||
message.get('id'),
|
||||
message.get('type'),
|
||||
message.get('time'),
|
||||
message.get('toType'),
|
||||
message.get('from'),
|
||||
message.get('to'),
|
||||
message.get('content')
|
||||
))
|
||||
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO message
|
||||
(id, type, time, toType, sender, receiver, content)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
''', message_data)
|
||||
|
||||
# 保存群组信息
|
||||
group_data = [(group, group) for group in self.groups]
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO "group"
|
||||
(id, name)
|
||||
VALUES (?, ?)
|
||||
''', group_data)
|
||||
|
||||
# 保存成员信息
|
||||
member_data = []
|
||||
for group_id, group_users in self.members.items():
|
||||
for user in group_users:
|
||||
member_data.append((group_id, user))
|
||||
|
||||
self.save_to_sqlite('''
|
||||
INSERT OR IGNORE INTO "member"
|
||||
(groupId, userId)
|
||||
VALUES (?, ?)
|
||||
''', member_data)
|
||||
|
||||
def process_chat_session(self, year: int, month: int, day: int, hour: int) -> bool:
|
||||
"""
|
||||
处理完整的聊天会话流程
|
||||
|
||||
Args:
|
||||
year: 年份
|
||||
month: 月份
|
||||
day: 日期
|
||||
hour: 小时
|
||||
|
||||
Returns:
|
||||
处理是否成功
|
||||
"""
|
||||
self.users = []
|
||||
self.groups = []
|
||||
self.members = {}
|
||||
self.messages = []
|
||||
self.files = {}
|
||||
try:
|
||||
self.cache_path = f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/"
|
||||
# 解析聊天历史
|
||||
self.parse_chat_history(year, month, day, hour)
|
||||
|
||||
# 保存到数据库
|
||||
self.save_all_to_sqlite()
|
||||
self.save_information_to_sqlite(f"{year}-{month:02d}-{day:02d}-{hour:02d}", len(self.messages),
|
||||
len(self.files), len(self.users), len(self.groups))
|
||||
print(f"成功处理 {year}-{month:02d}-{day:02d}-{hour:02d} 的聊天会话")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"处理聊天会话失败: {e}")
|
||||
return False
|
||||
|
||||
def get_chat_history(self, start_time_str: str, end_time_str: str) -> list:
|
||||
"""
|
||||
获取指定时间的聊天历史记录
|
||||
|
||||
Args:
|
||||
start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS"
|
||||
end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS"
|
||||
|
||||
Returns:
|
||||
聊天历史记录列表
|
||||
"""
|
||||
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT message.time, message.content
|
||||
FROM message
|
||||
WHERE (message.type = 'text'
|
||||
or message.type = 'mixed')
|
||||
AND message.time >= ?
|
||||
AND message.time <= ?
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
res = cursor.fetchall()
|
||||
return res
|
||||
|
||||
def get_history(self, start_time_str: str, end_time_str: str) -> list:
|
||||
"""
|
||||
获取指定时间的聊天历史记录
|
||||
|
||||
Args:
|
||||
start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS"
|
||||
end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS"
|
||||
|
||||
Returns:
|
||||
聊天历史记录列表
|
||||
"""
|
||||
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT count(1) as count
|
||||
FROM message
|
||||
WHERE message.time >= ?
|
||||
AND message.time <= ?
|
||||
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
res = cursor.fetchall()
|
||||
print(f"总消息数量: {res[0][0]}")
|
||||
cursor.execute('''
|
||||
SELECT 1 as count
|
||||
FROM message
|
||||
WHERE message.time >= ?
|
||||
AND message.time <= ?
|
||||
group by message.sender
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
res = cursor.fetchall()
|
||||
print(f"发送者数量: {len(res)}")
|
||||
cursor.execute('''
|
||||
SELECT *
|
||||
FROM message
|
||||
|
||||
WHERE message.type in ('text', 'mixed', 'chatrecord')
|
||||
AND message.time >= ?
|
||||
AND message.time <= ?
|
||||
AND message.toType == 1
|
||||
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
res = cursor.fetchall()
|
||||
return res
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭数据库连接"""
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""析构函数,确保资源被正确释放"""
|
||||
self.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chat_session = ChatSessionManager()
|
||||
# for day in range(22, 30+1):
|
||||
# for hour in range(0, 24):
|
||||
# chat_session.process_chat_session(2025, 9, day, hour)
|
||||
# for day in range(1, 29+1):
|
||||
# for hour in range(0, 24):
|
||||
# chat_session.process_chat_session(2025, 10, day, hour)
|
||||
# start = "2025-10-15 00:00:00"
|
||||
# end = "2025-10-30 23:59:59"
|
||||
# data = chat_session.get_history(start, end)
|
||||
# group_messages = {}
|
||||
# for i in data:
|
||||
# group_id = i[-1]
|
||||
# if group_id not in group_messages:
|
||||
# group_messages[group_id] = []
|
||||
# group_messages[group_id].append((i[5], i[2], i[3]))
|
||||
# group_qa = {}
|
||||
# for group_id in group_messages:
|
||||
# print(f"群ID: {group_id}")
|
||||
# print(f"消息数量: {len(group_messages[group_id])}")
|
||||
# content = ""
|
||||
# if len(group_messages[group_id]) < 2:
|
||||
# continue
|
||||
# elif len(group_messages[group_id]) == 2:
|
||||
# print(group_messages[group_id])
|
||||
# else:
|
||||
# for msg in sorted(group_messages[group_id], key=lambda i: i[1]):
|
||||
# content += json.dumps({
|
||||
# "user": msg[0],
|
||||
# "time": msg[1],
|
||||
# "content": msg[2]
|
||||
# },ensure_ascii=False) + ",\n"
|
||||
#
|
||||
# template = f"""
|
||||
# 以下是群中的消息记录
|
||||
#
|
||||
# ```json
|
||||
# [
|
||||
# {content}
|
||||
# ]
|
||||
# ```
|
||||
# """ + """
|
||||
# 找到其中的问题和回答,如果没有回答就只整理问题,问题是与产品填报系统,上市公司,证券政策相关的问题
|
||||
# 问题可能不只由一句话构成,回答也可能不只由一句话构成
|
||||
# 多个问题需要分开放置
|
||||
# 输出格式
|
||||
#
|
||||
# ```json
|
||||
# [
|
||||
# {
|
||||
# "question": [{
|
||||
# "user": "用户1",
|
||||
# "time": "时间戳",
|
||||
# "content": "问题"
|
||||
# }],
|
||||
# "answer": [
|
||||
# {
|
||||
# "user": "用户1",
|
||||
# "time": "时间戳",
|
||||
# "content": "回答"
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
# ]
|
||||
# ```
|
||||
# """
|
||||
# answer = agent.chat_deepseek(template)
|
||||
# answer = answer.replace("```json", "").replace("```", "") if "```json" in answer else answer
|
||||
# try:
|
||||
# answer = json.loads(answer)
|
||||
# except:
|
||||
# print(answer)
|
||||
# answer = []
|
||||
# group_qa[group_id] = answer
|
||||
# print(group_qa)
|
||||
# with open(f"group_qa-{start.split(' ')[0]}-{end.split(' ')[0]}.json", "w", encoding="utf-8") as f:
|
||||
# f.write(json.dumps(group_qa, ensure_ascii=False))
|
||||
# push_message(f"群聊问答-{start.split(' ')[0]}-{end.split(' ')[0]}", "问题提取完成", "")
|
||||
with open(f"group_qa.json", "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
cursor = chat_session.conn.cursor()
|
||||
a = cursor.execute('''
|
||||
select *
|
||||
from message
|
||||
''').fetchall()
|
||||
|
||||
mapping = {}
|
||||
for i in a:
|
||||
mapping[i[2] + "-" + i[5]] = i[0]
|
||||
dar = []
|
||||
for r in data:
|
||||
qas = data.get(r)
|
||||
|
||||
for qa in qas:
|
||||
Q = []
|
||||
A = []
|
||||
q = qa["question"]
|
||||
for qq in q:
|
||||
if qq["time"] + "-" + qq["user"] not in mapping:
|
||||
print(qq)
|
||||
else:
|
||||
Q.append(mapping[qq["time"] + "-" + qq["user"]])
|
||||
a = qa["answer"]
|
||||
for aa in a:
|
||||
if aa["time"] + "-" + aa["user"] not in mapping:
|
||||
print(aa)
|
||||
else:
|
||||
A.append(mapping[aa["time"] + "-" + aa["user"]])
|
||||
dar.append((r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False)))
|
||||
a = cursor.executemany('''
|
||||
insert into qa (groupId, question, answer)
|
||||
values (?, ?, ?)
|
||||
''', dar)
|
||||
chat_session.conn.commit()
|
||||
Reference in New Issue
Block a user