import datetime import json import os.path import sqlite3 from typing import List, Tuple import pytz from file import ObjectFileManager, LocalFileManager TZ = pytz.timezone('Asia/Shanghai') def convert_to_utc(time_str, time_format='%Y-%m-%d %H:%M:%S', source_timezone=None): # 解析时间字符串为datetime对象 local_dt = datetime.datetime.strptime(time_str, time_format) # 如果指定了源时区,则将本地时间转换为UTC时间 if source_timezone: tz = pytz.timezone(source_timezone) local_dt = tz.localize(local_dt) utc_dt = local_dt.astimezone(pytz.UTC) else: # 如果没有指定源时区,则假设时间字符串已经是UTC时间 utc_dt = pytz.UTC.localize(local_dt) return utc_dt class ChatSessionManager: """聊天会话管理器类""" def __init__(self, chat_history_dir: str = "../chat_history"): """ 初始化聊天会话管理器 Args: chat_history_dir: 聊天历史文件目录 """ self.chat_history_dir = chat_history_dir self.files_dir = os.path.join(chat_history_dir, "files") self.messages_dir = os.path.join(chat_history_dir, "messages") self.db_path = os.path.join("", "data.sqlite") self.cache_path = None # 初始化文件管理器 self.obj = ObjectFileManager() self.lfm = LocalFileManager(self.files_dir) self.users = [] self.groups = [] self.members = {} self.messages = [] self.files = {} # 初始化数据库连接 self.conn = sqlite3.connect(self.db_path) self.cursor = self.conn.cursor() # 确保目录存在 os.makedirs(self.files_dir, exist_ok=True) os.makedirs(self.messages_dir, exist_ok=True) def download_chat_history(self, year: int, month: int, day: int, hour: int) -> bool: """ 下载指定时间的聊天历史 Args: year: 年份 month: 月份 day: 日期 hour: 小时 Returns: 下载是否成功 """ message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json" print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d}") try: file_list = self.obj.get_file_list(f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/") print(f"共计{len(file_list)}个文件") history = {} count = 1 for file_info in file_list: path = file_info["path"].replace("s3://", "") file_content = self.obj.get_file(path) print(f"正在处理第{count}个文件: {path}") count += 1 if path.endswith(".json"): data = json.loads(file_content) history[data["msgid"]] = data else: # 保存非JSON文件 filename = path.split("/")[-1] with open(os.path.join(self.files_dir, filename), "wb") as f: f.write(file_content) # 保存聊天历史JSON文件 output_path = os.path.join(self.messages_dir, message_history_name) with open(output_path, "w", encoding="utf-8") as f: json.dump(history, f, ensure_ascii=False, indent=2) print(f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} complete") return True except Exception as e: error_file = os.path.join(self.messages_dir, f"{message_history_name}.unfinished") with open(error_file, "w", encoding="utf-8") as f: f.write(str(e)) print( f"Downloading chat history for {year}-{month:02d}-{day:02d}-{hour:02d} failed: please check the {message_history_name}.unfinished") return False def _parse_message(self, data, is_mixed: bool = False, idx: int = 0): msg_type = data["msgtype"] if msg_type == "text": content = data['text']["content"] elif msg_type in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]: file_data = data[msg_type] suffixs = [f"_{idx}", ".png", ".jpg", ".gif", ".mp4", "_.png", "_.jpg", "_.gif", "_.mp4", ("_" + file_data["filename"]) if msg_type == "file" else ""] file_path = "" for suffix in suffixs: temp_path = f"{data['msgid']}_mixed{suffix}" if is_mixed else f"{data['msgid']}_{msg_type}{suffix}" if self.lfm.exists(temp_path): file_path = temp_path break if file_path != "": ext_mapping = { "file": (file_data.get('fileext', ''), "filesize"), "image": ("png", file_data.get("filesize", 0)), "video": ("mp4", file_data.get("filesize", 0)), "voice": ("wav", file_data.get("voice_size", 0)), "emotion": ("gif" if file_data.get("type", 0) == 1 else "png", file_data.get("imagesize", 0)), "meeting_voice_call": ("wav", self.lfm.get_size(file_path)), } file_info = { "id": file_data.get("md5sum", self.lfm.get_md5(file_path)), "size": ext_mapping[msg_type][1], "extension": ext_mapping[msg_type][0], "content": file_path, "original": self.cache_path + file_path, "type": msg_type } self.files[file_info["id"]] = file_info content = file_info["id"] else: file_path = f"{data['msgid']}_{msg_type}_0" file_info = { "id": file_data.get("md5sum", file_path), "size": file_data.get('filesize', 0), "extension": file_data.get('fileext', "unknown"), "content": file_path, "original": self.cache_path + file_path, "type": msg_type } self.files[file_info["id"]] = file_info content = file_info["id"] elif msg_type == "link": data = data[msg_type] content = f'{data["description"]}{data["title"]}({data["link_url"]})' else: print(f"未知消息类型: {msg_type}") content = json.dumps(data, ensure_ascii=False) return content def _parse_mixed_message(self, param, idx: int): tp = param["msgtype"] content = self._parse_message(param, True, idx) if tp in ["file", "image", "video", "voice", "emotion", "meeting_voice_call"]: self.files[content]["content"] = f'{param["msgid"]}_mixed_{idx}' idx += 1 return idx, content def parse_chat_history(self, year: int, month: int, day: int, hour: int): """ 解析聊天历史数据 Args: year: 年份 month: 月份 day: 日期 hour: 小时 Returns: (用户列表, 群组列表, 成员映射, 消息列表, 文件映射) """ message_history_name = f"{year}_{month:02d}_{day:02d}_{hour:02d}.json" history_path = os.path.join(self.messages_dir, message_history_name) # 如果文件不存在,先下载 if not os.path.exists(history_path): self.download_chat_history(year, month, day, hour) with open(history_path, "r", encoding="utf-8") as f: message_history = json.load(f) try: for message_data in message_history.values(): # 只处理发送的消息 if message_data.get("action") != "send": continue # 提取基本信息 from_user = message_data["from"] to_users = message_data["tolist"] room_id = message_data.get("roomid", "") # 更新用户列表 self.users.extend([from_user] + to_users) self.users = list(set(self.users)) # 确定接收者类型 to_type = 1 if room_id else 0 receiver = room_id if to_type == 1 else to_users[0] # 处理群组信息 if room_id: if room_id not in self.members: self.members[room_id] = to_users + [from_user] else: self.members[room_id].extend(to_users + [from_user]) self.members[room_id] = list(set(self.members[room_id])) if room_id not in self.groups: self.groups.append(room_id) # 构建消息对象 message = { "id": message_data["msgid"], "type": message_data["msgtype"], "time": message_data["msgtime"], "toType": to_type, "from": from_user, "to": receiver } # 根据消息类型处理内容 msg_type = message_data["msgtype"] if msg_type == "mixed": temp = [] idx = 0 for t in message_data["mixed"]["item"]: idx, content = self._parse_mixed_message({"msgid": message_data["msgid"], "msgtype": t["type"], t["type"]: json.loads(t["content"])}, idx) temp.append([t["type"], content]) message["content"] = json.dumps(temp, ensure_ascii=False) else: message["content"] = self._parse_message(message_data) self.messages.append(message) except Exception as e: print(f"解析聊天记录失败: {e}") raise e def save_to_sqlite(self, sql: str, data: List[Tuple]) -> None: """ 批量保存数据到SQLite数据库 Args: sql: SQL语句 data: 数据列表 """ if not data: return print(f"开始插入 {len(data)} 条记录") self.cursor.executemany(sql, data) self.conn.commit() print(f"成功插入 {len(data)} 条记录") def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int, group_num: int) -> None: """ 保存会话信息到SQLite数据库 Args: date_time: 会话时间 message_num: 消息数量 file_num: 文件数量 user_num: 用户数量 group_num: 群组数量 """ self.save_to_sqlite(''' INSERT OR IGNORE INTO "information" (datetime, message, file, user, "group") VALUES (?, ?, ?, ?, ?) ''', [(date_time, message_num, file_num, user_num, group_num)]) def save_all_to_sqlite(self) -> None: """ 保存所有聊天数据到SQLite数据库 Args: users: 用户列表 groups: 群组列表 members: 成员映射 messages: 消息列表 files: 文件映射 """ # 保存文件信息 file_data = [] for file_info in self.files.values(): file_data.append(( file_info.get('id'), file_info.get('type'), file_info.get('content'), file_info.get('original', ''), file_info.get('extension', ''), file_info.get('size', 0) )) self.save_to_sqlite(''' INSERT OR IGNORE INTO file (id, type, content, original, extension, size) VALUES (?, ?, ?, ?, ?, ?) ''', file_data) # 保存用户信息 user_data = [(user, user) for user in self.users] self.save_to_sqlite(''' INSERT OR IGNORE INTO user (id, name) VALUES (?, ?) ''', user_data) # 保存消息信息 message_data = [] for message in self.messages: message_data.append(( message.get('id'), message.get('type'), message.get('time'), message.get('toType'), message.get('from'), message.get('to'), message.get('content') )) self.save_to_sqlite(''' INSERT OR IGNORE INTO message (id, type, time, toType, sender, receiver, content) VALUES (?, ?, ?, ?, ?, ?, ?) ''', message_data) # 保存群组信息 group_data = [(group, group) for group in self.groups] self.save_to_sqlite(''' INSERT OR IGNORE INTO "group" (id, name) VALUES (?, ?) ''', group_data) # 保存成员信息 member_data = [] for group_id, group_users in self.members.items(): for user in group_users: member_data.append((group_id, user)) self.save_to_sqlite(''' INSERT OR IGNORE INTO "member" (groupId, userId) VALUES (?, ?) ''', member_data) def process_chat_session(self, year: int, month: int, day: int, hour: int) -> bool: """ 处理完整的聊天会话流程 Args: year: 年份 month: 月份 day: 日期 hour: 小时 Returns: 处理是否成功 """ self.users = [] self.groups = [] self.members = {} self.messages = [] self.files = {} try: self.cache_path = f"zysl-log/wework/chat_msg/{year}/{month}/{day}/{hour}/" # 解析聊天历史 self.parse_chat_history(year, month, day, hour) # 保存到数据库 self.save_all_to_sqlite() self.save_information_to_sqlite(f"{year}-{month:02d}-{day:02d}-{hour:02d}", len(self.messages), len(self.files), len(self.users), len(self.groups)) print(f"成功处理 {year}-{month:02d}-{day:02d}-{hour:02d} 的聊天会话") return True except Exception as e: print(f"处理聊天会话失败: {e}") return False def get_chat_history(self, start_time_str: str, end_time_str: str) -> list: """ 获取指定时间的聊天历史记录 Args: start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS" end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS" Returns: 聊天历史记录列表 """ start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp() end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp() cursor = self.conn.cursor() cursor.execute(''' SELECT message.time, message.content FROM message WHERE (message.type = 'text' or message.type = 'mixed') AND message.time >= ? AND message.time <= ? ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) res = cursor.fetchall() return res def get_history(self, start_time_str: str, end_time_str: str) -> list: """ 获取指定时间的聊天历史记录 Args: start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS" end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS" Returns: 聊天历史记录列表 """ start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp() end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp() cursor = self.conn.cursor() cursor.execute(''' SELECT count(1) as count FROM message WHERE message.time >= ? AND message.time <= ? ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) res = cursor.fetchall() print(f"总消息数量: {res[0][0]}") cursor.execute(''' SELECT 1 as count FROM message WHERE message.time >= ? AND message.time <= ? group by message.sender ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) res = cursor.fetchall() print(f"发送者数量: {len(res)}") cursor.execute(''' SELECT * FROM message WHERE message.type in ('text', 'mixed', 'chatrecord') AND message.time >= ? AND message.time <= ? AND message.toType == 1 ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) res = cursor.fetchall() return res def close(self) -> None: """关闭数据库连接""" if self.conn: self.conn.close() def __del__(self) -> None: """析构函数,确保资源被正确释放""" self.close() if __name__ == '__main__': chat_session = ChatSessionManager() # for day in range(22, 30+1): # for hour in range(0, 24): # chat_session.process_chat_session(2025, 9, day, hour) # for day in range(1, 29+1): # for hour in range(0, 24): # chat_session.process_chat_session(2025, 10, day, hour) # start = "2025-10-15 00:00:00" # end = "2025-10-30 23:59:59" # data = chat_session.get_history(start, end) # group_messages = {} # for i in data: # group_id = i[-1] # if group_id not in group_messages: # group_messages[group_id] = [] # group_messages[group_id].append((i[5], i[2], i[3])) # group_qa = {} # for group_id in group_messages: # print(f"群ID: {group_id}") # print(f"消息数量: {len(group_messages[group_id])}") # content = "" # if len(group_messages[group_id]) < 2: # continue # elif len(group_messages[group_id]) == 2: # print(group_messages[group_id]) # else: # for msg in sorted(group_messages[group_id], key=lambda i: i[1]): # content += json.dumps({ # "user": msg[0], # "time": msg[1], # "content": msg[2] # },ensure_ascii=False) + ",\n" # # template = f""" # 以下是群中的消息记录 # # ```json # [ # {content} # ] # ``` # """ + """ # 找到其中的问题和回答,如果没有回答就只整理问题,问题是与产品填报系统,上市公司,证券政策相关的问题 # 问题可能不只由一句话构成,回答也可能不只由一句话构成 # 多个问题需要分开放置 # 输出格式 # # ```json # [ # { # "question": [{ # "user": "用户1", # "time": "时间戳", # "content": "问题" # }], # "answer": [ # { # "user": "用户1", # "time": "时间戳", # "content": "回答" # } # ] # } # ] # ``` # """ # answer = agent.chat_deepseek(template) # answer = answer.replace("```json", "").replace("```", "") if "```json" in answer else answer # try: # answer = json.loads(answer) # except: # print(answer) # answer = [] # group_qa[group_id] = answer # print(group_qa) # with open(f"group_qa-{start.split(' ')[0]}-{end.split(' ')[0]}.json", "w", encoding="utf-8") as f: # f.write(json.dumps(group_qa, ensure_ascii=False)) # push_message(f"群聊问答-{start.split(' ')[0]}-{end.split(' ')[0]}", "问题提取完成", "") with open(f"group_qa.json", "r", encoding="utf-8") as f: data = json.load(f) cursor = chat_session.conn.cursor() a = cursor.execute(''' select * from message ''').fetchall() mapping = {} for i in a: mapping[i[2] + "-" + i[5]] = i[0] dar = [] for r in data: qas = data.get(r) for qa in qas: Q = [] A = [] q = qa["question"] for qq in q: if qq["time"] + "-" + qq["user"] not in mapping: print(qq) else: Q.append(mapping[qq["time"] + "-" + qq["user"]]) a = qa["answer"] for aa in a: if aa["time"] + "-" + aa["user"] not in mapping: print(aa) else: A.append(mapping[aa["time"] + "-" + aa["user"]]) dar.append((r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False))) a = cursor.executemany(''' insert into qa (groupId, question, answer) values (?, ?, ?) ''', dar) chat_session.conn.commit()