This commit is contained in:
lychang
2025-11-05 01:43:15 +08:00
parent d10ec1f911
commit 43cb0ff431
18 changed files with 4111 additions and 2579 deletions

View File

@@ -3,7 +3,12 @@ import json
import os.path
import sqlite3
from typing import List, Tuple
import logging
import os
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
import pytz
from file import ObjectFileManager, LocalFileManager
@@ -27,6 +32,133 @@ def convert_to_utc(time_str, time_format='%Y-%m-%d %H:%M:%S', source_timezone=No
return utc_dt
class AIAgent:
"""AI代理类,整合了各种AI功能"""
def __init__(self, deepseek_api_key: str = "sk-c2f04b4d635b4df0aace0432f3c6d6f4",
max_retries: int = 3, timeout: int = 30):
"""初始化AI助手
Args:
deepseek_api_key: DeepSeek API密钥
max_retries: 最大重试次数
timeout: 请求超时时间(秒)
"""
self.deepseek_api_key = deepseek_api_key
self.timeout = timeout
# 配置日志
self._setup_logging()
# 创建带重试机制的会话
self.session = self._create_session(max_retries)
# 配置常量
self.BASE_REPORT_URL = "https://a.cmdp.cn/basiceg/v1/json/tpldev"
self.CACHE_DIR = "cache"
def _setup_logging(self):
"""配置日志"""
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
self.logger = logging.getLogger(__name__)
@staticmethod
def _create_session(max_retries: int) -> requests.Session:
"""创建带重试机制的会话
Args:
max_retries: 最大重试次数
Returns:
配置好的会话对象
"""
session = requests.Session()
# 配置重试策略
retry_strategy = Retry(
total=max_retries,
status_forcelist=[429, 500, 502, 503, 504],
backoff_factor=1
)
# 为HTTP和HTTPS适配器添加重试机制
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def _make_request(self, method: str, url: str, **kwargs):
"""发送HTTP请求(带错误处理)
Args:
method: HTTP方法
url: 请求URL
**kwargs: 其他请求参数
Returns:
JSON响应或None
"""
try:
# 设置默认超时
if 'timeout' not in kwargs:
kwargs['timeout'] = self.timeout
response = self.session.request(method, url, **kwargs)
response.raise_for_status()
self.logger.info(f"请求成功: {url}")
return response.json()
except requests.exceptions.RequestException as e:
self.logger.error(f"请求失败: {url} - {str(e)}")
return {}
except json.JSONDecodeError as e:
self.logger.error(f"JSON解析失败: {url} - {str(e)}")
return {}
def chat_deepseek(self, message: str, role: str = "", model: str = "deepseek-chat") -> str:
"""与DeepSeek聊天
Args:
message: 用户消息
role: 系统角色
model: 使用的模型
Returns:
AI回复内容
"""
url = "https://api.deepseek.com/chat/completions"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.deepseek_api_key}"
}
messages = []
if role:
messages.append({"role": "system", "content": role})
messages.append({"role": "user", "content": message})
body = {
"model": model,
"messages": messages,
"stream": False
}
result = self._make_request("POST", url, json=body, headers=headers)
if result and "choices" in result:
return result.get('choices', [{}])[0]['message']['content']
self.logger.warning("DeepSeek API调用失败或返回空结果")
return ""
agent = AIAgent(max_retries=1, timeout=180)
class ChatSessionManager:
"""聊天会话管理器类"""
@@ -40,7 +172,7 @@ class ChatSessionManager:
self.chat_history_dir = chat_history_dir
self.files_dir = os.path.join(chat_history_dir, "files")
self.messages_dir = os.path.join(chat_history_dir, "messages")
self.db_path = os.path.join("", "data.sqlite")
self.db_path = os.path.join("db", "data.sqlite")
self.cache_path = None
# 初始化文件管理器
self.obj = ObjectFileManager()
@@ -293,13 +425,6 @@ class ChatSessionManager:
def save_all_to_sqlite(self) -> None:
"""
保存所有聊天数据到SQLite数据库
Args:
users: 用户列表
groups: 群组列表
members: 成员映射
messages: 消息列表
files: 文件映射
"""
# 保存文件信息
file_data = []
@@ -425,50 +550,106 @@ class ChatSessionManager:
res = cursor.fetchall()
return res
def get_history(self, start_time_str: str, end_time_str: str) -> list:
@staticmethod
def analyze_qa_content(group_messages) -> dict:
group_qa = {}
for group_id in group_messages:
print(f"群ID: {group_id}")
print(f"消息数量: {len(group_messages[group_id])}")
content = ""
if len(group_messages[group_id]) <= 2:
continue
else:
for msg in sorted(group_messages[group_id], key=lambda i: i[1]):
content += json.dumps({
"user": msg[1],
"time": msg[2],
"content": msg[3],
"message_id": msg[4]
}, ensure_ascii=False) + ",\n"
template = f"""
以下是群中的消息记录
```json
[
{content}
]
```
""" + """
找到其中的问题和回答,如果没有回答就只整理问题,问题是与产品填报系统,上市公司,证券政策相关的问题
问题可能不只由一句话构成,回答也可能不只由一句话构成
多个问题需要分开放置
输出格式
```json
[
{
"question": ["message_id",...],
"answer": ["message_id",...]
}
]
```
"""
answer = agent.chat_deepseek(template)
answer = answer.replace("```json", "").replace("```", "") if "```json" in answer else answer
try:
answer = json.loads(answer)
except:
print(answer)
answer = []
group_qa[group_id] = answer
return group_qa
def save_qa_to_sqlite(self, group_qa: dict):
"""
获取指定时间的聊天历史记录
将问答数据保存到 SQLite 数据库
Args:
group_qa: 问答数据字典,键为群ID,值为问答列表
"""
data = []
cursor = self.conn.cursor()
for r in group_qa:
qas = group_qa.get(r, {})
for qa in qas:
Q = qa["question"]
A = qa["answer"]
data.append((r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False)))
cursor.executemany('''
insert into qa (groupId, question, answer)
values (?, ?, ?)
''', data)
self.conn.commit()
def update_qa_data(self, start_time_str: str, end_time_str: str):
"""
更新指定时间范围内的问答内容
Args:
start_time_str: 开始时间,格式为 "YYYY-MM-DD HH:MM:SS"
end_time_str: 结束时间,格式为 "YYYY-MM-DD HH:MM:SS"
Returns:
聊天历史记录列表
"""
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
cursor = self.conn.cursor()
cursor.execute('''
SELECT count(1) as count
SELECT receiver, sender, time, content, id
FROM message
WHERE message.time >= ?
AND message.time <= ?
AND toType = 1
AND type in ('text', 'mixed', 'chatrecord')
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
print(f"总消息数量: {res[0][0]}")
cursor.execute('''
SELECT 1 as count
FROM message
WHERE message.time >= ?
AND message.time <= ?
group by message.sender
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
print(f"发送者数量: {len(res)}")
cursor.execute('''
SELECT *
FROM message
WHERE message.type in ('text', 'mixed', 'chatrecord')
AND message.time >= ?
AND message.time <= ?
AND message.toType == 1
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
return res
data = cursor.fetchall()
group_messages = {}
for i in data:
group_id = i[0]
if group_id not in group_messages:
group_messages[group_id] = []
group_messages[group_id].append(i)
group_qa = self.analyze_qa_content(group_messages)
self.analyze_qa_content(group_qa)
def close(self) -> None:
"""关闭数据库连接"""
@@ -482,116 +663,6 @@ class ChatSessionManager:
if __name__ == '__main__':
chat_session = ChatSessionManager()
# for day in range(22, 30+1):
# for hour in range(0, 24):
# chat_session.process_chat_session(2025, 9, day, hour)
# for day in range(1, 29+1):
# for hour in range(0, 24):
# chat_session.process_chat_session(2025, 10, day, hour)
# start = "2025-10-15 00:00:00"
# end = "2025-10-30 23:59:59"
# data = chat_session.get_history(start, end)
# group_messages = {}
# for i in data:
# group_id = i[-1]
# if group_id not in group_messages:
# group_messages[group_id] = []
# group_messages[group_id].append((i[5], i[2], i[3]))
# group_qa = {}
# for group_id in group_messages:
# print(f"群ID: {group_id}")
# print(f"消息数量: {len(group_messages[group_id])}")
# content = ""
# if len(group_messages[group_id]) < 2:
# continue
# elif len(group_messages[group_id]) == 2:
# print(group_messages[group_id])
# else:
# for msg in sorted(group_messages[group_id], key=lambda i: i[1]):
# content += json.dumps({
# "user": msg[0],
# "time": msg[1],
# "content": msg[2]
# },ensure_ascii=False) + ",\n"
#
# template = f"""
# 以下是群中的消息记录
#
# ```json
# [
# {content}
# ]
# ```
# """ + """
# 找到其中的问题和回答,如果没有回答就只整理问题,问题是与产品填报系统,上市公司,证券政策相关的问题
# 问题可能不只由一句话构成,回答也可能不只由一句话构成
# 多个问题需要分开放置
# 输出格式
#
# ```json
# [
# {
# "question": [{
# "user": "用户1",
# "time": "时间戳",
# "content": "问题"
# }],
# "answer": [
# {
# "user": "用户1",
# "time": "时间戳",
# "content": "回答"
# }
# ]
# }
# ]
# ```
# """
# answer = agent.chat_deepseek(template)
# answer = answer.replace("```json", "").replace("```", "") if "```json" in answer else answer
# try:
# answer = json.loads(answer)
# except:
# print(answer)
# answer = []
# group_qa[group_id] = answer
# print(group_qa)
# with open(f"group_qa-{start.split(' ')[0]}-{end.split(' ')[0]}.json", "w", encoding="utf-8") as f:
# f.write(json.dumps(group_qa, ensure_ascii=False))
# push_message(f"群聊问答-{start.split(' ')[0]}-{end.split(' ')[0]}", "问题提取完成", "")
with open(f"group_qa.json", "r", encoding="utf-8") as f:
data = json.load(f)
cursor = chat_session.conn.cursor()
a = cursor.execute('''
select *
from message
''').fetchall()
mapping = {}
for i in a:
mapping[i[2] + "-" + i[5]] = i[0]
dar = []
for r in data:
qas = data.get(r)
for qa in qas:
Q = []
A = []
q = qa["question"]
for qq in q:
if qq["time"] + "-" + qq["user"] not in mapping:
print(qq)
else:
Q.append(mapping[qq["time"] + "-" + qq["user"]])
a = qa["answer"]
for aa in a:
if aa["time"] + "-" + aa["user"] not in mapping:
print(aa)
else:
A.append(mapping[aa["time"] + "-" + aa["user"]])
dar.append((r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False)))
a = cursor.executemany('''
insert into qa (groupId, question, answer)
values (?, ?, ?)
''', dar)
chat_session.conn.commit()
start = "2025-10-15 00:00:00"
end = "2025-10-15 23:59:59"
chat_session.update_qa_data(start, end)