update code

This commit is contained in:
lychang
2025-11-14 01:21:48 +08:00
parent 00545db202
commit 95652ed5be
4 changed files with 187 additions and 132 deletions

91
app.py
View File

@@ -1,48 +1,11 @@
from datetime import datetime, timedelta from datetime import datetime, timedelta
from flask import Flask, render_template, jsonify, request from flask import Flask, render_template, jsonify, request
import sqlite3
import json import json
import os import os
import time import time
from chat_session import agent, ChatSessionManager from chat_session import agent, ChatSessionManager
from tools import TimeUtils, now, from_string, from_datetime, format_timestamp from tools import TimeUtils, now, from_string, from_datetime, format_timestamp, DatabaseManager
class DatabaseManager:
"""数据库管理类"""
def __init__(self, db_path='db/data'):
self.db_path = db_path
def get_connection(self):
"""获取数据库连接"""
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def execute_query(self, query, params=None, fetch_all=True):
"""执行查询并返回结果"""
conn = self.get_connection()
cursor = conn.cursor()
try:
if params:
cursor.execute(query, params)
else:
cursor.execute(query)
if fetch_all:
result = cursor.fetchall()
else:
result = cursor.fetchone()
conn.close()
return result
except Exception as e:
print(e)
conn.close()
raise e
class MessageRender: class MessageRender:
@@ -99,9 +62,13 @@ class MessageRender:
return message_data return message_data
class ReportService(DatabaseManager): class ReportService:
"""报告服务类""" """报告服务类"""
def __init__(self, db: DatabaseManager):
self.db = db
self.execute_query = self.db.execute_query
def get_message_count_by_time_range(self, start_timestamp, end_timestamp): def get_message_count_by_time_range(self, start_timestamp, end_timestamp):
"""获取指定时间范围内的消息数量""" """获取指定时间范围内的消息数量"""
result = self.execute_query(''' result = self.execute_query('''
@@ -124,7 +91,6 @@ class ReportService(DatabaseManager):
def get_qa_count_by_time_range(self, start_timestamp, end_timestamp): def get_qa_count_by_time_range(self, start_timestamp, end_timestamp):
"""获取指定时间范围内的QA记录数量""" """获取指定时间范围内的QA记录数量"""
print(start_timestamp, end_timestamp)
result = self.execute_query(''' result = self.execute_query('''
SELECT COUNT(*) as count SELECT COUNT(*) as count
FROM qa FROM qa
@@ -135,7 +101,6 @@ class ReportService(DatabaseManager):
def get_non_answer_qa_count_by_time_range(self, start_timestamp, end_timestamp): def get_non_answer_qa_count_by_time_range(self, start_timestamp, end_timestamp):
"""获取指定时间范围内的QA记录数量""" """获取指定时间范围内的QA记录数量"""
print(start_timestamp, end_timestamp)
result = self.execute_query(''' result = self.execute_query('''
SELECT COUNT(*) as count SELECT COUNT(*) as count
FROM qa FROM qa
@@ -502,10 +467,10 @@ class ReportService(DatabaseManager):
app = Flask(__name__) app = Flask(__name__)
app.template_folder = 'templates' app.template_folder = 'templates'
# 初始化管理器 # 初始化管理器
db_manager = DatabaseManager() db_manager = DatabaseManager('db/data', online_db_uri="zysl-log/wework/chat_msg/db/data.db")
report_service = ReportService() report_service = ReportService(db_manager)
message_processor = MessageRender() message_processor = MessageRender()
chat_session = ChatSessionManager() chat_session = ChatSessionManager(db_manager)
def api_response(success=True, data=None, error=None, status_code=200): def api_response(success=True, data=None, error=None, status_code=200):
@@ -541,35 +506,36 @@ def weekly_report_page():
def daily_report_page(): def daily_report_page():
"""日报页面""" """日报页面"""
return render_template('daily_report.html') return render_template('daily_report.html')
@app.route('/api/db/datetime')
def db_status():
@app.route('/db/update') return api_response(success=True, data={'message': chat_session.status()})
@app.route('/api/db/update')
def db_update(): def db_update():
current_time = time.localtime(time.time() - 24 * 60 * 60) current_time = time.localtime(time.time() - 24 * 60 * 60)
day = current_time.tm_mday day = current_time.tm_mday
month = current_time.tm_mon month = current_time.tm_mon
year = current_time.tm_year year = current_time.tm_year
if chat_session.status() == f"{year}-{month:02d}-{day:02d}-23": if chat_session.status() == f"{year}-{month:02d}-{day:02d}-23":
return api_response(success=False, error="数据库已更新", status_code=400) return api_response(success=False, error="数据库已更新", status_code=201)
for hour in range(24): for hour in range(24):
chat_session.process_chat_session(year, month, day, hour) chat_session.process_chat_session(year, month, day, hour)
start = f"{year}-{month:02d}-{day:02d} 00:00:00" start = f"{year}-{month:02d}-{day:02d} 00:00:00"
end = f"{year}-{month:02d}-{day:02d} 23:59:59" end = f"{year}-{month:02d}-{day:02d} 23:59:59"
chat_session.update_qa_data(start, end) chat_session.update_qa_data(start, end)
chat_session.upload_db() db_manager.upload_db()
return api_response(success=True, data={'message': '数据库更新成功'}) return api_response(success=True, data={'message': '数据库更新成功'})
@app.route('/db/upload') @app.route('/api/db/upload')
def db_upload(): def db_upload():
chat_session.upload_db() db_manager.upload_db()
return api_response(success=True, data={'message': '数据库上传成功'}) return api_response(success=True, data={'message': '数据库上传成功'})
@app.route('/db/download') @app.route('/api/db/download')
def db_download(): def db_download():
chat_session.download_db() db_manager.download_db()
return api_response(success=True, data={'message': '数据库下载成功'}) return api_response(success=True, data={'message': '数据库下载成功'})
@@ -857,7 +823,7 @@ def generate_daily_report():
# 获取QA记录数量 # 获取QA记录数量
today_qa_count = report_service.get_qa_count_by_time_range(start_timestamp, end_timestamp) today_qa_count = report_service.get_qa_count_by_time_range(start_timestamp, end_timestamp)
yesterday_qa_count = report_service.get_qa_count_by_time_range(yesterday_start, yesterday_end) yesterday_qa_count = report_service.get_qa_count_by_time_range(yesterday_start, yesterday_end)
non_qa_count = report_service.get_non_answer_qa_count_by_time_range(start_timestamp, end_timestamp)
qa_change_rate = report_service.calculate_change_rate( qa_change_rate = report_service.calculate_change_rate(
today_qa_count, yesterday_qa_count today_qa_count, yesterday_qa_count
) )
@@ -895,20 +861,24 @@ def generate_daily_report():
``` ```
## **3. 今日热点与时段** ## **3. 今日热点与时段**
* **热点问题TOP 3**:
### **热点问题TOP 3**:
""" """
for i, topic in enumerate(hot_topics, 1): for i, topic in enumerate(hot_topics, 1):
daily_report += f" {i}. {topic}\n" daily_report += f" {i}. {topic}\n"
daily_report += f"""* **活跃时段**:消息高峰分布 daily_report += f"""### **活跃时段**:消息高峰分布
``` ```
{hourly_activity_chart} {hourly_activity_chart}
``` ```
## **4. 待办事项** ## **4. 待办事项**
* 关注活跃群组的后续讨论,确保所有疑问被解答。
* 明日重点关注:根据今日热点预测明日可能的问题。 {"有"+str(non_qa_count)+"条未被回答的问题,关注活跃群组的后续讨论,确保所有疑问被解答。" if non_qa_count > 0 else "无未被回答的问题,继续保持活跃群组的讨论。"}
明日重点关注:根据今日热点预测明日可能的问题。
""" """
return api_response(data={ return api_response(data={
@@ -1041,9 +1011,10 @@ def generate_weekly_report():
if __name__ == '__main__': if __name__ == '__main__':
# 检查数据库文件是否存在 # 检查数据库文件是否存在
if not os.path.exists('db/data'): if not os.path.exists('db/data'):
print(f"错误: 数据库文件 db/data.sqlite 不存在") print(f"错误: 数据库文件不存在")
chat_session.download_db()
print("正在下载云端数据库") print("正在下载云端数据库")
db_manager.download_db()
print("下载完成")
else: else:
print("启动聊天网页服务...") print("启动聊天网页服务...")
print("访问地址: http://127.0.0.1:5000") print("访问地址: http://127.0.0.1:5000")

View File

@@ -1,7 +1,6 @@
import datetime import datetime
import json import json
import os.path import os.path
import sqlite3
from typing import List, Tuple from typing import List, Tuple
import logging import logging
import os import os
@@ -11,6 +10,7 @@ from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry from urllib3.util.retry import Retry
import pytz import pytz
from tools import DatabaseManager
from file import ObjectFileManager, LocalFileManager from file import ObjectFileManager, LocalFileManager
TZ = pytz.timezone('Asia/Shanghai') TZ = pytz.timezone('Asia/Shanghai')
@@ -162,47 +162,48 @@ agent = AIAgent(max_retries=1, timeout=180)
class ChatSessionManager: class ChatSessionManager:
"""聊天会话管理器类""" """聊天会话管理器类"""
def __init__(self, online_db_uri: str = "zysl-log/wework/chat_msg/db/data.db"): def __init__(self, db_manager: DatabaseManager):
""" """
初始化聊天会话管理器 初始化聊天会话管理器
Args: Args:
online_db_uri: 在线数据库URI db_manager: 数据库管理工具
""" """
self.online_db_uri = online_db_uri
self.files_dir = os.path.join("", "files") self.files_dir = os.path.join("", "files")
self.messages_dir = os.path.join("", "messages") self.messages_dir = os.path.join("", "messages")
self.db_path = os.path.join("db", "data")
self.cache_path = None self.cache_path = None
# 初始化文件管理器 # 初始化文件管理器
self.obj = ObjectFileManager() self.obj = ObjectFileManager()
self.lfm = LocalFileManager(self.files_dir) self.lfm = LocalFileManager(self.files_dir)
self.dbm = LocalFileManager("db") self.dbm = db_manager
self.users = [] self.users = []
self.groups = [] self.groups = []
self.members = {} self.members = {}
self.messages = [] self.messages = []
self.files = {} self.files = {}
# 初始化数据库连接
self.conn = sqlite3.connect(self.db_path)
self.cursor = self.conn.cursor()
# 确保目录存在 # 确保目录存在
os.makedirs(self.files_dir, exist_ok=True) os.makedirs(self.files_dir, exist_ok=True)
os.makedirs(self.messages_dir, exist_ok=True) os.makedirs(self.messages_dir, exist_ok=True)
def upload_db(self): def execute_query(self, query: str, params=None, fetch_all=True):
file = self.dbm.get_file("data") result = []
self.obj.upload_file(self.online_db_uri, file) if fetch_all:
rows = self.dbm.execute_query(query, params)
for row in rows:
result.append(list(row))
else:
rows = self.dbm.execute_query(query, params, False)
result = list(rows)
return result
def download_db(self): def execute_event(self, query: str, params=None):
file = self.obj.get_file(self.online_db_uri) return self.dbm.execute_event(query, params)
l = self.dbm.upload_file("data", file)
def status(self): def status(self):
cursor = self.conn.cursor()
cursor.execute("SELECT datetime FROM information order by datetime desc limit 1") row = self.dbm.execute_query("SELECT datetime FROM information order by datetime desc limit 1", fetch_all=False)
row = cursor.fetchone()
if row: if row:
return row[0] return row[0]
else: else:
@@ -418,8 +419,7 @@ class ChatSessionManager:
return return
print(f"开始插入 {len(data)} 条记录") print(f"开始插入 {len(data)} 条记录")
self.cursor.executemany(sql, data) self.dbm.execute_event(sql, data)
self.conn.commit()
print(f"成功插入 {len(data)} 条记录") print(f"成功插入 {len(data)} 条记录")
def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int, def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int,
@@ -556,8 +556,8 @@ class ChatSessionManager:
""" """
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp() start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp() end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
cursor = self.conn.cursor()
cursor.execute(''' res = self.dbm.execute_query('''
SELECT message.time, message.content SELECT message.time, message.content
FROM message FROM message
WHERE (message.type = 'text' WHERE (message.type = 'text'
@@ -565,7 +565,6 @@ class ChatSessionManager:
AND message.time >= ? AND message.time >= ?
AND message.time <= ? AND message.time <= ?
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
res = cursor.fetchall()
return res return res
@staticmethod @staticmethod
@@ -637,11 +636,7 @@ class ChatSessionManager:
group_qa: 问答数据字典,键为群ID,值为问答列表 group_qa: 问答数据字典,键为群ID,值为问答列表
""" """
data = [] data = []
cursor = self.conn.cursor() index_id = self.execute_query(f"SELECT id FROM qa order by id desc limit 1",fetch_all=False)[0]
index_id = cursor.execute('''select id
from qa
order by id desc
limit 1''').fetchone()[0]
for r in group_qa: for r in group_qa:
qas = group_qa.get(r, {}) qas = group_qa.get(r, {})
@@ -652,11 +647,10 @@ class ChatSessionManager:
data.append((index_id, r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False), data.append((index_id, r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False),
qa["start_time"], qa["start_time"],
qa["end_time"])) qa["end_time"]))
cursor.executemany(''' self.execute_event('''
insert into qa (id, groupId, question, answer, start_time, end_time) insert into qa (id, groupId, question, answer, start_time, end_time)
values (?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?)
''', data) ''', data)
self.conn.commit()
def update_qa_data(self, start_time_str: str, end_time_str: str): def update_qa_data(self, start_time_str: str, end_time_str: str):
""" """
@@ -668,8 +662,7 @@ class ChatSessionManager:
""" """
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp() start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp() end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
cursor = self.conn.cursor() data = self.execute_query('''
cursor.execute('''
SELECT receiver, sender, time, content, id SELECT receiver, sender, time, content, id
FROM message FROM message
WHERE message.time >= ? WHERE message.time >= ?
@@ -677,7 +670,6 @@ class ChatSessionManager:
AND toType = 1 AND toType = 1
AND type in ('text', 'mixed', 'chatrecord') AND type in ('text', 'mixed', 'chatrecord')
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000"))) ''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
data = cursor.fetchall()
group_messages = {} group_messages = {}
for i in data: for i in data:
group_id = i[0] group_id = i[0]
@@ -686,25 +678,3 @@ class ChatSessionManager:
group_messages[group_id].append(i) group_messages[group_id].append(i)
group_qa = self.analyze_qa_content(group_messages) group_qa = self.analyze_qa_content(group_messages)
self.save_qa_to_sqlite(group_qa) self.save_qa_to_sqlite(group_qa)
def close(self) -> None:
"""关闭数据库连接"""
if self.conn:
self.conn.close()
def __del__(self) -> None:
"""析构函数,确保资源被正确释放"""
self.close()
if __name__ == '__main__':
chat_session = ChatSessionManager()
chat_session.upload_db()
chat_session.download_db()
# for day in range(11,13):
# for hour in range(24):
# chat_session.process_chat_session(2025,11,day,hour)
# start = "2025-11-11 00:00:00"
# end = "2025-11-13 00:00:00"
#
# chat_session.update_qa_data(start, end)

View File

@@ -38,11 +38,57 @@ function markdownToHtml(markdown) {
// 替换图片 // 替换图片
html = html.replace(/!\[(.*?)\]\((.*?)\)/g, '<img src="$2" alt="$1" style="max-width: 100%;">'); html = html.replace(/!\[(.*?)\]\((.*?)\)/g, '<img src="$2" alt="$1" style="max-width: 100%;">');
// 替换表格(简单实现) // 替换表格(增强实现)
html = html.replace(/\|(.+)\|/g, function(match) { html = html.replace(/\|(.+)\|\n\|([-|: ]+)\|\n((?:\|.+\|\n)*)/g, function(match, headerRow, separatorRow, bodyRows) {
// 解析表头
const headers = headerRow.split('|').filter(cell => cell.trim());
// 解析分隔行,确定对齐方式
const separators = separatorRow.split('|').filter(cell => cell.trim());
const alignments = separators.map(sep => {
if (sep.startsWith(':') && sep.endsWith(':')) return 'center';
if (sep.endsWith(':')) return 'right';
return 'left';
});
// 构建表头HTML
let tableHtml = '<table style="border-collapse: collapse; width: 100%; margin: 10px 0; border: 1px solid #ddd;">';
tableHtml += '<thead><tr style="background-color: #f5f5f5;">';
headers.forEach((header, index) => {
const alignment = alignments[index] || 'left';
tableHtml += `<th style="border: 1px solid #ddd; padding: 8px; text-align: ${alignment}; font-weight: bold;">${header.trim()}</th>`;
});
tableHtml += '</tr></thead>';
// 解析表格主体
const bodyRowsArray = bodyRows.trim().split('\n');
if (bodyRowsArray.length > 0) {
tableHtml += '<tbody>';
bodyRowsArray.forEach(row => {
const cells = row.split('|').filter(cell => cell.trim());
if (cells.length > 0) {
tableHtml += '<tr>';
cells.forEach((cell, index) => {
const alignment = alignments[index] || 'left';
tableHtml += `<td style="border: 1px solid #ddd; padding: 8px; text-align: ${alignment};">${cell.trim()}</td>`;
});
tableHtml += '</tr>';
}
});
tableHtml += '</tbody>';
}
tableHtml += '</table>';
return tableHtml;
});
// 处理简单的单行表格(向后兼容)
html = html.replace(/^\|(.+)\|$/gm, function(match) {
const cells = match.split('|').filter(cell => cell.trim()); const cells = match.split('|').filter(cell => cell.trim());
if (cells.length > 0) { if (cells.length > 0) {
return '<tr><td>' + cells.join('</td><td>') + '</td></tr>'; return '<table style="border-collapse: collapse; width: 100%; margin: 10px 0; border: 1px solid #ddd;"><tr>' +
cells.map(cell => `<td style="border: 1px solid #ddd; padding: 8px;">${cell.trim()}</td>`).join('') +
'</tr></table>';
} }
return match; return match;
}); });

View File

@@ -2,6 +2,74 @@ import time
import datetime import datetime
from typing import Union, Optional from typing import Union, Optional
import sqlite3
from file import ObjectFileManager, LocalFileManager
class DatabaseManager:
"""数据库管理类"""
def __init__(self, db_path, online_db_uri=None):
self.db_path = db_path
self.online_db_uri = online_db_uri
self.obj = ObjectFileManager()
self.lfm = LocalFileManager("db")
def upload_db(self):
file = self.lfm.get_file("data")
self.obj.upload_file(self.online_db_uri, file)
def download_db(self):
file = self.obj.get_file(self.online_db_uri)
self.lfm.upload_file("data", file)
def get_connection(self):
"""获取数据库连接"""
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def execute_event(self, query, params=None):
conn = self.get_connection()
cursor = conn.cursor()
try:
if params:
cursor.executemany(query, params)
else:
cursor.execute(query)
conn.commit()
conn.close()
return True
except Exception as e:
print(e)
conn.close()
return False
def execute_query(self, query, params=None, fetch_all=True):
"""执行查询并返回结果"""
conn = self.get_connection()
cursor = conn.cursor()
try:
if params:
cursor.execute(query, params)
else:
cursor.execute(query)
if fetch_all:
result = cursor.fetchall()
else:
result = cursor.fetchone()
conn.close()
return result
except Exception as e:
print(e)
conn.close()
raise e
class TimeUtils: class TimeUtils:
""" """