update code
This commit is contained in:
91
app.py
91
app.py
@@ -1,48 +1,11 @@
|
||||
from datetime import datetime, timedelta
|
||||
from flask import Flask, render_template, jsonify, request
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from chat_session import agent, ChatSessionManager
|
||||
from tools import TimeUtils, now, from_string, from_datetime, format_timestamp
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
"""数据库管理类"""
|
||||
|
||||
def __init__(self, db_path='db/data'):
|
||||
self.db_path = db_path
|
||||
|
||||
def get_connection(self):
|
||||
"""获取数据库连接"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def execute_query(self, query, params=None, fetch_all=True):
|
||||
"""执行查询并返回结果"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
if params:
|
||||
cursor.execute(query, params)
|
||||
else:
|
||||
cursor.execute(query)
|
||||
|
||||
if fetch_all:
|
||||
result = cursor.fetchall()
|
||||
else:
|
||||
result = cursor.fetchone()
|
||||
|
||||
conn.close()
|
||||
return result
|
||||
except Exception as e:
|
||||
print(e)
|
||||
conn.close()
|
||||
raise e
|
||||
from tools import TimeUtils, now, from_string, from_datetime, format_timestamp, DatabaseManager
|
||||
|
||||
|
||||
class MessageRender:
|
||||
@@ -99,9 +62,13 @@ class MessageRender:
|
||||
return message_data
|
||||
|
||||
|
||||
class ReportService(DatabaseManager):
|
||||
class ReportService:
|
||||
"""报告服务类"""
|
||||
|
||||
def __init__(self, db: DatabaseManager):
|
||||
self.db = db
|
||||
self.execute_query = self.db.execute_query
|
||||
|
||||
def get_message_count_by_time_range(self, start_timestamp, end_timestamp):
|
||||
"""获取指定时间范围内的消息数量"""
|
||||
result = self.execute_query('''
|
||||
@@ -124,7 +91,6 @@ class ReportService(DatabaseManager):
|
||||
|
||||
def get_qa_count_by_time_range(self, start_timestamp, end_timestamp):
|
||||
"""获取指定时间范围内的QA记录数量"""
|
||||
print(start_timestamp, end_timestamp)
|
||||
result = self.execute_query('''
|
||||
SELECT COUNT(*) as count
|
||||
FROM qa
|
||||
@@ -135,7 +101,6 @@ class ReportService(DatabaseManager):
|
||||
|
||||
def get_non_answer_qa_count_by_time_range(self, start_timestamp, end_timestamp):
|
||||
"""获取指定时间范围内的QA记录数量"""
|
||||
print(start_timestamp, end_timestamp)
|
||||
result = self.execute_query('''
|
||||
SELECT COUNT(*) as count
|
||||
FROM qa
|
||||
@@ -502,10 +467,10 @@ class ReportService(DatabaseManager):
|
||||
app = Flask(__name__)
|
||||
app.template_folder = 'templates'
|
||||
# 初始化管理器
|
||||
db_manager = DatabaseManager()
|
||||
report_service = ReportService()
|
||||
db_manager = DatabaseManager('db/data', online_db_uri="zysl-log/wework/chat_msg/db/data.db")
|
||||
report_service = ReportService(db_manager)
|
||||
message_processor = MessageRender()
|
||||
chat_session = ChatSessionManager()
|
||||
chat_session = ChatSessionManager(db_manager)
|
||||
|
||||
|
||||
def api_response(success=True, data=None, error=None, status_code=200):
|
||||
@@ -541,35 +506,36 @@ def weekly_report_page():
|
||||
def daily_report_page():
|
||||
"""日报页面"""
|
||||
return render_template('daily_report.html')
|
||||
|
||||
|
||||
@app.route('/db/update')
|
||||
@app.route('/api/db/datetime')
|
||||
def db_status():
|
||||
return api_response(success=True, data={'message': chat_session.status()})
|
||||
@app.route('/api/db/update')
|
||||
def db_update():
|
||||
current_time = time.localtime(time.time() - 24 * 60 * 60)
|
||||
day = current_time.tm_mday
|
||||
month = current_time.tm_mon
|
||||
year = current_time.tm_year
|
||||
if chat_session.status() == f"{year}-{month:02d}-{day:02d}-23":
|
||||
return api_response(success=False, error="数据库已更新", status_code=400)
|
||||
return api_response(success=False, error="数据库已更新", status_code=201)
|
||||
for hour in range(24):
|
||||
chat_session.process_chat_session(year, month, day, hour)
|
||||
start = f"{year}-{month:02d}-{day:02d} 00:00:00"
|
||||
end = f"{year}-{month:02d}-{day:02d} 23:59:59"
|
||||
|
||||
chat_session.update_qa_data(start, end)
|
||||
chat_session.upload_db()
|
||||
db_manager.upload_db()
|
||||
return api_response(success=True, data={'message': '数据库更新成功'})
|
||||
|
||||
|
||||
@app.route('/db/upload')
|
||||
@app.route('/api/db/upload')
|
||||
def db_upload():
|
||||
chat_session.upload_db()
|
||||
db_manager.upload_db()
|
||||
return api_response(success=True, data={'message': '数据库上传成功'})
|
||||
|
||||
|
||||
@app.route('/db/download')
|
||||
@app.route('/api/db/download')
|
||||
def db_download():
|
||||
chat_session.download_db()
|
||||
db_manager.download_db()
|
||||
return api_response(success=True, data={'message': '数据库下载成功'})
|
||||
|
||||
|
||||
@@ -857,7 +823,7 @@ def generate_daily_report():
|
||||
# 获取QA记录数量
|
||||
today_qa_count = report_service.get_qa_count_by_time_range(start_timestamp, end_timestamp)
|
||||
yesterday_qa_count = report_service.get_qa_count_by_time_range(yesterday_start, yesterday_end)
|
||||
|
||||
non_qa_count = report_service.get_non_answer_qa_count_by_time_range(start_timestamp, end_timestamp)
|
||||
qa_change_rate = report_service.calculate_change_rate(
|
||||
today_qa_count, yesterday_qa_count
|
||||
)
|
||||
@@ -895,20 +861,24 @@ def generate_daily_report():
|
||||
```
|
||||
|
||||
## **3. 今日热点与时段**
|
||||
* **热点问题TOP 3**:
|
||||
|
||||
### **热点问题TOP 3**:
|
||||
|
||||
"""
|
||||
|
||||
for i, topic in enumerate(hot_topics, 1):
|
||||
daily_report += f" {i}. {topic}\n"
|
||||
|
||||
daily_report += f"""* **活跃时段**:消息高峰分布
|
||||
daily_report += f"""### **活跃时段**:消息高峰分布
|
||||
```
|
||||
{hourly_activity_chart}
|
||||
```
|
||||
|
||||
## **4. 待办事项**
|
||||
* 关注活跃群组的后续讨论,确保所有疑问被解答。
|
||||
* 明日重点关注:根据今日热点预测明日可能的问题。
|
||||
|
||||
{"有"+str(non_qa_count)+"条未被回答的问题,关注活跃群组的后续讨论,确保所有疑问被解答。" if non_qa_count > 0 else "无未被回答的问题,继续保持活跃群组的讨论。"}
|
||||
|
||||
明日重点关注:根据今日热点预测明日可能的问题。
|
||||
"""
|
||||
|
||||
return api_response(data={
|
||||
@@ -1041,9 +1011,10 @@ def generate_weekly_report():
|
||||
if __name__ == '__main__':
|
||||
# 检查数据库文件是否存在
|
||||
if not os.path.exists('db/data'):
|
||||
print(f"错误: 数据库文件 db/data.sqlite 不存在")
|
||||
chat_session.download_db()
|
||||
print(f"错误: 数据库文件不存在")
|
||||
print("正在下载云端数据库")
|
||||
db_manager.download_db()
|
||||
print("下载完成")
|
||||
else:
|
||||
print("启动聊天网页服务...")
|
||||
print("访问地址: http://127.0.0.1:5000")
|
||||
|
||||
108
chat_session.py
108
chat_session.py
@@ -1,7 +1,6 @@
|
||||
import datetime
|
||||
import json
|
||||
import os.path
|
||||
import sqlite3
|
||||
from typing import List, Tuple
|
||||
import logging
|
||||
import os
|
||||
@@ -11,6 +10,7 @@ from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
import pytz
|
||||
|
||||
from tools import DatabaseManager
|
||||
from file import ObjectFileManager, LocalFileManager
|
||||
|
||||
TZ = pytz.timezone('Asia/Shanghai')
|
||||
@@ -162,47 +162,48 @@ agent = AIAgent(max_retries=1, timeout=180)
|
||||
class ChatSessionManager:
|
||||
"""聊天会话管理器类"""
|
||||
|
||||
def __init__(self, online_db_uri: str = "zysl-log/wework/chat_msg/db/data.db"):
|
||||
def __init__(self, db_manager: DatabaseManager):
|
||||
"""
|
||||
初始化聊天会话管理器
|
||||
|
||||
Args:
|
||||
online_db_uri: 在线数据库URI
|
||||
db_manager: 数据库管理工具
|
||||
"""
|
||||
self.online_db_uri = online_db_uri
|
||||
self.files_dir = os.path.join("", "files")
|
||||
self.messages_dir = os.path.join("", "messages")
|
||||
self.db_path = os.path.join("db", "data")
|
||||
self.cache_path = None
|
||||
# 初始化文件管理器
|
||||
self.obj = ObjectFileManager()
|
||||
self.lfm = LocalFileManager(self.files_dir)
|
||||
self.dbm = LocalFileManager("db")
|
||||
self.dbm = db_manager
|
||||
self.users = []
|
||||
self.groups = []
|
||||
self.members = {}
|
||||
self.messages = []
|
||||
self.files = {}
|
||||
# 初始化数据库连接
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
# 确保目录存在
|
||||
os.makedirs(self.files_dir, exist_ok=True)
|
||||
os.makedirs(self.messages_dir, exist_ok=True)
|
||||
|
||||
def upload_db(self):
|
||||
file = self.dbm.get_file("data")
|
||||
self.obj.upload_file(self.online_db_uri, file)
|
||||
def execute_query(self, query: str, params=None, fetch_all=True):
|
||||
result = []
|
||||
if fetch_all:
|
||||
rows = self.dbm.execute_query(query, params)
|
||||
for row in rows:
|
||||
result.append(list(row))
|
||||
else:
|
||||
rows = self.dbm.execute_query(query, params, False)
|
||||
result = list(rows)
|
||||
return result
|
||||
|
||||
def download_db(self):
|
||||
file = self.obj.get_file(self.online_db_uri)
|
||||
l = self.dbm.upload_file("data", file)
|
||||
def execute_event(self, query: str, params=None):
|
||||
return self.dbm.execute_event(query, params)
|
||||
|
||||
def status(self):
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("SELECT datetime FROM information order by datetime desc limit 1")
|
||||
row = cursor.fetchone()
|
||||
|
||||
row = self.dbm.execute_query("SELECT datetime FROM information order by datetime desc limit 1", fetch_all=False)
|
||||
|
||||
if row:
|
||||
return row[0]
|
||||
else:
|
||||
@@ -418,8 +419,7 @@ class ChatSessionManager:
|
||||
return
|
||||
|
||||
print(f"开始插入 {len(data)} 条记录")
|
||||
self.cursor.executemany(sql, data)
|
||||
self.conn.commit()
|
||||
self.dbm.execute_event(sql, data)
|
||||
print(f"成功插入 {len(data)} 条记录")
|
||||
|
||||
def save_information_to_sqlite(self, date_time: str, message_num: int, file_num: int, user_num: int,
|
||||
@@ -556,16 +556,15 @@ class ChatSessionManager:
|
||||
"""
|
||||
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT message.time, message.content
|
||||
FROM message
|
||||
WHERE (message.type = 'text'
|
||||
or message.type = 'mixed')
|
||||
AND message.time >= ?
|
||||
AND message.time <= ?
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
res = cursor.fetchall()
|
||||
|
||||
res = self.dbm.execute_query('''
|
||||
SELECT message.time, message.content
|
||||
FROM message
|
||||
WHERE (message.type = 'text'
|
||||
or message.type = 'mixed')
|
||||
AND message.time >= ?
|
||||
AND message.time <= ?
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
@@ -637,11 +636,7 @@ class ChatSessionManager:
|
||||
group_qa: 问答数据字典,键为群ID,值为问答列表
|
||||
"""
|
||||
data = []
|
||||
cursor = self.conn.cursor()
|
||||
index_id = cursor.execute('''select id
|
||||
from qa
|
||||
order by id desc
|
||||
limit 1''').fetchone()[0]
|
||||
index_id = self.execute_query(f"SELECT id FROM qa order by id desc limit 1",fetch_all=False)[0]
|
||||
|
||||
for r in group_qa:
|
||||
qas = group_qa.get(r, {})
|
||||
@@ -652,11 +647,10 @@ class ChatSessionManager:
|
||||
data.append((index_id, r, json.dumps(Q, ensure_ascii=False), json.dumps(A, ensure_ascii=False),
|
||||
qa["start_time"],
|
||||
qa["end_time"]))
|
||||
cursor.executemany('''
|
||||
self.execute_event('''
|
||||
insert into qa (id, groupId, question, answer, start_time, end_time)
|
||||
values (?, ?, ?, ?, ?, ?)
|
||||
''', data)
|
||||
self.conn.commit()
|
||||
|
||||
def update_qa_data(self, start_time_str: str, end_time_str: str):
|
||||
"""
|
||||
@@ -668,16 +662,14 @@ class ChatSessionManager:
|
||||
"""
|
||||
start_time = convert_to_utc(start_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
end_time = convert_to_utc(end_time_str, source_timezone="Asia/Shanghai").timestamp()
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT receiver, sender, time, content, id
|
||||
FROM message
|
||||
WHERE message.time >= ?
|
||||
AND message.time <= ?
|
||||
AND toType = 1
|
||||
AND type in ('text', 'mixed', 'chatrecord')
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
data = cursor.fetchall()
|
||||
data = self.execute_query('''
|
||||
SELECT receiver, sender, time, content, id
|
||||
FROM message
|
||||
WHERE message.time >= ?
|
||||
AND message.time <= ?
|
||||
AND toType = 1
|
||||
AND type in ('text', 'mixed', 'chatrecord')
|
||||
''', (str(start_time).replace(".0", "000"), str(end_time).replace(".0", "000")))
|
||||
group_messages = {}
|
||||
for i in data:
|
||||
group_id = i[0]
|
||||
@@ -686,25 +678,3 @@ class ChatSessionManager:
|
||||
group_messages[group_id].append(i)
|
||||
group_qa = self.analyze_qa_content(group_messages)
|
||||
self.save_qa_to_sqlite(group_qa)
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭数据库连接"""
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""析构函数,确保资源被正确释放"""
|
||||
self.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
chat_session = ChatSessionManager()
|
||||
chat_session.upload_db()
|
||||
chat_session.download_db()
|
||||
# for day in range(11,13):
|
||||
# for hour in range(24):
|
||||
# chat_session.process_chat_session(2025,11,day,hour)
|
||||
# start = "2025-11-11 00:00:00"
|
||||
# end = "2025-11-13 00:00:00"
|
||||
#
|
||||
# chat_session.update_qa_data(start, end)
|
||||
|
||||
@@ -38,11 +38,57 @@ function markdownToHtml(markdown) {
|
||||
// 替换图片
|
||||
html = html.replace(/!\[(.*?)\]\((.*?)\)/g, '<img src="$2" alt="$1" style="max-width: 100%;">');
|
||||
|
||||
// 替换表格(简单实现)
|
||||
html = html.replace(/\|(.+)\|/g, function(match) {
|
||||
// 替换表格(增强实现)
|
||||
html = html.replace(/\|(.+)\|\n\|([-|: ]+)\|\n((?:\|.+\|\n)*)/g, function(match, headerRow, separatorRow, bodyRows) {
|
||||
// 解析表头
|
||||
const headers = headerRow.split('|').filter(cell => cell.trim());
|
||||
|
||||
// 解析分隔行,确定对齐方式
|
||||
const separators = separatorRow.split('|').filter(cell => cell.trim());
|
||||
const alignments = separators.map(sep => {
|
||||
if (sep.startsWith(':') && sep.endsWith(':')) return 'center';
|
||||
if (sep.endsWith(':')) return 'right';
|
||||
return 'left';
|
||||
});
|
||||
|
||||
// 构建表头HTML
|
||||
let tableHtml = '<table style="border-collapse: collapse; width: 100%; margin: 10px 0; border: 1px solid #ddd;">';
|
||||
tableHtml += '<thead><tr style="background-color: #f5f5f5;">';
|
||||
headers.forEach((header, index) => {
|
||||
const alignment = alignments[index] || 'left';
|
||||
tableHtml += `<th style="border: 1px solid #ddd; padding: 8px; text-align: ${alignment}; font-weight: bold;">${header.trim()}</th>`;
|
||||
});
|
||||
tableHtml += '</tr></thead>';
|
||||
|
||||
// 解析表格主体
|
||||
const bodyRowsArray = bodyRows.trim().split('\n');
|
||||
if (bodyRowsArray.length > 0) {
|
||||
tableHtml += '<tbody>';
|
||||
bodyRowsArray.forEach(row => {
|
||||
const cells = row.split('|').filter(cell => cell.trim());
|
||||
if (cells.length > 0) {
|
||||
tableHtml += '<tr>';
|
||||
cells.forEach((cell, index) => {
|
||||
const alignment = alignments[index] || 'left';
|
||||
tableHtml += `<td style="border: 1px solid #ddd; padding: 8px; text-align: ${alignment};">${cell.trim()}</td>`;
|
||||
});
|
||||
tableHtml += '</tr>';
|
||||
}
|
||||
});
|
||||
tableHtml += '</tbody>';
|
||||
}
|
||||
|
||||
tableHtml += '</table>';
|
||||
return tableHtml;
|
||||
});
|
||||
|
||||
// 处理简单的单行表格(向后兼容)
|
||||
html = html.replace(/^\|(.+)\|$/gm, function(match) {
|
||||
const cells = match.split('|').filter(cell => cell.trim());
|
||||
if (cells.length > 0) {
|
||||
return '<tr><td>' + cells.join('</td><td>') + '</td></tr>';
|
||||
return '<table style="border-collapse: collapse; width: 100%; margin: 10px 0; border: 1px solid #ddd;"><tr>' +
|
||||
cells.map(cell => `<td style="border: 1px solid #ddd; padding: 8px;">${cell.trim()}</td>`).join('') +
|
||||
'</tr></table>';
|
||||
}
|
||||
return match;
|
||||
});
|
||||
|
||||
68
tools.py
68
tools.py
@@ -2,6 +2,74 @@ import time
|
||||
import datetime
|
||||
from typing import Union, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
from file import ObjectFileManager, LocalFileManager
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
"""数据库管理类"""
|
||||
|
||||
def __init__(self, db_path, online_db_uri=None):
|
||||
self.db_path = db_path
|
||||
self.online_db_uri = online_db_uri
|
||||
self.obj = ObjectFileManager()
|
||||
self.lfm = LocalFileManager("db")
|
||||
|
||||
def upload_db(self):
|
||||
file = self.lfm.get_file("data")
|
||||
self.obj.upload_file(self.online_db_uri, file)
|
||||
|
||||
def download_db(self):
|
||||
file = self.obj.get_file(self.online_db_uri)
|
||||
self.lfm.upload_file("data", file)
|
||||
|
||||
def get_connection(self):
|
||||
"""获取数据库连接"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def execute_event(self, query, params=None):
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
if params:
|
||||
cursor.executemany(query, params)
|
||||
else:
|
||||
cursor.execute(query)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
conn.close()
|
||||
return False
|
||||
|
||||
def execute_query(self, query, params=None, fetch_all=True):
|
||||
"""执行查询并返回结果"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
if params:
|
||||
cursor.execute(query, params)
|
||||
else:
|
||||
cursor.execute(query)
|
||||
|
||||
if fetch_all:
|
||||
result = cursor.fetchall()
|
||||
else:
|
||||
result = cursor.fetchone()
|
||||
|
||||
conn.close()
|
||||
return result
|
||||
except Exception as e:
|
||||
print(e)
|
||||
conn.close()
|
||||
raise e
|
||||
|
||||
|
||||
class TimeUtils:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user