541 lines
13 KiB
Go
541 lines
13 KiB
Go
// Package panorama 灵瞰 - 可观测与度量层
|
|
//
|
|
// 居高临下，俯瞰全局。此层不参与业务交互，
|
|
// 却如"天眼"般记录每一帧交互的耗时、置信度与资源消耗，
|
|
// 让黑盒群体智能变为白盒可解释的图景。
|
|
package panorama
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"synapse/pkg/types"
|
|
)
|
|
|
|
// =============================================================================
|
|
// Tracing - 链路追踪
|
|
// =============================================================================
|
|
|
|
// Trace 追踪链路
|
|
type Trace struct {
|
|
TraceID string `json:"trace_id"`
|
|
RootSpan *Span `json:"root_span"`
|
|
Spans map[string]*Span `json:"spans"`
|
|
StartTime time.Time `json:"start_time"`
|
|
EndTime *time.Time `json:"end_time,omitempty"`
|
|
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
|
}
|
|
|
|
// Span 追踪跨度
|
|
type Span struct {
|
|
SpanID string `json:"span_id"`
|
|
TraceID string `json:"trace_id"`
|
|
ParentID string `json:"parent_id,omitempty"`
|
|
Name string `json:"name"`
|
|
Service string `json:"service"`
|
|
AgentID string `json:"agent_id"`
|
|
StartTime time.Time `json:"start_time"`
|
|
EndTime *time.Time `json:"end_time,omitempty"`
|
|
Duration time.Duration `json:"duration_ms"`
|
|
Status SpanStatus `json:"status"`
|
|
Tags map[string]string `json:"tags,omitempty"`
|
|
Logs []SpanLog `json:"logs,omitempty"`
|
|
Attributes map[string]interface{} `json:"attributes,omitempty"`
|
|
}
|
|
|
|
// SpanStatus 跨度状态
|
|
type SpanStatus string
|
|
|
|
const (
|
|
SpanStatusPending SpanStatus = "pending"
|
|
SpanStatusRunning SpanStatus = "running"
|
|
SpanStatusSuccess SpanStatus = "success"
|
|
SpanStatusFailed SpanStatus = "failed"
|
|
SpanStatusCancelled SpanStatus = "cancelled"
|
|
)
|
|
|
|
// SpanLog 跨度日志
|
|
type SpanLog struct {
|
|
Timestamp time.Time `json:"timestamp"`
|
|
Message string `json:"message"`
|
|
Fields map[string]interface{} `json:"fields,omitempty"`
|
|
}
|
|
|
|
// Tracer 追踪器
|
|
type Tracer struct {
|
|
mu sync.RWMutex
|
|
traces map[string]*Trace
|
|
config *TracerConfig
|
|
}
|
|
|
|
// TracerConfig 追踪器配置
|
|
type TracerConfig struct {
|
|
MaxTraces int
|
|
MaxSpansPerTrace int
|
|
}
|
|
|
|
// DefaultTracerConfig 返回默认追踪器配置
|
|
func DefaultTracerConfig() *TracerConfig {
|
|
return &TracerConfig{
|
|
MaxTraces: 1000,
|
|
MaxSpansPerTrace: 100,
|
|
}
|
|
}
|
|
|
|
// NewTracer 创建追踪器
|
|
func NewTracer(config *TracerConfig) *Tracer {
|
|
if config == nil {
|
|
config = DefaultTracerConfig()
|
|
}
|
|
return &Tracer{
|
|
traces: make(map[string]*Trace),
|
|
config: config,
|
|
}
|
|
}
|
|
|
|
// StartTrace 开始追踪
|
|
func (t *Tracer) StartTrace(traceID, name string, agentID string) *Trace {
|
|
if traceID == "" {
|
|
traceID = fmt.Sprintf("trace-%d", time.Now().UnixNano())
|
|
}
|
|
|
|
rootSpan := &Span{
|
|
SpanID: fmt.Sprintf("span-%d", time.Now().UnixNano()),
|
|
TraceID: traceID,
|
|
Name: name,
|
|
Service: "synapse",
|
|
AgentID: agentID,
|
|
StartTime: time.Now(),
|
|
Status: SpanStatusRunning,
|
|
Tags: make(map[string]string),
|
|
Logs: make([]SpanLog, 0),
|
|
Attributes: make(map[string]interface{}),
|
|
}
|
|
|
|
trace := &Trace{
|
|
TraceID: traceID,
|
|
RootSpan: rootSpan,
|
|
Spans: map[string]*Span{rootSpan.SpanID: rootSpan},
|
|
StartTime: time.Now(),
|
|
Metadata: make(map[string]interface{}),
|
|
}
|
|
|
|
t.mu.Lock()
|
|
t.traces[traceID] = trace
|
|
t.mu.Unlock()
|
|
|
|
log.Printf("[灵瞰] 开始追踪: %s (%s)", traceID, name)
|
|
return trace
|
|
}
|
|
|
|
// StartSpan 开始跨度
|
|
func (t *Tracer) StartSpan(traceID, parentSpanID, name, agentID string) (*Span, error) {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
|
|
trace, ok := t.traces[traceID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("trace %s not found", traceID)
|
|
}
|
|
|
|
span := &Span{
|
|
SpanID: fmt.Sprintf("span-%d", time.Now().UnixNano()),
|
|
TraceID: traceID,
|
|
ParentID: parentSpanID,
|
|
Name: name,
|
|
Service: "synapse",
|
|
AgentID: agentID,
|
|
StartTime: time.Now(),
|
|
Status: SpanStatusRunning,
|
|
Tags: make(map[string]string),
|
|
Logs: make([]SpanLog, 0),
|
|
Attributes: make(map[string]interface{}),
|
|
}
|
|
|
|
trace.Spans[span.SpanID] = span
|
|
return span, nil
|
|
}
|
|
|
|
// EndSpan 结束跨度
|
|
func (t *Tracer) EndSpan(traceID, spanID string, status SpanStatus, err error) error {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
|
|
trace, ok := t.traces[traceID]
|
|
if !ok {
|
|
return fmt.Errorf("trace %s not found", traceID)
|
|
}
|
|
|
|
span, ok := trace.Spans[spanID]
|
|
if !ok {
|
|
return fmt.Errorf("span %s not found", spanID)
|
|
}
|
|
|
|
now := time.Now()
|
|
span.EndTime = &now
|
|
span.Duration = now.Sub(span.StartTime)
|
|
span.Status = status
|
|
|
|
if err != nil {
|
|
span.Logs = append(span.Logs, SpanLog{
|
|
Timestamp: now,
|
|
Message: err.Error(),
|
|
Fields: map[string]interface{}{"error": err.Error()},
|
|
})
|
|
}
|
|
|
|
// 如果是根跨度，结束整个追踪
|
|
if span == trace.RootSpan {
|
|
trace.EndTime = &now
|
|
}
|
|
|
|
log.Printf("[灵瞰] 结束跨度: %s (%s) - %v", spanID, span.Name, span.Duration)
|
|
return nil
|
|
}
|
|
|
|
// AddSpanLog 添加日志
|
|
func (t *Tracer) AddSpanLog(traceID, spanID, message string, fields map[string]interface{}) error {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
|
|
trace, ok := t.traces[traceID]
|
|
if !ok {
|
|
return fmt.Errorf("trace %s not found", traceID)
|
|
}
|
|
|
|
span, ok := trace.Spans[spanID]
|
|
if !ok {
|
|
return fmt.Errorf("span %s not found", spanID)
|
|
}
|
|
|
|
span.Logs = append(span.Logs, SpanLog{
|
|
Timestamp: time.Now(),
|
|
Message: message,
|
|
Fields: fields,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetTrace 获取追踪
|
|
func (t *Tracer) GetTrace(traceID string) (*Trace, error) {
|
|
t.mu.RLock()
|
|
defer t.mu.RUnlock()
|
|
|
|
trace, ok := t.traces[traceID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("trace %s not found", traceID)
|
|
}
|
|
return trace, nil
|
|
}
|
|
|
|
// =============================================================================
|
|
// Metrics - 指标收集
|
|
// =============================================================================
|
|
|
|
// MetricType 指标类型
|
|
type MetricType string
|
|
|
|
const (
|
|
MetricCounter MetricType = "counter"
|
|
MetricGauge MetricType = "gauge"
|
|
MetricHistogram MetricType = "histogram"
|
|
)
|
|
|
|
// Metric 指标
|
|
type Metric struct {
|
|
Name string `json:"name"`
|
|
Type MetricType `json:"type"`
|
|
Value float64 `json:"value"`
|
|
Labels map[string]string `json:"labels"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
// MetricsCollector 指标收集器
|
|
type MetricsCollector struct {
|
|
mu sync.RWMutex
|
|
metrics map[string]*Metric
|
|
histories map[string][]Metric
|
|
config *MetricsConfig
|
|
}
|
|
|
|
// MetricsConfig 指标配置
|
|
type MetricsConfig struct {
|
|
MaxHistory int
|
|
}
|
|
|
|
// DefaultMetricsConfig 返回默认指标配置
|
|
func DefaultMetricsConfig() *MetricsConfig {
|
|
return &MetricsConfig{
|
|
MaxHistory: 1000,
|
|
}
|
|
}
|
|
|
|
// NewMetricsCollector 创建指标收集器
|
|
func NewMetricsCollector(config *MetricsConfig) *MetricsCollector {
|
|
if config == nil {
|
|
config = DefaultMetricsConfig()
|
|
}
|
|
return &MetricsCollector{
|
|
metrics: make(map[string]*Metric),
|
|
histories: make(map[string][]Metric),
|
|
config: config,
|
|
}
|
|
}
|
|
|
|
// RecordMetric 记录指标
|
|
func (mc *MetricsCollector) RecordMetric(name string, metricType MetricType, value float64, labels map[string]string) {
|
|
mc.mu.Lock()
|
|
defer mc.mu.Unlock()
|
|
|
|
key := name + labelKey(labels)
|
|
metric := &Metric{
|
|
Name: name,
|
|
Type: metricType,
|
|
Value: value,
|
|
Labels: labels,
|
|
Timestamp: time.Now(),
|
|
}
|
|
|
|
mc.metrics[key] = metric
|
|
|
|
// 记录历史
|
|
if history, ok := mc.histories[key]; ok {
|
|
if len(history) >= mc.config.MaxHistory {
|
|
history = history[1:]
|
|
}
|
|
mc.histories[key] = append(history, *metric)
|
|
} else {
|
|
mc.histories[key] = []Metric{*metric}
|
|
}
|
|
}
|
|
|
|
// IncrementCounter 计数器递增
|
|
func (mc *MetricsCollector) IncrementCounter(name string, labels map[string]string, delta float64) {
|
|
mc.mu.Lock()
|
|
defer mc.mu.Unlock()
|
|
|
|
key := name + labelKey(labels)
|
|
metric, ok := mc.metrics[key]
|
|
if !ok {
|
|
metric = &Metric{
|
|
Name: name,
|
|
Type: MetricCounter,
|
|
Value: 0,
|
|
Labels: labels,
|
|
Timestamp: time.Now(),
|
|
}
|
|
mc.metrics[key] = metric
|
|
}
|
|
metric.Value += delta
|
|
metric.Timestamp = time.Now()
|
|
}
|
|
|
|
// SetGauge 设置计量值
|
|
func (mc *MetricsCollector) SetGauge(name string, labels map[string]string, value float64) {
|
|
mc.RecordMetric(name, MetricGauge, value, labels)
|
|
}
|
|
|
|
// GetMetric 获取指标
|
|
func (mc *MetricsCollector) GetMetric(name string, labels map[string]string) (*Metric, error) {
|
|
mc.mu.RLock()
|
|
defer mc.mu.RUnlock()
|
|
|
|
key := name + labelKey(labels)
|
|
metric, ok := mc.metrics[key]
|
|
if !ok {
|
|
return nil, fmt.Errorf("metric %s not found", name)
|
|
}
|
|
return metric, nil
|
|
}
|
|
|
|
// GetAllMetrics 获取所有指标
|
|
func (mc *MetricsCollector) GetAllMetrics() []Metric {
|
|
mc.mu.RLock()
|
|
defer mc.mu.RUnlock()
|
|
|
|
metrics := make([]Metric, 0, len(mc.metrics))
|
|
for _, m := range mc.metrics {
|
|
metrics = append(metrics, *m)
|
|
}
|
|
return metrics
|
|
}
|
|
|
|
// GetMetricHistory 获取指标历史
|
|
func (mc *MetricsCollector) GetMetricHistory(name string, labels map[string]string) ([]Metric, error) {
|
|
mc.mu.RLock()
|
|
defer mc.mu.RUnlock()
|
|
|
|
key := name + labelKey(labels)
|
|
history, ok := mc.histories[key]
|
|
if !ok {
|
|
return nil, fmt.Errorf("metric %s not found", name)
|
|
}
|
|
|
|
result := make([]Metric, len(history))
|
|
copy(result, history)
|
|
return result, nil
|
|
}
|
|
|
|
func labelKey(labels map[string]string) string {
|
|
if len(labels) == 0 {
|
|
return ""
|
|
}
|
|
data, _ := json.Marshal(labels)
|
|
return string(data)
|
|
}
|
|
|
|
// =============================================================================
|
|
// Health Check - 健康检查
|
|
// =============================================================================
|
|
|
|
// HealthStatus 健康状态
|
|
type HealthStatus string
|
|
|
|
const (
|
|
HealthStatusHealthy HealthStatus = "healthy"
|
|
HealthStatusDegraded HealthStatus = "degraded"
|
|
HealthStatusUnhealthy HealthStatus = "unhealthy"
|
|
HealthStatusUnknown HealthStatus = "unknown"
|
|
)
|
|
|
|
// HealthCheck 健康检查结果
|
|
type HealthCheck struct {
|
|
Name string `json:"name"`
|
|
Status HealthStatus `json:"status"`
|
|
Message string `json:"message"`
|
|
Details map[string]interface{} `json:"details,omitempty"`
|
|
CheckedAt time.Time `json:"checked_at"`
|
|
}
|
|
|
|
// HealthChecker 健康检查器
|
|
type HealthChecker struct {
|
|
mu sync.RWMutex
|
|
checks map[string]HealthCheckable
|
|
config *HealthConfig
|
|
}
|
|
|
|
// HealthCheckable 可健康检查的接口
|
|
type HealthCheckable interface {
|
|
HealthCheck(ctx context.Context) *HealthCheck
|
|
}
|
|
|
|
// HealthConfig 健康检查配置
|
|
type HealthConfig struct {
|
|
CheckInterval time.Duration
|
|
}
|
|
|
|
// DefaultHealthConfig 返回默认健康检查配置
|
|
func DefaultHealthConfig() *HealthConfig {
|
|
return &HealthConfig{
|
|
CheckInterval: 30 * time.Second,
|
|
}
|
|
}
|
|
|
|
// NewHealthChecker 创建健康检查器
|
|
func NewHealthChecker(config *HealthConfig) *HealthChecker {
|
|
if config == nil {
|
|
config = DefaultHealthConfig()
|
|
}
|
|
return &HealthChecker{
|
|
checks: make(map[string]HealthCheckable),
|
|
config: config,
|
|
}
|
|
}
|
|
|
|
// RegisterCheck 注册健康检查
|
|
func (hc *HealthChecker) RegisterCheck(name string, checkable HealthCheckable) {
|
|
hc.mu.Lock()
|
|
defer hc.mu.Unlock()
|
|
hc.checks[name] = checkable
|
|
}
|
|
|
|
// CheckAll 检查所有组件
|
|
func (hc *HealthChecker) CheckAll(ctx context.Context) map[string]*HealthCheck {
|
|
hc.mu.RLock()
|
|
defer hc.mu.RUnlock()
|
|
|
|
results := make(map[string]*HealthCheck)
|
|
for name, checkable := range hc.checks {
|
|
results[name] = checkable.HealthCheck(ctx)
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
// OverallStatus 总体健康状态
|
|
func (hc *HealthChecker) OverallStatus(ctx context.Context) HealthStatus {
|
|
results := hc.CheckAll(ctx)
|
|
|
|
allHealthy := true
|
|
anyUnhealthy := false
|
|
|
|
for _, result := range results {
|
|
switch result.Status {
|
|
case HealthStatusUnhealthy:
|
|
anyUnhealthy = true
|
|
case HealthStatusDegraded:
|
|
allHealthy = false
|
|
case HealthStatusUnknown:
|
|
allHealthy = false
|
|
}
|
|
}
|
|
|
|
if anyUnhealthy {
|
|
return HealthStatusUnhealthy
|
|
}
|
|
if !allHealthy {
|
|
return HealthStatusDegraded
|
|
}
|
|
return HealthStatusHealthy
|
|
}
|
|
|
|
// =============================================================================
|
|
// Panorama - 灵瞰整体入口
|
|
// =============================================================================
|
|
|
|
// Panorama 灵瞰 - 可观测与度量
|
|
type Panorama struct {
|
|
Tracer *Tracer
|
|
Metrics *MetricsCollector
|
|
Health *HealthChecker
|
|
}
|
|
|
|
// NewPanorama 创建灵瞰
|
|
func NewPanorama() *Panorama {
|
|
return &Panorama{
|
|
Tracer: NewTracer(nil),
|
|
Metrics: NewMetricsCollector(nil),
|
|
Health: NewHealthChecker(nil),
|
|
}
|
|
}
|
|
|
|
// RecordMessage 记录消息（自动增加指标）
|
|
func (p *Panorama) RecordMessage(msg *types.UnifiedSynapseMessage, direction string, duration time.Duration) {
|
|
labels := map[string]string{
|
|
"type": msg.Type,
|
|
"direction": direction,
|
|
"sender": msg.Sender.AgentID,
|
|
}
|
|
|
|
p.Metrics.IncrementCounter("synapse_messages_total", labels, 1)
|
|
p.Metrics.RecordMetric("synapse_message_duration_seconds", MetricHistogram,
|
|
duration.Seconds(), labels)
|
|
}
|
|
|
|
// StartTaskTrace 开始任务追踪
|
|
func (p *Panorama) StartTaskTrace(taskID, taskType, agentID string) string {
|
|
trace := p.Tracer.StartTrace(taskID, fmt.Sprintf("task:%s", taskType), agentID)
|
|
return trace.TraceID
|
|
}
|
|
|
|
// ToJSON 导出为 JSON
|
|
func (t *Trace) ToJSON() string {
|
|
data, _ := json.MarshalIndent(t, "", " ")
|
|
return string(data)
|
|
}
|