// Package panorama 灵瞰 - 可观测与度量层 // // 居高临下,俯瞰全局。此层不参与业务交互, // 却如"天眼"般记录每一帧交互的耗时、置信度与资源消耗, // 让黑盒群体智能变为白盒可解释的图景。 package panorama import ( "context" "encoding/json" "fmt" "log" "sync" "time" "synapse/pkg/types" ) // ============================================================================= // Tracing - 链路追踪 // ============================================================================= // Trace 追踪链路 type Trace struct { TraceID string `json:"trace_id"` RootSpan *Span `json:"root_span"` Spans map[string]*Span `json:"spans"` StartTime time.Time `json:"start_time"` EndTime *time.Time `json:"end_time,omitempty"` Metadata map[string]interface{} `json:"metadata,omitempty"` } // Span 追踪跨度 type Span struct { SpanID string `json:"span_id"` TraceID string `json:"trace_id"` ParentID string `json:"parent_id,omitempty"` Name string `json:"name"` Service string `json:"service"` AgentID string `json:"agent_id"` StartTime time.Time `json:"start_time"` EndTime *time.Time `json:"end_time,omitempty"` Duration time.Duration `json:"duration_ms"` Status SpanStatus `json:"status"` Tags map[string]string `json:"tags,omitempty"` Logs []SpanLog `json:"logs,omitempty"` Attributes map[string]interface{} `json:"attributes,omitempty"` } // SpanStatus 跨度状态 type SpanStatus string const ( SpanStatusPending SpanStatus = "pending" SpanStatusRunning SpanStatus = "running" SpanStatusSuccess SpanStatus = "success" SpanStatusFailed SpanStatus = "failed" SpanStatusCancelled SpanStatus = "cancelled" ) // SpanLog 跨度日志 type SpanLog struct { Timestamp time.Time `json:"timestamp"` Message string `json:"message"` Fields map[string]interface{} `json:"fields,omitempty"` } // Tracer 追踪器 type Tracer struct { mu sync.RWMutex traces map[string]*Trace config *TracerConfig } // TracerConfig 追踪器配置 type TracerConfig struct { MaxTraces int MaxSpansPerTrace int } // DefaultTracerConfig 返回默认追踪器配置 func DefaultTracerConfig() *TracerConfig { return &TracerConfig{ MaxTraces: 1000, MaxSpansPerTrace: 100, } } // NewTracer 创建追踪器 func NewTracer(config *TracerConfig) *Tracer { if config == nil { config = DefaultTracerConfig() } return &Tracer{ traces: make(map[string]*Trace), config: config, } } // StartTrace 开始追踪 func (t *Tracer) StartTrace(traceID, name string, agentID string) *Trace { if traceID == "" { traceID = fmt.Sprintf("trace-%d", time.Now().UnixNano()) } rootSpan := &Span{ SpanID: fmt.Sprintf("span-%d", time.Now().UnixNano()), TraceID: traceID, Name: name, Service: "synapse", AgentID: agentID, StartTime: time.Now(), Status: SpanStatusRunning, Tags: make(map[string]string), Logs: make([]SpanLog, 0), Attributes: make(map[string]interface{}), } trace := &Trace{ TraceID: traceID, RootSpan: rootSpan, Spans: map[string]*Span{rootSpan.SpanID: rootSpan}, StartTime: time.Now(), Metadata: make(map[string]interface{}), } t.mu.Lock() t.traces[traceID] = trace t.mu.Unlock() log.Printf("[灵瞰] 开始追踪: %s (%s)", traceID, name) return trace } // StartSpan 开始跨度 func (t *Tracer) StartSpan(traceID, parentSpanID, name, agentID string) (*Span, error) { t.mu.Lock() defer t.mu.Unlock() trace, ok := t.traces[traceID] if !ok { return nil, fmt.Errorf("trace %s not found", traceID) } span := &Span{ SpanID: fmt.Sprintf("span-%d", time.Now().UnixNano()), TraceID: traceID, ParentID: parentSpanID, Name: name, Service: "synapse", AgentID: agentID, StartTime: time.Now(), Status: SpanStatusRunning, Tags: make(map[string]string), Logs: make([]SpanLog, 0), Attributes: make(map[string]interface{}), } trace.Spans[span.SpanID] = span return span, nil } // EndSpan 结束跨度 func (t *Tracer) EndSpan(traceID, spanID string, status SpanStatus, err error) error { t.mu.Lock() defer t.mu.Unlock() trace, ok := t.traces[traceID] if !ok { return fmt.Errorf("trace %s not found", traceID) } span, ok := trace.Spans[spanID] if !ok { return fmt.Errorf("span %s not found", spanID) } now := time.Now() span.EndTime = &now span.Duration = now.Sub(span.StartTime) span.Status = status if err != nil { span.Logs = append(span.Logs, SpanLog{ Timestamp: now, Message: err.Error(), Fields: map[string]interface{}{"error": err.Error()}, }) } // 如果是根跨度,结束整个追踪 if span == trace.RootSpan { trace.EndTime = &now } log.Printf("[灵瞰] 结束跨度: %s (%s) - %v", spanID, span.Name, span.Duration) return nil } // AddSpanLog 添加日志 func (t *Tracer) AddSpanLog(traceID, spanID, message string, fields map[string]interface{}) error { t.mu.Lock() defer t.mu.Unlock() trace, ok := t.traces[traceID] if !ok { return fmt.Errorf("trace %s not found", traceID) } span, ok := trace.Spans[spanID] if !ok { return fmt.Errorf("span %s not found", spanID) } span.Logs = append(span.Logs, SpanLog{ Timestamp: time.Now(), Message: message, Fields: fields, }) return nil } // GetTrace 获取追踪 func (t *Tracer) GetTrace(traceID string) (*Trace, error) { t.mu.RLock() defer t.mu.RUnlock() trace, ok := t.traces[traceID] if !ok { return nil, fmt.Errorf("trace %s not found", traceID) } return trace, nil } // ============================================================================= // Metrics - 指标收集 // ============================================================================= // MetricType 指标类型 type MetricType string const ( MetricCounter MetricType = "counter" MetricGauge MetricType = "gauge" MetricHistogram MetricType = "histogram" ) // Metric 指标 type Metric struct { Name string `json:"name"` Type MetricType `json:"type"` Value float64 `json:"value"` Labels map[string]string `json:"labels"` Timestamp time.Time `json:"timestamp"` } // MetricsCollector 指标收集器 type MetricsCollector struct { mu sync.RWMutex metrics map[string]*Metric histories map[string][]Metric config *MetricsConfig } // MetricsConfig 指标配置 type MetricsConfig struct { MaxHistory int } // DefaultMetricsConfig 返回默认指标配置 func DefaultMetricsConfig() *MetricsConfig { return &MetricsConfig{ MaxHistory: 1000, } } // NewMetricsCollector 创建指标收集器 func NewMetricsCollector(config *MetricsConfig) *MetricsCollector { if config == nil { config = DefaultMetricsConfig() } return &MetricsCollector{ metrics: make(map[string]*Metric), histories: make(map[string][]Metric), config: config, } } // RecordMetric 记录指标 func (mc *MetricsCollector) RecordMetric(name string, metricType MetricType, value float64, labels map[string]string) { mc.mu.Lock() defer mc.mu.Unlock() key := name + labelKey(labels) metric := &Metric{ Name: name, Type: metricType, Value: value, Labels: labels, Timestamp: time.Now(), } mc.metrics[key] = metric // 记录历史 if history, ok := mc.histories[key]; ok { if len(history) >= mc.config.MaxHistory { history = history[1:] } mc.histories[key] = append(history, *metric) } else { mc.histories[key] = []Metric{*metric} } } // IncrementCounter 计数器递增 func (mc *MetricsCollector) IncrementCounter(name string, labels map[string]string, delta float64) { mc.mu.Lock() defer mc.mu.Unlock() key := name + labelKey(labels) metric, ok := mc.metrics[key] if !ok { metric = &Metric{ Name: name, Type: MetricCounter, Value: 0, Labels: labels, Timestamp: time.Now(), } mc.metrics[key] = metric } metric.Value += delta metric.Timestamp = time.Now() } // SetGauge 设置计量值 func (mc *MetricsCollector) SetGauge(name string, labels map[string]string, value float64) { mc.RecordMetric(name, MetricGauge, value, labels) } // GetMetric 获取指标 func (mc *MetricsCollector) GetMetric(name string, labels map[string]string) (*Metric, error) { mc.mu.RLock() defer mc.mu.RUnlock() key := name + labelKey(labels) metric, ok := mc.metrics[key] if !ok { return nil, fmt.Errorf("metric %s not found", name) } return metric, nil } // GetAllMetrics 获取所有指标 func (mc *MetricsCollector) GetAllMetrics() []Metric { mc.mu.RLock() defer mc.mu.RUnlock() metrics := make([]Metric, 0, len(mc.metrics)) for _, m := range mc.metrics { metrics = append(metrics, *m) } return metrics } // GetMetricHistory 获取指标历史 func (mc *MetricsCollector) GetMetricHistory(name string, labels map[string]string) ([]Metric, error) { mc.mu.RLock() defer mc.mu.RUnlock() key := name + labelKey(labels) history, ok := mc.histories[key] if !ok { return nil, fmt.Errorf("metric %s not found", name) } result := make([]Metric, len(history)) copy(result, history) return result, nil } func labelKey(labels map[string]string) string { if len(labels) == 0 { return "" } data, _ := json.Marshal(labels) return string(data) } // ============================================================================= // Health Check - 健康检查 // ============================================================================= // HealthStatus 健康状态 type HealthStatus string const ( HealthStatusHealthy HealthStatus = "healthy" HealthStatusDegraded HealthStatus = "degraded" HealthStatusUnhealthy HealthStatus = "unhealthy" HealthStatusUnknown HealthStatus = "unknown" ) // HealthCheck 健康检查结果 type HealthCheck struct { Name string `json:"name"` Status HealthStatus `json:"status"` Message string `json:"message"` Details map[string]interface{} `json:"details,omitempty"` CheckedAt time.Time `json:"checked_at"` } // HealthChecker 健康检查器 type HealthChecker struct { mu sync.RWMutex checks map[string]HealthCheckable config *HealthConfig } // HealthCheckable 可健康检查的接口 type HealthCheckable interface { HealthCheck(ctx context.Context) *HealthCheck } // HealthConfig 健康检查配置 type HealthConfig struct { CheckInterval time.Duration } // DefaultHealthConfig 返回默认健康检查配置 func DefaultHealthConfig() *HealthConfig { return &HealthConfig{ CheckInterval: 30 * time.Second, } } // NewHealthChecker 创建健康检查器 func NewHealthChecker(config *HealthConfig) *HealthChecker { if config == nil { config = DefaultHealthConfig() } return &HealthChecker{ checks: make(map[string]HealthCheckable), config: config, } } // RegisterCheck 注册健康检查 func (hc *HealthChecker) RegisterCheck(name string, checkable HealthCheckable) { hc.mu.Lock() defer hc.mu.Unlock() hc.checks[name] = checkable } // CheckAll 检查所有组件 func (hc *HealthChecker) CheckAll(ctx context.Context) map[string]*HealthCheck { hc.mu.RLock() defer hc.mu.RUnlock() results := make(map[string]*HealthCheck) for name, checkable := range hc.checks { results[name] = checkable.HealthCheck(ctx) } return results } // OverallStatus 总体健康状态 func (hc *HealthChecker) OverallStatus(ctx context.Context) HealthStatus { results := hc.CheckAll(ctx) allHealthy := true anyUnhealthy := false for _, result := range results { switch result.Status { case HealthStatusUnhealthy: anyUnhealthy = true case HealthStatusDegraded: allHealthy = false case HealthStatusUnknown: allHealthy = false } } if anyUnhealthy { return HealthStatusUnhealthy } if !allHealthy { return HealthStatusDegraded } return HealthStatusHealthy } // ============================================================================= // Panorama - 灵瞰整体入口 // ============================================================================= // Panorama 灵瞰 - 可观测与度量 type Panorama struct { Tracer *Tracer Metrics *MetricsCollector Health *HealthChecker } // NewPanorama 创建灵瞰 func NewPanorama() *Panorama { return &Panorama{ Tracer: NewTracer(nil), Metrics: NewMetricsCollector(nil), Health: NewHealthChecker(nil), } } // RecordMessage 记录消息(自动增加指标) func (p *Panorama) RecordMessage(msg *types.UnifiedSynapseMessage, direction string, duration time.Duration) { labels := map[string]string{ "type": msg.Type, "direction": direction, "sender": msg.Sender.AgentID, } p.Metrics.IncrementCounter("synapse_messages_total", labels, 1) p.Metrics.RecordMetric("synapse_message_duration_seconds", MetricHistogram, duration.Seconds(), labels) } // StartTaskTrace 开始任务追踪 func (p *Panorama) StartTaskTrace(taskID, taskType, agentID string) string { trace := p.Tracer.StartTrace(taskID, fmt.Sprintf("task:%s", taskType), agentID) return trace.TraceID } // ToJSON 导出为 JSON func (t *Trace) ToJSON() string { data, _ := json.MarshalIndent(t, "", " ") return string(data) }