325 lines
13 KiB
Go
325 lines
13 KiB
Go
package server
|
||
|
||
import (
|
||
"context"
|
||
"log"
|
||
"math"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
"github.com/jmoiron/sqlx"
|
||
"vdi-cloud/platform/internal/store"
|
||
)
|
||
|
||
type alertRule struct {
|
||
ID string `json:"id" db:"id"`
|
||
Name string `json:"name" db:"name" binding:"required,max=80"`
|
||
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline terminal_cpu terminal_memory terminal_temperature"`
|
||
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
|
||
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
|
||
Enabled bool `json:"enabled" db:"enabled"`
|
||
UpdatedAt time.Time `json:"-" db:"updated_at"`
|
||
}
|
||
type alertRecord struct {
|
||
ID string `json:"id" db:"id"`
|
||
RuleID string `json:"ruleId" db:"rule_id"`
|
||
RuleName string `json:"ruleName" db:"rule_name"`
|
||
Metric string `json:"metric" db:"metric"`
|
||
TargetID string `json:"targetId" db:"target_id"`
|
||
TargetName string `json:"targetName" db:"target_name"`
|
||
Value float64 `json:"value" db:"value"`
|
||
Threshold float64 `json:"threshold" db:"threshold"`
|
||
Status string `json:"status" db:"status"`
|
||
StartedAt time.Time `json:"startedAt" db:"started_at"`
|
||
OpenedAt time.Time `json:"openedAt" db:"opened_at"`
|
||
ResolvedAt *time.Time `json:"resolvedAt" db:"resolved_at"`
|
||
ResolveReason string `json:"resolveReason" db:"resolve_reason"`
|
||
AcknowledgedAt *time.Time `json:"acknowledgedAt" db:"acknowledged_at"`
|
||
AcknowledgedBy string `json:"acknowledgedBy" db:"acknowledged_by"`
|
||
DataFresh *bool `json:"dataFresh" db:"data_fresh"`
|
||
}
|
||
type resourcePoint struct {
|
||
Time time.Time `json:"time" db:"time"`
|
||
CPU *float64 `json:"cpu" db:"cpu"`
|
||
Memory *float64 `json:"memory" db:"memory"`
|
||
Storage *float64 `json:"storage" db:"storage"`
|
||
Samples int `json:"samples" db:"samples"`
|
||
}
|
||
|
||
func (s *Server) resourceSeries(c *gin.Context) {
|
||
from, to, ok := timeRange(c, 7)
|
||
if !ok {
|
||
s.fail(c, 400, "invalid_range", "请选择有效时间范围，最长 7 天")
|
||
return
|
||
}
|
||
id := c.Query("resourceId")
|
||
if id == "" || len(id) > 100 {
|
||
s.fail(c, 400, "invalid_request", "请选择节点")
|
||
return
|
||
}
|
||
bucket := 60
|
||
if to.Sub(from) > 6*time.Hour {
|
||
bucket = 300
|
||
}
|
||
if to.Sub(from) > 24*time.Hour {
|
||
bucket = 1800
|
||
}
|
||
items := []resourcePoint{}
|
||
err := s.DB.SelectContext(c.Request.Context(), &items, `SELECT date_bin(make_interval(secs=>$4),sampled_at,'2000-01-01'::timestamptz) time,
|
||
avg(cpu)*100 cpu,avg(memory)*100 memory,avg(storage)*100 storage,count(*) samples FROM resource_samples
|
||
WHERE resource_id=$1 AND sampled_at>=$2 AND sampled_at<=$3 GROUP BY 1 ORDER BY 1`, id, from, to, bucket)
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"items": items, "bucketSeconds": bucket, "retentionDays": 30})
|
||
}
|
||
func (s *Server) alertRules(c *gin.Context) {
|
||
items := []alertRule{}
|
||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT * FROM alert_rules ORDER BY name,id"); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
list(c, items)
|
||
}
|
||
func observabilityLock(ctx context.Context, tx *sqlx.Tx) error {
|
||
_, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(76511102026)")
|
||
return err
|
||
}
|
||
func (s *Server) saveAlertRule(c *gin.Context) {
|
||
var in alertRule
|
||
if !s.decode(c, &in) {
|
||
return
|
||
}
|
||
in.Name = strings.TrimSpace(in.Name)
|
||
offline := in.Metric == "terminal_offline" || in.Metric == "node_unavailable"
|
||
maximum := 100.0
|
||
if in.Metric == "terminal_temperature" {
|
||
maximum = 150
|
||
}
|
||
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > maximum)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
|
||
s.fail(c, 400, "invalid_request", "使用率阈值为 1～100%，SoC 温度为 1～150°C；离线阈值为 90～86400 秒且不额外设置持续时间")
|
||
return
|
||
}
|
||
ctx := c.Request.Context()
|
||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
defer tx.Rollback()
|
||
if err = observabilityLock(ctx, tx); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
in.ID = c.Param("id")
|
||
if in.ID == "" {
|
||
in.ID = store.ID("rule_")
|
||
_, err = tx.ExecContext(ctx, "INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES($1,$2,$3,$4,$5,$6)", in.ID, in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled)
|
||
} else {
|
||
var id string
|
||
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5,updated_at=now() WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
|
||
if err == nil {
|
||
_, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='rule_changed' WHERE rule_id=$1 AND status='firing'", in.ID)
|
||
}
|
||
if err == nil {
|
||
_, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1", in.ID)
|
||
}
|
||
}
|
||
if err == nil {
|
||
err = accountAudit(ctx, tx, s.admin(c).Username, "save_alert_rule", in.ID, c.GetString("requestId"))
|
||
}
|
||
if err == nil {
|
||
err = tx.Commit()
|
||
}
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(200, in)
|
||
}
|
||
func (s *Server) alerts(c *gin.Context) {
|
||
status := c.Query("status")
|
||
if status != "" && status != "firing" && status != "resolved" {
|
||
s.fail(c, 400, "invalid_request", "告警状态无效")
|
||
return
|
||
}
|
||
page, size := pageParams(c)
|
||
filter := " FROM alerts WHERE ($1='' OR status=$1) AND (target_name ILIKE $2 OR rule_name ILIKE $2)"
|
||
args := []any{status, "%" + c.Query("keyword") + "%"}
|
||
var total, firing, unacknowledged int
|
||
if err := s.DB.GetContext(c.Request.Context(), &total, "SELECT count(*)"+filter, args...); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if err := s.DB.QueryRowContext(c.Request.Context(), "SELECT count(*),count(*) FILTER(WHERE acknowledged_at IS NULL) FROM alerts WHERE status='firing'").Scan(&firing, &unacknowledged); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
items := []alertRecord{}
|
||
args = append(args, size, (page-1)*size)
|
||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *,"+terminalAlertFreshSQL+" AS data_fresh"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"items": items, "total": total, "page": page, "pageSize": size, "firingCount": firing, "unacknowledgedCount": unacknowledged, "requestId": c.GetString("requestId")})
|
||
}
|
||
func (s *Server) acknowledgeAlert(c *gin.Context) {
|
||
ctx := c.Request.Context()
|
||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
defer tx.Rollback()
|
||
var row alertRecord
|
||
if err = tx.GetContext(ctx, &row, "SELECT * FROM alerts WHERE id=$1 FOR UPDATE", c.Param("id")); err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
if row.Status != "firing" {
|
||
s.fail(c, 409, "alert_resolved", "告警已经恢复")
|
||
return
|
||
}
|
||
if row.AcknowledgedAt == nil {
|
||
_, err = tx.ExecContext(ctx, "UPDATE alerts SET acknowledged_at=now(),acknowledged_by=$1 WHERE id=$2", s.admin(c).Username, row.ID)
|
||
if err == nil {
|
||
err = accountAudit(ctx, tx, s.admin(c).Username, "acknowledge_alert", row.ID, c.GetString("requestId"))
|
||
}
|
||
}
|
||
if err == nil {
|
||
err = tx.Commit()
|
||
}
|
||
if err != nil {
|
||
s.dbError(c, err)
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"accepted": true})
|
||
}
|
||
|
||
type observation struct {
|
||
ID string `db:"id"`
|
||
Name string `db:"name"`
|
||
Value *float64 `db:"value"`
|
||
Unavailable bool `db:"unavailable"`
|
||
SampledAt *time.Time `db:"sampled_at"`
|
||
}
|
||
|
||
// One transaction serializes rule edits, expiry and condition evaluation across replicas.
|
||
func (s *Server) observe(ctx context.Context) error {
|
||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer tx.Rollback()
|
||
var acquired bool
|
||
if err = tx.GetContext(ctx, &acquired, "SELECT pg_try_advisory_xact_lock(76511102026)"); err != nil || !acquired {
|
||
return err
|
||
}
|
||
if _, err = tx.ExecContext(ctx, `UPDATE session_history SET state='interrupted',ended_at=last_reported_at+interval '90 seconds',end_reason='heartbeat_timeout'
|
||
WHERE ended_at IS NULL AND last_reported_at<now()-interval '90 seconds'`); err != nil {
|
||
return err
|
||
}
|
||
rules := []alertRule{}
|
||
if err = tx.SelectContext(ctx, &rules, "SELECT * FROM alert_rules WHERE enabled ORDER BY id"); err != nil {
|
||
return err
|
||
}
|
||
for _, rule := range rules {
|
||
targets := []observation{}
|
||
query := `SELECT id,name,extract(epoch FROM(now()-last_seen))::double precision value,false unavailable FROM terminals WHERE enabled AND token_hash IS NOT NULL AND last_seen IS NOT NULL AND status!='unregistered'`
|
||
if rule.Metric == "node_unavailable" {
|
||
query = `SELECT id,name,extract(epoch FROM(now()-updated_at))::double precision value,(status!='online') unavailable FROM resources WHERE provider='proxmox'`
|
||
}
|
||
column := map[string]string{"node_cpu": "cpu", "node_memory": "memory", "node_storage": "storage"}[rule.Metric]
|
||
if column != "" {
|
||
query = `SELECT r.id,r.name,CASE WHEN r.status='online' AND r.updated_at>now()-interval '90 seconds' THEN s.` + column + `*100 ELSE NULL END value,false unavailable
|
||
FROM resources r LEFT JOIN LATERAL(SELECT ` + column + ` FROM resource_samples WHERE resource_id=r.id AND sampled_at>now()-interval '90 seconds' ORDER BY sampled_at DESC LIMIT 1) s ON true WHERE r.provider='proxmox'`
|
||
}
|
||
terminalColumn := terminalAlertColumn(rule.Metric)
|
||
if terminalColumn != "" {
|
||
query = terminalAlertQuery(terminalColumn)
|
||
}
|
||
if err = tx.SelectContext(ctx, &targets, query); err != nil {
|
||
return err
|
||
}
|
||
ids := []string{}
|
||
for _, target := range targets {
|
||
ids = append(ids, target.ID)
|
||
if target.Value == nil {
|
||
if terminalColumn != "" {
|
||
// Unknown data interrupts the pending duration, while an open
|
||
// incident remains open until a fresh normal sample arrives.
|
||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2 AND NOT EXISTS(SELECT 1 FROM alerts WHERE rule_id=$1 AND target_id=$2 AND status='firing')", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if *target.Value < rule.Threshold && !target.Unavailable {
|
||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='condition_cleared' WHERE rule_id=$1 AND target_id=$2 AND status='firing'", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if terminalColumn != "" {
|
||
if err = evaluateTerminalHigh(ctx, tx, rule, target, terminalColumn); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "INSERT INTO alert_states(rule_id,target_id) VALUES($1,$2) ON CONFLICT DO NOTHING", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
hold := rule.HoldSeconds
|
||
if rule.Metric == "node_unavailable" && target.Unavailable {
|
||
hold = int(math.Ceil(rule.Threshold))
|
||
if err = tx.GetContext(ctx, target.Value, "SELECT extract(epoch FROM(now()-first_seen))::double precision FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err = tx.ExecContext(ctx, `INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at)
|
||
SELECT $1,$2,$3,$4,$5,$6,$7,$8,first_seen FROM alert_states WHERE rule_id=$2 AND target_id=$5 AND first_seen<=now()-make_interval(secs=>$9)
|
||
ON CONFLICT(rule_id,target_id) WHERE status='firing' DO UPDATE SET value=EXCLUDED.value`, store.ID("alert_"), rule.ID, rule.Name, rule.Metric, target.ID, target.Name, *target.Value, rule.Threshold, hold); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='target_unavailable' WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[])) AND status='firing'", rule.ID, ids); err != nil {
|
||
return err
|
||
}
|
||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[]))", rule.ID, ids); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
// Bounded retention. Current incidents and active session records are preserved.
|
||
for _, query := range []string{"DELETE FROM login_limits WHERE window_started<now()-interval '1 day'", "DELETE FROM resource_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM terminal_metric_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM session_history WHERE ended_at<now()-interval '90 days'", "DELETE FROM alerts WHERE resolved_at<now()-interval '90 days'"} {
|
||
if _, err = tx.ExecContext(ctx, query); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return tx.Commit()
|
||
}
|
||
func (s *Server) RunObservability(ctx context.Context) {
|
||
ticker := time.NewTicker(15 * time.Second)
|
||
defer ticker.Stop()
|
||
for {
|
||
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||
if err := s.observe(readCtx); err != nil {
|
||
log.Printf("observability evaluation failed: %v", err)
|
||
}
|
||
cancel()
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}
|