package main import ( "context" "encoding/json" "fmt" "log" "net/http" "os" "os/signal" "sort" "strconv" "sync" "syscall" "time" ) // --------------------------------------------------------------------------- // Distributed collector — HTTP dashboard that ingests events from workers // --------------------------------------------------------------------------- // // Endpoints: // POST /events ingest a wireBatch from a worker container // GET /api/state JSON snapshot for the live dashboard // GET /api/events?cursor= JSON event log slice past the cursor // GET / the dashboard HTML page // // Data model: // workers keyed by "/" — same uiStats shape the // TUI uses. A 1-second tick goroutine computes throughput deltas and p95 // over a sliding window of recent latencies. type collectorWorker struct { uiStats containerID string workerName string } type collectorState struct { mu sync.Mutex startAt time.Time // keyed by "/"; iteration order preserved via slice keys []string workers map[string]*collectorWorker events []eventLine nextCursor int64 inconsistencies []eventLine nextInconsCursor int64 // global series for the top-of-dashboard charts p95Series []float64 okPerSecSeries []float64 failPerSecSeries []float64 missingPerSecSeries []float64 globalLat []time.Duration prevTotalOps int64 prevTotalFail int64 prevTotalMissing int64 } func newCollectorState() *collectorState { return &collectorState{ startAt: time.Now(), workers: make(map[string]*collectorWorker), } } // ---- ingest ------------------------------------------------------------- func (c *collectorState) ingest(b wireBatch) { c.mu.Lock() defer c.mu.Unlock() for _, ev := range b.Events { switch ev.Type { case "op": c.applyOp(b.ContainerID, ev) case "sub": c.applySub(b.ContainerID, ev) case "ledger": c.applyLedger(b.ContainerID, ev) case "info": c.appendEvent(ev.Level, ev.At, ev.Text) } } } func (c *collectorState) workerByKey(containerID, workerName string) *collectorWorker { key := containerID + "/" + workerName w, ok := c.workers[key] if !ok { w = &collectorWorker{containerID: containerID, workerName: workerName} c.workers[key] = w c.keys = append(c.keys, key) sort.Strings(c.keys) } return w } func (c *collectorState) applyOp(containerID string, ev wireEvent) { w := c.workerByKey(containerID, ev.Worker) if w.role == "" { w.role = roleFromOp(ev.Op) } w.ops++ if !ev.OK { // Surface every individual failure in the dashboard event log so // the operator can see what's actually breaking, not just the // summarized OUTAGE START/END boundaries. c.appendEvent("err", ev.At, fmt.Sprintf("[%s/%s] %s FAIL: %s", containerID, ev.Worker, ev.Op, trim(ev.Err, 200))) w.fail++ w.consecFail++ if !w.inOutage { w.inOutage = true w.outageStart = ev.At w.outageOpsLost = 1 c.appendEvent("err", ev.At, fmt.Sprintf("[%s/%s] OUTAGE START", containerID, ev.Worker)) } else { w.outageOpsLost++ } return } w.ok++ if w.inOutage { w.outages = append(w.outages, outageRec{ start: w.outageStart, end: ev.At, opsLost: w.outageOpsLost, }) w.inOutage = false w.consecFail = 0 w.outageOpsLost = 0 } if ev.DurUS > 0 { dur := time.Duration(ev.DurUS) * time.Microsecond w.recentLat = appendBoundedDur(w.recentLat, dur, maxRecentLat) c.globalLat = appendBoundedDur(c.globalLat, dur, maxGlobalLat) } } func (c *collectorState) applySub(containerID string, ev wireEvent) { w := c.workerByKey(containerID, ev.Worker) if w.role == "" { w.role = "subscriber" } w.msgs++ if w.lastSeq != 0 && ev.Seq > w.lastSeq+1 { gap := ev.Seq - w.lastSeq - 1 w.gaps += gap } if !w.lastSeqAt.IsZero() { if d := ev.At.Sub(w.lastSeqAt); d > w.longestGap { w.longestGap = d } } w.lastSeq = ev.Seq w.lastSeqAt = ev.At } func (c *collectorState) applyLedger(containerID string, ev wireEvent) { w := c.workerByKey(containerID, ev.Worker) if w.role == "" { w.role = "reader" } switch ev.Kind { case "ok": w.verified++ case "missing": w.missing++ c.appendInconsistency("err", ev.At, fmt.Sprintf("[%s/%s] LEDGER MISSING seq=%d %s", containerID, ev.Worker, ev.Seq, ev.Detail)) case "mismatch": w.mismatch++ c.appendInconsistency("err", ev.At, fmt.Sprintf("[%s/%s] LEDGER MISMATCH seq=%d %s", containerID, ev.Worker, ev.Seq, ev.Detail)) } } // roleFromOp infers a worker's role from the first op name we see, since // workers don't currently transmit role with each event. Best-effort only. func roleFromOp(op string) string { switch op { case "WRITE": return "writer" case "RW": return "readwriter" case "GET", "PING": return "reader" case "PUB": return "publisher" case "SUBSCRIBE": return "subscriber" case "BLOAT": return "bloater" } return "" } func (c *collectorState) appendEvent(level string, at time.Time, text string) { if at.IsZero() { at = time.Now() } if level == "" { level = "info" } if len(c.events) >= maxEvents { copy(c.events, c.events[1:]) c.events = c.events[:len(c.events)-1] } c.events = append(c.events, eventLine{at: at, level: level, text: text}) c.nextCursor++ } func (c *collectorState) appendInconsistency(level string, at time.Time, text string) { if at.IsZero() { at = time.Now() } if level == "" { level = "info" } if len(c.inconsistencies) >= maxEvents { copy(c.inconsistencies, c.inconsistencies[1:]) c.inconsistencies = c.inconsistencies[:len(c.inconsistencies)-1] } c.inconsistencies = append(c.inconsistencies, eventLine{at: at, level: level, text: text}) c.nextInconsCursor++ } // ---- tick (1Hz) --------------------------------------------------------- func (c *collectorState) tick() { c.mu.Lock() defer c.mu.Unlock() if len(c.globalLat) > 0 { cp := make([]time.Duration, len(c.globalLat)) copy(cp, c.globalLat) sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] }) idx := 95 * len(cp) / 100 if idx >= len(cp) { idx = len(cp) - 1 } p95ms := float64(cp[idx]) / float64(time.Millisecond) c.p95Series = appendBoundedFloat(c.p95Series, p95ms, maxSeriesPoints) } else { c.p95Series = appendBoundedFloat(c.p95Series, 0, maxSeriesPoints) } var totalOps, totalFail, totalMissing int64 for _, w := range c.workers { totalOps += w.ops + w.msgs totalFail += w.fail totalMissing += w.missing } okDelta := (totalOps - totalFail) - (c.prevTotalOps - c.prevTotalFail) if okDelta < 0 { okDelta = 0 } failDelta := totalFail - c.prevTotalFail if failDelta < 0 { failDelta = 0 } missingDelta := totalMissing - c.prevTotalMissing if missingDelta < 0 { missingDelta = 0 } c.prevTotalOps = totalOps c.prevTotalFail = totalFail c.prevTotalMissing = totalMissing c.okPerSecSeries = appendBoundedFloat(c.okPerSecSeries, float64(okDelta), maxSeriesPoints) c.failPerSecSeries = appendBoundedFloat(c.failPerSecSeries, float64(failDelta), maxSeriesPoints) c.missingPerSecSeries = appendBoundedFloat(c.missingPerSecSeries, float64(missingDelta), maxSeriesPoints) for _, w := range c.workers { current := w.ops + w.msgs delta := current - w.prevOps if delta < 0 { delta = 0 } w.prevOps = current w.opsHist = appendBoundedFloat(w.opsHist, float64(delta), maxSparkSamples) } } // ---- snapshot view ------------------------------------------------------ type stateView struct { StartAt time.Time `json:"start_at"` Elapsed string `json:"elapsed"` Containers int `json:"containers"` InOutage int `json:"in_outage"` Workers []workerView `json:"workers"` Charts chartsView `json:"charts"` } type workerView struct { Container string `json:"container"` Name string `json:"name"` Role string `json:"role"` Ops int64 `json:"ops"` OK int64 `json:"ok"` Fail int64 `json:"fail"` UptimePct float64 `json:"uptime_pct"` P50MS float64 `json:"p50_ms"` P95MS float64 `json:"p95_ms"` OpsHist []float64 `json:"ops_hist"` InOutage bool `json:"in_outage"` Outages int `json:"outages"` LongestMS float64 `json:"longest_outage_ms"` // reader-only Verified int64 `json:"verified,omitempty"` Missing int64 `json:"missing,omitempty"` Mismatch int64 `json:"mismatch,omitempty"` // subscriber-only Msgs int64 `json:"msgs,omitempty"` Gaps int64 `json:"gaps,omitempty"` LongestGapMS float64 `json:"longest_gap_ms,omitempty"` } type chartsView struct { P95 []float64 `json:"p95"` OkPerSec []float64 `json:"ok_per_sec"` FailPerSec []float64 `json:"fail_per_sec"` MissingPerSec []float64 `json:"missing_per_sec"` P95Peak float64 `json:"p95_peak"` OkPeak float64 `json:"ok_peak"` FailPeak float64 `json:"fail_peak"` MissingPeak float64 `json:"missing_peak"` } func (c *collectorState) snapshot() stateView { c.mu.Lock() defer c.mu.Unlock() containers := map[string]struct{}{} for _, k := range c.keys { w := c.workers[k] containers[w.containerID] = struct{}{} } view := stateView{ StartAt: c.startAt, Elapsed: time.Since(c.startAt).Round(time.Second).String(), Containers: len(containers), Charts: chartsView{ P95: append([]float64(nil), c.p95Series...), OkPerSec: append([]float64(nil), c.okPerSecSeries...), FailPerSec: append([]float64(nil), c.failPerSecSeries...), MissingPerSec: append([]float64(nil), c.missingPerSecSeries...), P95Peak: maxFloat(c.p95Series), OkPeak: maxFloat(c.okPerSecSeries), FailPeak: maxFloat(c.failPerSecSeries), MissingPeak: maxFloat(c.missingPerSecSeries), }, } for _, k := range c.keys { w := c.workers[k] uptime := 100.0 if w.ops > 0 { uptime = float64(w.ok) / float64(w.ops) * 100 } var longest time.Duration for _, o := range w.outages { if d := o.end.Sub(o.start); d > longest { longest = d } } if w.inOutage { view.InOutage++ } wv := workerView{ Container: w.containerID, Name: w.workerName, Role: w.role, Ops: w.ops, OK: w.ok, Fail: w.fail, UptimePct: uptime, P50MS: float64(percentileLat(w.recentLat, 50)) / float64(time.Millisecond), P95MS: float64(percentileLat(w.recentLat, 95)) / float64(time.Millisecond), OpsHist: append([]float64(nil), w.opsHist...), InOutage: w.inOutage, Outages: len(w.outages), LongestMS: float64(longest) / float64(time.Millisecond), } switch w.role { case "reader": wv.Verified = w.verified wv.Missing = w.missing wv.Mismatch = w.mismatch case "subscriber": wv.Msgs = w.msgs wv.Gaps = w.gaps wv.LongestGapMS = float64(w.longestGap) / float64(time.Millisecond) } view.Workers = append(view.Workers, wv) } return view } type eventOut struct { ID int64 `json:"id"` At string `json:"at"` Level string `json:"level"` Text string `json:"text"` } func (c *collectorState) eventsSince(cursor int64) ([]eventOut, int64) { c.mu.Lock() defer c.mu.Unlock() // nextCursor counts events ever inserted; events slice may have dropped older. // Compute base id of the first slot in the current slice: baseID := c.nextCursor - int64(len(c.events)) if cursor < baseID { cursor = baseID } startIdx := int(cursor - baseID) if startIdx < 0 { startIdx = 0 } if startIdx > len(c.events) { startIdx = len(c.events) } out := make([]eventOut, 0, len(c.events)-startIdx) for i := startIdx; i < len(c.events); i++ { e := c.events[i] out = append(out, eventOut{ ID: baseID + int64(i), At: e.at.Format("15:04:05.000"), Level: e.level, Text: e.text, }) } return out, c.nextCursor } func (c *collectorState) inconsistenciesSince(cursor int64) ([]eventOut, int64) { c.mu.Lock() defer c.mu.Unlock() baseID := c.nextInconsCursor - int64(len(c.inconsistencies)) if cursor < baseID { cursor = baseID } startIdx := int(cursor - baseID) if startIdx < 0 { startIdx = 0 } if startIdx > len(c.inconsistencies) { startIdx = len(c.inconsistencies) } out := make([]eventOut, 0, len(c.inconsistencies)-startIdx) for i := startIdx; i < len(c.inconsistencies); i++ { e := c.inconsistencies[i] out = append(out, eventOut{ ID: baseID + int64(i), At: e.at.Format("15:04:05.000"), Level: e.level, Text: e.text, }) } return out, c.nextInconsCursor } // ---- HTTP server -------------------------------------------------------- func runCollector(port int) int { state := newCollectorState() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() go func() { t := time.NewTicker(tickInterval) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: state.tick() } } }() mux := http.NewServeMux() mux.HandleFunc("/events", state.handleIngest) mux.HandleFunc("/api/state", state.handleState) mux.HandleFunc("/api/events", state.handleEvents) mux.HandleFunc("/api/inconsistencies", state.handleInconsistencies) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = w.Write([]byte(dashboardHTML)) }) srv := &http.Server{ Addr: fmt.Sprintf(":%d", port), Handler: mux, ReadHeaderTimeout: 5 * time.Second, } go func() { log.Printf("collector listening on :%d", port) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Fatalf("collector: %v", err) } }() <-ctx.Done() log.Printf("collector shutting down…") shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _ = srv.Shutdown(shutdownCtx) return 0 } func (c *collectorState) handleIngest(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "POST only", http.StatusMethodNotAllowed) return } var b wireBatch if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&b); err != nil { http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest) return } c.ingest(b) w.WriteHeader(http.StatusNoContent) } func (c *collectorState) handleState(w http.ResponseWriter, _ *http.Request) { view := c.snapshot() w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(view) } func (c *collectorState) handleEvents(w http.ResponseWriter, r *http.Request) { cursor := int64(0) if v := r.URL.Query().Get("cursor"); v != "" { if n, err := strconv.ParseInt(v, 10, 64); err == nil { cursor = n } } events, next := c.eventsSince(cursor) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "events": events, "next_cursor": next, }) } func (c *collectorState) handleInconsistencies(w http.ResponseWriter, r *http.Request) { cursor := int64(0) if v := r.URL.Query().Get("cursor"); v != "" { if n, err := strconv.ParseInt(v, 10, 64); err == nil { cursor = n } } events, next := c.inconsistenciesSince(cursor) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "events": events, "next_cursor": next, }) } // ---- embedded dashboard HTML ------------------------------------------- const dashboardHTML = ` valkey-ha chaos

valkey-ha chaos

p95 latency (rolling 2 min)

ok ops/sec (rolling 2 min)

fail ops/sec (rolling 2 min)

missing ledger/sec (rolling 2 min)

workers

CONTAINERNAMEROLE OPSOKFAIL UPTIMEP50P95 SPARKOUTAGESSTATUSNOTES

client errors

ledger inconsistencies

`