valkey + gitea + long build

valkey/.gitignore

@@ -0,0 +1 @@
+valkey-test

valkey/ca.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB8TCCAZagAwIBAgIQbwx3ZUCaU9IQr+ynltMGsTAKBggqhkjOPQQDAjA2MRUw
+EwYDVQQKEwxaZXJvcHMgSW5mcmExHTAbBgNVBAMTFFplcm9wcyBJbmZyYSBSb290
+IENBMB4XDTI1MDkwMTA3NDMzOVoXDTI3MDkwMTA3NDMxM1owVjELMAkGA1UEBhMC
+Q1oxDzANBgNVBAoTBlplcm9wczEQMA4GA1UECxMHU2VydmljZTEkMCIGA1UEAxMb
+WmVyb3BzIFNlcnZpY2UgSW50ZXJtZWRpYXRlMFkwEwYHKoZIzj0CAQYIKoZIzj0D
+AQcDQgAEeJwBKQfdfdRR7QJWVLegIXvn1k8SqUo8dp8bY2Fj9PUSNsTZ4yDsgpGo
+Zummp6P92mOookbowR1nc0d8TyRYs6NmMGQwDgYDVR0PAQH/BAQDAgEGMBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFLhO0kbsvwnGvuBp+gBy82mzsMbzMB8G
+A1UdIwQYMBaAFJIVumuZisDXAdV9359xveeP3HkhMAoGCCqGSM49BAMCA0kAMEYC
+IQCanJOFmOCQjnC/0oGQTHKMmmfPcCXp82XOYDnfY6YEGAIhAPYpIizvP5RpLymG
+fn7wyg9VTA7W2wxF0ToJUXOKJO5O
+-----END CERTIFICATE-----

valkey/collector.go

@@ -0,0 +1,897 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"sort"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// ---------------------------------------------------------------------------
+// Distributed collector — HTTP dashboard that ingests events from workers
+// ---------------------------------------------------------------------------
+//
+// Endpoints:
+//   POST /events             ingest a wireBatch from a worker container
+//   GET  /api/state          JSON snapshot for the live dashboard
+//   GET  /api/events?cursor= JSON event log slice past the cursor
+//   GET  /                   the dashboard HTML page
+//
+// Data model:
+//   workers keyed by "<containerID>/<workerName>" — same uiStats shape the
+//   TUI uses. A 1-second tick goroutine computes throughput deltas and p95
+//   over a sliding window of recent latencies.
+
+type collectorWorker struct {
+	uiStats
+	containerID string
+	workerName  string
+}
+
+type collectorState struct {
+	mu sync.Mutex
+
+	startAt time.Time
+
+	// keyed by "<container>/<worker>"; iteration order preserved via slice
+	keys    []string
+	workers map[string]*collectorWorker
+
+	events     []eventLine
+	nextCursor int64
+
+	inconsistencies     []eventLine
+	nextInconsCursor    int64
+
+	// global series for the top-of-dashboard charts
+	p95Series           []float64
+	okPerSecSeries      []float64
+	failPerSecSeries    []float64
+	missingPerSecSeries []float64
+	globalLat           []time.Duration
+	prevTotalOps        int64
+	prevTotalFail       int64
+	prevTotalMissing    int64
+}
+
+func newCollectorState() *collectorState {
+	return &collectorState{
+		startAt: time.Now(),
+		workers: make(map[string]*collectorWorker),
+	}
+}
+
+// ---- ingest -------------------------------------------------------------
+
+func (c *collectorState) ingest(b wireBatch) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for _, ev := range b.Events {
+		switch ev.Type {
+		case "op":
+			c.applyOp(b.ContainerID, ev)
+		case "sub":
+			c.applySub(b.ContainerID, ev)
+		case "ledger":
+			c.applyLedger(b.ContainerID, ev)
+		case "info":
+			c.appendEvent(ev.Level, ev.At, ev.Text)
+		}
+	}
+}
+
+func (c *collectorState) workerByKey(containerID, workerName string) *collectorWorker {
+	key := containerID + "/" + workerName
+	w, ok := c.workers[key]
+	if !ok {
+		w = &collectorWorker{containerID: containerID, workerName: workerName}
+		c.workers[key] = w
+		c.keys = append(c.keys, key)
+		sort.Strings(c.keys)
+	}
+	return w
+}
+
+func (c *collectorState) applyOp(containerID string, ev wireEvent) {
+	w := c.workerByKey(containerID, ev.Worker)
+	if w.role == "" {
+		w.role = roleFromOp(ev.Op)
+	}
+	w.ops++
+	if !ev.OK {
+		// Surface every individual failure in the dashboard event log so
+		// the operator can see what's actually breaking, not just the
+		// summarized OUTAGE START/END boundaries.
+		c.appendEvent("err", ev.At,
+			fmt.Sprintf("[%s/%s] %s FAIL: %s",
+				containerID, ev.Worker, ev.Op, trim(ev.Err, 200)))
+		w.fail++
+		w.consecFail++
+		if !w.inOutage {
+			w.inOutage = true
+			w.outageStart = ev.At
+			w.outageOpsLost = 1
+			c.appendEvent("err", ev.At,
+				fmt.Sprintf("[%s/%s] OUTAGE START", containerID, ev.Worker))
+		} else {
+			w.outageOpsLost++
+		}
+		return
+	}
+	w.ok++
+	if w.inOutage {
+		w.outages = append(w.outages, outageRec{
+			start: w.outageStart, end: ev.At, opsLost: w.outageOpsLost,
+		})
+		w.inOutage = false
+		w.consecFail = 0
+		w.outageOpsLost = 0
+	}
+	if ev.DurUS > 0 {
+		dur := time.Duration(ev.DurUS) * time.Microsecond
+		w.recentLat = appendBoundedDur(w.recentLat, dur, maxRecentLat)
+		c.globalLat = appendBoundedDur(c.globalLat, dur, maxGlobalLat)
+	}
+}
+
+func (c *collectorState) applySub(containerID string, ev wireEvent) {
+	w := c.workerByKey(containerID, ev.Worker)
+	if w.role == "" {
+		w.role = "subscriber"
+	}
+	w.msgs++
+	if w.lastSeq != 0 && ev.Seq > w.lastSeq+1 {
+		gap := ev.Seq - w.lastSeq - 1
+		w.gaps += gap
+	}
+	if !w.lastSeqAt.IsZero() {
+		if d := ev.At.Sub(w.lastSeqAt); d > w.longestGap {
+			w.longestGap = d
+		}
+	}
+	w.lastSeq = ev.Seq
+	w.lastSeqAt = ev.At
+}
+
+func (c *collectorState) applyLedger(containerID string, ev wireEvent) {
+	w := c.workerByKey(containerID, ev.Worker)
+	if w.role == "" {
+		w.role = "reader"
+	}
+	switch ev.Kind {
+	case "ok":
+		w.verified++
+	case "missing":
+		w.missing++
+		c.appendInconsistency("err", ev.At,
+			fmt.Sprintf("[%s/%s] LEDGER MISSING seq=%d  %s",
+				containerID, ev.Worker, ev.Seq, ev.Detail))
+	case "mismatch":
+		w.mismatch++
+		c.appendInconsistency("err", ev.At,
+			fmt.Sprintf("[%s/%s] LEDGER MISMATCH seq=%d  %s",
+				containerID, ev.Worker, ev.Seq, ev.Detail))
+	}
+}
+
+// roleFromOp infers a worker's role from the first op name we see, since
+// workers don't currently transmit role with each event. Best-effort only.
+func roleFromOp(op string) string {
+	switch op {
+	case "WRITE":
+		return "writer"
+	case "RW":
+		return "readwriter"
+	case "GET", "PING":
+		return "reader"
+	case "PUB":
+		return "publisher"
+	case "SUBSCRIBE":
+		return "subscriber"
+	case "BLOAT":
+		return "bloater"
+	}
+	return ""
+}
+
+func (c *collectorState) appendEvent(level string, at time.Time, text string) {
+	if at.IsZero() {
+		at = time.Now()
+	}
+	if level == "" {
+		level = "info"
+	}
+	if len(c.events) >= maxEvents {
+		copy(c.events, c.events[1:])
+		c.events = c.events[:len(c.events)-1]
+	}
+	c.events = append(c.events, eventLine{at: at, level: level, text: text})
+	c.nextCursor++
+}
+
+func (c *collectorState) appendInconsistency(level string, at time.Time, text string) {
+	if at.IsZero() {
+		at = time.Now()
+	}
+	if level == "" {
+		level = "info"
+	}
+	if len(c.inconsistencies) >= maxEvents {
+		copy(c.inconsistencies, c.inconsistencies[1:])
+		c.inconsistencies = c.inconsistencies[:len(c.inconsistencies)-1]
+	}
+	c.inconsistencies = append(c.inconsistencies, eventLine{at: at, level: level, text: text})
+	c.nextInconsCursor++
+}
+
+// ---- tick (1Hz) ---------------------------------------------------------
+
+func (c *collectorState) tick() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if len(c.globalLat) > 0 {
+		cp := make([]time.Duration, len(c.globalLat))
+		copy(cp, c.globalLat)
+		sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] })
+		idx := 95 * len(cp) / 100
+		if idx >= len(cp) {
+			idx = len(cp) - 1
+		}
+		p95ms := float64(cp[idx]) / float64(time.Millisecond)
+		c.p95Series = appendBoundedFloat(c.p95Series, p95ms, maxSeriesPoints)
+	} else {
+		c.p95Series = appendBoundedFloat(c.p95Series, 0, maxSeriesPoints)
+	}
+
+	var totalOps, totalFail, totalMissing int64
+	for _, w := range c.workers {
+		totalOps += w.ops + w.msgs
+		totalFail += w.fail
+		totalMissing += w.missing
+	}
+	okDelta := (totalOps - totalFail) - (c.prevTotalOps - c.prevTotalFail)
+	if okDelta < 0 {
+		okDelta = 0
+	}
+	failDelta := totalFail - c.prevTotalFail
+	if failDelta < 0 {
+		failDelta = 0
+	}
+	missingDelta := totalMissing - c.prevTotalMissing
+	if missingDelta < 0 {
+		missingDelta = 0
+	}
+	c.prevTotalOps = totalOps
+	c.prevTotalFail = totalFail
+	c.prevTotalMissing = totalMissing
+	c.okPerSecSeries = appendBoundedFloat(c.okPerSecSeries, float64(okDelta), maxSeriesPoints)
+	c.failPerSecSeries = appendBoundedFloat(c.failPerSecSeries, float64(failDelta), maxSeriesPoints)
+	c.missingPerSecSeries = appendBoundedFloat(c.missingPerSecSeries, float64(missingDelta), maxSeriesPoints)
+
+	for _, w := range c.workers {
+		current := w.ops + w.msgs
+		delta := current - w.prevOps
+		if delta < 0 {
+			delta = 0
+		}
+		w.prevOps = current
+		w.opsHist = appendBoundedFloat(w.opsHist, float64(delta), maxSparkSamples)
+	}
+}
+
+// ---- snapshot view ------------------------------------------------------
+
+type stateView struct {
+	StartAt    time.Time         `json:"start_at"`
+	Elapsed    string            `json:"elapsed"`
+	Containers int               `json:"containers"`
+	InOutage   int               `json:"in_outage"`
+	Workers    []workerView      `json:"workers"`
+	Charts     chartsView        `json:"charts"`
+}
+
+type workerView struct {
+	Container  string  `json:"container"`
+	Name       string  `json:"name"`
+	Role       string  `json:"role"`
+	Ops        int64   `json:"ops"`
+	OK         int64   `json:"ok"`
+	Fail       int64   `json:"fail"`
+	UptimePct  float64 `json:"uptime_pct"`
+	P50MS      float64 `json:"p50_ms"`
+	P95MS      float64 `json:"p95_ms"`
+	OpsHist    []float64 `json:"ops_hist"`
+	InOutage   bool    `json:"in_outage"`
+	Outages    int     `json:"outages"`
+	LongestMS  float64 `json:"longest_outage_ms"`
+	// reader-only
+	Verified int64 `json:"verified,omitempty"`
+	Missing  int64 `json:"missing,omitempty"`
+	Mismatch int64 `json:"mismatch,omitempty"`
+	// subscriber-only
+	Msgs         int64   `json:"msgs,omitempty"`
+	Gaps         int64   `json:"gaps,omitempty"`
+	LongestGapMS float64 `json:"longest_gap_ms,omitempty"`
+}
+
+type chartsView struct {
+	P95           []float64 `json:"p95"`
+	OkPerSec      []float64 `json:"ok_per_sec"`
+	FailPerSec    []float64 `json:"fail_per_sec"`
+	MissingPerSec []float64 `json:"missing_per_sec"`
+	P95Peak       float64   `json:"p95_peak"`
+	OkPeak        float64   `json:"ok_peak"`
+	FailPeak      float64   `json:"fail_peak"`
+	MissingPeak   float64   `json:"missing_peak"`
+}
+
+func (c *collectorState) snapshot() stateView {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	containers := map[string]struct{}{}
+	for _, k := range c.keys {
+		w := c.workers[k]
+		containers[w.containerID] = struct{}{}
+	}
+
+	view := stateView{
+		StartAt:    c.startAt,
+		Elapsed:    time.Since(c.startAt).Round(time.Second).String(),
+		Containers: len(containers),
+		Charts: chartsView{
+			P95:           append([]float64(nil), c.p95Series...),
+			OkPerSec:      append([]float64(nil), c.okPerSecSeries...),
+			FailPerSec:    append([]float64(nil), c.failPerSecSeries...),
+			MissingPerSec: append([]float64(nil), c.missingPerSecSeries...),
+			P95Peak:       maxFloat(c.p95Series),
+			OkPeak:        maxFloat(c.okPerSecSeries),
+			FailPeak:      maxFloat(c.failPerSecSeries),
+			MissingPeak:   maxFloat(c.missingPerSecSeries),
+		},
+	}
+	for _, k := range c.keys {
+		w := c.workers[k]
+		uptime := 100.0
+		if w.ops > 0 {
+			uptime = float64(w.ok) / float64(w.ops) * 100
+		}
+		var longest time.Duration
+		for _, o := range w.outages {
+			if d := o.end.Sub(o.start); d > longest {
+				longest = d
+			}
+		}
+		if w.inOutage {
+			view.InOutage++
+		}
+		wv := workerView{
+			Container: w.containerID,
+			Name:      w.workerName,
+			Role:      w.role,
+			Ops:       w.ops,
+			OK:        w.ok,
+			Fail:      w.fail,
+			UptimePct: uptime,
+			P50MS:     float64(percentileLat(w.recentLat, 50)) / float64(time.Millisecond),
+			P95MS:     float64(percentileLat(w.recentLat, 95)) / float64(time.Millisecond),
+			OpsHist:   append([]float64(nil), w.opsHist...),
+			InOutage:  w.inOutage,
+			Outages:   len(w.outages),
+			LongestMS: float64(longest) / float64(time.Millisecond),
+		}
+		switch w.role {
+		case "reader":
+			wv.Verified = w.verified
+			wv.Missing = w.missing
+			wv.Mismatch = w.mismatch
+		case "subscriber":
+			wv.Msgs = w.msgs
+			wv.Gaps = w.gaps
+			wv.LongestGapMS = float64(w.longestGap) / float64(time.Millisecond)
+		}
+		view.Workers = append(view.Workers, wv)
+	}
+	return view
+}
+
+type eventOut struct {
+	ID    int64  `json:"id"`
+	At    string `json:"at"`
+	Level string `json:"level"`
+	Text  string `json:"text"`
+}
+
+func (c *collectorState) eventsSince(cursor int64) ([]eventOut, int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// nextCursor counts events ever inserted; events slice may have dropped older.
+	// Compute base id of the first slot in the current slice:
+	baseID := c.nextCursor - int64(len(c.events))
+	if cursor < baseID {
+		cursor = baseID
+	}
+	startIdx := int(cursor - baseID)
+	if startIdx < 0 {
+		startIdx = 0
+	}
+	if startIdx > len(c.events) {
+		startIdx = len(c.events)
+	}
+	out := make([]eventOut, 0, len(c.events)-startIdx)
+	for i := startIdx; i < len(c.events); i++ {
+		e := c.events[i]
+		out = append(out, eventOut{
+			ID:    baseID + int64(i),
+			At:    e.at.Format("15:04:05.000"),
+			Level: e.level,
+			Text:  e.text,
+		})
+	}
+	return out, c.nextCursor
+}
+
+func (c *collectorState) inconsistenciesSince(cursor int64) ([]eventOut, int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	baseID := c.nextInconsCursor - int64(len(c.inconsistencies))
+	if cursor < baseID {
+		cursor = baseID
+	}
+	startIdx := int(cursor - baseID)
+	if startIdx < 0 {
+		startIdx = 0
+	}
+	if startIdx > len(c.inconsistencies) {
+		startIdx = len(c.inconsistencies)
+	}
+	out := make([]eventOut, 0, len(c.inconsistencies)-startIdx)
+	for i := startIdx; i < len(c.inconsistencies); i++ {
+		e := c.inconsistencies[i]
+		out = append(out, eventOut{
+			ID:    baseID + int64(i),
+			At:    e.at.Format("15:04:05.000"),
+			Level: e.level,
+			Text:  e.text,
+		})
+	}
+	return out, c.nextInconsCursor
+}
+
+// ---- HTTP server --------------------------------------------------------
+
+func runCollector(port int) int {
+	state := newCollectorState()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	go func() {
+		t := time.NewTicker(tickInterval)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				state.tick()
+			}
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/events", state.handleIngest)
+	mux.HandleFunc("/api/state", state.handleState)
+	mux.HandleFunc("/api/events", state.handleEvents)
+	mux.HandleFunc("/api/inconsistencies", state.handleInconsistencies)
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(dashboardHTML))
+	})
+
+	srv := &http.Server{
+		Addr:              fmt.Sprintf(":%d", port),
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	go func() {
+		log.Printf("collector listening on :%d", port)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatalf("collector: %v", err)
+		}
+	}()
+
+	<-ctx.Done()
+	log.Printf("collector shutting down…")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = srv.Shutdown(shutdownCtx)
+	return 0
+}
+
+func (c *collectorState) handleIngest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "POST only", http.StatusMethodNotAllowed)
+		return
+	}
+	var b wireBatch
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&b); err != nil {
+		http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest)
+		return
+	}
+	c.ingest(b)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (c *collectorState) handleState(w http.ResponseWriter, _ *http.Request) {
+	view := c.snapshot()
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(view)
+}
+
+func (c *collectorState) handleEvents(w http.ResponseWriter, r *http.Request) {
+	cursor := int64(0)
+	if v := r.URL.Query().Get("cursor"); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+			cursor = n
+		}
+	}
+	events, next := c.eventsSince(cursor)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{
+		"events":      events,
+		"next_cursor": next,
+	})
+}
+
+func (c *collectorState) handleInconsistencies(w http.ResponseWriter, r *http.Request) {
+	cursor := int64(0)
+	if v := r.URL.Query().Get("cursor"); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+			cursor = n
+		}
+	}
+	events, next := c.inconsistenciesSince(cursor)
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]any{
+		"events":      events,
+		"next_cursor": next,
+	})
+}
+
+// ---- embedded dashboard HTML -------------------------------------------
+
+const dashboardHTML = `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>valkey-ha chaos</title>
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<style>
+  :root {
+    --bg: #0c0c10;
+    --panel: #15151c;
+    --border: #2a2a35;
+    --text: #e6e6ea;
+    --dim: #6c6c7a;
+    --accent: #d75faf;
+    --ok: #5fd75f;
+    --warn: #ffaf5f;
+    --err: #ff5f5f;
+    --chart: #af87ff;
+    --chart-fail: #ff5f5f;
+  }
+  * { box-sizing: border-box; }
+  body { font-family: ui-monospace, 'JetBrains Mono', Menlo, Consolas, monospace;
+         background: var(--bg); color: var(--text); margin: 0; padding: 16px;
+         font-size: 13px; line-height: 1.45; }
+  h1 { color: var(--accent); margin: 0 0 4px; font-size: 18px; }
+  .header-meta { color: var(--dim); margin-bottom: 14px; }
+  .badge { padding: 1px 6px; border-radius: 3px; font-weight: 600; }
+  .badge.ok   { background: #1c3a1c; color: var(--ok); }
+  .badge.err  { background: #401818; color: var(--err); }
+  .badge.warn { background: #402a18; color: var(--warn); }
+  .grid { display: grid; grid-template-columns: 1fr 1fr 1fr 1fr; gap: 14px; margin-bottom: 16px; }
+  @media (max-width: 1400px) { .grid { grid-template-columns: 1fr 1fr; } }
+  @media (max-width: 700px)  { .grid { grid-template-columns: 1fr; } }
+  .panel { background: var(--panel); border: 1px solid var(--border);
+           border-radius: 4px; padding: 10px 12px; }
+  .panel h3 { margin: 0 0 6px; color: #5fafff; font-size: 12px;
+              text-transform: uppercase; letter-spacing: 0.06em; }
+  .panel .meta { color: var(--dim); font-size: 11px; margin-bottom: 4px; }
+  table { border-collapse: collapse; width: 100%; font-size: 12px; }
+  th, td { padding: 4px 8px; text-align: left; vertical-align: middle;
+           border-bottom: 1px solid var(--border); }
+  th { color: #5fafff; font-weight: 600; text-transform: uppercase;
+       font-size: 10px; letter-spacing: 0.05em; }
+  td.num { text-align: right; font-variant-numeric: tabular-nums; }
+  tr.outage td:first-child { border-left: 3px solid var(--err); }
+  .container-tag { color: var(--accent); }
+  .events { background: var(--panel); border: 1px solid var(--border);
+            border-radius: 4px; padding: 8px 10px; max-height: 50vh;
+            overflow-y: auto; font-size: 12px; }
+  .events .ev { padding: 1px 0; white-space: pre-wrap; word-break: break-word; }
+  .events .ev .ts { color: var(--dim); margin-right: 6px; }
+  .events .ev.err  { color: var(--err); }
+  .events .ev.warn { color: var(--warn); }
+  .events .ev.info { color: var(--text); }
+  svg { display: block; width: 100%; height: 80px; }
+  polyline { fill: none; stroke: var(--chart); stroke-width: 1.4; }
+  .area    { fill: var(--chart); fill-opacity: 0.18; stroke: none; }
+  .spark   { font-family: ui-monospace, monospace; color: var(--chart); }
+  .dim { color: var(--dim); }
+  .filter { margin-bottom: 8px; }
+  .filter input { background: var(--panel); color: var(--text);
+                  border: 1px solid var(--border); padding: 4px 8px;
+                  border-radius: 3px; font: inherit; }
+  .footer { color: var(--dim); margin-top: 12px; font-size: 11px; }
+</style>
+</head>
+<body>
+<h1>valkey-ha chaos</h1>
+<div class="header-meta" id="header">connecting…</div>
+
+<div class="grid">
+  <div class="panel">
+    <h3>p95 latency (rolling 2 min)</h3>
+    <div class="meta" id="lat-meta">—</div>
+    <svg id="chart-lat" viewBox="0 0 100 30" preserveAspectRatio="none"></svg>
+  </div>
+  <div class="panel">
+    <h3>ok ops/sec (rolling 2 min)</h3>
+    <div class="meta" id="tp-meta">—</div>
+    <svg id="chart-tp" viewBox="0 0 100 30" preserveAspectRatio="none"></svg>
+  </div>
+  <div class="panel">
+    <h3>fail ops/sec (rolling 2 min)</h3>
+    <div class="meta" id="fail-meta">—</div>
+    <svg id="chart-fail" viewBox="0 0 100 30" preserveAspectRatio="none"></svg>
+  </div>
+  <div class="panel">
+    <h3>missing ledger/sec (rolling 2 min)</h3>
+    <div class="meta" id="miss-meta">—</div>
+    <svg id="chart-miss" viewBox="0 0 100 30" preserveAspectRatio="none"></svg>
+  </div>
+</div>
+
+<div class="panel" style="margin-bottom:16px">
+  <h3>workers</h3>
+  <div class="filter">
+    <input id="filter" placeholder="filter by container or worker name" />
+  </div>
+  <table id="workers">
+    <thead><tr>
+      <th>CONTAINER</th><th>NAME</th><th>ROLE</th>
+      <th class="num">OPS</th><th class="num">OK</th><th class="num">FAIL</th>
+      <th class="num">UPTIME</th><th class="num">P50</th><th class="num">P95</th>
+      <th>SPARK</th><th class="num">OUTAGES</th><th>STATUS</th><th>NOTES</th>
+    </tr></thead>
+    <tbody></tbody>
+  </table>
+</div>
+
+<div>
+  <h3 style="color:#5fafff;font-size:12px;text-transform:uppercase;
+             letter-spacing:0.06em;margin:0 0 6px">client errors</h3>
+  <div class="events" id="events"></div>
+</div>
+
+<div style="margin-top:16px">
+  <h3 style="color:#5fafff;font-size:12px;text-transform:uppercase;
+             letter-spacing:0.06em;margin:0 0 6px">ledger inconsistencies</h3>
+  <div class="events" id="inconsistencies"></div>
+</div>
+
+<div class="footer" id="footer">collector dashboard · poll 1s</div>
+
+<script>
+const sparkChars = [' ','▁','▂','▃','▄','▅','▆','▇','█'];
+function spark(values, width) {
+  if (!values || !values.length) return '';
+  const max = Math.max(...values);
+  if (max <= 0) return ' '.repeat(width);
+  let out = '';
+  for (let col = 0; col < width; col++) {
+    const idx = Math.min(values.length - 1, Math.floor(col * values.length / width));
+    const v = values[idx];
+    const ratio = v / max;
+    const i = Math.max(0, Math.min(8, Math.floor(ratio * 8)));
+    out += sparkChars[i];
+  }
+  return out;
+}
+
+function drawChart(svgID, values, color) {
+  const stroke = color || 'var(--chart)';
+  const svg = document.getElementById(svgID);
+  svg.innerHTML = '';
+  if (!values || !values.length) return;
+  const max = Math.max(1, ...values);
+  const w = 100, h = 30;
+  const step = w / Math.max(1, values.length - 1);
+  let path = '';
+  for (let i = 0; i < values.length; i++) {
+    const x = i * step;
+    const y = h - (values[i] / max) * h;
+    path += (i === 0 ? 'M' : 'L') + x.toFixed(2) + ',' + y.toFixed(2) + ' ';
+  }
+  const ns = 'http://www.w3.org/2000/svg';
+  const area = document.createElementNS(ns, 'path');
+  area.setAttribute('fill', stroke);
+  area.setAttribute('fill-opacity', '0.18');
+  area.setAttribute('stroke', 'none');
+  area.setAttribute('d', path + 'L' + w + ',' + h + ' L0,' + h + ' Z');
+  svg.appendChild(area);
+  const line = document.createElementNS(ns, 'path');
+  line.setAttribute('d', path);
+  line.setAttribute('fill', 'none');
+  line.setAttribute('stroke', stroke);
+  line.setAttribute('stroke-width', '1.2');
+  svg.appendChild(line);
+}
+
+function fmt(n) { return n == null ? '-' : n.toLocaleString(); }
+function fmtMS(v) {
+  if (v == null || v === 0) return '-';
+  if (v < 1) return (v * 1000).toFixed(0) + 'µs';
+  if (v < 1000) return v.toFixed(1) + 'ms';
+  return (v / 1000).toFixed(2) + 's';
+}
+
+let cursor = 0;
+let inconsCursor = 0;
+let lastWorkers = [];
+let filterStr = '';
+
+document.getElementById('filter').addEventListener('input', e => {
+  filterStr = e.target.value.toLowerCase();
+  renderTable(lastWorkers);
+});
+
+function renderTable(workers) {
+  const tbody = document.querySelector('#workers tbody');
+  tbody.innerHTML = '';
+  for (const w of workers) {
+    if (filterStr) {
+      const hay = (w.container + ' ' + w.name + ' ' + w.role).toLowerCase();
+      if (!hay.includes(filterStr)) continue;
+    }
+    const tr = document.createElement('tr');
+    if (w.in_outage) tr.classList.add('outage');
+    const status = w.in_outage
+      ? '<span class="badge err">OUTAGE</span>'
+      : (w.role === 'subscriber' && w.msgs === 0
+          ? '<span class="badge warn">NO MSGS</span>'
+          : '<span class="badge ok">OK</span>');
+    let notes = '';
+    if (w.role === 'reader') {
+      notes = '<span class="dim">v=' + fmt(w.verified) +
+              ' miss=' + fmt(w.missing) +
+              ' mis=' + fmt(w.mismatch) + '</span>';
+    } else if (w.role === 'subscriber') {
+      notes = '<span class="dim">msgs=' + fmt(w.msgs) +
+              ' gaps=' + fmt(w.gaps) +
+              ' longest_gap=' + fmtMS(w.longest_gap_ms) + '</span>';
+    }
+    tr.innerHTML =
+      '<td><span class="container-tag">' + w.container + '</span></td>' +
+      '<td>' + w.name + '</td>' +
+      '<td>' + (w.role || '?') + '</td>' +
+      '<td class="num">' + fmt(w.ops) + '</td>' +
+      '<td class="num">' + fmt(w.ok) + '</td>' +
+      '<td class="num">' + fmt(w.fail) + '</td>' +
+      '<td class="num">' + (w.uptime_pct != null ? w.uptime_pct.toFixed(2) + '%' : '-') + '</td>' +
+      '<td class="num">' + fmtMS(w.p50_ms) + '</td>' +
+      '<td class="num">' + fmtMS(w.p95_ms) + '</td>' +
+      '<td><span class="spark">' + spark(w.ops_hist, 14) + '</span></td>' +
+      '<td class="num">' + fmt(w.outages) + '</td>' +
+      '<td>' + status + '</td>' +
+      '<td>' + notes + '</td>';
+    tbody.appendChild(tr);
+  }
+}
+
+async function tickState() {
+  try {
+    const s = await (await fetch('/api/state')).json();
+    const tag = s.in_outage > 0
+      ? '<span class="badge err">' + s.in_outage + ' worker(s) in outage</span>'
+      : '<span class="badge ok">healthy</span>';
+    document.getElementById('header').innerHTML =
+      s.containers + ' container(s) · ' + s.workers.length + ' worker(s) · elapsed ' +
+      s.elapsed + ' · ' + tag;
+    document.getElementById('lat-meta').textContent =
+      'peak ' + s.charts.p95_peak.toFixed(1) + 'ms · last ' +
+      (s.charts.p95.length ? s.charts.p95[s.charts.p95.length-1].toFixed(1) + 'ms' : '-');
+    document.getElementById('tp-meta').textContent =
+      'peak ' + Math.round(s.charts.ok_peak) + ' · last ' +
+      (s.charts.ok_per_sec.length ? Math.round(s.charts.ok_per_sec[s.charts.ok_per_sec.length-1]) : '-');
+    document.getElementById('fail-meta').textContent =
+      'peak ' + Math.round(s.charts.fail_peak) + ' · last ' +
+      (s.charts.fail_per_sec.length ? Math.round(s.charts.fail_per_sec[s.charts.fail_per_sec.length-1]) : '-');
+    document.getElementById('miss-meta').textContent =
+      'peak ' + Math.round(s.charts.missing_peak) + ' · last ' +
+      (s.charts.missing_per_sec.length ? Math.round(s.charts.missing_per_sec[s.charts.missing_per_sec.length-1]) : '-');
+    drawChart('chart-lat', s.charts.p95);
+    drawChart('chart-tp', s.charts.ok_per_sec);
+    drawChart('chart-fail', s.charts.fail_per_sec, 'var(--chart-fail)');
+    drawChart('chart-miss', s.charts.missing_per_sec, 'var(--chart-fail)');
+    lastWorkers = s.workers;
+    renderTable(s.workers);
+  } catch (e) {
+    document.getElementById('header').textContent = 'fetch error: ' + e;
+  }
+}
+
+async function tickEvents() {
+  try {
+    const r = await (await fetch('/api/events?cursor=' + cursor)).json();
+    cursor = r.next_cursor;
+    const div = document.getElementById('events');
+    const wasAtBottom = div.scrollHeight - div.scrollTop - div.clientHeight < 30;
+    for (const e of r.events) {
+      const d = document.createElement('div');
+      d.className = 'ev ' + (e.level || 'info');
+      const ts = document.createElement('span');
+      ts.className = 'ts';
+      ts.textContent = e.at;
+      d.appendChild(ts);
+      d.appendChild(document.createTextNode(e.text));
+      div.appendChild(d);
+    }
+    while (div.children.length > 1000) div.removeChild(div.firstChild);
+    if (wasAtBottom) div.scrollTop = div.scrollHeight;
+  } catch (e) { /* noop */ }
+}
+
+async function tickInconsistencies() {
+  try {
+    const r = await (await fetch('/api/inconsistencies?cursor=' + inconsCursor)).json();
+    inconsCursor = r.next_cursor;
+    const div = document.getElementById('inconsistencies');
+    const wasAtBottom = div.scrollHeight - div.scrollTop - div.clientHeight < 30;
+    for (const e of r.events) {
+      const d = document.createElement('div');
+      d.className = 'ev ' + (e.level || 'info');
+      const ts = document.createElement('span');
+      ts.className = 'ts';
+      ts.textContent = e.at;
+      d.appendChild(ts);
+      d.appendChild(document.createTextNode(e.text));
+      div.appendChild(d);
+    }
+    while (div.children.length > 1000) div.removeChild(div.firstChild);
+    if (wasAtBottom) div.scrollTop = div.scrollHeight;
+  } catch (e) { /* noop */ }
+}
+
+setInterval(tickState, 1000);
+setInterval(tickEvents, 1000);
+setInterval(tickInconsistencies, 1000);
+tickState();
+tickEvents();
+tickInconsistencies();
+</script>
+</body>
+</html>
+`

valkey/config-basic.yaml

@@ -0,0 +1,102 @@
+# Intermediate chaos workload: 1 readwriter + 1 TLS reader per valkey.
+#
+# Per target:
+#   - readwriter on plaintext :6379 at 10ms (~100 ops/sec)
+#   - reader     on TLS       :6380 at 5ms  (~200 ops/sec)
+#
+# ⇒ ~300 ops/sec per target, ~1.8k ops/sec total per container, and both
+# the plaintext and TLS ingress paths get exercised on every valkey.
+#
+# Per-worker passwords reference env vars that the deployment yaml
+# (zerops-basic-import.yaml) wires in from each valkey service.
+#
+# NB: :6380 is the master TLS port; both valkey:single and valkey:ha
+# masters serve GETs there, so a reader against :6380 works on both
+# topologies. If a single-mode valkey ever stops exposing :6380, the
+# corresponding ro-* worker will surface as a connect error — that's
+# signal, not a bug here.
+
+workers:
+  - name: rw-single1
+    role: readwriter
+    host: valkeysingle1.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYSINGLE1_PASSWORD}
+    interval: 10ms
+  - name: ro-single1
+    role: reader
+    host: valkeysingle1.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYSINGLE1_PASSWORD}
+    interval: 5ms
+
+  - name: rw-single2
+    role: readwriter
+    host: valkeysingle2.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYSINGLE2_PASSWORD}
+    interval: 10ms
+  - name: ro-single2
+    role: reader
+    host: valkeysingle2.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYSINGLE2_PASSWORD}
+    interval: 5ms
+
+  - name: rw-single3
+    role: readwriter
+    host: valkeysingle3.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYSINGLE3_PASSWORD}
+    interval: 10ms
+  - name: ro-single3
+    role: reader
+    host: valkeysingle3.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYSINGLE3_PASSWORD}
+    interval: 5ms
+
+  - name: rw-ha1
+    role: readwriter
+    host: valkeyha1.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYHA1_PASSWORD}
+    interval: 10ms
+  - name: ro-ha1
+    role: reader
+    host: valkeyha1.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYHA1_PASSWORD}
+    interval: 5ms
+
+  - name: rw-ha2
+    role: readwriter
+    host: valkeyha2.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYHA2_PASSWORD}
+    interval: 10ms
+  - name: ro-ha2
+    role: reader
+    host: valkeyha2.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYHA2_PASSWORD}
+    interval: 5ms
+
+  - name: rw-ha3
+    role: readwriter
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${VALKEY_VALKEYHA3_PASSWORD}
+    interval: 10ms
+  - name: ro-ha3
+    role: reader
+    host: valkeyha3.zerops
+    port: 6380
+    tls: true
+    password: ${VALKEY_VALKEYHA3_PASSWORD}
+    interval: 5ms

valkey/config.yaml

@@ -0,0 +1,135 @@
+# Valkey HA chaos/soak config — spicy edition.
+#
+# 13 workers per container. With Zerops scaling to 10 replicas, that's
+# ~130 concurrent connections × short intervals = serious load on
+# HAProxy, Sentinel, and the Valkey master.
+#
+# Each $ZEROPS_Number replica gets its own DB and its own channel suffix
+# (handled in main.go::applyZeropsOverrides), so the 10 containers don't
+# step on each other's keyspace or pub/sub traffic.
+#
+# Topology:
+#   6379 = read/write plaintext (master)
+#   6380 = read/write TLS       (master)
+#   7000 = read-only plaintext  (replicas via HAProxy)
+#   7001 = read-only TLS        (replicas via HAProxy)
+#
+# Roughly half the workers run over TLS so each ingress path gets
+# exercised under load.
+#
+# Workload mix:
+#   - 1× readwriter (write + read-after-write on master)
+#   - 2× pure writer (separate TCP sessions — stress the master)
+#   - 1× master-side reader (does master serve GETs under load?)
+#   - 4× replica-side reader (stress replicas + verify replication catches up)
+#   - 2× publisher on two distinct channels (one slow, one tight)
+#   - 2× master-side subscriber (one per channel)
+#   - 1× replica-side subscriber (topology probe — should receive 0 msgs)
+
+workers:
+  # ---- write path (3 connections) ----------------------------------------
+  - name: rw-main
+    role: readwriter
+    host: valkeyha.zerops
+    port: 6379
+    db: 0
+    interval: 20ms
+
+  - name: rw-burst-1
+    role: writer
+    host: valkeyha.zerops
+    port: 6380
+    tls: true
+    interval: 3ms
+
+  - name: rw-burst-2
+    role: writer
+    host: valkeyha.zerops
+    port: 6379
+    interval: 3ms
+
+  # ---- master-side reader (sanity that master also serves reads) ---------
+  - name: ro-master
+    role: reader
+    host: valkeyha.zerops
+    port: 6380
+    tls: true
+    interval: 5ms
+
+  # ---- replica-side reader fan-out (4 connections) -----------------------
+  - name: ro-1
+    role: reader
+    host: valkeyha.zerops
+    port: 7000
+    interval: 2ms
+  - name: ro-2
+    role: reader
+    host: valkeyha.zerops
+    port: 7001
+    tls: true
+    interval: 2ms
+  - name: ro-3
+    role: reader
+    host: valkeyha.zerops
+    port: 7000
+    interval: 3ms
+  - name: ro-4
+    role: reader
+    host: valkeyha.zerops
+    port: 7001
+    tls: true
+    interval: 3ms
+
+  # ---- pub/sub: two channels at different cadences -----------------------
+  - name: pub-bus
+    role: publisher
+    host: valkeyha.zerops
+    port: 6379
+    channel: "vk:chaos:bus"
+    interval: 5ms
+
+  - name: pub-fast
+    role: publisher
+    host: valkeyha.zerops
+    port: 6380
+    tls: true
+    channel: "vk:chaos:fast"
+    interval: 2ms
+
+  - name: sub-bus
+    role: subscriber
+    host: valkeyha.zerops
+    port: 6380
+    tls: true
+    channel: "vk:chaos:bus"
+
+  - name: sub-fast
+    role: subscriber
+    host: valkeyha.zerops
+    port: 6379
+    channel: "vk:chaos:fast"
+
+  # ---- memory bloater ----------------------------------------------------
+  # Writes 8 KB of random bytes to a fresh key every 10ms with NO TTL.
+  # ~800 KB/sec per container × 10 containers = ~8 MB/sec growth on the
+  # master. Watch used_memory climb. Keys land under vk:chaos:bloat:<seq>
+  # so they don't pollute the durability ledger. To reset, FLUSHDB the
+  # affected DB (or each per-replica DB if running in Zerops).
+  # - name: bloat-1
+  #   role: bloater
+  #   host: valkeyha.zerops
+  #   port: 6379
+  #   interval: 10ms
+  #   valueBytes: 8192
+
+  # ---- topology probe ----------------------------------------------------
+  # Subscriber on the read-only VIP. Valkey 7.2 + Sentinel does not
+  # replicate pub/sub from master to replicas, so this should stay at
+  # msgs=0 the entire run. If it ever ticks up, replication semantics
+  # changed.
+  - name: sub-replica
+    role: subscriber
+    host: valkeyha.zerops
+    port: 7001
+    tls: true
+    channel: "vk:chaos:bus"

valkey/go.mod

@@ -0,0 +1,36 @@
+module valkey-test
+
+go 1.24.2
+
+require (
+	github.com/charmbracelet/bubbles v1.0.0
+	github.com/charmbracelet/bubbletea v1.3.10
+	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/redis/go-redis/v9 v9.7.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/charmbracelet/colorprofile v0.4.1 // indirect
+	github.com/charmbracelet/x/ansi v0.11.6 // indirect
+	github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.9.0 // indirect
+	github.com/clipperhouse/stringish v0.1.1 // indirect
+	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.19 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/text v0.3.8 // indirect
+)

valkey/go.sum

@@ -0,0 +1,64 @@
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
+github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E=
+github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
+github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
+github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
+github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
+github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
+github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
+github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
+github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
+github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
+github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
+github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
+github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
+github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
+github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
+github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
+github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
+github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U=
+github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
+github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
+github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
+github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

valkey/main.go

@@ -0,0 +1,639 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+	"gopkg.in/yaml.v3"
+)
+
+// tlsRootCAs, when non-nil, is used as the RootCAs pool for every TLS
+// connection. Set by --tls-ca for local testing against a Valkey whose
+// cert is signed by a non-system CA.
+var tlsRootCAs *x509.CertPool
+
+type WorkerConfig struct {
+	Name       string        `yaml:"name"`
+	Role       string        `yaml:"role"`
+	Host       string        `yaml:"host"`
+	Port       int           `yaml:"port"`
+	Password   string        `yaml:"password"`
+	DB         int           `yaml:"db"`
+	TLS        bool          `yaml:"tls"`
+	Interval   time.Duration `yaml:"interval"`
+	Channel    string        `yaml:"channel"`
+	ValueBytes int           `yaml:"valueBytes"` // bloater role only
+}
+
+type Config struct {
+	Host     string `yaml:"host,omitempty"`
+	Port     int    `yaml:"port,omitempty"`
+	Password string `yaml:"password,omitempty"`
+	DB       int    `yaml:"db,omitempty"`
+	TLS      bool   `yaml:"tls,omitempty"`
+
+	Workers []WorkerConfig `yaml:"workers,omitempty"`
+}
+
+func loadConfig(path string) (*Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read config: %w", err)
+	}
+	data = []byte(os.ExpandEnv(string(data)))
+	var cfg Config
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("parse config: %w", err)
+	}
+	applyZeropsOverrides(&cfg)
+	return &cfg, nil
+}
+
+// applyZeropsOverrides applies env-var overrides on top of the parsed
+// config so the same config.yaml works locally and in Zerops:
+//
+//   $VALKEY_PASSWORD  → overrides every worker's password (so the secret
+//                       lives in the Zerops env, not in committed YAML).
+//
+//   $ZEROPS_Number    → ordinal injected per Zerops replica. Distributes
+//                       10 containers across distinct DBs and pub/sub
+//                       channels so they don't collide on shared cluster:
+//                         cfg.Workers[*].DB      = ordinal mod 16
+//                         cfg.Workers[*].Channel = "<channel>:<ord mod 16>"
+//                       The mod-16 wrap is needed because Valkey defaults
+//                       to DB 0–15 and Zerops's ordinal grows across
+//                       redeploys (a fresh deploy can start well above 16).
+//
+// Each override is independent: an unset env var is a no-op, leaving the
+// YAML value in place.
+func applyZeropsOverrides(cfg *Config) {
+	if pw := os.Getenv("VALKEY_PASSWORD"); pw != "" {
+		for i := range cfg.Workers {
+			cfg.Workers[i].Password = pw
+		}
+	}
+	if n := os.Getenv("ZEROPS_Number"); n != "" {
+		if raw, err := strconv.Atoi(n); err == nil {
+			dbN := raw % 16
+			suffix := strconv.Itoa(dbN)
+			for i := range cfg.Workers {
+				cfg.Workers[i].DB = dbN
+				if cfg.Workers[i].Channel != "" {
+					cfg.Workers[i].Channel = cfg.Workers[i].Channel + ":" + suffix
+				}
+			}
+		}
+	}
+}
+
+// applyFlagOverrides applies --host / --password / --tls-ca on top of the
+// loaded config so the local binary can target a remote Valkey without
+// editing config.yaml. --tls-ca loads the PEM into the package-level
+// tlsRootCAs pool and forces TLS on every worker.
+func applyFlagOverrides(cfg *Config, host string, port int, password string, tlsEnable bool, tlsCAPath string) error {
+	if host != "" {
+		cfg.Host = host
+		for i := range cfg.Workers {
+			cfg.Workers[i].Host = host
+		}
+	}
+	if port != 0 {
+		cfg.Port = port
+		for i := range cfg.Workers {
+			cfg.Workers[i].Port = port
+		}
+	}
+	if password != "" {
+		cfg.Password = password
+		for i := range cfg.Workers {
+			cfg.Workers[i].Password = password
+		}
+	}
+	if tlsCAPath != "" {
+		pem, err := os.ReadFile(tlsCAPath)
+		if err != nil {
+			return fmt.Errorf("read tls-ca: %w", err)
+		}
+		pool := x509.NewCertPool()
+		if !pool.AppendCertsFromPEM(pem) {
+			return fmt.Errorf("tls-ca: no certificates parsed from %s", tlsCAPath)
+		}
+		tlsRootCAs = pool
+	}
+	if tlsEnable || tlsCAPath != "" {
+		cfg.TLS = true
+		for i := range cfg.Workers {
+			cfg.Workers[i].TLS = true
+		}
+	}
+	return nil
+}
+
+func (c *Config) primary() (*redis.Options, error) {
+	if c.Host != "" && c.Port != 0 {
+		return optionsFromConn(c.Host, c.Port, c.Password, c.DB, c.TLS), nil
+	}
+	if len(c.Workers) > 0 {
+		w := c.Workers[0]
+		return optionsFromConn(w.Host, w.Port, w.Password, w.DB, w.TLS), nil
+	}
+	return nil, fmt.Errorf("config has neither top-level host nor workers")
+}
+
+func optionsFromConn(host string, port int, password string, db int, useTLS bool) *redis.Options {
+	opts := &redis.Options{
+		Addr:     fmt.Sprintf("%s:%d", host, port),
+		Password: password,
+		DB:       db,
+	}
+	if useTLS {
+		opts.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: tlsRootCAs}
+	}
+	return opts
+}
+
+func redactPassword(pw string) string {
+	if pw == "" {
+		return "(none)"
+	}
+	return "(set)"
+}
+
+func envInt(name string, def int) int {
+	if v := os.Getenv(name); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			return n
+		}
+	}
+	return def
+}
+
+func main() {
+	configPath := flag.String("config", "config.yaml", "path to YAML config file")
+	continuous := flag.Bool("continuous", false, "single-worker rolling smoke loop")
+	chaosLocal := flag.Bool("chaos", false, "multi-worker chaos TUI (local, in-process)")
+	workerMode := flag.Bool("worker", false, "distributed worker — runs the chaos workload and ships events to --collector-url")
+	collectorMode := flag.Bool("collector", false, "distributed collector — HTTP dashboard that ingests events from workers")
+	collectorURL := flag.String("collector-url", os.Getenv("COLLECTOR_URL"), "collector base URL (worker mode); env: COLLECTOR_URL")
+	port := flag.Int("collector-port", envInt("PORT", 8080), "HTTP port (collector mode); env: PORT")
+	interval := flag.Duration("interval", time.Second, "tick interval for --continuous")
+	hostOverride := flag.String("host", "", "override host for all workers (local testing)")
+	portOverride := flag.Int("port", 0, "override port for all workers (local testing)")
+	passwordOverride := flag.String("password", "", "override password for all workers (local testing)")
+	tlsEnable := flag.Bool("tls", false, "enable TLS for all workers using system root CAs (local testing)")
+	tlsCAPath := flag.String("tls-ca", "", "path to TLS CA cert PEM; enables TLS for all workers (local testing)")
+	flag.Parse()
+
+	switch {
+	case *collectorMode:
+		os.Exit(runCollector(*port))
+	}
+
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		log.Fatalf("config error: %v", err)
+	}
+	if err := applyFlagOverrides(cfg, *hostOverride, *portOverride, *passwordOverride, *tlsEnable, *tlsCAPath); err != nil {
+		log.Fatalf("flag override error: %v", err)
+	}
+
+	switch {
+	case *chaosLocal:
+		if len(cfg.Workers) == 0 {
+			log.Fatalf("--chaos requires workers: list in %s", *configPath)
+		}
+		os.Exit(runChaos(cfg.Workers))
+	case *workerMode:
+		if len(cfg.Workers) == 0 {
+			log.Fatalf("--worker requires workers: list in %s", *configPath)
+		}
+		if *collectorURL == "" {
+			log.Fatalf("--worker requires --collector-url or $COLLECTOR_URL")
+		}
+		os.Exit(runWorker(cfg.Workers, *collectorURL))
+	}
+
+	primary, err := cfg.primary()
+	if err != nil {
+		log.Fatalf("config error: %v", err)
+	}
+	fmt.Printf("redis config: addr=%s db=%d tls=%t password=%s\n",
+		primary.Addr, primary.DB, primary.TLSConfig != nil, redactPassword(primary.Password))
+	rdb := redis.NewClient(primary)
+	defer rdb.Close()
+
+	if *continuous {
+		runContinuous(rdb, *interval)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	tests := []struct {
+		name string
+		fn   func(context.Context, *redis.Client) error
+	}{
+		{"PING", testPing},
+		{"INFO version", testInfo},
+		{"STRING SET/GET", testString},
+		{"INCR/DECR", testCounter},
+		{"EXPIRE/TTL", testExpire},
+		{"LIST ops", testList},
+		{"HASH ops", testHash},
+		{"SET ops", testSet},
+		{"SORTED SET ops", testZSet},
+		{"TRANSACTION", testTxn},
+		{"PIPELINE", testPipeline},
+		{"PUB/SUB", testPubSub},
+		{"FLUSHDB cleanup", testCleanup},
+	}
+
+	failed := 0
+	for _, t := range tests {
+		fmt.Printf("[ .. ] %s\n", t.name)
+		if err := t.fn(ctx, rdb); err != nil {
+			fmt.Printf("[FAIL] %s: %v\n", t.name, err)
+			failed++
+			continue
+		}
+		fmt.Printf("[ OK ] %s\n", t.name)
+	}
+
+	fmt.Printf("\n%d/%d tests passed\n", len(tests)-failed, len(tests))
+	if failed > 0 {
+		os.Exit(1)
+	}
+}
+
+func testPing(ctx context.Context, r *redis.Client) error {
+	pong, err := r.Ping(ctx).Result()
+	if err != nil {
+		return err
+	}
+	if pong != "PONG" {
+		return fmt.Errorf("expected PONG, got %q", pong)
+	}
+	return nil
+}
+
+func testInfo(ctx context.Context, r *redis.Client) error {
+	info, err := r.Info(ctx, "server").Result()
+	if err != nil {
+		return err
+	}
+	if len(info) == 0 {
+		return fmt.Errorf("empty INFO response")
+	}
+	return nil
+}
+
+func testString(ctx context.Context, r *redis.Client) error {
+	if err := r.Set(ctx, "vk:string", "hello-valkey", 0).Err(); err != nil {
+		return err
+	}
+	got, err := r.Get(ctx, "vk:string").Result()
+	if err != nil {
+		return err
+	}
+	if got != "hello-valkey" {
+		return fmt.Errorf("expected hello-valkey, got %q", got)
+	}
+	return nil
+}
+
+func testCounter(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:counter")
+	if _, err := r.Incr(ctx, "vk:counter").Result(); err != nil {
+		return err
+	}
+	if _, err := r.IncrBy(ctx, "vk:counter", 9).Result(); err != nil {
+		return err
+	}
+	v, err := r.Decr(ctx, "vk:counter").Result()
+	if err != nil {
+		return err
+	}
+	if v != 9 {
+		return fmt.Errorf("expected 9, got %d", v)
+	}
+	return nil
+}
+
+func testExpire(ctx context.Context, r *redis.Client) error {
+	if err := r.Set(ctx, "vk:ttl", "x", 5*time.Second).Err(); err != nil {
+		return err
+	}
+	ttl, err := r.TTL(ctx, "vk:ttl").Result()
+	if err != nil {
+		return err
+	}
+	if ttl <= 0 || ttl > 5*time.Second {
+		return fmt.Errorf("unexpected TTL: %v", ttl)
+	}
+	return nil
+}
+
+func testList(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:list")
+	if _, err := r.RPush(ctx, "vk:list", "a", "b", "c").Result(); err != nil {
+		return err
+	}
+	vals, err := r.LRange(ctx, "vk:list", 0, -1).Result()
+	if err != nil {
+		return err
+	}
+	if len(vals) != 3 || vals[0] != "a" || vals[2] != "c" {
+		return fmt.Errorf("unexpected list: %v", vals)
+	}
+	return nil
+}
+
+func testHash(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:hash")
+	if err := r.HSet(ctx, "vk:hash", "name", "valkey", "version", "7.2").Err(); err != nil {
+		return err
+	}
+	got, err := r.HGetAll(ctx, "vk:hash").Result()
+	if err != nil {
+		return err
+	}
+	if got["name"] != "valkey" || got["version"] != "7.2" {
+		return fmt.Errorf("unexpected hash: %v", got)
+	}
+	return nil
+}
+
+func testSet(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:set")
+	if _, err := r.SAdd(ctx, "vk:set", "x", "y", "z", "x").Result(); err != nil {
+		return err
+	}
+	n, err := r.SCard(ctx, "vk:set").Result()
+	if err != nil {
+		return err
+	}
+	if n != 3 {
+		return fmt.Errorf("expected 3 members, got %d", n)
+	}
+	return nil
+}
+
+func testZSet(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:zset")
+	_, err := r.ZAdd(ctx, "vk:zset",
+		redis.Z{Score: 1, Member: "one"},
+		redis.Z{Score: 2, Member: "two"},
+		redis.Z{Score: 3, Member: "three"},
+	).Result()
+	if err != nil {
+		return err
+	}
+	vals, err := r.ZRangeByScore(ctx, "vk:zset", &redis.ZRangeBy{Min: "1", Max: "2"}).Result()
+	if err != nil {
+		return err
+	}
+	if len(vals) != 2 || vals[0] != "one" {
+		return fmt.Errorf("unexpected zrange: %v", vals)
+	}
+	return nil
+}
+
+func testTxn(ctx context.Context, r *redis.Client) error {
+	r.Del(ctx, "vk:txn")
+	pipe := r.TxPipeline()
+	pipe.Set(ctx, "vk:txn", "1", 0)
+	pipe.Incr(ctx, "vk:txn")
+	pipe.Incr(ctx, "vk:txn")
+	if _, err := pipe.Exec(ctx); err != nil {
+		return err
+	}
+	v, err := r.Get(ctx, "vk:txn").Result()
+	if err != nil {
+		return err
+	}
+	if v != "3" {
+		return fmt.Errorf("expected 3, got %s", v)
+	}
+	return nil
+}
+
+func testPipeline(ctx context.Context, r *redis.Client) error {
+	pipe := r.Pipeline()
+	for i := 0; i < 5; i++ {
+		pipe.Set(ctx, fmt.Sprintf("vk:pipe:%d", i), i, 0)
+	}
+	if _, err := pipe.Exec(ctx); err != nil {
+		return err
+	}
+	v, err := r.Get(ctx, "vk:pipe:3").Result()
+	if err != nil {
+		return err
+	}
+	if v != "3" {
+		return fmt.Errorf("expected 3, got %s", v)
+	}
+	return nil
+}
+
+func testPubSub(ctx context.Context, r *redis.Client) error {
+	sub := r.Subscribe(ctx, "vk:channel")
+	defer sub.Close()
+
+	if _, err := sub.Receive(ctx); err != nil {
+		return fmt.Errorf("subscribe: %w", err)
+	}
+	ch := sub.Channel()
+
+	if err := r.Publish(ctx, "vk:channel", "ping").Err(); err != nil {
+		return err
+	}
+
+	select {
+	case msg := <-ch:
+		if msg.Payload != "ping" {
+			return fmt.Errorf("expected ping, got %q", msg.Payload)
+		}
+	case <-time.After(3 * time.Second):
+		return fmt.Errorf("timeout waiting for message")
+	}
+	return nil
+}
+
+func testCleanup(ctx context.Context, r *redis.Client) error {
+	keys, err := r.Keys(ctx, "vk:*").Result()
+	if err != nil {
+		return err
+	}
+	if len(keys) == 0 {
+		return nil
+	}
+	return r.Del(ctx, keys...).Err()
+}
+
+func runContinuous(r *redis.Client, interval time.Duration) {
+	stopCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	subCtx, cancelSub := context.WithCancel(stopCtx)
+	defer cancelSub()
+	sub := r.Subscribe(subCtx, "vk:cont:channel")
+	defer sub.Close()
+	if _, err := sub.Receive(subCtx); err != nil {
+		log.Fatalf("subscribe: %v", err)
+	}
+	subCh := sub.Channel()
+
+	ops := []struct {
+		name string
+		fn   func(context.Context, *redis.Client, int64) error
+	}{
+		{"PING", opPing},
+		{"SET/GET", opSetGet},
+		{"INCR", opIncr},
+		{"LIST", opList},
+		{"INFO", opInfo},
+		{"PUB/SUB", func(ctx context.Context, r *redis.Client, i int64) error {
+			return opPubSub(ctx, r, i, subCh)
+		}},
+	}
+
+	var cycle, okCount, failCount int64
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	fmt.Printf("continuous mode: addr=%s interval=%s (Ctrl+C to stop)\n", r.Options().Addr, interval)
+
+	for {
+		select {
+		case <-stopCtx.Done():
+			fmt.Printf("\nstopped: cycles=%d ok=%d failed=%d\n", cycle, okCount, failCount)
+			return
+		case <-ticker.C:
+		}
+		cycle++
+
+		results := make([]string, len(ops))
+		var failures []string
+		cycleStart := time.Now()
+		for i, op := range ops {
+			ctx, cancel := context.WithTimeout(stopCtx, interval)
+			start := time.Now()
+			err := op.fn(ctx, r, cycle)
+			cancel()
+			dur := time.Since(start)
+			if err != nil {
+				results[i] = fmt.Sprintf("%s=FAIL", op.name)
+				failures = append(failures, fmt.Sprintf("  %s (%s): %v", op.name, dur, err))
+			} else {
+				results[i] = fmt.Sprintf("%s=ok(%s)", op.name, dur.Round(time.Microsecond))
+			}
+		}
+
+		if len(failures) == 0 {
+			okCount++
+		} else {
+			failCount++
+		}
+
+		fmt.Printf("[%s] #%d %s | total=%s\n",
+			time.Now().Format("15:04:05"), cycle, joinResults(results), time.Since(cycleStart).Round(time.Microsecond))
+		for _, f := range failures {
+			fmt.Println(f)
+		}
+	}
+}
+
+func joinResults(parts []string) string {
+	out := ""
+	for i, p := range parts {
+		if i > 0 {
+			out += " "
+		}
+		out += p
+	}
+	return out
+}
+
+func opPing(ctx context.Context, r *redis.Client, _ int64) error {
+	pong, err := r.Ping(ctx).Result()
+	if err != nil {
+		return err
+	}
+	if pong != "PONG" {
+		return fmt.Errorf("expected PONG, got %q", pong)
+	}
+	return nil
+}
+
+func opSetGet(ctx context.Context, r *redis.Client, i int64) error {
+	val := fmt.Sprintf("v-%d", i)
+	if err := r.Set(ctx, "vk:cont:str", val, 10*time.Second).Err(); err != nil {
+		return err
+	}
+	got, err := r.Get(ctx, "vk:cont:str").Result()
+	if err != nil {
+		return err
+	}
+	if got != val {
+		return fmt.Errorf("set %q, got %q", val, got)
+	}
+	return nil
+}
+
+func opIncr(ctx context.Context, r *redis.Client, _ int64) error {
+	_, err := r.Incr(ctx, "vk:cont:counter").Result()
+	return err
+}
+
+func opList(ctx context.Context, r *redis.Client, i int64) error {
+	if err := r.RPush(ctx, "vk:cont:list", fmt.Sprintf("item-%d", i)).Err(); err != nil {
+		return err
+	}
+	if err := r.LTrim(ctx, "vk:cont:list", -10, -1).Err(); err != nil {
+		return err
+	}
+	_, err := r.LRange(ctx, "vk:cont:list", 0, -1).Result()
+	return err
+}
+
+func opInfo(ctx context.Context, r *redis.Client, _ int64) error {
+	info, err := r.Info(ctx, "server").Result()
+	if err != nil {
+		return err
+	}
+	if len(info) == 0 {
+		return fmt.Errorf("empty INFO")
+	}
+	return nil
+}
+
+func opPubSub(ctx context.Context, r *redis.Client, i int64, ch <-chan *redis.Message) error {
+	payload := fmt.Sprintf("ping-%d", i)
+	if err := r.Publish(ctx, "vk:cont:channel", payload).Err(); err != nil {
+		return err
+	}
+	for {
+		select {
+		case msg := <-ch:
+			if msg == nil {
+				return fmt.Errorf("channel closed")
+			}
+			if msg.Payload == payload {
+				return nil
+			}
+		case <-ctx.Done():
+			return fmt.Errorf("timeout waiting for %q", payload)
+		}
+	}
+}

valkey/test-all.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PORT="${PORT:-6379}"
+
+services=$(zcli service list 2>/dev/null \
+    | awk -F'│' '{gsub(/ /, "", $3); print $3}' \
+    | grep -E '^valkey' || true)
+
+if [[ -z "$services" ]]; then
+    echo "no valkey* services found" >&2
+    exit 1
+fi
+
+fail=0
+while IFS= read -r svc; do
+    echo "=== testing $svc ==="
+    password=$(zcli project env --service "$svc" 2>/dev/null \
+        | grep -E '^password=' \
+        | head -n1 \
+        | sed -E 's/^password="?([^"]*)"?$/\1/')
+    if [[ -z "$password" ]]; then
+        echo "!!! $svc: could not fetch password" >&2
+        fail=1
+        continue
+    fi
+    if ! go run ./... -password "$password" -host "${svc}.zerops" -port "$PORT"; then
+        echo "!!! $svc FAILED" >&2
+        fail=1
+    fi
+done <<< "$services"
+
+exit "$fail"

valkey/zerops-basic-import.yaml

@@ -0,0 +1,26 @@
+# Basic stress import: one chaosbasic container that hits all 6 valkeys
+# in the project with a single readwriter each, plaintext :6379, AND
+# serves the collector dashboard on :8080 (subdomain access on).
+#
+# Build/run config (incl. the startCommands that boot both the collector
+# and the worker in the same container) lives in zerops.yaml under
+# setup: chaosbasic.
+#
+# Use with:  zcli project service-import zerops-basic-import.yaml
+# Then:      zcli push --setup chaosbasic
+#
+# Prereqs in the same project (e.g. via zerops-mega-import.yaml):
+#   valkeysingle1 / valkeysingle2 / valkeysingle3  (valkey:single@7.2)
+#   valkeyha1     / valkeyha2     / valkeyha3      (valkey:ha@7.2)
+#
+# NB — config-basic.yaml uses literal ${VALKEY_..._PASSWORD} strings in
+# each worker's password field. applyZeropsOverrides today only honors
+# a single $VALKEY_PASSWORD env var; extend it to expand ${VAR}
+# references in each worker.Password against os.Getenv at load time
+# before this will authenticate against the per-host passwords below.
+
+services:
+  - hostname: chaosbasic
+    type: go@1
+    minContainers: 1
+    maxContainers: 1

valkey/zerops-chaos-import.yaml

@@ -0,0 +1,22 @@
+# Full project import for the chaos test infra:
+#   - Valkey HA cluster (the system under test)
+#   - chaosworker  (10x runner replicas; user-defined Go service)
+#   - chaoscollector (1x dashboard; user-defined Go service, subdomain on)
+#
+# Use with:  zcli project project-import zerops-chaos-import.yaml
+#
+# After import, push the runtime config from valkey/zerops.yaml to deploy
+# the app code to chaosworker and chaoscollector.
+#
+# Hostnames are alphanumeric to match Zerops conventions.
+services:
+  - hostname: chaoscollector
+    type: go@1
+    enableSubdomainAccess: true
+    minContainers: 1
+    maxContainers: 1
+
+  - hostname: chaosworker
+    type: go@1
+    minContainers: 10
+    maxContainers: 10

valkey/zerops-ha-import.yaml

@@ -0,0 +1,7 @@
+services:
+  - hostname: valkeyha
+    type: valkey@7.2
+    mode: HA
+    verticalAutoscaling:
+      minFreeRamPercent: 40
+      minRam: 4

valkey/zerops-mega-import.yaml

@@ -0,0 +1,13 @@
+services:
+  - hostname: valkeysingle1
+    type: valkey:single@7.2
+  - hostname: valkeysingle2
+    type: valkey:single@7.2
+  - hostname: valkeysingle3
+    type: valkey:single@7.2
+  - hostname: valkeyha1
+    type: valkey:ha@7.2
+  - hostname: valkeyha2
+    type: valkey:ha@7.2
+  - hostname: valkeyha3
+    type: valkey:ha@7.2

valkey/zerops-single-import.yaml

@@ -0,0 +1,4 @@
+services:
+  - hostname: valkeysingle
+    type: valkey@7.2
+    mode: NON_HA

valkey/zerops.yaml

@@ -0,0 +1,76 @@
+# zerops.yaml — runtime build/run config for the chaos test services.
+#
+# Two setups, both built from the same Go source under valkey/.
+#
+#   chaosworker     10 replicas, ships events to chaoscollector (no exposed port).
+#                   Reads $ZEROPS_Number → maps to Redis DB N and pub/sub
+#                   channel suffix ":N" so the 10 containers don't collide
+#                   in the keyspace or on the same channel.
+#
+#   chaoscollector  1 replica, exposes :8080 with subdomain access enabled,
+#                   ingests events from the workers and serves the dashboard.
+#
+# NB: validate the exact schema (field names, autoscaling block, prepare
+# commands) against the current Zerops docs before pushing. Field names
+# below follow the typical zerops.yaml conventions but vary by runtime
+# version.
+
+zerops:
+  - setup: chaosworker
+    build:
+      base: ubuntu/go@1
+      buildCommands:
+        - go build -o app .
+      deployFiles:
+        - app
+        - config.yaml
+      cache: true
+    run:
+      base: ubuntu@latest
+      start: ./app --worker --config config.yaml
+      envVariables:
+        COLLECTOR_URL: http://chaoscollector.zerops:8080
+        VALKEY_PASSWORD: ${valkeyha_password}
+
+  - setup: chaoscollector
+    build:
+      base: ubuntu/go@1
+      buildCommands:
+        - go build -o app .
+      deployFiles:
+        - app
+      cache: true
+    run:
+      base: ubuntu@latest
+      ports:
+        - port: 8080
+          httpSupport: true
+      start: ./app --collector --collector-port 8080
+
+  - setup: chaosbasic
+    build:
+      base: ubuntu/go@1
+      buildCommands:
+        - go build -o app .
+      deployFiles:
+        - app
+        - config-basic.yaml
+      cache: true
+    run:
+      base: ubuntu@latest
+      ports:
+        - port: 8080
+          httpSupport: true
+      startCommands:
+        - name: collector
+          command: ./app --collector --collector-port 8080
+        - name: worker
+          command: ./app --worker --config config-basic.yaml
+      envVariables:
+        COLLECTOR_URL: http://localhost:8080
+        VALKEY_VALKEYSINGLE1_PASSWORD: ${valkeysingle1_password}
+        VALKEY_VALKEYSINGLE2_PASSWORD: ${valkeysingle2_password}
+        VALKEY_VALKEYSINGLE3_PASSWORD: ${valkeysingle3_password}
+        VALKEY_VALKEYHA1_PASSWORD: ${valkeyha1_password}
+        VALKEY_VALKEYHA2_PASSWORD: ${valkeyha2_password}
+        VALKEY_VALKEYHA3_PASSWORD: ${valkeyha3_password}