tikinang/zcli-playground
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

valkey chaos

Browse Source
This commit is contained in:
tikinang
2026-06-11 10:52:17 +02:00
parent ec88214a2e
commit 6a43ba022c
7 changed files with 287 additions and 1005 deletions
Download Patch File Download Diff File Expand all files Collapse all files
valkey/chaos.go
+5 -703
View File
@@ -19,9 +19,6 @@ import (
"syscall"
"time"
"github.com/charmbracelet/bubbles/viewport"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/redis/go-redis/v9"
)
@@ -129,13 +126,9 @@ type outageRec struct {
// ---- event sink abstraction --------------------------------------------
//
// Workers don't know whether their events go to an in-process Bubble Tea
// program (local TUI) or a remote collector over HTTP. Two implementations:
//
// teaSink wraps *tea.Program — used by --chaos
// httpSink batches and POSTs to a collector — used by --worker
//
// Both deliver the same opResultMsg / subRecvMsg / ledgerCheckMsg shapes.
// Workers ship their events to a remote collector over HTTP via httpSink,
// which batches and POSTs them; the collector serves the dashboard. Events
// flow as opResultMsg / subRecvMsg / ledgerCheckMsg / eventMsg shapes.
type EventSink interface {
OpResult(opResultMsg)
@@ -145,16 +138,6 @@ type EventSink interface {
Close() error
}
// ---- tea sink (local TUI) ----------------------------------------------
type teaSink struct{ p *tea.Program }
func (s *teaSink) OpResult(m opResultMsg) { s.p.Send(m) }
func (s *teaSink) SubRecv(m subRecvMsg) { s.p.Send(m) }
func (s *teaSink) LedgerCheck(m ledgerCheckMsg) { s.p.Send(m) }
func (s *teaSink) Event(m eventMsg) { s.p.Send(m) }
func (s *teaSink) Close() error { return nil }
// ---- HTTP sink (Zerops worker) -----------------------------------------
type httpSink struct {
@@ -564,7 +547,7 @@ func (w *worker) runSubscriber(ctx context.Context) {
}
}
// ---- TUI message types --------------------------------------------------
// ---- event message types (worker → sink) -------------------------------
type opResultMsg struct {
worker string
@@ -593,9 +576,7 @@ type eventMsg struct {
text string
}
type tickMsg time.Time
// ---- TUI model ----------------------------------------------------------
// ---- shared stats types (mirrored server-side by the collector) ---------
type uiStats struct {
role string
@@ -629,591 +610,6 @@ type eventLine struct {
text string
}
type chaosModel struct {
workers []WorkerConfig
stats map[string]*uiStats
events []eventLine
eventVP viewport.Model
p95Series []float64
okPerSecSeries []float64
failPerSecSeries []float64
globalLat []time.Duration
prevTotalOps int64
prevTotalFail int64
width, height int
startAt time.Time
quitting bool
}
func newChaosModel(ws []WorkerConfig, startAt time.Time) *chaosModel {
stats := make(map[string]*uiStats, len(ws))
for _, w := range ws {
stats[w.Name] = &uiStats{
role: w.Role,
addr: fmt.Sprintf("%s:%d", w.Host, w.Port),
}
}
vp := viewport.New(80, 8)
vp.SetContent(dimStyle.Render("(no events yet)"))
return &chaosModel{
workers: ws,
stats: stats,
eventVP: vp,
startAt: startAt,
}
}
func (m *chaosModel) Init() tea.Cmd {
return tea.Tick(tickInterval, func(t time.Time) tea.Msg { return tickMsg(t) })
}
func (m *chaosModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.height = msg.Height
m.recomputeLayout()
return m, nil
case tea.KeyMsg:
switch msg.String() {
case "q", "ctrl+c", "esc":
m.quitting = true
return m, tea.Quit
case "up", "k":
m.eventVP.LineUp(1)
case "down", "j":
m.eventVP.LineDown(1)
case "G", "end":
m.eventVP.GotoBottom()
case "g", "home":
m.eventVP.GotoTop()
case "pgup", "b":
m.eventVP.HalfViewUp()
case "pgdown", "f", " ":
m.eventVP.HalfViewDown()
}
return m, nil
case opResultMsg:
m.applyOpResult(msg)
return m, nil
case subRecvMsg:
m.applySubRecv(msg)
return m, nil
case ledgerCheckMsg:
m.applyLedgerCheck(msg)
return m, nil
case eventMsg:
m.appendEvent(msg.level, msg.at, msg.text)
return m, nil
case tickMsg:
m.onTick()
return m, tea.Tick(tickInterval, func(t time.Time) tea.Msg { return tickMsg(t) })
}
return m, nil
}
func (m *chaosModel) applyOpResult(msg opResultMsg) {
s, ok := m.stats[msg.worker]
if !ok {
return
}
s.ops++
if msg.err != nil {
s.fail++
s.consecFail++
if !s.inOutage {
s.inOutage = true
s.outageStart = msg.at
s.outageOpsLost = 1
m.appendEvent("err", msg.at,
fmt.Sprintf("[%s] OUTAGE START — %s: %s", msg.worker, msg.op, trim(msg.err.Error(), 80)))
} else {
s.outageOpsLost++
}
return
}
s.ok++
if s.inOutage {
d := msg.at.Sub(s.outageStart)
s.outages = append(s.outages, outageRec{
start: s.outageStart, end: msg.at, opsLost: s.outageOpsLost,
})
m.appendEvent("info", msg.at,
fmt.Sprintf("[%s] OUTAGE END — duration=%s ops_lost=%d",
msg.worker, d.Round(time.Millisecond), s.outageOpsLost))
s.inOutage = false
s.consecFail = 0
s.outageOpsLost = 0
}
if msg.dur > 0 {
s.recentLat = appendBoundedDur(s.recentLat, msg.dur, maxRecentLat)
m.globalLat = appendBoundedDur(m.globalLat, msg.dur, maxGlobalLat)
}
}
func (m *chaosModel) applySubRecv(msg subRecvMsg) {
s, ok := m.stats[msg.worker]
if !ok {
return
}
s.msgs++
if s.lastSeq != 0 && msg.seq > s.lastSeq+1 {
gap := msg.seq - s.lastSeq - 1
s.gaps += gap
m.appendEvent("warn", msg.at,
fmt.Sprintf("[%s] PUBSUB GAP last=%d now=%d missed=%d",
msg.worker, s.lastSeq, msg.seq, gap))
}
if !s.lastSeqAt.IsZero() {
if d := msg.at.Sub(s.lastSeqAt); d > s.longestGap {
s.longestGap = d
}
}
s.lastSeq = msg.seq
s.lastSeqAt = msg.at
}
func (m *chaosModel) applyLedgerCheck(msg ledgerCheckMsg) {
s, ok := m.stats[msg.worker]
if !ok {
return
}
switch msg.kind {
case "ok":
s.verified++
case "missing":
s.missing++
m.appendEvent("err", time.Now(),
fmt.Sprintf("[%s] LEDGER MISSING seq=%d %s", msg.worker, msg.seq, msg.detail))
case "mismatch":
s.mismatch++
m.appendEvent("err", time.Now(),
fmt.Sprintf("[%s] LEDGER MISMATCH seq=%d %s", msg.worker, msg.seq, msg.detail))
}
}
func (m *chaosModel) appendEvent(level string, at time.Time, text string) {
if len(m.events) >= maxEvents {
copy(m.events, m.events[1:])
m.events = m.events[:len(m.events)-1]
}
m.events = append(m.events, eventLine{at: at, level: level, text: text})
atBottom := m.eventVP.AtBottom()
m.eventVP.SetContent(m.renderEventContent())
if atBottom {
m.eventVP.GotoBottom()
}
}
func (m *chaosModel) onTick() {
if len(m.globalLat) > 0 {
cp := make([]time.Duration, len(m.globalLat))
copy(cp, m.globalLat)
sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] })
idx := 95 * len(cp) / 100
if idx >= len(cp) {
idx = len(cp) - 1
}
p95ms := float64(cp[idx]) / float64(time.Millisecond)
m.p95Series = appendBoundedFloat(m.p95Series, p95ms, maxSeriesPoints)
} else {
m.p95Series = appendBoundedFloat(m.p95Series, 0, maxSeriesPoints)
}
var totalOps, totalFail int64
for _, s := range m.stats {
totalOps += s.ops + s.msgs
totalFail += s.fail
}
okDelta := (totalOps - totalFail) - (m.prevTotalOps - m.prevTotalFail)
if okDelta < 0 {
okDelta = 0
}
failDelta := totalFail - m.prevTotalFail
if failDelta < 0 {
failDelta = 0
}
m.prevTotalOps = totalOps
m.prevTotalFail = totalFail
m.okPerSecSeries = appendBoundedFloat(m.okPerSecSeries, float64(okDelta), maxSeriesPoints)
m.failPerSecSeries = appendBoundedFloat(m.failPerSecSeries, float64(failDelta), maxSeriesPoints)
for _, s := range m.stats {
current := s.ops + s.msgs
delta := current - s.prevOps
if delta < 0 {
delta = 0
}
s.prevOps = current
s.opsHist = appendBoundedFloat(s.opsHist, float64(delta), maxSparkSamples)
}
}
func (m *chaosModel) recomputeLayout() {
headerH := 1
tableTitleH := 1
tableRows := len(m.workers) + 2
chartTitleH := 1
chartH := 8
chartFooterH := 1
eventTitleH := 1
footerH := 1
used := headerH + tableTitleH + tableRows + chartTitleH + chartH + chartFooterH + eventTitleH + footerH + 2
eventH := m.height - used
if eventH < 4 {
eventH = 4
}
w := m.width - 2
if w < 20 {
w = 20
}
m.eventVP.Width = w
m.eventVP.Height = eventH
m.eventVP.SetContent(m.renderEventContent())
}
// ---- TUI rendering ------------------------------------------------------
var (
titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("212"))
headStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("39"))
dimStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("241"))
okStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("42"))
warnStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("214"))
errStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("196"))
chartStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("141"))
)
func (m *chaosModel) View() string {
if m.quitting {
return "stopping…\n"
}
if m.width == 0 || m.height == 0 {
return "loading…\n"
}
if m.width < 90 || m.height < 25 {
return fmt.Sprintf("terminal too small: %dx%d (need 90x25 minimum). press q to quit.\n",
m.width, m.height)
}
header := m.renderHeader()
table := m.renderTable()
charts := m.renderCharts()
events := m.renderEvents()
footer := dimStyle.Render(" q quit · ↑/↓ scroll · pgup/pgdn half-page · g/G top/bottom")
return strings.Join([]string{header, "", table, "", charts, "", events, footer}, "\n")
}
func (m *chaosModel) renderHeader() string {
elapsed := time.Since(m.startAt).Round(time.Second)
var inOutage int
for _, s := range m.stats {
if s.inOutage {
inOutage++
}
}
tag := okStyle.Render("healthy")
if inOutage > 0 {
tag = errStyle.Render(fmt.Sprintf("%d worker(s) in outage", inOutage))
}
return titleStyle.Render(" valkey-ha chaos") +
dimStyle.Render(fmt.Sprintf(" · %d workers · elapsed %s · ", len(m.workers), elapsed)) +
tag
}
func (m *chaosModel) renderTable() string {
var b strings.Builder
b.WriteString(headStyle.Render(" workers") + "\n")
hdr := fmt.Sprintf(" %-14s %-11s %7s %7s %6s %7s %8s %8s %-12s %s",
"NAME", "ROLE", "OPS", "OK", "FAIL", "UPTIME", "P50", "P95", "OPS/SEC", "STATUS")
b.WriteString(dimStyle.Render(hdr) + "\n")
for _, w := range m.workers {
s := m.stats[w.Name]
switch s.role {
case "subscriber":
line := fmt.Sprintf(" %-14s %-11s msgs=%-7d gaps=%-3d longest_gap=%-7s %s",
w.Name, "subscriber", s.msgs, s.gaps,
shortDur(s.longestGap), m.statusBadge(s))
b.WriteString(line + "\n")
default:
uptime := 100.0
if s.ops > 0 {
uptime = float64(s.ok) / float64(s.ops) * 100
}
p50 := percentileLat(s.recentLat, 50)
p95 := percentileLat(s.recentLat, 95)
spark := chartStyle.Render(sparkline(s.opsHist, 12))
extra := ""
if s.role == "reader" {
extra = dimStyle.Render(fmt.Sprintf(" v=%d miss=%d mis=%d",
s.verified, s.missing, s.mismatch))
}
line := fmt.Sprintf(" %-14s %-11s %7d %7d %6d %6.2f%% %8s %8s %s %s%s",
w.Name, s.role, s.ops, s.ok, s.fail, uptime,
shortDur(p50), shortDur(p95),
spark, m.statusBadge(s), extra)
b.WriteString(line + "\n")
}
}
return strings.TrimRight(b.String(), "\n")
}
func (m *chaosModel) statusBadge(s *uiStats) string {
if s.inOutage {
return errStyle.Render("OUTAGE")
}
if s.role == "subscriber" && s.msgs == 0 && time.Since(m.startAt) > 5*time.Second {
return warnStyle.Render("NO MSGS")
}
return okStyle.Render("OK")
}
func (m *chaosModel) renderCharts() string {
chartH := 8
chartW := (m.width - 4) / 3 // 3 charts with 2× " " separators between
if chartW < 20 {
chartW = 20
}
p95Max := maxFloat(m.p95Series)
if p95Max < 1 {
p95Max = 1
}
p95Lines := renderBarChart(m.p95Series, chartW, chartH, p95Max)
p95Title := headStyle.Render(" p95 latency") +
dimStyle.Render(fmt.Sprintf(" peak %.1fms · last %s",
p95Max, lastFloatLabel(m.p95Series, "ms")))
p95Box := p95Title + "\n" + chartStyle.Render(strings.Join(p95Lines, "\n"))
tpMax := maxFloat(m.okPerSecSeries)
if tpMax < 5 {
tpMax = 5
}
tpLines := renderBarChart(m.okPerSecSeries, chartW, chartH, tpMax)
tpTitle := headStyle.Render(" ok ops/sec") +
dimStyle.Render(fmt.Sprintf(" peak %.0f · last %s",
tpMax, lastFloatLabel(m.okPerSecSeries, "")))
tpBox := tpTitle + "\n" + chartStyle.Render(strings.Join(tpLines, "\n"))
failMax := maxFloat(m.failPerSecSeries)
if failMax < 1 {
failMax = 1
}
failLines := renderBarChart(m.failPerSecSeries, chartW, chartH, failMax)
failTitle := headStyle.Render(" fail ops/sec") +
dimStyle.Render(fmt.Sprintf(" peak %.0f · last %s",
failMax, lastFloatLabel(m.failPerSecSeries, "")))
failBox := failTitle + "\n" + errStyle.Render(strings.Join(failLines, "\n"))
return lipgloss.JoinHorizontal(lipgloss.Top, p95Box, " ", tpBox, " ", failBox)
}
func (m *chaosModel) renderEvents() string {
title := headStyle.Render(" events") +
dimStyle.Render(fmt.Sprintf(" (%d, scroll with ↑/↓)", len(m.events)))
return title + "\n" + m.eventVP.View()
}
func (m *chaosModel) renderEventContent() string {
if len(m.events) == 0 {
return dimStyle.Render("(no events yet)")
}
var b strings.Builder
for _, e := range m.events {
ts := dimStyle.Render(e.at.Format("15:04:05.000"))
var styled string
switch e.level {
case "err":
styled = errStyle.Render(e.text)
case "warn":
styled = warnStyle.Render(e.text)
default:
styled = e.text
}
b.WriteString(ts + " " + styled + "\n")
}
return strings.TrimRight(b.String(), "\n")
}
// ---- chart primitives ---------------------------------------------------
var sparkChars = []rune(" ▁▂▃▄▅▆▇█")
func sparkline(values []float64, width int) string {
if width <= 0 {
return ""
}
if len(values) == 0 {
return strings.Repeat(" ", width)
}
maxV := 0.0
for _, v := range values {
if v > maxV {
maxV = v
}
}
var b strings.Builder
n := len(values)
for col := 0; col < width; col++ {
if maxV == 0 {
b.WriteRune(' ')
continue
}
idx := col * n / width
if idx >= n {
idx = n - 1
}
v := values[idx]
if v < 0 {
v = 0
}
if v > maxV {
v = maxV
}
i := int(v / maxV * float64(len(sparkChars)-1))
if i < 0 {
i = 0
}
if i >= len(sparkChars) {
i = len(sparkChars) - 1
}
b.WriteRune(sparkChars[i])
}
return b.String()
}
func renderBarChart(values []float64, width, height int, maxV float64) []string {
rows := make([][]rune, height)
for i := range rows {
rows[i] = []rune(strings.Repeat(" ", width))
}
flush := func() []string {
out := make([]string, height)
for i := range rows {
out[i] = string(rows[i])
}
return out
}
if width <= 0 || height <= 0 || maxV <= 0 || len(values) == 0 {
return flush()
}
n := len(values)
subSteps := []rune(" ▁▂▃▄▅▆▇")
for col := 0; col < width; col++ {
idx := col * n / width
if idx >= n {
idx = n - 1
}
v := values[idx]
if v < 0 {
v = 0
}
if v > maxV {
v = maxV
}
cells := int(v / maxV * float64(height) * 8)
full := cells / 8
rem := cells % 8
for r := 0; r < full && r < height; r++ {
rows[height-1-r][col] = '█'
}
if full < height && rem > 0 {
rows[height-1-full][col] = subSteps[rem]
}
}
return flush()
}
func lastFloatLabel(s []float64, unit string) string {
if len(s) == 0 {
return "-"
}
v := s[len(s)-1]
if unit == "ms" {
return fmt.Sprintf("%.1f%s", v, unit)
}
return fmt.Sprintf("%.0f%s", v, unit)
}
// ---- runChaos (local TUI) ----------------------------------------------
func runChaos(ws []WorkerConfig) int {
for _, w := range ws {
if err := validateWorker(w); err != nil {
fmt.Fprintf(os.Stderr, "worker %q: %v\n", w.Name, err)
return 2
}
}
logFile, ferr := os.Create("chaos-debug.log")
if ferr == nil {
defer logFile.Close()
log.SetOutput(logFile)
} else {
log.SetOutput(io.Discard)
}
defer log.SetOutput(os.Stderr)
workerCtx, cancel := context.WithCancel(context.Background())
cc := &chaosCtx{
ledger: newLedger(),
startAt: time.Now(),
}
workers := buildWorkers(ws, cc)
defer func() {
for _, w := range workers {
_ = w.rdb.Close()
}
}()
model := newChaosModel(ws, cc.startAt)
p := tea.NewProgram(model, tea.WithAltScreen())
cc.sink = &teaSink{p: p}
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
go func() {
select {
case <-sigCh:
p.Quit()
case <-workerCtx.Done():
}
}()
var wg sync.WaitGroup
for _, w := range workers {
wg.Add(1)
go func(w *worker) {
defer wg.Done()
w.run(workerCtx)
}(w)
}
if _, err := p.Run(); err != nil {
fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
}
cancel()
waitWorkers(&wg, 5*time.Second)
fmt.Println("\nrunning final durability sweep…")
fd := finalSweep(workers, cc)
printFinalReport(model, time.Since(cc.startAt), fd)
if fd.missing > 0 || fd.mismatch > 0 {
return 1
}
return 0
}
// ---- runWorker (Zerops worker container) -------------------------------
func runWorker(ws []WorkerConfig, collectorURL string) int {
@@ -1369,84 +765,6 @@ func finalSweep(workers []*worker, cc *chaosCtx) finalDur {
return fd
}
func printFinalReport(m *chaosModel, dur time.Duration, fd finalDur) {
fmt.Println("=== valkey-ha chaos report ===")
fmt.Printf("duration: %s\n\n", dur.Round(time.Millisecond))
fmt.Println("per-worker:")
for _, w := range m.workers {
s := m.stats[w.Name]
switch s.role {
case "subscriber":
fmt.Printf(" %-14s [sub] msgs=%d gaps=%d longest_gap=%s\n",
w.Name, s.msgs, s.gaps, s.longestGap.Round(time.Millisecond))
default:
uptime := 100.0
if s.ops > 0 {
uptime = float64(s.ok) / float64(s.ops) * 100
}
var longest time.Duration
for _, o := range s.outages {
if d := o.end.Sub(o.start); d > longest {
longest = d
}
}
extra := ""
if s.role == "reader" {
extra = fmt.Sprintf(" verified=%d missing=%d mismatch=%d",
s.verified, s.missing, s.mismatch)
}
fmt.Printf(" %-14s [%-10s] ops=%d ok=%d fail=%d uptime=%.2f%% p50=%s p95=%s outages=%d longest=%s%s\n",
w.Name, s.role, s.ops, s.ok, s.fail, uptime,
shortDur(percentileLat(s.recentLat, 50)),
shortDur(percentileLat(s.recentLat, 95)),
len(s.outages), longest.Round(time.Millisecond),
extra)
}
}
fmt.Println("\ndurability:")
fmt.Printf(" ledger: acked=%d verified=%d missing=%d mismatch=%d\n",
fd.acked, fd.verified, fd.missing, fd.mismatch)
if len(fd.missingSeqs) > 0 {
n := 20
if len(fd.missingSeqs) < n {
n = len(fd.missingSeqs)
}
fmt.Printf(" missing seqs (first %d): %v\n", n, fd.missingSeqs[:n])
}
delta := fd.serverCounter - fd.localAcked
fmt.Printf(" counter: local_acked=%d server_value=%d delta=%d\n",
fd.localAcked, fd.serverCounter, delta)
type tagged struct {
w string
rec outageRec
}
var all []tagged
for _, w := range m.workers {
for _, o := range m.stats[w.Name].outages {
all = append(all, tagged{w: w.Name, rec: o})
}
}
sort.Slice(all, func(i, j int) bool { return all[i].rec.start.Before(all[j].rec.start) })
if len(all) > 0 {
fmt.Println("\noutages (chronological):")
for _, t := range all {
fmt.Printf(" %s -> %s (%s) worker=%s ops_lost=%d\n",
t.rec.start.Format("15:04:05.000"),
t.rec.end.Format("15:04:05.000"),
t.rec.end.Sub(t.rec.start).Round(time.Millisecond),
t.w, t.rec.opsLost)
}
}
fmt.Println("\nnotes:")
fmt.Println(" - Valkey 7.2 + Sentinel does NOT replicate pub/sub from master to replicas.")
fmt.Println(" A subscriber on the read-only VIP receiving 0 messages is expected.")
fmt.Println(" - debug log: ./chaos-debug.log")
}
// ---- helpers ------------------------------------------------------------
func appendBoundedDur(s []time.Duration, v time.Duration, max int) []time.Duration {
@@ -1496,22 +814,6 @@ func maxDur(a, b time.Duration) time.Duration {
return b
}
func shortDur(d time.Duration) string {
if d <= 0 {
return "-"
}
if d < time.Microsecond {
return "0"
}
if d < time.Millisecond {
return fmt.Sprintf("%dµs", d.Microseconds())
}
if d < time.Second {
return fmt.Sprintf("%.1fms", float64(d)/float64(time.Millisecond))
}
return d.Round(time.Millisecond).String()
}
func trim(s string, n int) string {
if len(s) <= n {
return s