seaweed testing, gitea actions

dckr-test/app.txt

@@ -0,0 +1 @@
+dummy

dckr-test/zerops.yml

@@ -0,0 +1,8 @@
+zerops:
+  - setup: dckr
+    run:
+      base: docker@26.1
+      volume:
+        hostname: data1
+        mountPath: /mnt/data
+      start: sleep infinity

gitea/README.md

@@ -1,57 +0,0 @@
-# Gitea on Zerops
-
-Self-hosted Gitea backed by HA Postgres and SeaweedFS shared storage.
-
-## Prepare secrets
-
-Gitea validates its security secrets at startup and tries to rewrite `app.ini`
-if they're missing or malformed. Zerops' `<@generateRandomString>` preprocessor
-produces alphanumeric values that don't match Gitea's required formats
-(base64url-32 bytes for `JWT_SECRET` / `LFS_JWT_SECRET`, a signed JWT for
-`INTERNAL_TOKEN`), so generate them with the Gitea binary itself:
-
-```sh
-./generate-secrets.sh
-```
-
-It downloads the matching Gitea release and prints four `KEY=VALUE` lines.
-Paste each value into the gitea service's **secret env vars** in the Zerops GUI
-(or pre-fill `zerops-service-import.yaml` before importing).
-
-`DB_PASSWORD` is generated by the Zerops preprocessor (`<@generateRandomString>`)
-during import — cross-service expansion like `${db_password}` is not evaluated
-inside the import.yaml, so the gitea service generates its own value and the
-Postgres service must be initialized with the same one.
-
-## Bootstrap
-
-1. Paste `zerops-service-import.yaml` into the Zerops GUI:
-   project → **Import service** → fill in the four secrets from step above.
-2. Push the gitea config and build recipe:
-   ```sh
-   zcli push gitea
-   ```
-3. Create the admin user from the running container (service console):
-   ```sh
-   sudo -u git -E /usr/local/bin/gitea admin user create \
-     --config /etc/gitea/app.ini \
-     --admin \
-     --username admin \
-     --email you@example.com \
-     --password 'choose-a-strong-one' \
-     --must-change-password=false
-   ```
-4. Log in at the service's HTTPS URL and add your SSH public key under
-   **Settings → SSH/GPG Keys**.
-
-## SSH access
-
-Gitea's built-in SSH listens on **port 2222** in the container. Zerops custom
-domains only carry HTTPS, so SSH needs one of:
-
-- **Public IPv4/IPv6 on the gitea service** — Zerops GUI → service →
-  *Public Routing* → add IP, open port 2222. Map `:2222 → :2222`.
-
-After SSH works, set `SSH_DOMAIN` / `ROOT_URL` overrides if you use a custom
-hostname, otherwise Gitea will print clone URLs using the autogenerated
-Zerops subdomain.

gitea/app.ini

@@ -1,56 +0,0 @@
-APP_NAME = Gitea
-RUN_USER = git
-RUN_MODE = prod
-WORK_PATH = /mnt/volume/gitea
-
-[server]
-PROTOCOL         = http
-DOMAIN           = %%GITEA_DOMAIN%%
-ROOT_URL         = %%GITEA_ROOT_URL%%
-HTTP_ADDR        = 0.0.0.0
-HTTP_PORT        = 3000
-SSH_DOMAIN       = %%GITEA_DOMAIN%%
-START_SSH_SERVER = true
-SSH_PORT         = 2222
-SSH_LISTEN_PORT  = 2222
-LFS_START_SERVER = true
-APP_DATA_PATH    = /mnt/volume/gitea/data
-DISABLE_SSH      = false
-LFS_JWT_SECRET   = %%LFS_JWT_SECRET%%
-
-[database]
-DB_TYPE  = postgres
-HOST     = %%DB_HOST%%:%%DB_PORT%%
-NAME     = %%DB_NAME%%
-USER     = %%DB_USER%%
-PASSWD   = %%DB_PASSWORD%%
-SSL_MODE = disable
-SCHEMA   = public
-
-[repository]
-ROOT = /mnt/volume/gitea/data/gitea-repositories
-
-[lfs]
-PATH = /mnt/volume/gitea/data/lfs
-
-[log]
-ROOT_PATH = /mnt/volume/gitea/log
-MODE      = console
-LEVEL     = info
-
-[service]
-DISABLE_REGISTRATION              = true
-REQUIRE_SIGNIN_VIEW               = false
-DEFAULT_KEEP_EMAIL_PRIVATE        = true
-DEFAULT_ALLOW_CREATE_ORGANIZATION = true
-
-[security]
-INSTALL_LOCK   = true
-SECRET_KEY     = %%SECRET_KEY%%
-INTERNAL_TOKEN = %%INTERNAL_TOKEN%%
-
-[oauth2]
-JWT_SECRET = %%JWT_SECRET%%
-
-[session]
-PROVIDER = file

gitea/generate-secrets.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-# Generates JWT_SECRET, SECRET_KEY and INTERNAL_TOKEN in the exact format Gitea expects.
-# Run once, then paste the three lines into Zerops as secret env vars for the gitea service.
-set -euo pipefail
-
-GITEA_VERSION="${GITEA_VERSION:-1.26.1}"
-BIN=$(mktemp)
-trap 'rm -f "$BIN"' EXIT
-
-wget -qO "$BIN" "https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64"
-chmod +x "$BIN"
-
-echo "JWT_SECRET=$("$BIN" generate secret JWT_SECRET)"
-echo "LFS_JWT_SECRET=$("$BIN" generate secret LFS_JWT_SECRET)"
-echo "SECRET_KEY=$("$BIN" generate secret SECRET_KEY)"
-echo "INTERNAL_TOKEN=$("$BIN" generate secret INTERNAL_TOKEN)"

gitea/zerops-service-import.yaml

@@ -1,22 +0,0 @@
-#zeropsPreprocessor=on
-services:
-  - hostname: db
-    type: postgresql:ha@18
-    priority: 10
-    verticalAutoscaling:
-      minFreeRamGB: 0.25
-  - hostname: volume
-    type: seaweedfs:ha@3
-    priority: 10
-  - hostname: gitea
-    type: ubuntu@24.04
-    envSecrets:
-      DB_PASSWORD: <@generateRandomString(<32>)>
-      JWT_SECRET: <fill-from-generate-secrets.sh>
-      LFS_JWT_SECRET: <fill-from-generate-secrets.sh>
-      SECRET_KEY: <fill-from-generate-secrets.sh>
-      INTERNAL_TOKEN: <fill-from-generate-secrets.sh>
-    minContainers: 1
-    maxContainers: 1
-    mount:
-      - volume

gitea/zerops.yaml

@@ -1,44 +0,0 @@
-zerops:
-  - setup: gitea
-    build:
-      base: ubuntu@24.04
-      deployFiles: app.ini
-    run:
-      base: ubuntu@24.04
-      ports:
-        - port: 3000
-          httpSupport: true
-        - port: 2222
-      envVariables:
-        GITEA_VERSION: 1.26.1
-        GITEA_WORK_DIR: /mnt/volume/gitea
-        GITEA_DOMAIN: git.matejpavlicek.cz
-        GITEA_ROOT_URL: https://git.matejpavlicek.cz
-        DB_HOST: db
-        DB_PORT: 5432
-        DB_NAME: giteadb
-        DB_USER: gitea
-      envReplace:
-        delimiter: "%%"
-        target:
-          - app.ini
-      prepareCommands:
-        - sudo apt-get update
-        - sudo apt-get install -y --no-install-recommends git ca-certificates gettext-base wget gpg
-        - wget -q -O /tmp/gitea https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64
-        - wget -q -O /tmp/gitea.asc https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64.asc
-        - gpg --keyserver hkps://keys.openpgp.org --recv 7C9E68152594688862D62AF62D9AE806EC1592E2
-        - gpg --verify /tmp/gitea.asc /tmp/gitea
-        - sudo install -m 755 /tmp/gitea /usr/local/bin/gitea
-        - sudo groupadd --system git
-        - sudo useradd --system --gid git --shell /bin/bash --home-dir /home/git --create-home git
-        - sudo mkdir -p /etc/gitea
-        - sudo chown root:git /etc/gitea
-        - sudo chmod 770 /etc/gitea
-      initCommands:
-        - until mountpoint -q /mnt/volume; do sleep 1; done
-        - sudo mkdir -p /mnt/volume/gitea/{custom,data,indexers,public,log}
-        - sudo chown -R git:git /mnt/volume/gitea
-        - sudo chmod -R 750 /mnt/volume/gitea
-        - sudo install -m 660 -o root -g git /var/www/app.ini /etc/gitea/app.ini
-      start: sudo -u git -E /usr/local/bin/gitea web --config /etc/gitea/app.ini

godummy/go.mod

@@ -1,3 +1,3 @@
 module github.com/tikinang/godummy
 
-go 1.23.1
+go 1.26.5

godummy/main.go

@@ -1,14 +1,56 @@
 package main
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var BuildTime string
+
+const ramChunkSize = 100 * 1024 * 1024 // 0.1 GiB
+
+var (
+	ramMu     sync.Mutex
+	ramChunks [][]byte
 )
 
 func main() {
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
-		_, _ = fmt.Fprintln(w, "OK")
+		_, _ = fmt.Fprintln(w, BuildTime)
+	})
+	http.HandleFunc("/ram", func(w http.ResponseWriter, r *http.Request) {
+		chunk := make([]byte, ramChunkSize)
+		for i := range chunk {
+			chunk[i] = 1
+		}
+		ramMu.Lock()
+		ramChunks = append(ramChunks, chunk)
+		total := len(ramChunks)
+		ramMu.Unlock()
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintf(w, "allocated 0.1 GiB, total held: %.1f GiB (%d chunks)\n", float64(total)*0.1, total)
+	})
+	http.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) {
+		iterations := 1_000_000
+		if n, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && n > 0 {
+			iterations = n
+		}
+
+		start := time.Now()
+		sum := sha256.Sum256([]byte("zerops"))
+		for i := 0; i < iterations; i++ {
+			sum = sha256.Sum256(sum[:])
+		}
+		elapsed := time.Since(start)
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintf(w, "computed %x in %s (%d iterations)\n", sum, elapsed, iterations)
 	})
 	_ = http.ListenAndServe(":8080", nil)
 }

godummy/zerops.yml

@@ -1,15 +1,13 @@
 zerops:
   - setup: app
     build:
-      os: ubuntu
-      base: go@latest
+      base: ubuntu/go@latest
       buildCommands:
-        - go build -o app main.go
+        - go build -ldflags "-X main.BuildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" -o app main.go
       cache: true
       deployFiles:
         - app
     run:
-      os: ubuntu
       base: ubuntu@latest
       ports:
         - port: 8080

ls-stress/go.mod

@@ -0,0 +1,17 @@
+module sqlite-ls
+
+go 1.25.0
+
+require modernc.org/sqlite v1.56.0
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	modernc.org/libc v1.74.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

ls-stress/go.sum

@@ -0,0 +1,50 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
+modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
+modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
+modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

ls-stress/import-2.yml

@@ -0,0 +1,23 @@
+project:
+  name: ls-stress-2
+services:
+  - hostname: data1
+    type: local-storage:single@1
+  - hostname: data2
+    type: local-storage:single@1
+  - hostname: c1a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c1b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3

ls-stress/import-3.yml

@@ -0,0 +1,23 @@
+project:
+  name: ls-stress-3
+services:
+  - hostname: data1
+    type: local-storage:single@1
+  - hostname: data2
+    type: local-storage:single@1
+  - hostname: c1a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c1b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3

ls-stress/import-4.yml

@@ -0,0 +1,23 @@
+project:
+  name: ls-stress-4
+services:
+  - hostname: data1
+    type: local-storage:single@1
+  - hostname: data2
+    type: local-storage:single@1
+  - hostname: c1a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c1b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3

ls-stress/import-data2.yml

@@ -0,0 +1,3 @@
+services:
+  - hostname: data2
+    type: local-storage:single@1

ls-stress/import-disk-test.yml

@@ -0,0 +1,11 @@
+project:
+  name: ls-disk-test
+services:
+  - hostname: data1
+    type: local-storage:single@1
+  - hostname: c1a
+    type: go@1
+    minContainers: 2
+    maxContainers: 2
+  - hostname: dckr
+    type: docker@26.1

ls-stress/main.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"crypto/rand"
+	"database/sql"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+var (
+	db       *sql.DB
+	hostname string
+	dbFile   string
+)
+
+func main() {
+	hostname, _ = os.Hostname()
+
+	dbPath := os.Getenv("DB_PATH")
+	if dbPath == "" {
+		dbPath = "/mnt/data/poc.db"
+	}
+	dbFile = dbPath
+
+	var err error
+	db, err = sql.Open("sqlite", "file:"+dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS entries (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		container TEXT NOT NULL,
+		msg TEXT NOT NULL,
+		created_at TEXT NOT NULL,
+		payload BLOB
+	)`); err != nil {
+		log.Fatal(err)
+	}
+	// the payload column was added after the first deploy, older DB files need it retrofitted
+	if _, err := db.Exec(`ALTER TABLE entries ADD COLUMN payload BLOB`); err != nil && !strings.Contains(err.Error(), "duplicate column") {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/", handleStatus)
+	http.HandleFunc("/write", handleWrite)
+	http.HandleFunc("/read", handleRead)
+	http.HandleFunc("/checkpoint", handleCheckpoint)
+
+	log.Printf("listening on :8080, db=%s, container=%s", dbPath, hostname)
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}
+
+func respond(w http.ResponseWriter, status int, data any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(data)
+}
+
+func handleStatus(w http.ResponseWriter, r *http.Request) {
+	var count int64
+	var journalMode string
+	if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	db.QueryRow("PRAGMA journal_mode").Scan(&journalMode)
+	fileSize := func(path string) int64 {
+		info, err := os.Stat(path)
+		if err != nil {
+			return 0
+		}
+		return info.Size()
+	}
+	respond(w, 200, map[string]any{
+		"container":    hostname,
+		"entries":      count,
+		"journalMode":  journalMode,
+		"dbSizeBytes":  fileSize(dbFile),
+		"walSizeBytes": fileSize(dbFile + "-wal"),
+	})
+}
+
+func handleWrite(w http.ResponseWriter, r *http.Request) {
+	msg := r.URL.Query().Get("msg")
+	if msg == "" {
+		msg = "hello"
+	}
+	intParam := func(key string, def, max int) int {
+		v, err := strconv.Atoi(r.URL.Query().Get(key))
+		if err != nil || v < 1 {
+			return def
+		}
+		return min(v, max)
+	}
+	n := intParam("n", 1, 10000)
+	size := intParam("size", 0, 1<<20)
+
+	tx, err := db.Begin()
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer tx.Rollback()
+	stmt, err := tx.Prepare("INSERT INTO entries (container, msg, created_at, payload) VALUES (?, ?, ?, ?)")
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer stmt.Close()
+
+	var lastId int64
+	now := time.Now().UTC().Format(time.RFC3339Nano)
+	for i := 0; i < n; i++ {
+		var payload []byte
+		if size > 0 {
+			payload = make([]byte, size)
+			rand.Read(payload)
+		}
+		res, err := stmt.Exec(hostname, msg, now, payload)
+		if err != nil {
+			respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+			return
+		}
+		lastId, _ = res.LastInsertId()
+	}
+	if err := tx.Commit(); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	respond(w, 200, map[string]any{"container": hostname, "inserted": n, "lastId": lastId})
+}
+
+func handleCheckpoint(w http.ResponseWriter, r *http.Request) {
+	var busy, logFrames, checkpointed int64
+	if err := db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	respond(w, 200, map[string]any{"container": hostname, "busy": busy, "logFrames": logFrames, "checkpointed": checkpointed})
+}
+
+func handleRead(w http.ResponseWriter, r *http.Request) {
+	var count int64
+	if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	rows, err := db.Query("SELECT id, container, msg, created_at FROM entries ORDER BY id DESC LIMIT 5")
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer rows.Close()
+	type entry struct {
+		Id        int64  `json:"id"`
+		Container string `json:"container"`
+		Msg       string `json:"msg"`
+		CreatedAt string `json:"createdAt"`
+	}
+	last := []entry{}
+	for rows.Next() {
+		var e entry
+		if err := rows.Scan(&e.Id, &e.Container, &e.Msg, &e.CreatedAt); err != nil {
+			respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+			return
+		}
+		last = append(last, e)
+	}
+	respond(w, 200, map[string]any{"container": hostname, "entries": count, "last": last})
+}

ls-stress/zerops-project-import.yml

@@ -0,0 +1,37 @@
+project:
+  name: ls-delete-stress
+services:
+  - hostname: data1
+    type: local-storage:single@1
+  - hostname: data2
+    type: local-storage:single@1
+  - hostname: c1a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c1b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c1c
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2a
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2b
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: c2c
+    type: go@1
+    minContainers: 3
+    maxContainers: 3
+  - hostname: cache
+    type: valkey@7.2
+    mode: NON_HA
+  - hostname: db
+    type: postgresql@16
+    mode: NON_HA

ls-stress/zerops.yml

@@ -0,0 +1,39 @@
+zerops:
+  - setup: c1a
+    build: &build
+      base: go@1
+      buildCommands:
+        - go build -o app main.go
+      deployFiles: app
+    run: &run-data1
+      base: go@1
+      ports:
+        - port: 8080
+          httpSupport: true
+      volume:
+        hostname: data1
+        mountPath: /mnt/data
+      start: ./app
+  - setup: c1b
+    build: *build
+    run: *run-data1
+  - setup: c1c
+    build: *build
+    run: *run-data1
+  - setup: c2a
+    build: *build
+    run: &run-data2
+      base: go@1
+      ports:
+        - port: 8080
+          httpSupport: true
+      volume:
+        hostname: data2
+        mountPath: /mnt/data
+      start: ./app
+  - setup: c2b
+    build: *build
+    run: *run-data2
+  - setup: c2c
+    build: *build
+    run: *run-data2

manybuilds/import.yaml

@@ -0,0 +1,121 @@
+services:
+  - hostname: app1
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app1
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app2
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app2
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app3
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app3
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app4
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app4
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app5
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app5
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app6
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app6
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app7
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app7
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true
+  - hostname: app8
+    override: true
+    type: ubuntu/golang@1
+    zeropsYaml:
+      zerops:
+        - setup: app8
+          build:
+            envVariables:
+              KOKOS: AHOJ-8
+            base: ubuntu/golang@1
+            buildCommands:
+              - echo $KOKOS
+              - echo $ZEROPS_BUILD_TOKEN
+              - sleep 15
+            cache: true

manybuilds/main.go

@@ -0,0 +1,11 @@
+package main
+
+import "net/http"
+
+func main() {
+	http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+		writer.WriteHeader(http.StatusOK)
+		_, _ = writer.Write([]byte("Hello World"))
+	})
+	_ = http.ListenAndServe(":8080", nil)
+}

manybuilds/zerops.yaml

@@ -0,0 +1,14 @@
+zerops:
+  - setup: app
+    build:
+      base: ubuntu/golang@1
+      deployFiles: main
+      buildCommands:
+        - go build main.go
+      cache: true
+    run:
+      base: ubuntu/golang@1
+      ports:
+        - port: 8080
+          httpSupport: true
+      start: ./main

seaweed-testing/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+package-lock.json
+app

seaweed-testing/go.mod

@@ -0,0 +1,3 @@
+module github.com/tikinang/godummy
+
+go 1.26.5

seaweed-testing/main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var BuildTime string
+
+const ramChunkSize = 100 * 1024 * 1024 // 0.1 GiB
+
+var (
+	ramMu     sync.Mutex
+	ramChunks [][]byte
+)
+
+func main() {
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintln(w, BuildTime)
+	})
+	http.HandleFunc("/ram", func(w http.ResponseWriter, r *http.Request) {
+		chunk := make([]byte, ramChunkSize)
+		for i := range chunk {
+			chunk[i] = 1
+		}
+		ramMu.Lock()
+		ramChunks = append(ramChunks, chunk)
+		total := len(ramChunks)
+		ramMu.Unlock()
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintf(w, "allocated 0.1 GiB, total held: %.1f GiB (%d chunks)\n", float64(total)*0.1, total)
+	})
+	http.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) {
+		iterations := 1_000_000
+		if n, err := strconv.Atoi(r.URL.Query().Get("n")); err == nil && n > 0 {
+			iterations = n
+		}
+
+		start := time.Now()
+		sum := sha256.Sum256([]byte("zerops"))
+		for i := 0; i < iterations; i++ {
+			sum = sha256.Sum256(sum[:])
+		}
+		elapsed := time.Since(start)
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprintf(w, "computed %x in %s (%d iterations)\n", sum, elapsed, iterations)
+	})
+	_ = http.ListenAndServe(":8080", nil)
+}

seaweed-testing/zerops-service-import.yml

@@ -0,0 +1,6 @@
+services:
+  - hostname: app
+    type: ubuntu@24.04
+    maxContainers: 1
+    buildFromGit: https://github.com/tikinang/godummy
+    enableSubdomainAccess: true

seaweed-testing/zerops.yml

@@ -0,0 +1,22 @@
+zerops:
+  - setup: test
+    build:
+      base: ubuntu/go@latest
+      buildCommands:
+        - go build -ldflags "-X main.BuildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" -o app main.go
+      cache: true
+      deployFiles:
+        - app
+    run:
+      base: ubuntu@latest
+      ports:
+        - port: 8080
+          httpSupport: true
+      startCommands:
+        - name: webserver
+          command: /var/www/app
+        - name: mount
+          initCommands:
+            - sudo mkdir -p /mnt/vol
+            - sudo chown -R zerops:zerops /mnt/vol
+          command: sudo /opt/zerops/bin/weed-3-85 mount -filer node-stable-1.db.sharedstorage.zerops:8888 -dir /mnt/vol

sqlite-ls/go.mod

@@ -0,0 +1,17 @@
+module sqlite-ls
+
+go 1.25.0
+
+require modernc.org/sqlite v1.56.0
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	modernc.org/libc v1.74.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

sqlite-ls/go.sum

@@ -0,0 +1,50 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
+modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
+modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
+modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

sqlite-ls/main.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"crypto/rand"
+	"database/sql"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+var (
+	db       *sql.DB
+	hostname string
+	dbFile   string
+)
+
+func main() {
+	hostname, _ = os.Hostname()
+
+	dbPath := os.Getenv("DB_PATH")
+	if dbPath == "" {
+		dbPath = "/mnt/data/poc.db"
+	}
+	dbFile = dbPath
+
+	var err error
+	db, err = sql.Open("sqlite", "file:"+dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS entries (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		container TEXT NOT NULL,
+		msg TEXT NOT NULL,
+		created_at TEXT NOT NULL,
+		payload BLOB
+	)`); err != nil {
+		log.Fatal(err)
+	}
+	// the payload column was added after the first deploy, older DB files need it retrofitted
+	if _, err := db.Exec(`ALTER TABLE entries ADD COLUMN payload BLOB`); err != nil && !strings.Contains(err.Error(), "duplicate column") {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/", handleStatus)
+	http.HandleFunc("/write", handleWrite)
+	http.HandleFunc("/read", handleRead)
+	http.HandleFunc("/checkpoint", handleCheckpoint)
+
+	log.Printf("listening on :8080, db=%s, container=%s", dbPath, hostname)
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}
+
+func respond(w http.ResponseWriter, status int, data any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(data)
+}
+
+func handleStatus(w http.ResponseWriter, r *http.Request) {
+	var count int64
+	var journalMode string
+	if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	db.QueryRow("PRAGMA journal_mode").Scan(&journalMode)
+	fileSize := func(path string) int64 {
+		info, err := os.Stat(path)
+		if err != nil {
+			return 0
+		}
+		return info.Size()
+	}
+	respond(w, 200, map[string]any{
+		"container":    hostname,
+		"entries":      count,
+		"journalMode":  journalMode,
+		"dbSizeBytes":  fileSize(dbFile),
+		"walSizeBytes": fileSize(dbFile + "-wal"),
+	})
+}
+
+func handleWrite(w http.ResponseWriter, r *http.Request) {
+	msg := r.URL.Query().Get("msg")
+	if msg == "" {
+		msg = "hello"
+	}
+	intParam := func(key string, def, max int) int {
+		v, err := strconv.Atoi(r.URL.Query().Get(key))
+		if err != nil || v < 1 {
+			return def
+		}
+		return min(v, max)
+	}
+	n := intParam("n", 1, 10000)
+	size := intParam("size", 0, 1<<20)
+
+	tx, err := db.Begin()
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer tx.Rollback()
+	stmt, err := tx.Prepare("INSERT INTO entries (container, msg, created_at, payload) VALUES (?, ?, ?, ?)")
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer stmt.Close()
+
+	var lastId int64
+	now := time.Now().UTC().Format(time.RFC3339Nano)
+	for i := 0; i < n; i++ {
+		var payload []byte
+		if size > 0 {
+			payload = make([]byte, size)
+			rand.Read(payload)
+		}
+		res, err := stmt.Exec(hostname, msg, now, payload)
+		if err != nil {
+			respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+			return
+		}
+		lastId, _ = res.LastInsertId()
+	}
+	if err := tx.Commit(); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	respond(w, 200, map[string]any{"container": hostname, "inserted": n, "lastId": lastId})
+}
+
+func handleCheckpoint(w http.ResponseWriter, r *http.Request) {
+	var busy, logFrames, checkpointed int64
+	if err := db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	respond(w, 200, map[string]any{"container": hostname, "busy": busy, "logFrames": logFrames, "checkpointed": checkpointed})
+}
+
+func handleRead(w http.ResponseWriter, r *http.Request) {
+	var count int64
+	if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	rows, err := db.Query("SELECT id, container, msg, created_at FROM entries ORDER BY id DESC LIMIT 5")
+	if err != nil {
+		respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+		return
+	}
+	defer rows.Close()
+	type entry struct {
+		Id        int64  `json:"id"`
+		Container string `json:"container"`
+		Msg       string `json:"msg"`
+		CreatedAt string `json:"createdAt"`
+	}
+	last := []entry{}
+	for rows.Next() {
+		var e entry
+		if err := rows.Scan(&e.Id, &e.Container, &e.Msg, &e.CreatedAt); err != nil {
+			respond(w, 500, map[string]any{"container": hostname, "error": err.Error()})
+			return
+		}
+		last = append(last, e)
+	}
+	respond(w, 200, map[string]any{"container": hostname, "entries": count, "last": last})
+}

sqlite-ls/zerops-project-import.yml

@@ -0,0 +1,9 @@
+project:
+  name: sqlite-ls-poc
+services:
+  - hostname: data
+    type: local-storage:single@1
+  - hostname: api
+    type: go@1
+    minContainers: 2
+    maxContainers: 2

sqlite-ls/zerops-project2-import.yml

@@ -0,0 +1,9 @@
+project:
+  name: sqlite-ls-poc2
+services:
+  - hostname: data
+    type: local-storage:single@1
+  - hostname: api
+    type: go@1
+    minContainers: 2
+    maxContainers: 2

sqlite-ls/zerops.yml

@@ -0,0 +1,16 @@
+zerops:
+  - setup: api
+    build:
+      base: go@1
+      buildCommands:
+        - go build -o app main.go
+      deployFiles: app
+    run:
+      base: go@1
+      ports:
+        - port: 8080
+          httpSupport: true
+      volume:
+        hostname: data
+        mountPath: /mnt/data
+      start: ./app

valkey/.gitignore

@@ -1 +1,3 @@
 valkey-test
+app
+main

valkey/chaos.go

@@ -19,9 +19,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/charmbracelet/bubbles/viewport"
-	tea "github.com/charmbracelet/bubbletea"
-	"github.com/charmbracelet/lipgloss"
 	"github.com/redis/go-redis/v9"
 )
 
@@ -129,13 +126,9 @@ type outageRec struct {
 
 // ---- event sink abstraction --------------------------------------------
 //
-// Workers don't know whether their events go to an in-process Bubble Tea
-// program (local TUI) or a remote collector over HTTP. Two implementations:
-//
-//   teaSink   wraps *tea.Program — used by --chaos
-//   httpSink  batches and POSTs to a collector — used by --worker
-//
-// Both deliver the same opResultMsg / subRecvMsg / ledgerCheckMsg shapes.
+// Workers ship their events to a remote collector over HTTP via httpSink,
+// which batches and POSTs them; the collector serves the dashboard. Events
+// flow as opResultMsg / subRecvMsg / ledgerCheckMsg / eventMsg shapes.
 
 type EventSink interface {
 	OpResult(opResultMsg)
@@ -145,16 +138,6 @@ type EventSink interface {
 	Close() error
 }
 
-// ---- tea sink (local TUI) ----------------------------------------------
-
-type teaSink struct{ p *tea.Program }
-
-func (s *teaSink) OpResult(m opResultMsg)         { s.p.Send(m) }
-func (s *teaSink) SubRecv(m subRecvMsg)           { s.p.Send(m) }
-func (s *teaSink) LedgerCheck(m ledgerCheckMsg)   { s.p.Send(m) }
-func (s *teaSink) Event(m eventMsg)               { s.p.Send(m) }
-func (s *teaSink) Close() error                   { return nil }
-
 // ---- HTTP sink (Zerops worker) -----------------------------------------
 
 type httpSink struct {
@@ -564,7 +547,7 @@ func (w *worker) runSubscriber(ctx context.Context) {
 	}
 }
 
-// ---- TUI message types --------------------------------------------------
+// ---- event message types (worker → sink) -------------------------------
 
 type opResultMsg struct {
 	worker string
@@ -593,9 +576,7 @@ type eventMsg struct {
 	text  string
 }
 
-type tickMsg time.Time
-
-// ---- TUI model ----------------------------------------------------------
+// ---- shared stats types (mirrored server-side by the collector) ---------
 
 type uiStats struct {
 	role string
@@ -629,591 +610,6 @@ type eventLine struct {
 	text  string
 }
 
-type chaosModel struct {
-	workers []WorkerConfig
-	stats   map[string]*uiStats
-
-	events  []eventLine
-	eventVP viewport.Model
-
-	p95Series        []float64
-	okPerSecSeries   []float64
-	failPerSecSeries []float64
-
-	globalLat     []time.Duration
-	prevTotalOps  int64
-	prevTotalFail int64
-
-	width, height int
-	startAt       time.Time
-	quitting      bool
-}
-
-func newChaosModel(ws []WorkerConfig, startAt time.Time) *chaosModel {
-	stats := make(map[string]*uiStats, len(ws))
-	for _, w := range ws {
-		stats[w.Name] = &uiStats{
-			role: w.Role,
-			addr: fmt.Sprintf("%s:%d", w.Host, w.Port),
-		}
-	}
-	vp := viewport.New(80, 8)
-	vp.SetContent(dimStyle.Render("(no events yet)"))
-	return &chaosModel{
-		workers: ws,
-		stats:   stats,
-		eventVP: vp,
-		startAt: startAt,
-	}
-}
-
-func (m *chaosModel) Init() tea.Cmd {
-	return tea.Tick(tickInterval, func(t time.Time) tea.Msg { return tickMsg(t) })
-}
-
-func (m *chaosModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
-	switch msg := msg.(type) {
-	case tea.WindowSizeMsg:
-		m.width = msg.Width
-		m.height = msg.Height
-		m.recomputeLayout()
-		return m, nil
-	case tea.KeyMsg:
-		switch msg.String() {
-		case "q", "ctrl+c", "esc":
-			m.quitting = true
-			return m, tea.Quit
-		case "up", "k":
-			m.eventVP.LineUp(1)
-		case "down", "j":
-			m.eventVP.LineDown(1)
-		case "G", "end":
-			m.eventVP.GotoBottom()
-		case "g", "home":
-			m.eventVP.GotoTop()
-		case "pgup", "b":
-			m.eventVP.HalfViewUp()
-		case "pgdown", "f", " ":
-			m.eventVP.HalfViewDown()
-		}
-		return m, nil
-	case opResultMsg:
-		m.applyOpResult(msg)
-		return m, nil
-	case subRecvMsg:
-		m.applySubRecv(msg)
-		return m, nil
-	case ledgerCheckMsg:
-		m.applyLedgerCheck(msg)
-		return m, nil
-	case eventMsg:
-		m.appendEvent(msg.level, msg.at, msg.text)
-		return m, nil
-	case tickMsg:
-		m.onTick()
-		return m, tea.Tick(tickInterval, func(t time.Time) tea.Msg { return tickMsg(t) })
-	}
-	return m, nil
-}
-
-func (m *chaosModel) applyOpResult(msg opResultMsg) {
-	s, ok := m.stats[msg.worker]
-	if !ok {
-		return
-	}
-	s.ops++
-	if msg.err != nil {
-		s.fail++
-		s.consecFail++
-		if !s.inOutage {
-			s.inOutage = true
-			s.outageStart = msg.at
-			s.outageOpsLost = 1
-			m.appendEvent("err", msg.at,
-				fmt.Sprintf("[%s] OUTAGE START — %s: %s", msg.worker, msg.op, trim(msg.err.Error(), 80)))
-		} else {
-			s.outageOpsLost++
-		}
-		return
-	}
-	s.ok++
-	if s.inOutage {
-		d := msg.at.Sub(s.outageStart)
-		s.outages = append(s.outages, outageRec{
-			start: s.outageStart, end: msg.at, opsLost: s.outageOpsLost,
-		})
-		m.appendEvent("info", msg.at,
-			fmt.Sprintf("[%s] OUTAGE END — duration=%s ops_lost=%d",
-				msg.worker, d.Round(time.Millisecond), s.outageOpsLost))
-		s.inOutage = false
-		s.consecFail = 0
-		s.outageOpsLost = 0
-	}
-	if msg.dur > 0 {
-		s.recentLat = appendBoundedDur(s.recentLat, msg.dur, maxRecentLat)
-		m.globalLat = appendBoundedDur(m.globalLat, msg.dur, maxGlobalLat)
-	}
-}
-
-func (m *chaosModel) applySubRecv(msg subRecvMsg) {
-	s, ok := m.stats[msg.worker]
-	if !ok {
-		return
-	}
-	s.msgs++
-	if s.lastSeq != 0 && msg.seq > s.lastSeq+1 {
-		gap := msg.seq - s.lastSeq - 1
-		s.gaps += gap
-		m.appendEvent("warn", msg.at,
-			fmt.Sprintf("[%s] PUBSUB GAP last=%d now=%d missed=%d",
-				msg.worker, s.lastSeq, msg.seq, gap))
-	}
-	if !s.lastSeqAt.IsZero() {
-		if d := msg.at.Sub(s.lastSeqAt); d > s.longestGap {
-			s.longestGap = d
-		}
-	}
-	s.lastSeq = msg.seq
-	s.lastSeqAt = msg.at
-}
-
-func (m *chaosModel) applyLedgerCheck(msg ledgerCheckMsg) {
-	s, ok := m.stats[msg.worker]
-	if !ok {
-		return
-	}
-	switch msg.kind {
-	case "ok":
-		s.verified++
-	case "missing":
-		s.missing++
-		m.appendEvent("err", time.Now(),
-			fmt.Sprintf("[%s] LEDGER MISSING seq=%d  %s", msg.worker, msg.seq, msg.detail))
-	case "mismatch":
-		s.mismatch++
-		m.appendEvent("err", time.Now(),
-			fmt.Sprintf("[%s] LEDGER MISMATCH seq=%d  %s", msg.worker, msg.seq, msg.detail))
-	}
-}
-
-func (m *chaosModel) appendEvent(level string, at time.Time, text string) {
-	if len(m.events) >= maxEvents {
-		copy(m.events, m.events[1:])
-		m.events = m.events[:len(m.events)-1]
-	}
-	m.events = append(m.events, eventLine{at: at, level: level, text: text})
-	atBottom := m.eventVP.AtBottom()
-	m.eventVP.SetContent(m.renderEventContent())
-	if atBottom {
-		m.eventVP.GotoBottom()
-	}
-}
-
-func (m *chaosModel) onTick() {
-	if len(m.globalLat) > 0 {
-		cp := make([]time.Duration, len(m.globalLat))
-		copy(cp, m.globalLat)
-		sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] })
-		idx := 95 * len(cp) / 100
-		if idx >= len(cp) {
-			idx = len(cp) - 1
-		}
-		p95ms := float64(cp[idx]) / float64(time.Millisecond)
-		m.p95Series = appendBoundedFloat(m.p95Series, p95ms, maxSeriesPoints)
-	} else {
-		m.p95Series = appendBoundedFloat(m.p95Series, 0, maxSeriesPoints)
-	}
-
-	var totalOps, totalFail int64
-	for _, s := range m.stats {
-		totalOps += s.ops + s.msgs
-		totalFail += s.fail
-	}
-	okDelta := (totalOps - totalFail) - (m.prevTotalOps - m.prevTotalFail)
-	if okDelta < 0 {
-		okDelta = 0
-	}
-	failDelta := totalFail - m.prevTotalFail
-	if failDelta < 0 {
-		failDelta = 0
-	}
-	m.prevTotalOps = totalOps
-	m.prevTotalFail = totalFail
-	m.okPerSecSeries = appendBoundedFloat(m.okPerSecSeries, float64(okDelta), maxSeriesPoints)
-	m.failPerSecSeries = appendBoundedFloat(m.failPerSecSeries, float64(failDelta), maxSeriesPoints)
-
-	for _, s := range m.stats {
-		current := s.ops + s.msgs
-		delta := current - s.prevOps
-		if delta < 0 {
-			delta = 0
-		}
-		s.prevOps = current
-		s.opsHist = appendBoundedFloat(s.opsHist, float64(delta), maxSparkSamples)
-	}
-}
-
-func (m *chaosModel) recomputeLayout() {
-	headerH := 1
-	tableTitleH := 1
-	tableRows := len(m.workers) + 2
-	chartTitleH := 1
-	chartH := 8
-	chartFooterH := 1
-	eventTitleH := 1
-	footerH := 1
-	used := headerH + tableTitleH + tableRows + chartTitleH + chartH + chartFooterH + eventTitleH + footerH + 2
-	eventH := m.height - used
-	if eventH < 4 {
-		eventH = 4
-	}
-	w := m.width - 2
-	if w < 20 {
-		w = 20
-	}
-	m.eventVP.Width = w
-	m.eventVP.Height = eventH
-	m.eventVP.SetContent(m.renderEventContent())
-}
-
-// ---- TUI rendering ------------------------------------------------------
-
-var (
-	titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("212"))
-	headStyle  = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("39"))
-	dimStyle   = lipgloss.NewStyle().Foreground(lipgloss.Color("241"))
-	okStyle    = lipgloss.NewStyle().Foreground(lipgloss.Color("42"))
-	warnStyle  = lipgloss.NewStyle().Foreground(lipgloss.Color("214"))
-	errStyle   = lipgloss.NewStyle().Foreground(lipgloss.Color("196"))
-	chartStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("141"))
-)
-
-func (m *chaosModel) View() string {
-	if m.quitting {
-		return "stopping…\n"
-	}
-	if m.width == 0 || m.height == 0 {
-		return "loading…\n"
-	}
-	if m.width < 90 || m.height < 25 {
-		return fmt.Sprintf("terminal too small: %dx%d (need 90x25 minimum). press q to quit.\n",
-			m.width, m.height)
-	}
-
-	header := m.renderHeader()
-	table := m.renderTable()
-	charts := m.renderCharts()
-	events := m.renderEvents()
-	footer := dimStyle.Render(" q quit · ↑/↓ scroll · pgup/pgdn half-page · g/G top/bottom")
-
-	return strings.Join([]string{header, "", table, "", charts, "", events, footer}, "\n")
-}
-
-func (m *chaosModel) renderHeader() string {
-	elapsed := time.Since(m.startAt).Round(time.Second)
-	var inOutage int
-	for _, s := range m.stats {
-		if s.inOutage {
-			inOutage++
-		}
-	}
-	tag := okStyle.Render("healthy")
-	if inOutage > 0 {
-		tag = errStyle.Render(fmt.Sprintf("%d worker(s) in outage", inOutage))
-	}
-	return titleStyle.Render(" valkey-ha chaos") +
-		dimStyle.Render(fmt.Sprintf("  ·  %d workers  ·  elapsed %s  ·  ", len(m.workers), elapsed)) +
-		tag
-}
-
-func (m *chaosModel) renderTable() string {
-	var b strings.Builder
-	b.WriteString(headStyle.Render(" workers") + "\n")
-	hdr := fmt.Sprintf("  %-14s %-11s %7s %7s %6s %7s %8s %8s  %-12s %s",
-		"NAME", "ROLE", "OPS", "OK", "FAIL", "UPTIME", "P50", "P95", "OPS/SEC", "STATUS")
-	b.WriteString(dimStyle.Render(hdr) + "\n")
-
-	for _, w := range m.workers {
-		s := m.stats[w.Name]
-		switch s.role {
-		case "subscriber":
-			line := fmt.Sprintf("  %-14s %-11s msgs=%-7d gaps=%-3d longest_gap=%-7s   %s",
-				w.Name, "subscriber", s.msgs, s.gaps,
-				shortDur(s.longestGap), m.statusBadge(s))
-			b.WriteString(line + "\n")
-		default:
-			uptime := 100.0
-			if s.ops > 0 {
-				uptime = float64(s.ok) / float64(s.ops) * 100
-			}
-			p50 := percentileLat(s.recentLat, 50)
-			p95 := percentileLat(s.recentLat, 95)
-			spark := chartStyle.Render(sparkline(s.opsHist, 12))
-			extra := ""
-			if s.role == "reader" {
-				extra = dimStyle.Render(fmt.Sprintf(" v=%d miss=%d mis=%d",
-					s.verified, s.missing, s.mismatch))
-			}
-			line := fmt.Sprintf("  %-14s %-11s %7d %7d %6d %6.2f%% %8s %8s  %s  %s%s",
-				w.Name, s.role, s.ops, s.ok, s.fail, uptime,
-				shortDur(p50), shortDur(p95),
-				spark, m.statusBadge(s), extra)
-			b.WriteString(line + "\n")
-		}
-	}
-	return strings.TrimRight(b.String(), "\n")
-}
-
-func (m *chaosModel) statusBadge(s *uiStats) string {
-	if s.inOutage {
-		return errStyle.Render("OUTAGE")
-	}
-	if s.role == "subscriber" && s.msgs == 0 && time.Since(m.startAt) > 5*time.Second {
-		return warnStyle.Render("NO MSGS")
-	}
-	return okStyle.Render("OK")
-}
-
-func (m *chaosModel) renderCharts() string {
-	chartH := 8
-	chartW := (m.width - 4) / 3 // 3 charts with 2× "  " separators between
-	if chartW < 20 {
-		chartW = 20
-	}
-
-	p95Max := maxFloat(m.p95Series)
-	if p95Max < 1 {
-		p95Max = 1
-	}
-	p95Lines := renderBarChart(m.p95Series, chartW, chartH, p95Max)
-	p95Title := headStyle.Render(" p95 latency") +
-		dimStyle.Render(fmt.Sprintf("  peak %.1fms · last %s",
-			p95Max, lastFloatLabel(m.p95Series, "ms")))
-	p95Box := p95Title + "\n" + chartStyle.Render(strings.Join(p95Lines, "\n"))
-
-	tpMax := maxFloat(m.okPerSecSeries)
-	if tpMax < 5 {
-		tpMax = 5
-	}
-	tpLines := renderBarChart(m.okPerSecSeries, chartW, chartH, tpMax)
-	tpTitle := headStyle.Render(" ok ops/sec") +
-		dimStyle.Render(fmt.Sprintf("  peak %.0f · last %s",
-			tpMax, lastFloatLabel(m.okPerSecSeries, "")))
-	tpBox := tpTitle + "\n" + chartStyle.Render(strings.Join(tpLines, "\n"))
-
-	failMax := maxFloat(m.failPerSecSeries)
-	if failMax < 1 {
-		failMax = 1
-	}
-	failLines := renderBarChart(m.failPerSecSeries, chartW, chartH, failMax)
-	failTitle := headStyle.Render(" fail ops/sec") +
-		dimStyle.Render(fmt.Sprintf("  peak %.0f · last %s",
-			failMax, lastFloatLabel(m.failPerSecSeries, "")))
-	failBox := failTitle + "\n" + errStyle.Render(strings.Join(failLines, "\n"))
-
-	return lipgloss.JoinHorizontal(lipgloss.Top, p95Box, "  ", tpBox, "  ", failBox)
-}
-
-func (m *chaosModel) renderEvents() string {
-	title := headStyle.Render(" events") +
-		dimStyle.Render(fmt.Sprintf("  (%d, scroll with ↑/↓)", len(m.events)))
-	return title + "\n" + m.eventVP.View()
-}
-
-func (m *chaosModel) renderEventContent() string {
-	if len(m.events) == 0 {
-		return dimStyle.Render("(no events yet)")
-	}
-	var b strings.Builder
-	for _, e := range m.events {
-		ts := dimStyle.Render(e.at.Format("15:04:05.000"))
-		var styled string
-		switch e.level {
-		case "err":
-			styled = errStyle.Render(e.text)
-		case "warn":
-			styled = warnStyle.Render(e.text)
-		default:
-			styled = e.text
-		}
-		b.WriteString(ts + " " + styled + "\n")
-	}
-	return strings.TrimRight(b.String(), "\n")
-}
-
-// ---- chart primitives ---------------------------------------------------
-
-var sparkChars = []rune(" ▁▂▃▄▅▆▇█")
-
-func sparkline(values []float64, width int) string {
-	if width <= 0 {
-		return ""
-	}
-	if len(values) == 0 {
-		return strings.Repeat(" ", width)
-	}
-	maxV := 0.0
-	for _, v := range values {
-		if v > maxV {
-			maxV = v
-		}
-	}
-	var b strings.Builder
-	n := len(values)
-	for col := 0; col < width; col++ {
-		if maxV == 0 {
-			b.WriteRune(' ')
-			continue
-		}
-		idx := col * n / width
-		if idx >= n {
-			idx = n - 1
-		}
-		v := values[idx]
-		if v < 0 {
-			v = 0
-		}
-		if v > maxV {
-			v = maxV
-		}
-		i := int(v / maxV * float64(len(sparkChars)-1))
-		if i < 0 {
-			i = 0
-		}
-		if i >= len(sparkChars) {
-			i = len(sparkChars) - 1
-		}
-		b.WriteRune(sparkChars[i])
-	}
-	return b.String()
-}
-
-func renderBarChart(values []float64, width, height int, maxV float64) []string {
-	rows := make([][]rune, height)
-	for i := range rows {
-		rows[i] = []rune(strings.Repeat(" ", width))
-	}
-	flush := func() []string {
-		out := make([]string, height)
-		for i := range rows {
-			out[i] = string(rows[i])
-		}
-		return out
-	}
-	if width <= 0 || height <= 0 || maxV <= 0 || len(values) == 0 {
-		return flush()
-	}
-	n := len(values)
-	subSteps := []rune(" ▁▂▃▄▅▆▇")
-	for col := 0; col < width; col++ {
-		idx := col * n / width
-		if idx >= n {
-			idx = n - 1
-		}
-		v := values[idx]
-		if v < 0 {
-			v = 0
-		}
-		if v > maxV {
-			v = maxV
-		}
-		cells := int(v / maxV * float64(height) * 8)
-		full := cells / 8
-		rem := cells % 8
-		for r := 0; r < full && r < height; r++ {
-			rows[height-1-r][col] = '█'
-		}
-		if full < height && rem > 0 {
-			rows[height-1-full][col] = subSteps[rem]
-		}
-	}
-	return flush()
-}
-
-func lastFloatLabel(s []float64, unit string) string {
-	if len(s) == 0 {
-		return "-"
-	}
-	v := s[len(s)-1]
-	if unit == "ms" {
-		return fmt.Sprintf("%.1f%s", v, unit)
-	}
-	return fmt.Sprintf("%.0f%s", v, unit)
-}
-
-// ---- runChaos (local TUI) ----------------------------------------------
-
-func runChaos(ws []WorkerConfig) int {
-	for _, w := range ws {
-		if err := validateWorker(w); err != nil {
-			fmt.Fprintf(os.Stderr, "worker %q: %v\n", w.Name, err)
-			return 2
-		}
-	}
-
-	logFile, ferr := os.Create("chaos-debug.log")
-	if ferr == nil {
-		defer logFile.Close()
-		log.SetOutput(logFile)
-	} else {
-		log.SetOutput(io.Discard)
-	}
-	defer log.SetOutput(os.Stderr)
-
-	workerCtx, cancel := context.WithCancel(context.Background())
-
-	cc := &chaosCtx{
-		ledger:  newLedger(),
-		startAt: time.Now(),
-	}
-
-	workers := buildWorkers(ws, cc)
-	defer func() {
-		for _, w := range workers {
-			_ = w.rdb.Close()
-		}
-	}()
-
-	model := newChaosModel(ws, cc.startAt)
-	p := tea.NewProgram(model, tea.WithAltScreen())
-	cc.sink = &teaSink{p: p}
-
-	sigCh := make(chan os.Signal, 1)
-	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		select {
-		case <-sigCh:
-			p.Quit()
-		case <-workerCtx.Done():
-		}
-	}()
-
-	var wg sync.WaitGroup
-	for _, w := range workers {
-		wg.Add(1)
-		go func(w *worker) {
-			defer wg.Done()
-			w.run(workerCtx)
-		}(w)
-	}
-
-	if _, err := p.Run(); err != nil {
-		fmt.Fprintf(os.Stderr, "tui error: %v\n", err)
-	}
-
-	cancel()
-	waitWorkers(&wg, 5*time.Second)
-
-	fmt.Println("\nrunning final durability sweep…")
-	fd := finalSweep(workers, cc)
-	printFinalReport(model, time.Since(cc.startAt), fd)
-
-	if fd.missing > 0 || fd.mismatch > 0 {
-		return 1
-	}
-	return 0
-}
-
 // ---- runWorker (Zerops worker container) -------------------------------
 
 func runWorker(ws []WorkerConfig, collectorURL string) int {
@@ -1283,7 +679,7 @@ func runWorker(ws []WorkerConfig, collectorURL string) int {
 func buildWorkers(ws []WorkerConfig, cc *chaosCtx) []*worker {
 	workers := make([]*worker, 0, len(ws))
 	for _, wc := range ws {
-		opts := optionsFromConn(wc.Host, wc.Port, wc.Password, wc.DB, wc.TLS)
+		opts := optionsFromConn(wc.Host, wc.Port, wc.Password, wc.DB, wc.TLS, "")
 		w := &worker{cfg: wc, rdb: redis.NewClient(opts), cc: cc}
 		if wc.Role == "bloater" && wc.ValueBytes > 0 {
 			w.bloatBuf = make([]byte, wc.ValueBytes)
@@ -1369,84 +765,6 @@ func finalSweep(workers []*worker, cc *chaosCtx) finalDur {
 	return fd
 }
 
-func printFinalReport(m *chaosModel, dur time.Duration, fd finalDur) {
-	fmt.Println("=== valkey-ha chaos report ===")
-	fmt.Printf("duration: %s\n\n", dur.Round(time.Millisecond))
-
-	fmt.Println("per-worker:")
-	for _, w := range m.workers {
-		s := m.stats[w.Name]
-		switch s.role {
-		case "subscriber":
-			fmt.Printf("  %-14s [sub]   msgs=%d gaps=%d longest_gap=%s\n",
-				w.Name, s.msgs, s.gaps, s.longestGap.Round(time.Millisecond))
-		default:
-			uptime := 100.0
-			if s.ops > 0 {
-				uptime = float64(s.ok) / float64(s.ops) * 100
-			}
-			var longest time.Duration
-			for _, o := range s.outages {
-				if d := o.end.Sub(o.start); d > longest {
-					longest = d
-				}
-			}
-			extra := ""
-			if s.role == "reader" {
-				extra = fmt.Sprintf(" verified=%d missing=%d mismatch=%d",
-					s.verified, s.missing, s.mismatch)
-			}
-			fmt.Printf("  %-14s [%-10s] ops=%d ok=%d fail=%d uptime=%.2f%% p50=%s p95=%s outages=%d longest=%s%s\n",
-				w.Name, s.role, s.ops, s.ok, s.fail, uptime,
-				shortDur(percentileLat(s.recentLat, 50)),
-				shortDur(percentileLat(s.recentLat, 95)),
-				len(s.outages), longest.Round(time.Millisecond),
-				extra)
-		}
-	}
-
-	fmt.Println("\ndurability:")
-	fmt.Printf("  ledger:  acked=%d verified=%d missing=%d mismatch=%d\n",
-		fd.acked, fd.verified, fd.missing, fd.mismatch)
-	if len(fd.missingSeqs) > 0 {
-		n := 20
-		if len(fd.missingSeqs) < n {
-			n = len(fd.missingSeqs)
-		}
-		fmt.Printf("           missing seqs (first %d): %v\n", n, fd.missingSeqs[:n])
-	}
-	delta := fd.serverCounter - fd.localAcked
-	fmt.Printf("  counter: local_acked=%d server_value=%d delta=%d\n",
-		fd.localAcked, fd.serverCounter, delta)
-
-	type tagged struct {
-		w   string
-		rec outageRec
-	}
-	var all []tagged
-	for _, w := range m.workers {
-		for _, o := range m.stats[w.Name].outages {
-			all = append(all, tagged{w: w.Name, rec: o})
-		}
-	}
-	sort.Slice(all, func(i, j int) bool { return all[i].rec.start.Before(all[j].rec.start) })
-	if len(all) > 0 {
-		fmt.Println("\noutages (chronological):")
-		for _, t := range all {
-			fmt.Printf("  %s -> %s (%s)  worker=%s ops_lost=%d\n",
-				t.rec.start.Format("15:04:05.000"),
-				t.rec.end.Format("15:04:05.000"),
-				t.rec.end.Sub(t.rec.start).Round(time.Millisecond),
-				t.w, t.rec.opsLost)
-		}
-	}
-
-	fmt.Println("\nnotes:")
-	fmt.Println("  - Valkey 7.2 + Sentinel does NOT replicate pub/sub from master to replicas.")
-	fmt.Println("    A subscriber on the read-only VIP receiving 0 messages is expected.")
-	fmt.Println("  - debug log: ./chaos-debug.log")
-}
-
 // ---- helpers ------------------------------------------------------------
 
 func appendBoundedDur(s []time.Duration, v time.Duration, max int) []time.Duration {
@@ -1496,22 +814,6 @@ func maxDur(a, b time.Duration) time.Duration {
 	return b
 }
 
-func shortDur(d time.Duration) string {
-	if d <= 0 {
-		return "-"
-	}
-	if d < time.Microsecond {
-		return "0"
-	}
-	if d < time.Millisecond {
-		return fmt.Sprintf("%dµs", d.Microseconds())
-	}
-	if d < time.Second {
-		return fmt.Sprintf("%.1fms", float64(d)/float64(time.Millisecond))
-	}
-	return d.Round(time.Millisecond).String()
-}
-
 func trim(s string, n int) string {
 	if len(s) <= n {
 		return s

valkey/config-sea-ha.yaml

@@ -0,0 +1,89 @@
+# Chaos workload for sea1, run against the HA valkeys.
+#
+# Ports (per Zerops valkey:ha): 6379 = master plaintext, 7000 = replica
+# plaintext. NOTE: the TLS ports (6380/7001) require mutual TLS (a client
+# certificate) on these migration valkeys — the tester only does server-side
+# trust (--tls-ca), so we use the plaintext ports with password auth, the same
+# path the seeding used.
+#
+# Auth: per-service password via ${valkeyhaN_password} (wired in zerops.yaml
+# setup `chaosworker-sea`).
+#
+# Targeting rationale:
+#   valkeyha3 (2 GiB) — full mix incl. write-heavy roles. The durability
+#                       ledger (vk:chaos:ledger:*) is written with NO TTL and
+#                       grows over time, so writes go only where there's RAM
+#                       headroom.
+#   valkeyha1/2 (0.5 GiB) — pub/sub only: exercises the master ingress and
+#                       pub/sub path with zero key growth (and no cross-host
+#                       read-after-write misses, since readers verify keys the
+#                       same container wrote).
+workers:
+  # ---- valkeyha3: full mix --------------------------------------------------
+  - name: rw-ha3
+    role: readwriter
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${valkeyha3_password}
+    interval: 10ms
+  - name: wr-ha3
+    role: writer
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${valkeyha3_password}
+    interval: 5ms
+  - name: rom-ha3
+    role: reader
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${valkeyha3_password}
+    interval: 5ms
+  - name: ror-ha3
+    role: reader
+    host: valkeyha3.zerops
+    port: 7000
+    password: ${valkeyha3_password}
+    interval: 3ms
+  - name: pub-ha3
+    role: publisher
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${valkeyha3_password}
+    channel: "vk:chaos:ha3"
+    interval: 5ms
+  - name: sub-ha3
+    role: subscriber
+    host: valkeyha3.zerops
+    port: 6379
+    password: ${valkeyha3_password}
+    channel: "vk:chaos:ha3"
+
+  # ---- valkeyha1: pub/sub only ---------------------------------------------
+  - name: pub-ha1
+    role: publisher
+    host: valkeyha1.zerops
+    port: 6379
+    password: ${valkeyha1_password}
+    channel: "vk:chaos:ha1"
+    interval: 5ms
+  - name: sub-ha1
+    role: subscriber
+    host: valkeyha1.zerops
+    port: 6379
+    password: ${valkeyha1_password}
+    channel: "vk:chaos:ha1"
+
+  # ---- valkeyha2: pub/sub only ---------------------------------------------
+  - name: pub-ha2
+    role: publisher
+    host: valkeyha2.zerops
+    port: 6379
+    password: ${valkeyha2_password}
+    channel: "vk:chaos:ha2"
+    interval: 5ms
+  - name: sub-ha2
+    role: subscriber
+    host: valkeyha2.zerops
+    port: 6379
+    password: ${valkeyha2_password}
+    channel: "vk:chaos:ha2"

valkey/go.mod

@@ -3,34 +3,11 @@ module valkey-test
 go 1.24.2
 
 require (
-	github.com/charmbracelet/bubbles v1.0.0
-	github.com/charmbracelet/bubbletea v1.3.10
-	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/redis/go-redis/v9 v9.7.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/charmbracelet/colorprofile v0.4.1 // indirect
-	github.com/charmbracelet/x/ansi v0.11.6 // indirect
-	github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
-	github.com/charmbracelet/x/term v0.2.2 // indirect
-	github.com/clipperhouse/displaywidth v0.9.0 // indirect
-	github.com/clipperhouse/stringish v0.1.1 // indirect
-	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
-	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-localereader v0.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.19 // indirect
-	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
-	github.com/muesli/cancelreader v0.2.2 // indirect
-	github.com/muesli/termenv v0.16.0 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.3.8 // indirect
 )

valkey/go.sum

@@ -1,63 +1,13 @@
-github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
-github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
-github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E=
-github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
-github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
-github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
-github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
-github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
-github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
-github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
-github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
-github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
-github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
-github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
-github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
-github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
-github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
-github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
-github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
-github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U=
-github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
-github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
-github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
-github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
-github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
-github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
-github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
-github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
-github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
-github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
-github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
-github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
-github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
 github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
-github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

valkey/main.go

@@ -139,29 +139,65 @@ func applyFlagOverrides(cfg *Config, host string, port int, password string, tls
 	return nil
 }
 
-func (c *Config) primary() (*redis.Options, error) {
-	if c.Host != "" && c.Port != 0 {
-		return optionsFromConn(c.Host, c.Port, c.Password, c.DB, c.TLS), nil
-	}
-	if len(c.Workers) > 0 {
-		w := c.Workers[0]
-		return optionsFromConn(w.Host, w.Port, w.Password, w.DB, w.TLS), nil
-	}
-	return nil, fmt.Errorf("config has neither top-level host nor workers")
-}
-
-func optionsFromConn(host string, port int, password string, db int, useTLS bool) *redis.Options {
+func optionsFromConn(host string, port int, password string, db int, useTLS bool, tlsServerName string) *redis.Options {
 	opts := &redis.Options{
 		Addr:     fmt.Sprintf("%s:%d", host, port),
 		Password: password,
 		DB:       db,
 	}
 	if useTLS {
-		opts.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: tlsRootCAs}
+		opts.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: tlsRootCAs, ServerName: tlsServerName}
 	}
 	return opts
 }
 
+// buildConnOptions resolves the Valkey connection for the --seed/--verify/
+// --flush/--inspect modes from either a --conn string or the override flags.
+// A conn string carries host/port/password/db/TLS, so it stands in for the
+// override flags; redis.ParseURL maps rediss:// to a TLS config. --tls-ca is
+// honoured in both paths so a local run can trust a non-system CA. --tls-servername
+// overrides the name TLS verifies against, which is needed when dialing a Zerops
+// public-port IP whose cert only carries internal DNS SANs (e.g. valkey1.zerops).
+func buildConnOptions(conn, host string, port int, password string, tlsEnable bool, tlsCAPath, tlsServerName string) (*redis.Options, error) {
+	if conn != "" {
+		parsed, err := redis.ParseURL(conn)
+		if err != nil {
+			return nil, fmt.Errorf("--conn parse error: %w", err)
+		}
+		if tlsCAPath != "" || tlsServerName != "" {
+			if parsed.TLSConfig == nil {
+				parsed.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12}
+			}
+		}
+		if tlsCAPath != "" {
+			pem, err := os.ReadFile(tlsCAPath)
+			if err != nil {
+				return nil, fmt.Errorf("read tls-ca: %w", err)
+			}
+			pool := x509.NewCertPool()
+			if !pool.AppendCertsFromPEM(pem) {
+				return nil, fmt.Errorf("tls-ca: no certificates parsed from %s", tlsCAPath)
+			}
+			parsed.TLSConfig.RootCAs = pool
+		}
+		if tlsServerName != "" {
+			parsed.TLSConfig.ServerName = tlsServerName
+		}
+		return parsed, nil
+	}
+	cfg := &Config{}
+	if err := applyFlagOverrides(cfg, host, port, password, tlsEnable, tlsCAPath); err != nil {
+		return nil, fmt.Errorf("flag override error: %w", err)
+	}
+	if cfg.Host == "" {
+		return nil, fmt.Errorf("requires --host (e.g. --host valkey.zerops over the zcli VPN) or --conn")
+	}
+	if cfg.Port == 0 {
+		cfg.Port = 6379
+	}
+	return optionsFromConn(cfg.Host, cfg.Port, cfg.Password, cfg.DB, cfg.TLS, tlsServerName), nil
+}
+
 func redactPassword(pw string) string {
 	if pw == "" {
 		return "(none)"
@@ -179,41 +215,46 @@ func envInt(name string, def int) int {
 }
 
 func main() {
-	configPath := flag.String("config", "config.yaml", "path to YAML config file")
-	continuous := flag.Bool("continuous", false, "single-worker rolling smoke loop")
-	chaosLocal := flag.Bool("chaos", false, "multi-worker chaos TUI (local, in-process)")
-	workerMode := flag.Bool("worker", false, "distributed worker — runs the chaos workload and ships events to --collector-url")
-	collectorMode := flag.Bool("collector", false, "distributed collector — HTTP dashboard that ingests events from workers")
+	configPath := flag.String("config", "config.yaml", "path to YAML config file (chaos --worker/--collector)")
+	workerMode := flag.Bool("worker", false, "chaos worker — runs the config.yaml workload and ships events to --collector-url")
+	collectorMode := flag.Bool("collector", false, "chaos collector — HTTP dashboard that ingests events from workers")
 	collectorURL := flag.String("collector-url", os.Getenv("COLLECTOR_URL"), "collector base URL (worker mode); env: COLLECTOR_URL")
 	port := flag.Int("collector-port", envInt("PORT", 8080), "HTTP port (collector mode); env: PORT")
-	interval := flag.Duration("interval", time.Second, "tick interval for --continuous")
-	hostOverride := flag.String("host", "", "override host for all workers (local testing)")
+	hostOverride := flag.String("host", "", "override host for the cli tool (--seed/--verify/etc and the default test suite)")
 	portOverride := flag.Int("port", 0, "override port for all workers (local testing)")
 	passwordOverride := flag.String("password", "", "override password for all workers (local testing)")
 	tlsEnable := flag.Bool("tls", false, "enable TLS for all workers using system root CAs (local testing)")
 	tlsCAPath := flag.String("tls-ca", "", "path to TLS CA cert PEM; enables TLS for all workers (local testing)")
+	tlsServerName := flag.String("tls-servername", "", "override the hostname TLS verifies against (e.g. valkey1.zerops) when dialing a Zerops public-port IP whose cert has no IP SANs")
+	seedConn := flag.String("conn", "", "Valkey connection string for the cli tool (default test suite, --watch, --seed/--verify/--flush/--inspect) — e.g. redis://default:pw@valkey1:6379 or rediss://...:6380 for TLS; overrides --host/--port/--password/--tls")
+	watchMode := flag.Bool("watch", false, "PING the target on --interval until Ctrl+C, printing each outage's start/recovery/duration and a summary on exit")
+	watchInterval := flag.Duration("interval", 500*time.Millisecond, "probe interval for --watch")
+	seedMode := flag.Bool("seed", false, "bulk-load one Valkey with deterministic data (local; use --host valkey.zerops over zcli VPN)")
+	verifyMode := flag.Bool("verify", false, "re-read seed:* keys and check they're 1:1 with the seed (same --seed-* flags)")
+	flushMode := flag.Bool("flush", false, "FLUSHDB the selected --db (clears the whole database)")
+	inspectMode := flag.Bool("inspect", false, "print every database and its key count")
+	seedMB := flag.Int("seed-mb", 500, "total payload in --seed/--verify mode, in MiB")
+	seedValueBytes := flag.Int("seed-value-bytes", 10*1024, "size of each value in --seed/--verify mode, in bytes")
+	seedBatch := flag.Int("seed-batch", 1000, "keys per pipeline batch in --seed/--verify mode")
+	seedSeed := flag.Uint64("seed-seed", 1, "PRNG seed for value generation; must match between --seed and --verify")
+	seedThrottleMB := flag.Float64("throttle-mb", 0, "throttle --seed write rate in MiB/s (0 = unlimited)")
+	dbOverride := flag.Int("db", -1, "Valkey DB number for --seed/--verify/--flush/--inspect (-1 = use --conn/config default)")
+	seedTimeout := flag.Duration("timeout", 5*time.Second, "dial/read/write timeout in --seed/--verify/--flush/--inspect mode (raise for slow VPN links)")
 	flag.Parse()
 
+	// ── Regime 1: chaos ──────────────────────────────────────────────────
+	// Driven solely by config.yaml. loadConfig applies the env templating that
+	// lets one config file drive the distributed fleet: ${var} expansion pulls
+	// secrets from the container env, and $ZEROPS_Number gives each replica a
+	// distinct DB + pub/sub channel suffix. No connection-flag overrides here.
 	switch {
 	case *collectorMode:
 		os.Exit(runCollector(*port))
-	}
-
-	cfg, err := loadConfig(*configPath)
-	if err != nil {
-		log.Fatalf("config error: %v", err)
-	}
-	if err := applyFlagOverrides(cfg, *hostOverride, *portOverride, *passwordOverride, *tlsEnable, *tlsCAPath); err != nil {
-		log.Fatalf("flag override error: %v", err)
-	}
-
-	switch {
-	case *chaosLocal:
-		if len(cfg.Workers) == 0 {
-			log.Fatalf("--chaos requires workers: list in %s", *configPath)
-		}
-		os.Exit(runChaos(cfg.Workers))
 	case *workerMode:
+		cfg, err := loadConfig(*configPath)
+		if err != nil {
+			log.Fatalf("config error: %v", err)
+		}
 		if len(cfg.Workers) == 0 {
 			log.Fatalf("--worker requires workers: list in %s", *configPath)
 		}
@@ -223,20 +264,146 @@ func main() {
 		os.Exit(runWorker(cfg.Workers, *collectorURL))
 	}
 
-	primary, err := cfg.primary()
+	// ── Regime 2: cli tool ───────────────────────────────────────────────
+	// One-time commands against a single Valkey, resolved from --conn or the
+	// split --host/--port/--password/--tls[-ca] flags. Covers the default test
+	// suite plus --seed/--verify/--flush/--inspect.
+	opts, err := buildConnOptions(*seedConn, *hostOverride, *portOverride, *passwordOverride, *tlsEnable, *tlsCAPath, *tlsServerName)
 	if err != nil {
-		log.Fatalf("config error: %v", err)
+		log.Fatalf("connection error: %v", err)
 	}
-	fmt.Printf("redis config: addr=%s db=%d tls=%t password=%s\n",
-		primary.Addr, primary.DB, primary.TLSConfig != nil, redactPassword(primary.Password))
-	rdb := redis.NewClient(primary)
+	// --db overrides whatever the conn string / flags resolved to, so the same
+	// target can be pointed at any logical database without editing the conn
+	// URL. -1 means "leave as-is".
+	if *dbOverride >= 0 {
+		opts.DB = *dbOverride
+	}
+	// Generous timeouts: the zcli VPN adds latency and can stall a dial well
+	// past go-redis's 5s default mid-run. PoolTimeout > DialTimeout so a slow
+	// dial doesn't surface as pool exhaustion first.
+	opts.DialTimeout = *seedTimeout
+	opts.ReadTimeout = *seedTimeout
+	opts.WriteTimeout = *seedTimeout
+	opts.PoolTimeout = *seedTimeout + 5*time.Second
+
+	switch {
+	case *inspectMode:
+		os.Exit(runInspect(opts))
+	case *flushMode:
+		os.Exit(runFlush(opts))
+	case *verifyMode:
+		os.Exit(runVerify(opts, *seedSeed, *seedMB*1024*1024, *seedValueBytes, *seedBatch))
+	case *seedMode:
+		os.Exit(runSeed(opts, *seedSeed, *seedMB*1024*1024, *seedValueBytes, *seedBatch, *seedThrottleMB*1024*1024))
+	case *watchMode:
+		os.Exit(runWatch(opts, *watchInterval))
+	default:
+		os.Exit(runBasicTests(opts))
+	}
+}
+
+// runWatch PINGs the target every interval until SIGINT/SIGTERM, reporting the
+// up→down and down→up edges so you can measure failover/outage windows. The
+// per-probe timeout is opts.ReadTimeout (set from --timeout); lower it for
+// tighter outage-edge resolution. Detection granularity ≈ max(interval, timeout).
+func runWatch(opts *redis.Options, interval time.Duration) int {
+	const tsLayout = "15:04:05.000"
+	fmt.Printf("watch: addr=%s db=%d tls=%t password=%s interval=%s timeout=%s (Ctrl+C to stop)\n",
+		opts.Addr, opts.DB, opts.TLSConfig != nil, redactPassword(opts.Password), interval, opts.ReadTimeout)
+
+	rdb := redis.NewClient(opts)
 	defer rdb.Close()
 
-	if *continuous {
-		runContinuous(rdb, *interval)
-		return
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	probe := func() error {
+		pctx, cancel := context.WithTimeout(ctx, opts.ReadTimeout)
+		defer cancel()
+		return rdb.Ping(pctx).Err()
 	}
 
+	var (
+		started     = time.Now()
+		up          = true
+		first       = true
+		outageStart time.Time
+		lastBeat    = time.Now()
+		outages     int
+		totalDown   time.Duration
+		longest     time.Duration
+	)
+	// endOutage closes the current outage, accumulates it, and prints recovery.
+	endOutage := func(at time.Time, recovered bool) {
+		d := at.Sub(outageStart)
+		totalDown += d
+		if d > longest {
+			longest = d
+		}
+		if recovered {
+			fmt.Printf("%s RECOVERED — outage lasted %s\n", at.Format(tsLayout), d.Round(time.Millisecond))
+		} else {
+			fmt.Printf("%s OUTAGE ONGOING at exit — %s so far\n", at.Format(tsLayout), d.Round(time.Millisecond))
+		}
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			now := time.Now()
+			if !up {
+				endOutage(now, false)
+			}
+			fmt.Printf("\nstopped after %s: outages=%d total_down=%s longest=%s\n",
+				time.Since(started).Round(time.Millisecond), outages,
+				totalDown.Round(time.Millisecond), longest.Round(time.Millisecond))
+			return 0
+		case <-ticker.C:
+		}
+
+		err := probe()
+		// SIGINT can land mid-probe; treat a cancelled probe as shutdown, not an outage.
+		if ctx.Err() != nil {
+			continue
+		}
+		nowUp := err == nil
+		now := time.Now()
+
+		switch {
+		case first:
+			first = false
+			up = nowUp
+			if nowUp {
+				fmt.Printf("%s up\n", now.Format(tsLayout))
+			} else {
+				outageStart, outages = now, outages+1
+				fmt.Printf("%s DOWN at start — %s\n", now.Format(tsLayout), trim(err.Error(), 120))
+			}
+		case up && !nowUp:
+			outageStart, outages = now, outages+1
+			fmt.Printf("%s OUTAGE START — %s\n", now.Format(tsLayout), trim(err.Error(), 120))
+		case !up && nowUp:
+			endOutage(now, true)
+		case up && nowUp && now.Sub(lastBeat) >= 10*time.Second:
+			fmt.Printf("%s … up (elapsed %s, outages %d)\n",
+				now.Format(tsLayout), time.Since(started).Round(time.Second), outages)
+			lastBeat = now
+		}
+		up = nowUp
+	}
+}
+
+// runBasicTests runs the one-shot smoke suite against a single Valkey resolved
+// from --conn or the split connection flags.
+func runBasicTests(opts *redis.Options) int {
+	fmt.Printf("redis config: addr=%s db=%d tls=%t password=%s\n",
+		opts.Addr, opts.DB, opts.TLSConfig != nil, redactPassword(opts.Password))
+	rdb := redis.NewClient(opts)
+	defer rdb.Close()
+
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -272,8 +439,9 @@ func main() {
 
 	fmt.Printf("\n%d/%d tests passed\n", len(tests)-failed, len(tests))
 	if failed > 0 {
-		os.Exit(1)
+		return 1
 	}
+	return 0
 }
 
 func testPing(ctx context.Context, r *redis.Client) error {
@@ -480,160 +648,3 @@ func testCleanup(ctx context.Context, r *redis.Client) error {
 	}
 	return r.Del(ctx, keys...).Err()
 }
-
-func runContinuous(r *redis.Client, interval time.Duration) {
-	stopCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
-	defer stop()
-
-	subCtx, cancelSub := context.WithCancel(stopCtx)
-	defer cancelSub()
-	sub := r.Subscribe(subCtx, "vk:cont:channel")
-	defer sub.Close()
-	if _, err := sub.Receive(subCtx); err != nil {
-		log.Fatalf("subscribe: %v", err)
-	}
-	subCh := sub.Channel()
-
-	ops := []struct {
-		name string
-		fn   func(context.Context, *redis.Client, int64) error
-	}{
-		{"PING", opPing},
-		{"SET/GET", opSetGet},
-		{"INCR", opIncr},
-		{"LIST", opList},
-		{"INFO", opInfo},
-		{"PUB/SUB", func(ctx context.Context, r *redis.Client, i int64) error {
-			return opPubSub(ctx, r, i, subCh)
-		}},
-	}
-
-	var cycle, okCount, failCount int64
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-
-	fmt.Printf("continuous mode: addr=%s interval=%s (Ctrl+C to stop)\n", r.Options().Addr, interval)
-
-	for {
-		select {
-		case <-stopCtx.Done():
-			fmt.Printf("\nstopped: cycles=%d ok=%d failed=%d\n", cycle, okCount, failCount)
-			return
-		case <-ticker.C:
-		}
-		cycle++
-
-		results := make([]string, len(ops))
-		var failures []string
-		cycleStart := time.Now()
-		for i, op := range ops {
-			ctx, cancel := context.WithTimeout(stopCtx, interval)
-			start := time.Now()
-			err := op.fn(ctx, r, cycle)
-			cancel()
-			dur := time.Since(start)
-			if err != nil {
-				results[i] = fmt.Sprintf("%s=FAIL", op.name)
-				failures = append(failures, fmt.Sprintf("  %s (%s): %v", op.name, dur, err))
-			} else {
-				results[i] = fmt.Sprintf("%s=ok(%s)", op.name, dur.Round(time.Microsecond))
-			}
-		}
-
-		if len(failures) == 0 {
-			okCount++
-		} else {
-			failCount++
-		}
-
-		fmt.Printf("[%s] #%d %s | total=%s\n",
-			time.Now().Format("15:04:05"), cycle, joinResults(results), time.Since(cycleStart).Round(time.Microsecond))
-		for _, f := range failures {
-			fmt.Println(f)
-		}
-	}
-}
-
-func joinResults(parts []string) string {
-	out := ""
-	for i, p := range parts {
-		if i > 0 {
-			out += " "
-		}
-		out += p
-	}
-	return out
-}
-
-func opPing(ctx context.Context, r *redis.Client, _ int64) error {
-	pong, err := r.Ping(ctx).Result()
-	if err != nil {
-		return err
-	}
-	if pong != "PONG" {
-		return fmt.Errorf("expected PONG, got %q", pong)
-	}
-	return nil
-}
-
-func opSetGet(ctx context.Context, r *redis.Client, i int64) error {
-	val := fmt.Sprintf("v-%d", i)
-	if err := r.Set(ctx, "vk:cont:str", val, 10*time.Second).Err(); err != nil {
-		return err
-	}
-	got, err := r.Get(ctx, "vk:cont:str").Result()
-	if err != nil {
-		return err
-	}
-	if got != val {
-		return fmt.Errorf("set %q, got %q", val, got)
-	}
-	return nil
-}
-
-func opIncr(ctx context.Context, r *redis.Client, _ int64) error {
-	_, err := r.Incr(ctx, "vk:cont:counter").Result()
-	return err
-}
-
-func opList(ctx context.Context, r *redis.Client, i int64) error {
-	if err := r.RPush(ctx, "vk:cont:list", fmt.Sprintf("item-%d", i)).Err(); err != nil {
-		return err
-	}
-	if err := r.LTrim(ctx, "vk:cont:list", -10, -1).Err(); err != nil {
-		return err
-	}
-	_, err := r.LRange(ctx, "vk:cont:list", 0, -1).Result()
-	return err
-}
-
-func opInfo(ctx context.Context, r *redis.Client, _ int64) error {
-	info, err := r.Info(ctx, "server").Result()
-	if err != nil {
-		return err
-	}
-	if len(info) == 0 {
-		return fmt.Errorf("empty INFO")
-	}
-	return nil
-}
-
-func opPubSub(ctx context.Context, r *redis.Client, i int64, ch <-chan *redis.Message) error {
-	payload := fmt.Sprintf("ping-%d", i)
-	if err := r.Publish(ctx, "vk:cont:channel", payload).Err(); err != nil {
-		return err
-	}
-	for {
-		select {
-		case msg := <-ch:
-			if msg == nil {
-				return fmt.Errorf("channel closed")
-			}
-			if msg.Payload == payload {
-				return nil
-			}
-		case <-ctx.Done():
-			return fmt.Errorf("timeout waiting for %q", payload)
-		}
-	}
-}

valkey/seed.go

@@ -0,0 +1,323 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// keyCount is the number of seed:<n> keys for a given target/value size.
+// Both seed and verify derive it the same way so they agree on the key set
+// without storing a manifest (which the "step 2" actions could mutate).
+func keyCount(targetBytes, valueBytes int) int64 {
+	if valueBytes <= 0 {
+		return 0
+	}
+	return int64((targetBytes + valueBytes - 1) / valueBytes)
+}
+
+// genValue deterministically fills buf with incompressible bytes derived from
+// (runSeed, idx). It's the single source of truth shared by seed and verify:
+// the same (runSeed, idx, len(buf)) always yields identical bytes, which is
+// what lets verify regenerate the expected value instead of remembering it.
+// ChaCha8 gives a fast, high-quality stream so the payload doesn't compress
+// or dedup and thus reflects real memory.
+func genValue(runSeed uint64, idx int64, buf []byte) {
+	var seed [32]byte
+	binary.LittleEndian.PutUint64(seed[0:8], runSeed)
+	binary.LittleEndian.PutUint64(seed[8:16], uint64(idx))
+	rand.NewChaCha8(seed).Read(buf)
+}
+
+// runSeed bulk-loads a single Valkey with deterministic random blobs until the
+// on-wire payload reaches targetBytes. Meant for local use over the zcli VPN
+// (e.g. --host valkey.zerops), so connection params come from the flags.
+//
+// Keys are seed:<n> (n in [0, keyCount)) each holding valueBytes generated by
+// genValue(runSeed, n). Writes are pipelined in batches of batch keys to keep
+// the round-trip count — and VPN latency — from dominating.
+//
+// throttleBytesPerSec, when > 0, caps the average write rate: after each batch
+// the loop sleeps until cumulative bytes / rate seconds have elapsed, so the
+// on-wire throughput converges to the cap without bursting a token bucket.
+func runSeed(opts *redis.Options, runSeed uint64, targetBytes, valueBytes, batch int, throttleBytesPerSec float64) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	rdb := redis.NewClient(opts)
+	defer rdb.Close()
+	if err := rdb.Ping(ctx).Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "seed: connect %s failed: %v\n", opts.Addr, err)
+		return 1
+	}
+
+	total := keyCount(targetBytes, valueBytes)
+	throttle := "unlimited"
+	if throttleBytesPerSec > 0 {
+		throttle = humanBytes(int(throttleBytesPerSec)) + "/s"
+	}
+	fmt.Printf("seed: addr=%s db=%d tls=%t target=%s value=%s keys=%d seed=%d batch=%d throttle=%s\n",
+		opts.Addr, opts.DB, opts.TLSConfig != nil,
+		humanBytes(targetBytes), humanBytes(valueBytes), total, runSeed, batch, throttle)
+
+	start, lastLog := time.Now(), time.Now()
+	var written, keys int64
+
+	for keys < total {
+		select {
+		case <-ctx.Done():
+			fmt.Printf("\nseed: interrupted — wrote %d/%d keys (%s)\n", keys, total, time.Since(start).Round(time.Second))
+			return 1
+		default:
+		}
+
+		pipe := rdb.Pipeline()
+		for i := 0; i < batch && keys < total; i++ {
+			// Fresh buffer per key: go-redis keeps the []byte by reference and
+			// only serializes it at Exec, so a reused buffer would store the
+			// batch's last value under every key.
+			buf := make([]byte, valueBytes)
+			genValue(runSeed, keys, buf)
+			pipe.Set(ctx, fmt.Sprintf("seed:%d", keys), buf, 0)
+			keys++
+			written += int64(valueBytes)
+		}
+		if _, err := pipe.Exec(ctx); err != nil {
+			fmt.Fprintf(os.Stderr, "seed: pipeline exec: %v\n", err)
+			return 1
+		}
+
+		// Throttle on cumulative progress: sleep until enough wall-clock has
+		// passed for everything written so far at the target rate. Interruptible
+		// so Ctrl+C doesn't wait out a long sleep.
+		if throttleBytesPerSec > 0 {
+			targetElapsed := time.Duration(float64(written) / throttleBytesPerSec * float64(time.Second))
+			if sleep := targetElapsed - time.Since(start); sleep > 0 {
+				select {
+				case <-ctx.Done():
+				case <-time.After(sleep):
+				}
+			}
+		}
+
+		if now := time.Now(); now.Sub(lastLog) >= time.Second {
+			rate := float64(written) / now.Sub(start).Seconds()
+			fmt.Printf("[%s] %s / %s  (%d/%d keys, %s/s)\n",
+				now.Format("15:04:05"), humanBytes(int(written)), humanBytes(targetBytes), keys, total, humanBytes(int(rate)))
+			lastLog = now
+		}
+	}
+
+	elapsed := time.Since(start)
+	fmt.Printf("seed: done — wrote %s across %d keys in %s (%s/s). verify with the same --seed-mb/--seed-value-bytes/--seed-seed.\n",
+		humanBytes(int(written)), keys, elapsed.Round(time.Millisecond), humanBytes(int(float64(written)/elapsed.Seconds())))
+	return 0
+}
+
+// runVerify regenerates every seed:<n> value and compares it byte-for-byte to
+// what's in Valkey, reporting missing/mismatched keys and any unexpected
+// extras (via DBSIZE). It must be run with the same runSeed/targetBytes/
+// valueBytes used to seed. Returns non-zero if the data is not 1:1.
+func runVerify(opts *redis.Options, runSeed uint64, targetBytes, valueBytes, batch int) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	rdb := redis.NewClient(opts)
+	defer rdb.Close()
+	if err := rdb.Ping(ctx).Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "verify: connect %s failed: %v\n", opts.Addr, err)
+		return 1
+	}
+
+	total := keyCount(targetBytes, valueBytes)
+	fmt.Printf("verify: addr=%s db=%d tls=%t expect keys=%d value=%s seed=%d\n",
+		opts.Addr, opts.DB, opts.TLSConfig != nil, total, humanBytes(valueBytes), runSeed)
+
+	start, lastLog := time.Now(), time.Now()
+	var checked, missing, mismatch int64
+	expect := make([]byte, valueBytes)
+
+	for base := int64(0); base < total; base += int64(batch) {
+		select {
+		case <-ctx.Done():
+			fmt.Printf("\nverify: interrupted at %d/%d keys\n", checked, total)
+			return 1
+		default:
+		}
+
+		end := base + int64(batch)
+		if end > total {
+			end = total
+		}
+		pipe := rdb.Pipeline()
+		cmds := make([]*redis.StringCmd, 0, end-base)
+		for idx := base; idx < end; idx++ {
+			cmds = append(cmds, pipe.Get(ctx, fmt.Sprintf("seed:%d", idx)))
+		}
+		if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
+			fmt.Fprintf(os.Stderr, "verify: pipeline exec: %v\n", err)
+			return 1
+		}
+
+		for i, cmd := range cmds {
+			idx := base + int64(i)
+			got, err := cmd.Bytes()
+			if err == redis.Nil {
+				missing++
+				if missing <= 10 {
+					fmt.Printf("  MISSING seed:%d\n", idx)
+				}
+				continue
+			}
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "verify: get seed:%d: %v\n", idx, err)
+				return 1
+			}
+			genValue(runSeed, idx, expect)
+			if !bytes.Equal(got, expect) {
+				mismatch++
+				if mismatch <= 10 {
+					fmt.Printf("  MISMATCH seed:%d (len got=%d want=%d)\n", idx, len(got), len(expect))
+				}
+			}
+			checked++
+		}
+
+		if now := time.Now(); now.Sub(lastLog) >= time.Second {
+			fmt.Printf("[%s] verified %d/%d keys\n", now.Format("15:04:05"), base+int64(len(cmds)), total)
+			lastLog = now
+		}
+	}
+
+	dbsize, err := rdb.DBSize(ctx).Result()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "verify: dbsize: %v\n", err)
+		return 1
+	}
+	extras := dbsize - total
+
+	fmt.Printf("verify: %d ok, %d missing, %d mismatched of %d expected; dbsize=%d (%+d vs expected) in %s\n",
+		checked-mismatch, missing, mismatch, total, dbsize, extras, time.Since(start).Round(time.Millisecond))
+
+	if missing == 0 && mismatch == 0 && extras == 0 {
+		fmt.Println("verify: PASS — data is 1:1 with the seed.")
+		return 0
+	}
+	fmt.Println("verify: FAIL — data diverged from the seed.")
+	return 1
+}
+
+// runFlush clears the whole database selected by opts.DB with FLUSHDB,
+// reporting how many keys it dropped. Only the one logical DB is touched —
+// other databases on the same server are left intact.
+func runFlush(opts *redis.Options) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	rdb := redis.NewClient(opts)
+	defer rdb.Close()
+	if err := rdb.Ping(ctx).Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "flush: connect %s failed: %v\n", opts.Addr, err)
+		return 1
+	}
+
+	before, err := rdb.DBSize(ctx).Result()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "flush: dbsize db%d: %v\n", opts.DB, err)
+		return 1
+	}
+	if err := rdb.FlushDB(ctx).Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "flush: flushdb db%d: %v\n", opts.DB, err)
+		return 1
+	}
+	fmt.Printf("flush: FLUSHDB db%d on %s — cleared %d keys\n", opts.DB, opts.Addr, before)
+	return 0
+}
+
+// runInspect prints every logical database and its key count. The number of
+// databases comes from CONFIG GET databases (default 16); per-DB key counts
+// come from a single INFO keyspace call, which lists only the non-empty DBs —
+// the rest are reported as 0. Empty DBs are still shown so the full layout is
+// visible at a glance.
+func runInspect(opts *redis.Options) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	rdb := redis.NewClient(opts)
+	defer rdb.Close()
+	if err := rdb.Ping(ctx).Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "inspect: connect %s failed: %v\n", opts.Addr, err)
+		return 1
+	}
+
+	numDBs := 16
+	if res, err := rdb.ConfigGet(ctx, "databases").Result(); err == nil {
+		if v, ok := res["databases"]; ok {
+			if n, err := strconv.Atoi(v); err == nil && n > 0 {
+				numDBs = n
+			}
+		}
+	}
+
+	info, err := rdb.Info(ctx, "keyspace").Result()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "inspect: info keyspace: %v\n", err)
+		return 1
+	}
+	// Lines look like: db0:keys=12,expires=0,avg_ttl=0
+	counts := make(map[int]int64)
+	for _, line := range strings.Split(info, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "db") {
+			continue
+		}
+		colon := strings.IndexByte(line, ':')
+		if colon < 0 {
+			continue
+		}
+		dbn, err := strconv.Atoi(line[2:colon])
+		if err != nil {
+			continue
+		}
+		for _, field := range strings.Split(line[colon+1:], ",") {
+			if rest, ok := strings.CutPrefix(field, "keys="); ok {
+				if k, err := strconv.ParseInt(rest, 10, 64); err == nil {
+					counts[dbn] = k
+				}
+			}
+		}
+	}
+
+	fmt.Printf("inspect: addr=%s tls=%t databases=%d\n", opts.Addr, opts.TLSConfig != nil, numDBs)
+	var total int64
+	for db := 0; db < numDBs; db++ {
+		k := counts[db]
+		total += k
+		fmt.Printf("  db%-2d  %d keys\n", db, k)
+	}
+	fmt.Printf("inspect: %d keys across %d databases\n", total, numDBs)
+	return 0
+}
+
+func humanBytes(n int) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%dB", n)
+	}
+	div, exp := int64(unit), 0
+	for v := n / unit; v >= unit; v /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f%cB", float64(n)/float64(div), "KMGTPE"[exp])
+}

valkey/zerops-chaos-sea-import.yaml

@@ -0,0 +1,17 @@
+# Chaos infra for sea1 (valkeys already exist). Adds the dashboard + a small
+# worker fleet. Worker fleet kept modest (max 3) for a LIGHT project.
+#
+# Import:  zcli project service-import zerops-chaos-sea-import.yaml -P <sea1>
+# Deploy:  zcli push -P <sea1> --service-id <collector> --setup chaoscollector
+#          zcli push -P <sea1> --service-id <worker>    --setup chaosworker-sea
+services:
+  - hostname: chaoscollector
+    type: go@1
+    enableSubdomainAccess: true
+    minContainers: 1
+    maxContainers: 1
+
+  - hostname: chaosworker
+    type: go@1
+    minContainers: 1
+    maxContainers: 3

valkey/zerops-ubuntu-import.yaml

@@ -0,0 +1,9 @@
+# Empty Ubuntu runtime service, started without code so we can deliver a
+# binary to it afterwards (via `zcli push --setup ubuntubin` or scp).
+#
+# Import with:  zcli project service-import zerops-ubuntu-import.yaml
+# Then deploy:  zcli push --setup ubuntubin
+services:
+  - hostname: ubuntubin
+    type: ubuntu@24.04
+    startWithoutCode: true

valkey/zerops.yaml

@@ -32,6 +32,18 @@ zerops:
         COLLECTOR_URL: http://chaoscollector.zerops:8080
         VALKEY_PASSWORD: ${valkeyha_password}
 
+  - setup: ubuntubin
+    build:
+      base: ubuntu/go@1
+      buildCommands:
+        - go build -o app .
+      deployFiles:
+        - app
+        - config.yaml
+      cache: true
+    run:
+      base: ubuntu@24.04
+
   - setup: chaoscollector
     build:
       base: ubuntu/go@1
@@ -74,3 +86,27 @@ zerops:
         VALKEY_VALKEYHA1_PASSWORD: ${valkeyha1_password}
         VALKEY_VALKEYHA2_PASSWORD: ${valkeyha2_password}
         VALKEY_VALKEYHA3_PASSWORD: ${valkeyha3_password}
+
+  # sea1 chaos worker against the HA valkeys, per-service auth over plaintext.
+  # The valkey TLS ports require mutual TLS (client cert), which the tester
+  # doesn't present, so the workers use the plaintext ports (config-sea-ha.yaml)
+  # with passwords. VALKEY_PASSWORD is intentionally NOT set so the per-worker
+  # ${valkeyhaN_password} values survive (applyZeropsOverrides only clobbers
+  # passwords when VALKEY_PASSWORD is present).
+  - setup: chaosworker-sea
+    build:
+      base: ubuntu/go@1
+      buildCommands:
+        - go build -o app .
+      deployFiles:
+        - app
+        - config-sea-ha.yaml
+      cache: true
+    run:
+      base: ubuntu@latest
+      start: ./app --worker --config config-sea-ha.yaml
+      envVariables:
+        COLLECTOR_URL: http://chaoscollector.zerops:8080
+        valkeyha1_password: ${valkeyha1_password}
+        valkeyha2_password: ${valkeyha2_password}
+        valkeyha3_password: ${valkeyha3_password}