From 6f63abecea73ac25ce406b02b2cc33a0d1d3c251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Pavl=C3=AD=C4=8Dek?= Date: Thu, 27 Aug 2026 09:14:21 +0200 Subject: [PATCH] move gitea to my-recipes, commit works --- dckr-test/app.txt | 1 + dckr-test/zerops.yml | 8 ++ gitea/README.md | 57 --------- gitea/app.ini | 56 --------- gitea/generate-secrets.sh | 16 --- gitea/zerops-service-import.yaml | 22 ---- gitea/zerops.yaml | 44 ------- ls-stress/go.mod | 17 +++ ls-stress/go.sum | 50 ++++++++ ls-stress/import-2.yml | 23 ++++ ls-stress/import-3.yml | 23 ++++ ls-stress/import-4.yml | 23 ++++ ls-stress/import-data2.yml | 3 + ls-stress/import-disk-test.yml | 11 ++ ls-stress/main.go | 178 +++++++++++++++++++++++++++ ls-stress/zerops-project-import.yml | 37 ++++++ ls-stress/zerops.yml | 39 ++++++ sqlite-ls/go.mod | 17 +++ sqlite-ls/go.sum | 50 ++++++++ sqlite-ls/main.go | 178 +++++++++++++++++++++++++++ sqlite-ls/zerops-project-import.yml | 9 ++ sqlite-ls/zerops-project2-import.yml | 9 ++ sqlite-ls/zerops.yml | 16 +++ 23 files changed, 692 insertions(+), 195 deletions(-) create mode 100644 dckr-test/app.txt create mode 100644 dckr-test/zerops.yml delete mode 100644 gitea/README.md delete mode 100644 gitea/app.ini delete mode 100755 gitea/generate-secrets.sh delete mode 100644 gitea/zerops-service-import.yaml delete mode 100644 gitea/zerops.yaml create mode 100644 ls-stress/go.mod create mode 100644 ls-stress/go.sum create mode 100644 ls-stress/import-2.yml create mode 100644 ls-stress/import-3.yml create mode 100644 ls-stress/import-4.yml create mode 100644 ls-stress/import-data2.yml create mode 100644 ls-stress/import-disk-test.yml create mode 100644 ls-stress/main.go create mode 100644 ls-stress/zerops-project-import.yml create mode 100644 ls-stress/zerops.yml create mode 100644 sqlite-ls/go.mod create mode 100644 sqlite-ls/go.sum create mode 100644 sqlite-ls/main.go create mode 100644 sqlite-ls/zerops-project-import.yml create mode 100644 sqlite-ls/zerops-project2-import.yml create mode 100644 sqlite-ls/zerops.yml diff --git a/dckr-test/app.txt b/dckr-test/app.txt new file mode 100644 index 0000000..421376d --- /dev/null +++ b/dckr-test/app.txt @@ -0,0 +1 @@ +dummy diff --git a/dckr-test/zerops.yml b/dckr-test/zerops.yml new file mode 100644 index 0000000..1757f1b --- /dev/null +++ b/dckr-test/zerops.yml @@ -0,0 +1,8 @@ +zerops: + - setup: dckr + run: + base: docker@26.1 + volume: + hostname: data1 + mountPath: /mnt/data + start: sleep infinity diff --git a/gitea/README.md b/gitea/README.md deleted file mode 100644 index ef65bd3..0000000 --- a/gitea/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Gitea on Zerops - -Self-hosted Gitea backed by HA Postgres and SeaweedFS shared storage. - -## Prepare secrets - -Gitea validates its security secrets at startup and tries to rewrite `app.ini` -if they're missing or malformed. Zerops' `<@generateRandomString>` preprocessor -produces alphanumeric values that don't match Gitea's required formats -(base64url-32 bytes for `JWT_SECRET` / `LFS_JWT_SECRET`, a signed JWT for -`INTERNAL_TOKEN`), so generate them with the Gitea binary itself: - -```sh -./generate-secrets.sh -``` - -It downloads the matching Gitea release and prints four `KEY=VALUE` lines. -Paste each value into the gitea service's **secret env vars** in the Zerops GUI -(or pre-fill `zerops-service-import.yaml` before importing). - -`DB_PASSWORD` is generated by the Zerops preprocessor (`<@generateRandomString>`) -during import — cross-service expansion like `${db_password}` is not evaluated -inside the import.yaml, so the gitea service generates its own value and the -Postgres service must be initialized with the same one. - -## Bootstrap - -1. Paste `zerops-service-import.yaml` into the Zerops GUI: - project → **Import service** → fill in the four secrets from step above. -2. Push the gitea config and build recipe: - ```sh - zcli push gitea - ``` -3. Create the admin user from the running container (service console): - ```sh - sudo -u git -E /usr/local/bin/gitea admin user create \ - --config /etc/gitea/app.ini \ - --admin \ - --username admin \ - --email you@example.com \ - --password 'choose-a-strong-one' \ - --must-change-password=false - ``` -4. Log in at the service's HTTPS URL and add your SSH public key under - **Settings → SSH/GPG Keys**. - -## SSH access - -Gitea's built-in SSH listens on **port 2222** in the container. Zerops custom -domains only carry HTTPS, so SSH needs one of: - -- **Public IPv4/IPv6 on the gitea service** — Zerops GUI → service → - *Public Routing* → add IP, open port 2222. Map `:2222 → :2222`. - -After SSH works, set `SSH_DOMAIN` / `ROOT_URL` overrides if you use a custom -hostname, otherwise Gitea will print clone URLs using the autogenerated -Zerops subdomain. diff --git a/gitea/app.ini b/gitea/app.ini deleted file mode 100644 index cbd9790..0000000 --- a/gitea/app.ini +++ /dev/null @@ -1,56 +0,0 @@ -APP_NAME = Gitea -RUN_USER = git -RUN_MODE = prod -WORK_PATH = /mnt/volume/gitea - -[server] -PROTOCOL = http -DOMAIN = %%GITEA_DOMAIN%% -ROOT_URL = %%GITEA_ROOT_URL%% -HTTP_ADDR = 0.0.0.0 -HTTP_PORT = 3000 -SSH_DOMAIN = %%GITEA_DOMAIN%% -START_SSH_SERVER = true -SSH_PORT = 2222 -SSH_LISTEN_PORT = 2222 -LFS_START_SERVER = true -APP_DATA_PATH = /mnt/volume/gitea/data -DISABLE_SSH = false -LFS_JWT_SECRET = %%LFS_JWT_SECRET%% - -[database] -DB_TYPE = postgres -HOST = %%DB_HOST%%:%%DB_PORT%% -NAME = %%DB_NAME%% -USER = %%DB_USER%% -PASSWD = %%DB_PASSWORD%% -SSL_MODE = disable -SCHEMA = public - -[repository] -ROOT = /mnt/volume/gitea/data/gitea-repositories - -[lfs] -PATH = /mnt/volume/gitea/data/lfs - -[log] -ROOT_PATH = /mnt/volume/gitea/log -MODE = console -LEVEL = info - -[service] -DISABLE_REGISTRATION = true -REQUIRE_SIGNIN_VIEW = false -DEFAULT_KEEP_EMAIL_PRIVATE = true -DEFAULT_ALLOW_CREATE_ORGANIZATION = true - -[security] -INSTALL_LOCK = true -SECRET_KEY = %%SECRET_KEY%% -INTERNAL_TOKEN = %%INTERNAL_TOKEN%% - -[oauth2] -JWT_SECRET = %%JWT_SECRET%% - -[session] -PROVIDER = file diff --git a/gitea/generate-secrets.sh b/gitea/generate-secrets.sh deleted file mode 100755 index efc5fc6..0000000 --- a/gitea/generate-secrets.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# Generates JWT_SECRET, SECRET_KEY and INTERNAL_TOKEN in the exact format Gitea expects. -# Run once, then paste the three lines into Zerops as secret env vars for the gitea service. -set -euo pipefail - -GITEA_VERSION="${GITEA_VERSION:-1.26.1}" -BIN=$(mktemp) -trap 'rm -f "$BIN"' EXIT - -wget -qO "$BIN" "https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64" -chmod +x "$BIN" - -echo "JWT_SECRET=$("$BIN" generate secret JWT_SECRET)" -echo "LFS_JWT_SECRET=$("$BIN" generate secret LFS_JWT_SECRET)" -echo "SECRET_KEY=$("$BIN" generate secret SECRET_KEY)" -echo "INTERNAL_TOKEN=$("$BIN" generate secret INTERNAL_TOKEN)" diff --git a/gitea/zerops-service-import.yaml b/gitea/zerops-service-import.yaml deleted file mode 100644 index b7c7fda..0000000 --- a/gitea/zerops-service-import.yaml +++ /dev/null @@ -1,22 +0,0 @@ -#zeropsPreprocessor=on -services: - - hostname: db - type: postgresql:ha@18 - priority: 10 - verticalAutoscaling: - minFreeRamGB: 0.25 - - hostname: volume - type: seaweedfs:ha@3 - priority: 10 - - hostname: gitea - type: ubuntu@24.04 - envSecrets: - DB_PASSWORD: <@generateRandomString(<32>)> - JWT_SECRET: - LFS_JWT_SECRET: - SECRET_KEY: - INTERNAL_TOKEN: - minContainers: 1 - maxContainers: 1 - mount: - - volume diff --git a/gitea/zerops.yaml b/gitea/zerops.yaml deleted file mode 100644 index bbd609d..0000000 --- a/gitea/zerops.yaml +++ /dev/null @@ -1,44 +0,0 @@ -zerops: - - setup: gitea - build: - base: ubuntu@24.04 - deployFiles: app.ini - run: - base: ubuntu@24.04 - ports: - - port: 3000 - httpSupport: true - - port: 2222 - envVariables: - GITEA_VERSION: 1.26.1 - GITEA_WORK_DIR: /mnt/volume/gitea - GITEA_DOMAIN: git.matejpavlicek.cz - GITEA_ROOT_URL: https://git.matejpavlicek.cz - DB_HOST: db - DB_PORT: 5432 - DB_NAME: giteadb - DB_USER: gitea - envReplace: - delimiter: "%%" - target: - - app.ini - prepareCommands: - - sudo apt-get update - - sudo apt-get install -y --no-install-recommends git ca-certificates gettext-base wget gpg - - wget -q -O /tmp/gitea https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64 - - wget -q -O /tmp/gitea.asc https://dl.gitea.com/gitea/${GITEA_VERSION}/gitea-${GITEA_VERSION}-linux-amd64.asc - - gpg --keyserver hkps://keys.openpgp.org --recv 7C9E68152594688862D62AF62D9AE806EC1592E2 - - gpg --verify /tmp/gitea.asc /tmp/gitea - - sudo install -m 755 /tmp/gitea /usr/local/bin/gitea - - sudo groupadd --system git - - sudo useradd --system --gid git --shell /bin/bash --home-dir /home/git --create-home git - - sudo mkdir -p /etc/gitea - - sudo chown root:git /etc/gitea - - sudo chmod 770 /etc/gitea - initCommands: - - until mountpoint -q /mnt/volume; do sleep 1; done - - sudo mkdir -p /mnt/volume/gitea/{custom,data,indexers,public,log} - - sudo chown -R git:git /mnt/volume/gitea - - sudo chmod -R 750 /mnt/volume/gitea - - sudo install -m 660 -o root -g git /var/www/app.ini /etc/gitea/app.ini - start: sudo -u git -E /usr/local/bin/gitea web --config /etc/gitea/app.ini diff --git a/ls-stress/go.mod b/ls-stress/go.mod new file mode 100644 index 0000000..9417cc5 --- /dev/null +++ b/ls-stress/go.mod @@ -0,0 +1,17 @@ +module sqlite-ls + +go 1.25.0 + +require modernc.org/sqlite v1.56.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/ls-stress/go.sum b/ls-stress/go.sum new file mode 100644 index 0000000..1932692 --- /dev/null +++ b/ls-stress/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/ls-stress/import-2.yml b/ls-stress/import-2.yml new file mode 100644 index 0000000..1801fdd --- /dev/null +++ b/ls-stress/import-2.yml @@ -0,0 +1,23 @@ +project: + name: ls-stress-2 +services: + - hostname: data1 + type: local-storage:single@1 + - hostname: data2 + type: local-storage:single@1 + - hostname: c1a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c1b + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2b + type: go@1 + minContainers: 3 + maxContainers: 3 diff --git a/ls-stress/import-3.yml b/ls-stress/import-3.yml new file mode 100644 index 0000000..4449a45 --- /dev/null +++ b/ls-stress/import-3.yml @@ -0,0 +1,23 @@ +project: + name: ls-stress-3 +services: + - hostname: data1 + type: local-storage:single@1 + - hostname: data2 + type: local-storage:single@1 + - hostname: c1a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c1b + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2b + type: go@1 + minContainers: 3 + maxContainers: 3 diff --git a/ls-stress/import-4.yml b/ls-stress/import-4.yml new file mode 100644 index 0000000..3882d8c --- /dev/null +++ b/ls-stress/import-4.yml @@ -0,0 +1,23 @@ +project: + name: ls-stress-4 +services: + - hostname: data1 + type: local-storage:single@1 + - hostname: data2 + type: local-storage:single@1 + - hostname: c1a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c1b + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2b + type: go@1 + minContainers: 3 + maxContainers: 3 diff --git a/ls-stress/import-data2.yml b/ls-stress/import-data2.yml new file mode 100644 index 0000000..7e02358 --- /dev/null +++ b/ls-stress/import-data2.yml @@ -0,0 +1,3 @@ +services: + - hostname: data2 + type: local-storage:single@1 diff --git a/ls-stress/import-disk-test.yml b/ls-stress/import-disk-test.yml new file mode 100644 index 0000000..c13a32e --- /dev/null +++ b/ls-stress/import-disk-test.yml @@ -0,0 +1,11 @@ +project: + name: ls-disk-test +services: + - hostname: data1 + type: local-storage:single@1 + - hostname: c1a + type: go@1 + minContainers: 2 + maxContainers: 2 + - hostname: dckr + type: docker@26.1 diff --git a/ls-stress/main.go b/ls-stress/main.go new file mode 100644 index 0000000..a88b6b0 --- /dev/null +++ b/ls-stress/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "crypto/rand" + "database/sql" + "encoding/json" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +var ( + db *sql.DB + hostname string + dbFile string +) + +func main() { + hostname, _ = os.Hostname() + + dbPath := os.Getenv("DB_PATH") + if dbPath == "" { + dbPath = "/mnt/data/poc.db" + } + dbFile = dbPath + + var err error + db, err = sql.Open("sqlite", "file:"+dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)") + if err != nil { + log.Fatal(err) + } + + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + container TEXT NOT NULL, + msg TEXT NOT NULL, + created_at TEXT NOT NULL, + payload BLOB + )`); err != nil { + log.Fatal(err) + } + // the payload column was added after the first deploy, older DB files need it retrofitted + if _, err := db.Exec(`ALTER TABLE entries ADD COLUMN payload BLOB`); err != nil && !strings.Contains(err.Error(), "duplicate column") { + log.Fatal(err) + } + + http.HandleFunc("/", handleStatus) + http.HandleFunc("/write", handleWrite) + http.HandleFunc("/read", handleRead) + http.HandleFunc("/checkpoint", handleCheckpoint) + + log.Printf("listening on :8080, db=%s, container=%s", dbPath, hostname) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func respond(w http.ResponseWriter, status int, data any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(data) +} + +func handleStatus(w http.ResponseWriter, r *http.Request) { + var count int64 + var journalMode string + if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + db.QueryRow("PRAGMA journal_mode").Scan(&journalMode) + fileSize := func(path string) int64 { + info, err := os.Stat(path) + if err != nil { + return 0 + } + return info.Size() + } + respond(w, 200, map[string]any{ + "container": hostname, + "entries": count, + "journalMode": journalMode, + "dbSizeBytes": fileSize(dbFile), + "walSizeBytes": fileSize(dbFile + "-wal"), + }) +} + +func handleWrite(w http.ResponseWriter, r *http.Request) { + msg := r.URL.Query().Get("msg") + if msg == "" { + msg = "hello" + } + intParam := func(key string, def, max int) int { + v, err := strconv.Atoi(r.URL.Query().Get(key)) + if err != nil || v < 1 { + return def + } + return min(v, max) + } + n := intParam("n", 1, 10000) + size := intParam("size", 0, 1<<20) + + tx, err := db.Begin() + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer tx.Rollback() + stmt, err := tx.Prepare("INSERT INTO entries (container, msg, created_at, payload) VALUES (?, ?, ?, ?)") + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer stmt.Close() + + var lastId int64 + now := time.Now().UTC().Format(time.RFC3339Nano) + for i := 0; i < n; i++ { + var payload []byte + if size > 0 { + payload = make([]byte, size) + rand.Read(payload) + } + res, err := stmt.Exec(hostname, msg, now, payload) + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + lastId, _ = res.LastInsertId() + } + if err := tx.Commit(); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + respond(w, 200, map[string]any{"container": hostname, "inserted": n, "lastId": lastId}) +} + +func handleCheckpoint(w http.ResponseWriter, r *http.Request) { + var busy, logFrames, checkpointed int64 + if err := db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + respond(w, 200, map[string]any{"container": hostname, "busy": busy, "logFrames": logFrames, "checkpointed": checkpointed}) +} + +func handleRead(w http.ResponseWriter, r *http.Request) { + var count int64 + if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + rows, err := db.Query("SELECT id, container, msg, created_at FROM entries ORDER BY id DESC LIMIT 5") + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer rows.Close() + type entry struct { + Id int64 `json:"id"` + Container string `json:"container"` + Msg string `json:"msg"` + CreatedAt string `json:"createdAt"` + } + last := []entry{} + for rows.Next() { + var e entry + if err := rows.Scan(&e.Id, &e.Container, &e.Msg, &e.CreatedAt); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + last = append(last, e) + } + respond(w, 200, map[string]any{"container": hostname, "entries": count, "last": last}) +} diff --git a/ls-stress/zerops-project-import.yml b/ls-stress/zerops-project-import.yml new file mode 100644 index 0000000..e4bf9a2 --- /dev/null +++ b/ls-stress/zerops-project-import.yml @@ -0,0 +1,37 @@ +project: + name: ls-delete-stress +services: + - hostname: data1 + type: local-storage:single@1 + - hostname: data2 + type: local-storage:single@1 + - hostname: c1a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c1b + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c1c + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2a + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2b + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: c2c + type: go@1 + minContainers: 3 + maxContainers: 3 + - hostname: cache + type: valkey@7.2 + mode: NON_HA + - hostname: db + type: postgresql@16 + mode: NON_HA diff --git a/ls-stress/zerops.yml b/ls-stress/zerops.yml new file mode 100644 index 0000000..8c0f4a4 --- /dev/null +++ b/ls-stress/zerops.yml @@ -0,0 +1,39 @@ +zerops: + - setup: c1a + build: &build + base: go@1 + buildCommands: + - go build -o app main.go + deployFiles: app + run: &run-data1 + base: go@1 + ports: + - port: 8080 + httpSupport: true + volume: + hostname: data1 + mountPath: /mnt/data + start: ./app + - setup: c1b + build: *build + run: *run-data1 + - setup: c1c + build: *build + run: *run-data1 + - setup: c2a + build: *build + run: &run-data2 + base: go@1 + ports: + - port: 8080 + httpSupport: true + volume: + hostname: data2 + mountPath: /mnt/data + start: ./app + - setup: c2b + build: *build + run: *run-data2 + - setup: c2c + build: *build + run: *run-data2 diff --git a/sqlite-ls/go.mod b/sqlite-ls/go.mod new file mode 100644 index 0000000..9417cc5 --- /dev/null +++ b/sqlite-ls/go.mod @@ -0,0 +1,17 @@ +module sqlite-ls + +go 1.25.0 + +require modernc.org/sqlite v1.56.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/sqlite-ls/go.sum b/sqlite-ls/go.sum new file mode 100644 index 0000000..1932692 --- /dev/null +++ b/sqlite-ls/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/sqlite-ls/main.go b/sqlite-ls/main.go new file mode 100644 index 0000000..a88b6b0 --- /dev/null +++ b/sqlite-ls/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "crypto/rand" + "database/sql" + "encoding/json" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +var ( + db *sql.DB + hostname string + dbFile string +) + +func main() { + hostname, _ = os.Hostname() + + dbPath := os.Getenv("DB_PATH") + if dbPath == "" { + dbPath = "/mnt/data/poc.db" + } + dbFile = dbPath + + var err error + db, err = sql.Open("sqlite", "file:"+dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)") + if err != nil { + log.Fatal(err) + } + + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + container TEXT NOT NULL, + msg TEXT NOT NULL, + created_at TEXT NOT NULL, + payload BLOB + )`); err != nil { + log.Fatal(err) + } + // the payload column was added after the first deploy, older DB files need it retrofitted + if _, err := db.Exec(`ALTER TABLE entries ADD COLUMN payload BLOB`); err != nil && !strings.Contains(err.Error(), "duplicate column") { + log.Fatal(err) + } + + http.HandleFunc("/", handleStatus) + http.HandleFunc("/write", handleWrite) + http.HandleFunc("/read", handleRead) + http.HandleFunc("/checkpoint", handleCheckpoint) + + log.Printf("listening on :8080, db=%s, container=%s", dbPath, hostname) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func respond(w http.ResponseWriter, status int, data any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(data) +} + +func handleStatus(w http.ResponseWriter, r *http.Request) { + var count int64 + var journalMode string + if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + db.QueryRow("PRAGMA journal_mode").Scan(&journalMode) + fileSize := func(path string) int64 { + info, err := os.Stat(path) + if err != nil { + return 0 + } + return info.Size() + } + respond(w, 200, map[string]any{ + "container": hostname, + "entries": count, + "journalMode": journalMode, + "dbSizeBytes": fileSize(dbFile), + "walSizeBytes": fileSize(dbFile + "-wal"), + }) +} + +func handleWrite(w http.ResponseWriter, r *http.Request) { + msg := r.URL.Query().Get("msg") + if msg == "" { + msg = "hello" + } + intParam := func(key string, def, max int) int { + v, err := strconv.Atoi(r.URL.Query().Get(key)) + if err != nil || v < 1 { + return def + } + return min(v, max) + } + n := intParam("n", 1, 10000) + size := intParam("size", 0, 1<<20) + + tx, err := db.Begin() + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer tx.Rollback() + stmt, err := tx.Prepare("INSERT INTO entries (container, msg, created_at, payload) VALUES (?, ?, ?, ?)") + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer stmt.Close() + + var lastId int64 + now := time.Now().UTC().Format(time.RFC3339Nano) + for i := 0; i < n; i++ { + var payload []byte + if size > 0 { + payload = make([]byte, size) + rand.Read(payload) + } + res, err := stmt.Exec(hostname, msg, now, payload) + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + lastId, _ = res.LastInsertId() + } + if err := tx.Commit(); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + respond(w, 200, map[string]any{"container": hostname, "inserted": n, "lastId": lastId}) +} + +func handleCheckpoint(w http.ResponseWriter, r *http.Request) { + var busy, logFrames, checkpointed int64 + if err := db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + respond(w, 200, map[string]any{"container": hostname, "busy": busy, "logFrames": logFrames, "checkpointed": checkpointed}) +} + +func handleRead(w http.ResponseWriter, r *http.Request) { + var count int64 + if err := db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&count); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + rows, err := db.Query("SELECT id, container, msg, created_at FROM entries ORDER BY id DESC LIMIT 5") + if err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + defer rows.Close() + type entry struct { + Id int64 `json:"id"` + Container string `json:"container"` + Msg string `json:"msg"` + CreatedAt string `json:"createdAt"` + } + last := []entry{} + for rows.Next() { + var e entry + if err := rows.Scan(&e.Id, &e.Container, &e.Msg, &e.CreatedAt); err != nil { + respond(w, 500, map[string]any{"container": hostname, "error": err.Error()}) + return + } + last = append(last, e) + } + respond(w, 200, map[string]any{"container": hostname, "entries": count, "last": last}) +} diff --git a/sqlite-ls/zerops-project-import.yml b/sqlite-ls/zerops-project-import.yml new file mode 100644 index 0000000..1db684d --- /dev/null +++ b/sqlite-ls/zerops-project-import.yml @@ -0,0 +1,9 @@ +project: + name: sqlite-ls-poc +services: + - hostname: data + type: local-storage:single@1 + - hostname: api + type: go@1 + minContainers: 2 + maxContainers: 2 diff --git a/sqlite-ls/zerops-project2-import.yml b/sqlite-ls/zerops-project2-import.yml new file mode 100644 index 0000000..4b5d3d2 --- /dev/null +++ b/sqlite-ls/zerops-project2-import.yml @@ -0,0 +1,9 @@ +project: + name: sqlite-ls-poc2 +services: + - hostname: data + type: local-storage:single@1 + - hostname: api + type: go@1 + minContainers: 2 + maxContainers: 2 diff --git a/sqlite-ls/zerops.yml b/sqlite-ls/zerops.yml new file mode 100644 index 0000000..83e1678 --- /dev/null +++ b/sqlite-ls/zerops.yml @@ -0,0 +1,16 @@ +zerops: + - setup: api + build: + base: go@1 + buildCommands: + - go build -o app main.go + deployFiles: app + run: + base: go@1 + ports: + - port: 8080 + httpSupport: true + volume: + hostname: data + mountPath: /mnt/data + start: ./app