Cloudron community app transparency reports

22#        paths, so they are migrated (ADR 0008) and only then deleted, once the app is proven healthy.
23# ------------------------------------------------------------------------------------------------
24CH_STORE=/var/lib/clickhouse
25MINIO_STORE=/var/lib/minio
26# CRITICAL ORDERING: nothing may create anything INSIDE either persistentDir before the boot legs in

23# ------------------------------------------------------------------------------------------------
24CH_STORE=/var/lib/clickhouse
25MINIO_STORE=/var/lib/minio
26# CRITICAL ORDERING: nothing may create anything INSIDE either persistentDir before the boot legs in
27# section 2b have run. Both the migration guard and the leg-3 rebuild decide what to do by asking

237"${CODE}/conf/cleanup-legacy-stores.sh" &
238 
239exec supervisord -c /etc/supervisor/supervisord.conf
240 

33start_app(){                       # $1 = container name, $2 = ch volume, $3 = minio volume, $4 = host port
34  $ENGINE run -d --name "$1" --network $NET -p "$4":3000 \
35    -v $VOL:/app/data -v "$2":/var/lib/clickhouse -v "$3":/var/lib/minio \
36    -e CLOUDRON=1 -e CLOUDRON_POSTGRESQL_URL="postgresql://langfuse:${PGPASS}@${PG}:5432/langfuse" \
37    -e CLOUDRON_REDIS_HOST=$RD -e CLOUDRON_REDIS_PORT=6379 -e CLOUDRON_REDIS_PASSWORD="$RPASS" \

46wait_health(){                     # $1 = host port, $2 = seconds
47  local i; for i in $(seq 1 "$2"); do
48    [ "$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$1/api/public/health")" = 200 ] && return 0
49    sleep 3
50  done; return 1

63TID="trace-clone-$(date +%s)"
64for n in 1 2 3 4 5; do
65  curl -s -o /dev/null -X POST http://localhost:3000/api/public/ingestion \
66    -H "Authorization: Basic $AUTH" -H "Content-Type: application/json" \
67    -d "{\"batch\":[{\"id\":\"ev-$n\",\"type\":\"trace-create\",\"timestamp\":\"2026-06-26T12:00:0$n.000Z\",\"body\":{\"id\":\"$TID-$n\",\"name\":\"clone-trace\",\"userId\":\"u1\"}}]}"

69seen=0
70for i in $(seq 1 30); do
71  curl -s -H "Authorization: Basic $AUTH" "http://localhost:3000/api/public/traces?limit=50" | grep -q "$TID-5" && { seen=1; break; }
72  sleep 2
73done

77case_ "2. backupCommand runs in a separate container while the app is LIVE"
78# This is the flock condition: the running server holds <store>/status throughout.
79BOUT=$($ENGINE run --rm -v $VOL:/app/data -v $CHVOL:/var/lib/clickhouse -v $MVOL:/var/lib/minio \
80        "$IMAGE" /app/code/conf/backup-clickhouse.sh 2>&1)
81brc=$?

108# on a base table would pass even when allow_different_table_def has re-normalised a view's dictionary
109# reference and the aggregates are wrong.
110got=$(curl -s -H "Authorization: Basic $AUTH" "http://localhost:3001/api/public/traces?limit=50")
111printf '%s' "$got" | grep -q "$TID-5" && ok "trace 5 readable from the clone via /api/public/traces" || bad "trace 5 missing from the clone"
112n=$(printf '%s' "$got" | grep -o "$TID-" | wc -l)

16 
17# --- A. version actually serving -------------------------------------------------------------
18V=$(curl -s "$H/api/public/health" | jq -r '.version // empty' 2>/dev/null)
19note "health reports version: ${V:-<none>}"
20 

20 
21# --- B. data preservation: trace count (the thing an update must never change) ----------------
22CNT=$(curl -s -H "Authorization: Basic $AUTH" "$H/api/public/traces?limit=1" | jq -r '.meta.totalItems // empty')
23if [ -n "$CNT" ]; then ok "traces API reachable, totalItems=${CNT}"; else bad "traces API gave no totalItems"; fi
24echo "TRACE_COUNT=${CNT}" > "/tmp/gate2-${LEG}.count"

26# --- 1. signup route rejects non-POST with 405 (upstream fix #15670) --------------------------
27# 4.2.0: `return;` with no response.  4.3.0: 405 + {"message":"Method not allowed"}
28SC=$(curl -s -o /tmp/gate2-su.json -w '%{http_code}' -X GET "$H/api/auth/signup" --max-time 20)
29MSG=$(jq -r '.message // empty' /tmp/gate2-su.json 2>/dev/null)
30if [ "$LEG" = after ]; then

44found=""; page=1
45while [ "$page" -le 20 ]; do
46  PJ=$(curl -sf -H "Authorization: Basic $AUTH" "$H/api/public/models?limit=100&page=${page}" 2>/dev/null) || break
47  n=$(echo "$PJ" | jq '.data | length' 2>/dev/null); [ "${n:-0}" -eq 0 ] && break
48  hit=$(echo "$PJ" | jq -c --arg m "$MODEL" '[.data[]|select(.modelName==$m)][0] // empty' 2>/dev/null)

69Q='{"view":"observations","metrics":[{"measure":"count","aggregation":"count"}],"dimensions":[{"field":"isRootObservation"}],"fromTimestamp":"2020-01-01T00:00:00Z","toTimestamp":"2030-01-01T00:00:00Z"}'
70QE=$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))' "$Q")
71S1=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Basic $AUTH" "$H/api/public/metrics?query=$QE" --max-time 30)
72# fail-lever: the same request with a KNOWN-GOOD dimension must succeed, proving the 400 is
73# specific to the new field and not just a broken endpoint.

74Q2=${Q//isRootObservation/name}
75QE2=$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))' "$Q2")
76S2=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Basic $AUTH" "$H/api/public/metrics?query=$QE2" --max-time 30)
77if [ "$S1" = 400 ] && [ "$S2" = 200 ]; then
78  ok "v1 metrics rejects isRootObservation (400) but accepts name (200) — containment holds"

21sleep 6
22$ENGINE run -d --name $APP --network $NET -p 3000:3000 -v $VOL:/app/data \
23  -v $CHVOL:/var/lib/clickhouse -v $MVOL:/var/lib/minio \
24  -e CLOUDRON=1 -e CLOUDRON_POSTGRESQL_URL="postgresql://langfuse:${PGPASS}@${PG}:5432/langfuse" \
25  -e CLOUDRON_REDIS_HOST=$RD -e CLOUDRON_REDIS_PORT=6379 -e CLOUDRON_REDIS_PASSWORD="$RPASS" \

31  -e LANGFUSE_INGESTION_QUEUE_DELAY_MS=0 -e LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1000 \
32  "$IMAGE" >/dev/null
33for i in $(seq 1 90); do [ "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:3000/api/public/health)" = 200 ] && break; sleep 3; done
34 
35AUTH=$(printf '%s:%s' "$PK" "$SK" | base64 -w0)

34 
35AUTH=$(printf '%s:%s' "$PK" "$SK" | base64 -w0)
36curl -s -H "Authorization: Basic $AUTH" http://localhost:3000/api/public/projects | grep -q testproj \
37  && ok "API key authenticates; project provisioned" || bad "API-key auth / project init failed"
38 

38 
39TID="trace-$(date +%s)-abc"
40code=$(curl -s -o /tmp/lf-ing.json -w '%{http_code}' -X POST http://localhost:3000/api/public/ingestion \
41  -H "Authorization: Basic $AUTH" -H "Content-Type: application/json" \
42  -d "{\"batch\":[{\"id\":\"ev-1\",\"type\":\"trace-create\",\"timestamp\":\"2026-06-26T12:00:00.000Z\",\"body\":{\"id\":\"$TID\",\"name\":\"smoke-trace\",\"userId\":\"u1\"}}]}")

45seen=0
46for i in $(seq 1 30); do
47  curl -s -H "Authorization: Basic $AUTH" "http://localhost:3000/api/public/traces?limit=10" | grep -q "$TID" && { seen=1; break; }
48  sleep 2
49done

50[ "$seen" = 1 ] && ok "trace read back via public API (web->redis->worker->ClickHouse)" || bad "trace not visible after 60s"
51 
52ch=$($ENGINE exec $APP sh -c '. /app/data/.secrets/secrets.env; curl -s "http://localhost:8123/?user=clickhouse&password=${CLICKHOUSE_PASSWORD}" --data-binary "SELECT count() FROM traces"' 2>/dev/null)
53{ [ -n "$ch" ] && [ "$ch" -ge 1 ] 2>/dev/null; } && ok "ClickHouse has >=1 trace row ($ch)" || bad "ClickHouse trace count=$ch"
54ev=$($ENGINE exec $APP sh -c 'find /var/lib/minio/langfuse/events -type f 2>/dev/null | wc -l')

52ch=$($ENGINE exec $APP sh -c '. /app/data/.secrets/secrets.env; curl -s "http://localhost:8123/?user=clickhouse&password=${CLICKHOUSE_PASSWORD}" --data-binary "SELECT count() FROM traces"' 2>/dev/null)
53{ [ -n "$ch" ] && [ "$ch" -ge 1 ] 2>/dev/null; } && ok "ClickHouse has >=1 trace row ($ch)" || bad "ClickHouse trace count=$ch"
54ev=$($ENGINE exec $APP sh -c 'find /var/lib/minio/langfuse/events -type f 2>/dev/null | wc -l')
55[ "${ev:-0}" -ge 1 ] && ok "event object written to bundled MinIO ($ev)" || bad "no event objects in MinIO"
56 

58[ "${stray:-0}" = 0 ] && ok "no churning store left inside the backed-up /app/data" \
59  || bad "something is still under /app/data/{clickhouse,minio} ($stray entries) — the syncer race is not fixed"
60chp=$($ENGINE exec $APP sh -c 'ls -d /var/lib/clickhouse/store >/dev/null 2>&1 && echo yes || echo no')
61[ "$chp" = yes ] && ok "ClickHouse store is on its persistentDir" || bad "ClickHouse store not at /var/lib/clickhouse"
62 

59  || bad "something is still under /app/data/{clickhouse,minio} ($stray entries) — the syncer race is not fixed"
60chp=$($ENGINE exec $APP sh -c 'ls -d /var/lib/clickhouse/store >/dev/null 2>&1 && echo yes || echo no')
61[ "$chp" = yes ] && ok "ClickHouse store is on its persistentDir" || bad "ClickHouse store not at /var/lib/clickhouse"
62 
63echo "=== ingest result: $fails failure(s) ==="

47  "${ENGINE}" run --rm --user root \
48    -v "$1:/app/data:Z" \
49    -v "$2/clickhouse:/var/lib/clickhouse:Z" \
50    -v "$2/minio:/var/lib/minio:Z" \
51    "${IMAGE}" /app/code/conf/migrate-stores.sh 2>&1

48    -v "$1:/app/data:Z" \
49    -v "$2/clickhouse:/var/lib/clickhouse:Z" \
50    -v "$2/minio:/var/lib/minio:Z" \
51    "${IMAGE}" /app/code/conf/migrate-stores.sh 2>&1
52}

154  # it does not ship is simply absent. Only then are the container-side hits for these exact paths
155  # dropped, and only for these exact paths, in the same spirit as the pinned SSH keys below.
156  RUNTIME_PATHS=(/etc/hosts /etc/resolv.conf /etc/hostname)
157  RUNFLAGS=(--network=none)                       # no DNS/hosts wiring either engine can avoid
158  [[ "$CRI" == *podman* ]] && RUNFLAGS+=(--no-hosts)   # podman only; docker always injects

225  # hashes whenever the base image digest changes.
226  declare -A PINNED_SSH=(
227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85

225  # hashes whenever the base image digest changes.
226  declare -A PINNED_SSH=(
227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85

226  declare -A PINNED_SSH=(
227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85
230  )

226  declare -A PINNED_SSH=(
227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85
230  )

227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85
230  )
231  ssh_listing="$("$CRI" run --rm --user 0 "${RUNFLAGS[@]}" --entrypoint /bin/bash "$IMAGE" \

227    [/etc/ssh/ssh_host_ecdsa_key]=677458f83d985da3fd7cdd208e90e4eac09da5be205425a5f96a6242dc985c33
228    [/etc/ssh/ssh_host_ed25519_key]=0c575ce8d9ba487b05cc473fad4b0650fb950181028e6ac19796f86f56f22a7a
229    [/etc/ssh/ssh_host_rsa_key]=ae0ea8087e90baf138d277ca52b6cf47b5010adc0e5bd84236713eee1b85de85
230  )
231  ssh_listing="$("$CRI" run --rm --user 0 "${RUNFLAGS[@]}" --entrypoint /bin/bash "$IMAGE" \

230  )
231  ssh_listing="$("$CRI" run --rm --user 0 "${RUNFLAGS[@]}" --entrypoint /bin/bash "$IMAGE" \
232                  -c 'for f in /etc/ssh/ssh_host_*_key; do [ -e "$f" ] && sha256sum "$f"; done' 2>/dev/null)"
233  found=0; pinned_ok=0
234  while IFS= read -r line; do

230  )
231  ssh_listing="$("$CRI" run --rm --user 0 "${RUNFLAGS[@]}" --entrypoint /bin/bash "$IMAGE" \
232                  -c 'for f in /etc/ssh/ssh_host_*_key; do [ -e "$f" ] && sha256sum "$f"; done' 2>/dev/null)"
233  found=0; pinned_ok=0
234  while IFS= read -r line; do

241      shp="$(printf '%s\n' "$shp" | grep -vF "$f:" || true)"   # drop ONLY this verified exact path
242    else
243      emit ssh-key "$f sha256=$h is NOT a pinned base host key (new, changed or extra: treat as a leak)"
244    fi
245  done <<< "$ssh_listing"

246  echo "  host keys: $found found, $pinned_ok pinned-ok, ${#PINNED_SSH[@]} expected"
247  [[ "$found" -eq "${#PINNED_SSH[@]}" && "$pinned_ok" -eq "${#PINNED_SSH[@]}" ]] \
248    || emit ssh-key "host key count mismatch: $found found, $pinned_ok pinned-ok, ${#PINNED_SSH[@]} expected"
249 
250  # DISTRO-GENERATED PLACEHOLDER KEYS. Some Debian packages mint a self-signed key in their

255  # Nothing else is exempt: a key at any other path is still a finding, including a second key in
256  # the same directory. Add a path here only after confirming the package does not reference it.
257  PLACEHOLDER_KEYS=(/etc/ssl/private/ssl-cert-snakeoil.key /etc/prosody/certs/localhost.key)
258  for k in "${PLACEHOLDER_KEYS[@]}"; do
259    if printf '%s' "$shp" | grep -qF "$k:"; then

56hc=0
57for i in $(seq 1 90); do
58  [ "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:3000/api/public/health 2>/dev/null)" = "200" ] && { hc=1; break; }
59  sleep 3
60done

62 
63# 2. all four services RUNNING
64st=$($ENGINE exec $APP supervisorctl -c /etc/supervisor/supervisord.conf status 2>/dev/null)
65for svc in clickhouse minio web worker; do echo "$st" | grep -qE "^${svc}\s+RUNNING" && ok "service ${svc} RUNNING" || bad "service ${svc} not RUNNING"; done
66 

92 
93# 6. ClickHouse UTC
94tz=$($ENGINE exec $APP sh -c '. /app/data/.secrets/secrets.env; curl -s "http://localhost:8123/?user=clickhouse&password=${CLICKHOUSE_PASSWORD}" --data-binary "SELECT timezone()"' 2>/dev/null)
95[ "$tz" = "UTC" ] && ok "ClickHouse timezone is UTC" || bad "ClickHouse timezone='$tz' (want UTC)"
96 

23set -euo pipefail
24 
25CH_STORE=/var/lib/clickhouse
26MINIO_STORE=/var/lib/minio
27DUMP=/app/data/clickhouse-backup

24 
25CH_STORE=/var/lib/clickhouse
26MINIO_STORE=/var/lib/minio
27DUMP=/app/data/clickhouse-backup
28MDUMP=/app/data/minio-backup

28MDUMP=/app/data/minio-backup
29SNAP=/app/data/.clickhouse-snapshot        # transient: built and removed within this command
30CONF=/etc/clickhouse-server/backups.xml
31STATUS=/app/data/.last-backup.log
32 

16 
17declare -a PENDING=()
18for dst in /var/lib/clickhouse /var/lib/minio; do
19  [ -e "${dst}/.migration-cleanup-pending" ] && PENDING+=( "${dst}" )
20done

24 
25waited=0
26until curl -sf --max-time 5 "${HEALTH_URL}" >/dev/null 2>&1; do
27  sleep 10
28  waited=$(( waited + 10 ))

148 
149CH_SRC=/app/data/clickhouse
150CH_DST=/var/lib/clickhouse
151MINIO_SRC=/app/data/minio
152MINIO_DST=/var/lib/minio

150CH_DST=/var/lib/clickhouse
151MINIO_SRC=/app/data/minio
152MINIO_DST=/var/lib/minio
153 
154# MinIO's sentinel is .minio.sys, the directory the server itself creates to mark a real store. The

20set -euo pipefail
21 
22CH_STORE=/var/lib/clickhouse
23MINIO_STORE=/var/lib/minio
24DUMP=/app/data/clickhouse-backup

21 
22CH_STORE=/var/lib/clickhouse
23MINIO_STORE=/var/lib/minio
24DUMP=/app/data/clickhouse-backup
25MDUMP=/app/data/minio-backup

83 
84# Background, NOT --daemon: --daemon conflicts with the console logger this package configures.
85su -s /bin/bash cloudron -c "exec clickhouse-server --config-file=/etc/clickhouse-server/config.xml" \
86  > /tmp/restore-ch.log 2>&1 &
87SVPID=$!

90 
91i=0
92until curl -sf http://localhost:8123/ping >/dev/null 2>&1; do
93  i=$(( i + 1 ))
94  if [ "${i}" -ge 90 ]; then

25export NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=1536}"
26 
27wait_for "clickhouse" curl -sf http://localhost:8123/ping
28wait_for "minio"      curl -sf http://localhost:9100/minio/health/live
29wait_for "postgres"   pg_isready -d "${DATABASE_URL}"

26 
27wait_for "clickhouse" curl -sf http://localhost:8123/ping
28wait_for "minio"      curl -sf http://localhost:9100/minio/health/live
29wait_for "postgres"   pg_isready -d "${DATABASE_URL}"
30REDISCLI_AUTH="${REDIS_AUTH}" wait_for "redis" redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" ping

19export NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=768}"
20 
21wait_for "clickhouse" curl -sf http://localhost:8123/ping
22wait_for "minio"      curl -sf http://localhost:9100/minio/health/live
23REDISCLI_AUTH="${REDIS_AUTH}" wait_for "redis" redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" ping

20 
21wait_for "clickhouse" curl -sf http://localhost:8123/ping
22wait_for "minio"      curl -sf http://localhost:9100/minio/health/live
23REDISCLI_AUTH="${REDIS_AUTH}" wait_for "redis" redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" ping
24