Interactive Services on TES: Commands¶
This page lists the commands of Run Interactive Services on
TES without the explanations. They use the
values of the virtual machine eoc-services of the BioHackathon Europe 2026:
replace the addresses with your own. All commands run on the virtual machine
in ~/eoc_services, except those marked laptop, which run on your
computer.
Install the scripts¶
# disable "!" history expansion while pasting
set +H
# create the directory layout and change into it
mkdir -p ~/eoc_services/{tes-hub,omop-tesk/pod,rstudio-tesk} && cd ~/eoc_services
# tunnel hub
cat > tes-hub/tes-hub.sh <<'__TES_SERVICES_EOF__'
#!/usr/bin/env bash
# Tunnel hub for services that run as GA4GH TES tasks (default: TESK @ ELIXIR-FI).
#
# TES cannot publish ports and TESK pods only get outbound HTTPS, so each task dials this
# chisel server through your HTTPS reverse proxy (websocket) and publishes its port
# as 127.0.0.1:<port> on this VM:
#
# TES pod --https--> HUB_URL (e.g. https://hub.example.org/tes-tunnel/)
# --> chisel HUB_LISTEN_HOST:8090 --> VM 127.0.0.1:<port>
#
# Each service owns a hub user that may only open its own ports. Pods get common.sh
# prepended to their script and call start_tunnel <remote>...
#
# Usage:
# ./tes-hub.sh setup <HUB_URL> [LISTEN_HOST] # once: chisel, hub key, proxy snippet, start
# ./tes-hub.sh start | stop | restart | status
# ./tes-hub.sh users # hub users and the remotes each may open
# ./tes-hub.sh add-user <user:password> <regex>... # add or replace a user
# ./tes-hub.sh env # HUB_URL, CHISEL_URL, HUB_FINGERPRINT for clients
#
# The hub and the users survive restarts; tasks reconnect by themselves. It does not
# survive a VM reboot: run start again.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
STATE_DIR="${HUB_STATE_DIR:-$HERE/.state}"
mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR"
CONFIG="$STATE_DIR/config"
# setup writes HUB_URL and HUB_LISTEN_HOST here; environment variables override them.
while IFS='=' read -r k v; do [[ -n "${!k:-}" ]] || eval "$k=$v"; done < <(cat "$CONFIG" 2>/dev/null)
# ---- configuration -----------------------------------------------------------------------
HUB_URL="${HUB_URL:-}" # public URL (proxy location)
HUB_LISTEN_HOST="${HUB_LISTEN_HOST:-127.0.0.1}" # where the proxy reaches the VM
HUB_LISTEN_PORT="${HUB_LISTEN_PORT:-8090}"
CHISEL_VERSION="${CHISEL_VERSION:-1.12.0}"
chisel_url() { echo "https://github.com/jpillora/chisel/releases/download/v${CHISEL_VERSION}/chisel_${CHISEL_VERSION}_linux_$1.gz"; }
# The hub runs on this VM, the clients in the TES pods: their CPUs may differ.
HUB_ARCH="${HUB_ARCH:-$(case "$(uname -m)" in aarch64|arm64) echo arm64 ;; *) echo amd64 ;; esac)}"
POD_ARCH="${POD_ARCH:-amd64}"
CHISEL_URL="$(chisel_url "$POD_ARCH")" # sent to the tasks
# ------------------------------------------------------------------------------------------
CHISEL="$STATE_DIR/bin/chisel"
AUTHFILE="$STATE_DIR/authfile.json"
PIDF="$STATE_DIR/hub.pid"
LOGF="$STATE_DIR/hub.log"
KEYF="$STATE_DIR/hub.key"
die() { echo "ERROR: $*" >&2; exit 1; }
running() { [[ -s "$PIDF" ]] && kill -0 "$(cat "$PIDF")" 2>/dev/null; }
fingerprint() { grep -o 'Fingerprint [^ ]*' "$LOGF" 2>/dev/null | tail -1 | cut -d' ' -f2 || true; }
cmd_setup() {
HUB_URL="${1:-$HUB_URL}"; HUB_LISTEN_HOST="${2:-$HUB_LISTEN_HOST}"
[[ "$HUB_URL" == https://*/ ]] || die "usage: $0 setup https://<proxy-host>/<path>/ [LISTEN_HOST]"
printf 'HUB_URL=%q\nHUB_LISTEN_HOST=%q\nHUB_LISTEN_PORT=%q\n' "$HUB_URL" "$HUB_LISTEN_HOST" "$HUB_LISTEN_PORT" > "$CONFIG"
if [[ ! -x "$CHISEL" ]]; then
mkdir -p "$STATE_DIR/bin"
curl -fsSL "$(chisel_url "$HUB_ARCH")" | gunzip > "$CHISEL" && chmod 755 "$CHISEL"
echo "installed chisel $("$CHISEL" --version 2>&1 | head -1) to $CHISEL"
fi
[[ -s "$KEYF" ]] || { "$CHISEL" server --keygen "$KEYF" >/dev/null; chmod 600 "$KEYF"; echo "generated hub key $KEYF"; }
[[ -f "$AUTHFILE" ]] || { echo '{}' > "$AUTHFILE"; chmod 600 "$AUTHFILE"; }
local host path
host="$(python3 -c 'import sys; from urllib.parse import urlparse; print(urlparse(sys.argv[1]).hostname)' "$HUB_URL")"
path="$(python3 -c 'import sys; from urllib.parse import urlparse; print(urlparse(sys.argv[1]).path or "/")' "$HUB_URL")"
cat <<EOF
Add this to the server block of $host on your proxy, then reload it:
location $path {
proxy_pass http://$HUB_LISTEN_HOST:$HUB_LISTEN_PORT/;
proxy_http_version 1.1;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host \$host;
proxy_read_timeout 1d;
proxy_send_timeout 1d;
proxy_buffering off;
}
EOF
cmd_start
}
cmd_start() {
[[ -x "$CHISEL" && -s "$KEYF" && -n "$HUB_URL" ]] || die "run: $0 setup <HUB_URL>"
if running; then echo "hub already running (pid $(cat "$PIDF"))"; return; fi
# setsid: the hub outlives this shell. exec: the pid file holds chisel's own pid.
setsid -f sh -c 'echo $$ > "$0"; exec "$@"' "$PIDF" "$CHISEL" server \
--host "$HUB_LISTEN_HOST" --port "$HUB_LISTEN_PORT" --reverse \
--authfile "$AUTHFILE" --keyfile "$KEYF" --keepalive 25s >>"$LOGF" 2>&1 < /dev/null
sleep 2
running || { tail -5 "$LOGF"; die "hub failed to start"; }
echo "hub listening on $HUB_LISTEN_HOST:$HUB_LISTEN_PORT (pid $(cat "$PIDF"), log: $LOGF)"
echo "fingerprint $(fingerprint)"
}
cmd_stop() { if running; then kill "$(cat "$PIDF")" && rm -f "$PIDF" && echo "hub stopped"; else echo "hub not running"; fi; }
cmd_status() {
[[ -n "$HUB_URL" ]] || die "hub not set up (run: $0 setup <HUB_URL>)"
if running; then echo "hub running (pid $(cat "$PIDF"))"; else echo "hub not running"; fi
# While the hub runs, chisel answers plain HTTP with "Not found"; anything else is the proxy.
if [[ "$(curl -s -m 10 "$HUB_URL" 2>/dev/null)" == "Not found" ]]; then echo "proxy route OK: $HUB_URL"
else echo "proxy route not working: $HUB_URL (hub down, or the proxy location is missing)"; fi
# Ports the users may publish, and which of them are connected right now.
local ports listening p
ports="$(python3 - "$AUTHFILE" <<'PY' 2>/dev/null || true
import json, re, sys
ports = {p for v in json.load(open(sys.argv[1])).values() for r in v if r.startswith("^R:")
for p in re.findall(r"\d{2,5}", r.split(":")[-1])}
print(" ".join(sorted(ports, key=int)))
PY
)"
listening="$(ss -tln 2>/dev/null)"
for p in $ports; do
[[ "$listening" == *"127.0.0.1:$p "* ]] && echo " tunnel up: 127.0.0.1:$p" || echo " not connected: 127.0.0.1:$p"
done
}
cmd_users() {
python3 - "$AUTHFILE" <<'PY'
import json, sys
for k, v in json.load(open(sys.argv[1])).items():
print(f"{k.split(':')[0]:10} {' '.join(v)}")
PY
}
# add-user <user:password> <regex>...: replaces any existing entry for that user.
# chisel reloads the authfile when it changes, so no restart is needed.
cmd_add_user() {
local auth="${1:-}"; shift || true
[[ "$auth" == *:* && $# -gt 0 ]] || die "usage: $0 add-user <user:password> <regex>..."
[[ -f "$AUTHFILE" ]] || die "run: $0 setup <HUB_URL>"
# Several services may run setup at the same time: one writer at a time.
(
flock -w 30 9 || die "authfile locked by another process"
python3 - "$AUTHFILE" "$auth" "$@" <<'PY'
import json, os, sys
p, auth, regexes = sys.argv[1], sys.argv[2], sys.argv[3:]
user = auth.split(":")[0] + ":"
a = {k: v for k, v in json.load(open(p)).items() if not k.startswith(user)}
a[auth] = regexes
# Write a new file and rename it, so chisel never reads a half-written authfile.
with open(p + ".tmp", "w") as f:
json.dump(a, f, indent=2)
os.chmod(p + ".tmp", 0o600)
os.replace(p + ".tmp", p)
PY
) 9>"$STATE_DIR/authfile.lock"
echo "hub user '${auth%%:*}' may open: $*"
}
cmd_env() {
[[ -n "$HUB_URL" ]] || die "run: $0 setup <HUB_URL>"
printf 'HUB_URL=%q\nCHISEL_URL=%q\nHUB_FINGERPRINT=%q\n' "$HUB_URL" "$CHISEL_URL" "$(fingerprint)"
}
case "${1:-}" in
setup) shift; cmd_setup "$@" ;;
start) cmd_start ;;
stop) cmd_stop ;;
restart) cmd_stop; sleep 1; cmd_start ;;
status) cmd_status ;;
users) cmd_users ;;
add-user) shift; cmd_add_user "$@" ;;
env) cmd_env ;;
*) sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 1 ;;
esac
__TES_SERVICES_EOF__
# code sent into every task: downloads chisel, opens the tunnel
cat > tes-hub/common.sh <<'__TES_SERVICES_EOF__'
# shellcheck shell=sh
# Pod-side tunnel helpers, prepended to the pod script by the launchers
# (tes-service.sh, ../omop-tesk/omop-tesk-fi.sh).
# Runs inside a TES pod: random UID, read-only image, outbound HTTPS only.
set -u
export HOME=/tmp
log() { echo "[$(date -u +%H:%M:%S) ${ROLE}] $*"; }
fetch() { # url dest; works with curl, wget or python3, whichever the image has
if command -v curl >/dev/null 2>&1; then curl -fsSL --retry 3 -o "$2" "$1"
elif command -v wget >/dev/null 2>&1; then wget -q -O "$2" "$1"
else python3 -c 'import sys, urllib.request; urllib.request.urlretrieve(*sys.argv[1:])' "$1" "$2"; fi
}
# Connects to the chisel hub on the VM (through the HTTPS proxy) and keeps the given
# remotes open. chisel reconnects by itself, so the tunnel survives hub/proxy restarts.
start_tunnel() {
if [ -z "${HUB_URL:-}" ]; then log "HUB_URL not set, running without tunnel"; return 0; fi
fetch "$CHISEL_URL" /tmp/chisel.gz && gunzip -f /tmp/chisel.gz && chmod 755 /tmp/chisel \
|| { log "ERROR: could not download chisel from $CHISEL_URL"; exit 2; }
log "tunnel to $HUB_URL: $*"
/tmp/chisel client --auth "$HUB_AUTH" --keepalive 25s --max-retry-interval 30s \
${HUB_FINGERPRINT:+--fingerprint "$HUB_FINGERPRINT"} "$HUB_URL" "$@" &
}
__TES_SERVICES_EOF__
# generic launcher for single-container services
cat > tes-hub/tes-service.sh <<'__TES_SERVICES_EOF__'
#!/usr/bin/env bash
# Run a web service (any container image) as a GA4GH TES task and make it reachable at
# 127.0.0.1:<HUB_PORT> on this VM through the tunnel hub (tes-hub.sh in this directory).
#
# A service is a directory next to tes-hub/ with two files:
# service.conf NAME, IMAGE, PORT (port in the pod); optional HUB_PORT (port on this VM,
# default PORT), POD_SHELL (/bin/sh), RAM_GB (1), CPU_CORES (1). Any other
# KEY=value is passed to the pod as an environment variable.
# pod.sh runs in the task after common.sh: starts the service on 127.0.0.1:$PORT,
# calls start_tunnel "R:127.0.0.1:$HUB_PORT:127.0.0.1:$PORT", stays in the
# foreground. Generated state goes to <service>/.state/.
#
# Usage: tes-service.sh <service-dir> <command>
# up start: hub user (once), task (unless running), keep-alive loop; waits for the tunnel
# down stop: keep-alive loop, lock the old pod out of the hub (new password + hub
# restart, because cancel does not stop the pod on TESK), cancel
# restart down + up
# status task, tunnel and keep-alive loop
# logs task logs (TESK fills them when the task ends)
# selftest short task without tunnel (pod.sh sees SELFTEST=1); fails unless SELFTEST_OK
# print the TES request JSON
# setup | submit | cancel | watch single steps, used by up/down
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
[[ $# -ge 2 ]] || { sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 1; }
if [[ -d "$1" ]]; then DIR="$(cd "$1" && pwd)"; elif [[ -d "$HERE/../$1" ]]; then DIR="$(cd "$HERE/../$1" && pwd)"
else echo "ERROR: no service directory '$1'" >&2; exit 1; fi
CMD="$2"; SELF="$0 $1"
[[ -f "$DIR/service.conf" && -f "$DIR/pod.sh" ]] || { echo "ERROR: $DIR needs service.conf and pod.sh" >&2; exit 1; }
STATE_DIR="$DIR/.state"
mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR"
[[ -f "$STATE_DIR/config" ]] && source "$STATE_DIR/config"
source "$DIR/service.conf"
CONF_KEYS="$(sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$DIR/service.conf")"
# ---- configuration -----------------------------------------------------------------------
TES_URL="${TES_URL:-https://csc-tesk-noauth.rahtiapp.fi}" # TESK @ ELIXIR-FI
TES_API="${TES_URL%/}/ga4gh/tes/v1"
HUB="${HUB:-$HERE/tes-hub.sh}" # shared tunnel hub
: "${NAME:?NAME missing in service.conf}" "${IMAGE:?IMAGE missing in service.conf}" "${PORT:?PORT missing in service.conf}"
HUB_PORT="${HUB_PORT:-$PORT}"
POD_SHELL="${POD_SHELL:-/bin/sh}"
RAM_GB="${RAM_GB:-1}"
CPU_CORES="${CPU_CORES:-1}"
HUB_AUTH="${HUB_AUTH:-}" # generated by setup
[[ "${HUB_AUTH%%:*}" == "$NAME" ]] || HUB_AUTH="" # e.g. .state copied from another service
WATCH_INTERVAL="${WATCH_INTERVAL:-60}"
# ------------------------------------------------------------------------------------------
die() { echo "ERROR: $*" >&2; exit 1; }
json_get() { python3 -c 'import json,sys; d=json.load(sys.stdin); print(eval(sys.argv[1], {"d": d}))' "$1"; }
ID_FILE="$STATE_DIR/task"
WATCH_PID="$STATE_DIR/watch.pid"
task_id() { [[ -s "$ID_FILE" ]] && cat "$ID_FILE" || true; }
need_id() { [[ -s "$ID_FILE" ]] || die "not submitted"; cat "$ID_FILE"; }
task_state() { curl -fsS -m 30 "$TES_API/tasks/$1?view=MINIMAL" | json_get 'd["state"]'; }
current_state() { local id; id="$(task_id)"; [[ -n "$id" ]] && { task_state "$id" 2>/dev/null || echo UNREACHABLE; } || echo NONE; }
is_done() { [[ "$1" =~ ^(NONE|COMPLETE|EXECUTOR_ERROR|SYSTEM_ERROR|CANCELED|PREEMPTED)$ ]]; }
tunnel_up() { [[ "$(ss -tln 2>/dev/null)" == *"127.0.0.1:$HUB_PORT "* ]]; }
# The pid must still be this service's loop: after a reboot, another process may have it.
watching() {
local pid; pid="$(cat "$WATCH_PID" 2>/dev/null)" || return 1
[[ -n "$pid" ]] && tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -qF -- "tes-service.sh $DIR watch"
}
print_logs() {
json_get '"state: " + d["state"] + "\n" + "\n".join((l.get("stdout") or "") + (l.get("stderr") or "") for t in d.get("logs",[]) for l in t.get("logs",[]))'
}
# build_task [selftest]
build_task() {
local selftest="${1:-}" k
[[ -n "$selftest" || -n "$HUB_AUTH" ]] || die "credentials missing (run: $SELF setup)"
local HUB_URL CHISEL_URL HUB_FINGERPRINT; eval "$("$HUB" env)"
# service.conf values (minus the launcher's own keys) become the pod's environment.
for k in $CONF_KEYS PORT HUB_PORT; do
[[ $k =~ ^(NAME|IMAGE|POD_SHELL|RAM_GB|CPU_CORES|TES_URL)$ ]] || printf '%s\0%s\0' "$k" "${!k}"
done > "$STATE_DIR/pod.env"
export HUB_AUTH HUB_URL CHISEL_URL HUB_FINGERPRINT NAME IMAGE POD_SHELL RAM_GB CPU_CORES
SELFTEST="${selftest:+1}" ENV_FILE="$STATE_DIR/pod.env" \
SCRIPT="$(cat "$HERE/common.sh" "$DIR/pod.sh")" python3 - <<'PY'
import json, os
e = os.environ
selftest = e["SELFTEST"] == "1"
f = open(e["ENV_FILE"]).read().split("\0")[:-1]
env = {"ROLE": e["NAME"], **dict(zip(f[::2], f[1::2]))}
env.update({"SELFTEST": "1"} if selftest else {
k: e[k] for k in ("HUB_AUTH", "HUB_URL", "HUB_FINGERPRINT", "CHISEL_URL")})
print(json.dumps({
"name": e["NAME"] + ("-selftest" if selftest else ""),
"description": f"{e['NAME']}, see tes-service.sh",
"tags": {"service": e["NAME"], "managed-by": "tes-service.sh"},
"resources": {"cpu_cores": int(e["CPU_CORES"]), "ram_gb": float(e["RAM_GB"]), "preemptible": False},
"executors": [{"image": e["IMAGE"], "command": [e["POD_SHELL"], "-c", e["SCRIPT"]], "env": env}],
}, indent=2))
PY
rm -f "$STATE_DIR/pod.env"
}
post_task() {
build_task "$@" | curl -fsS --connect-timeout 10 -m 60 -X POST -H 'Content-Type: application/json' \
--data-binary @- "$TES_API/tasks" | json_get 'd["id"]'
}
cmd_setup() {
if [[ -z "$HUB_AUTH" ]]; then
HUB_AUTH="$NAME:$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')"
printf 'HUB_AUTH=%q\n' "$HUB_AUTH" > "$STATE_DIR/config"; chmod 600 "$STATE_DIR/config"
echo "generated hub credentials in $STATE_DIR/config"
fi
"$HUB" add-user "$HUB_AUTH" "^R:127\\.0\\.0\\.1:$HUB_PORT\$"
}
# Lock out the pods of this service: a new hub password and a hub restart.
lock_out() { rm -f "$STATE_DIR/config"; HUB_AUTH=""; cmd_setup; "$HUB" restart; }
# One submission at a time (up, watch and submit may run together), and only if no task
# of this service runs, so that two tasks never compete for the port.
cmd_submit() {
[[ "$("$HUB" status)" == *"hub running"* ]] || die "hub not running (run: $HUB start)"
exec 9>"$STATE_DIR/submit.lock"
flock -w 120 9 || die "another submission is running"
local id state; state="$(current_state)"
is_done "$state" || { echo "task $(task_id) is $state, not submitting"; flock -u 9; return 0; }
id="$(post_task)" || { flock -u 9; die "submitting to $TES_API failed"; }
echo "$id" > "$ID_FILE"; echo "submitted $id"
flock -u 9
}
cmd_cancel() {
local id; id="$(task_id)"
[[ -n "$id" ]] && curl -fsS -m 30 -X POST "$TES_API/tasks/$id:cancel" >/dev/null && echo "canceled $id" || true
}
cmd_status() {
local id; id="$(task_id)"
echo "task: ${id:-NOT_SUBMITTED} ${id:+$(current_state)}"
tunnel_up && echo "tunnel up: http://localhost:$HUB_PORT" || echo "tunnel not connected"
watching && echo "keep-alive loop running (pid $(cat "$WATCH_PID"))" || echo "keep-alive loop not running"
}
cmd_selftest() {
local id i out
id="$(post_task selftest)" || die "submitting to $TES_API failed"; echo "selftest: $id"
for ((i = 0; i < 60; i++)); do
is_done "$(task_state "$id" 2>/dev/null || echo UNKNOWN)" && break
sleep 10
done
out="$(curl -fsS -m 60 "$TES_API/tasks/$id?view=FULL" | print_logs)"; echo "$out"
[[ "$out" == "state: COMPLETE"* && "$out" == *SELFTEST_OK* ]] || die "selftest failed"
}
cmd_watch() {
echo $$ > "$WATCH_PID"
trap 'rm -f "$WATCH_PID"' EXIT
echo "$(date -Is) keeping $NAME alive on $TES_URL (every ${WATCH_INTERVAL}s)"
local state
while true; do
state="$(current_state)"
if is_done "$state"; then
echo "$(date -Is) task $(task_id) is $state, resubmitting"
# A canceled task's pod may still hold the port: lock it out first.
if tunnel_up; then echo "$(date -Is) port $HUB_PORT still taken, locking the old pod out"; lock_out || true; fi
(cmd_submit) || true
fi
sleep "$WATCH_INTERVAL"
done
}
cmd_up() {
[[ -n "$HUB_AUTH" ]] || cmd_setup
local state i; state="$(current_state)"
if is_done "$state"; then cmd_submit; else echo "task $(task_id) is $state, not resubmitting"; fi
if ! watching; then
setsid -f "$HERE/tes-service.sh" "$DIR" watch >> "$STATE_DIR/watch.log" 2>&1 < /dev/null
echo "keep-alive loop started (log: $STATE_DIR/watch.log)"
fi
echo "waiting for the tunnel (usually 1-3 minutes; Ctrl-C is safe, the task keeps running)"
for ((i = 0; i < 60; i++)); do tunnel_up && break; sleep 10; done
tunnel_up || die "tunnel not up after 10 minutes, see: $SELF status"
echo "$NAME is up on this VM at 127.0.0.1:$HUB_PORT. On your laptop run:"
echo " ssh -N -L $HUB_PORT:127.0.0.1:$HUB_PORT $(id -un)@$(hostname -f)"
echo "and open http://localhost:$HUB_PORT"
}
cmd_down() {
if watching; then kill "$(cat "$WATCH_PID")" && rm -f "$WATCH_PID" && echo "keep-alive loop stopped"; fi
# Cancel marks the task CANCELED but the pod keeps running: lock it out of the hub first,
# so that an interrupted down never leaves the old pod connected.
lock_out
cmd_cancel
}
case "$CMD" in
up) cmd_up ;;
down) cmd_down ;;
restart) cmd_down; cmd_up ;;
setup) cmd_setup ;;
submit) cmd_submit ;;
status) cmd_status ;;
logs) id="$(need_id)"; curl -fsS -m 60 "$TES_API/tasks/$id?view=FULL" | print_logs ;;
cancel) cmd_cancel ;;
watch) cmd_watch ;;
selftest) cmd_selftest ;;
print) build_task ;;
*) die "unknown command '$CMD' (run $0 without arguments for help)" ;;
esac
__TES_SERVICES_EOF__
# Atlas stack launcher
cat > omop-tesk/omop-tesk-fi.sh <<'__TES_SERVICES_EOF__'
#!/usr/bin/env bash
# Run OHDSI Atlas + WebAPI + an OMOP CDM database as three GA4GH TES tasks on
# TESK @ ELIXIR-FI and make Atlas reachable at http://localhost:8080 on this VM.
#
# TES cannot publish ports and TESK pods only get outbound HTTPS, so every task dials the
# tunnel hub in ../tes-hub (chisel on this VM, behind your HTTPS reverse proxy):
#
# browser -> VM 127.0.0.1:8080 <=hub= Atlas task (nginx, /WebAPI/ -> hub:18081)
# VM 127.0.0.1:18081 <=hub= WebAPI task (DB via hub:15432)
# VM 127.0.0.1:15432 <=hub= DB task (postgres + Eunomia demo CDM)
#
# Browser on another machine: ssh -N -L 8080:127.0.0.1:8080 <user>@<this VM>
#
# Usage:
# ./omop-tesk-fi.sh up # start what isn't running and the keep-alive loop,
# # wait until Atlas works
# ./omop-tesk-fi.sh down # stop the loop, lock the old pods out of the hub, cancel all
# ./omop-tesk-fi.sh restart # down + up: a fresh stack
# ./omop-tesk-fi.sh status # tasks, tunnels, WebAPI and data source
# ./omop-tesk-fi.sh setup # once: credentials + hub user "omop" (needs tes-hub setup)
# ./omop-tesk-fi.sh submit [db|webapi|atlas|all] (default: all)
# ./omop-tesk-fi.sh logs <role> # executor logs (TESK fills them when a task ends)
# ./omop-tesk-fi.sh cancel [role|all]
# ./omop-tesk-fi.sh watch # the keep-alive loop (up starts it in the background)
# ./omop-tesk-fi.sh selftest db|atlas # short task without tunnel; fails unless SELFTEST_OK
# ./omop-tesk-fi.sh print <role> # print the TES request JSON
#
# SECURITY: csc-tesk-noauth is unauthenticated. Anyone can read a task with ?view=FULL,
# including the hub credentials and DB password in it. The hub only lets those credentials
# open the three ports above, and the DB holds synthetic demo data only. Don't use real data.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
STATE_DIR="${STATE_DIR:-$HERE/.state}"
mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR"
# setup writes generated values here; environment variables override them.
[[ -f "$STATE_DIR/config" ]] && source "$STATE_DIR/config"
# ---- configuration -----------------------------------------------------------------------
TES_URL="${TES_URL:-https://csc-tesk-noauth.rahtiapp.fi}" # TESK @ ELIXIR-FI
TES_API="${TES_URL%/}/ga4gh/tes/v1"
HUB="${HUB:-$HERE/../tes-hub/tes-hub.sh}" # shared tunnel hub
HUB_ATLAS_PORT="${HUB_ATLAS_PORT:-8080}" # -> http://localhost:8080
HUB_WEBAPI_PORT="${HUB_WEBAPI_PORT:-18081}"
HUB_DB_PORT="${HUB_DB_PORT:-15432}"
HUB_AUTH="${HUB_AUTH:-}" # user:pass, generated by setup
DB_PASSWORD="${DB_PASSWORD:-}" # generated by setup
EUNOMIA_URL="${EUNOMIA_URL:-https://raw.githubusercontent.com/OHDSI/EunomiaDatasets/main/datasets/GiBleed/GiBleed_5.3.zip}"
CDM_DDL_URL="${CDM_DDL_URL:-https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.1/inst/ddl/5.3/postgresql/OMOPCDM_postgresql_5.3_ddl.sql}"
DB_IMAGE="${DB_IMAGE:-postgres:16-alpine}"
# Pinned: WebAPI 3 (ohdsi/webapi:latest since 2026-11) starts differently, see pod/webapi.sh.
WEBAPI_IMAGE="${WEBAPI_IMAGE:-ohdsi/webapi:2.15.2}"
ATLAS_IMAGE="${ATLAS_IMAGE:-ohdsi/atlas:2.15.0}"
# Requests with ram_gb > 1 or any disk_gb never started on this TESK (2026-10-01), so
# WebAPI gets 1 GB with a small heap.
WEBAPI_RAM_GB="${WEBAPI_RAM_GB:-1}"
WEBAPI_JAVA_OPTS="${WEBAPI_JAVA_OPTS:--Xmx640m -XX:MaxMetaspaceSize=192m -Xss512k -Djava.security.egd=file:///dev/./urandom}"
WATCH_INTERVAL="${WATCH_INTERVAL:-60}"
ROLES=(db webapi atlas)
# ------------------------------------------------------------------------------------------
die() { echo "ERROR: $*" >&2; exit 1; }
need() { command -v "$1" >/dev/null || die "'$1' is required"; }
need curl; need python3
json_get() { python3 -c 'import json,sys; d=json.load(sys.stdin); print(eval(sys.argv[1], {"d": d}))' "$1"; }
id_file() { echo "$STATE_DIR/task_$1"; }
task_id() { [[ -s "$(id_file "$1")" ]] && cat "$(id_file "$1")" || true; }
task_state() { curl -fsS -m 30 "$TES_API/tasks/$1?view=MINIMAL" | json_get 'd["state"]'; }
WATCH_PID="$STATE_DIR/watch.pid"
# The pid must still be this stack's loop: after a reboot, another process may have it.
watching() {
local pid; pid="$(cat "$WATCH_PID" 2>/dev/null)" || return 1
[[ -n "$pid" ]] && tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -qF -- "$HERE/omop-tesk-fi.sh watch"
}
# One process at a time submits, cancels or rotates credentials (up, down and watch).
lock() { exec 9>"$STATE_DIR/lock"; flock -w 300 9 || die "another omop-tesk-fi.sh holds $STATE_DIR/lock"; }
unlock() { flock -u 9; }
check_role() { [[ " ${ROLES[*]} " == *" $1 "* ]] || die "unknown role '$1' (db|webapi|atlas)"; }
# build_task <role> [selftest]
build_task() {
local role="$1" selftest="${2:-}"
check_role "$role"
[[ -n "$HUB_AUTH" && -n "$DB_PASSWORD" ]] || die "credentials missing (run: $0 setup)"
local HUB_URL CHISEL_URL HUB_FINGERPRINT; eval "$("$HUB" env)"
ROLE="$role" SELFTEST="${selftest:+1}" \
TES_ENV="$(python3 - <<PY
import json
common = {"ROLE": "$role", "CHISEL_URL": "$CHISEL_URL", "HUB_URL": "$HUB_URL" if not "$selftest" else "",
"HUB_AUTH": "$HUB_AUTH", "HUB_FINGERPRINT": "$HUB_FINGERPRINT", "HUB_ATLAS_PORT": "$HUB_ATLAS_PORT",
"HUB_WEBAPI_PORT": "$HUB_WEBAPI_PORT", "HUB_DB_PORT": "$HUB_DB_PORT"}
if "$selftest": common.update(SELFTEST="1", HUB_AUTH="")
pg = "jdbc:postgresql://127.0.0.1:5432/postgres"
extra = {
"db": {"DB_PASSWORD": "$DB_PASSWORD", "EUNOMIA_URL": "$EUNOMIA_URL", "CDM_DDL_URL": "$CDM_DDL_URL"},
"webapi": {
"WEBAPI_JAVA_OPTS": "$WEBAPI_JAVA_OPTS", "SERVER_PORT": "8080",
"DATASOURCE_DRIVERCLASSNAME": "org.postgresql.Driver", "DATASOURCE_URL": pg,
"DATASOURCE_USERNAME": "postgres", "DATASOURCE_PASSWORD": "$DB_PASSWORD",
"DATASOURCE_OHDSI_SCHEMA": "webapi",
"SPRING_JPA_PROPERTIES_HIBERNATE_DEFAULT_SCHEMA": "webapi",
"SPRING_JPA_PROPERTIES_HIBERNATE_DIALECT": "org.hibernate.dialect.PostgreSQLDialect",
"SPRING_BATCH_REPOSITORY_TABLEPREFIX": "webapi.BATCH_",
"FLYWAY_DATASOURCE_DRIVERCLASSNAME": "org.postgresql.Driver", "FLYWAY_DATASOURCE_URL": pg,
"FLYWAY_DATASOURCE_USERNAME": "postgres", "FLYWAY_DATASOURCE_PASSWORD": "$DB_PASSWORD",
"FLYWAY_LOCATIONS": "classpath:db/migration/postgresql", "FLYWAY_SCHEMAS": "webapi",
"FLYWAY_PLACEHOLDERS_OHDSISCHEMA": "webapi", "FLYWAY_BASELINEONMIGRATE": "true",
"SECURITY_PROVIDER": "DisabledSecurity", "SECURITY_SSL_ENABLED": "false"},
"atlas": {},
}["$role"]
print(json.dumps({**common, **extra}))
PY
)" \
IMAGE="$(case $role in db) echo "$DB_IMAGE";; webapi) echo "$WEBAPI_IMAGE";; atlas) echo "$ATLAS_IMAGE";; esac)" \
RAM="$([[ $role == webapi ]] && echo "$WEBAPI_RAM_GB" || echo 1)" \
SCRIPT="$(cat "$(dirname "$HUB")/common.sh" "$HERE/pod/$role.sh")" \
python3 - <<'PY'
import json, os
e = os.environ
selftest = e["SELFTEST"] == "1"
print(json.dumps({
"name": f"omop-{e['ROLE']}" + ("-selftest" if selftest else ""),
"description": f"OMOP stack ({e['ROLE']}), see omop-tesk-fi.sh",
"tags": {"service": "omop", "role": e["ROLE"], "managed-by": "omop-tesk-fi.sh"},
"resources": {"cpu_cores": 1, "ram_gb": float(e["RAM"]), "preemptible": False},
"executors": [{
"image": e["IMAGE"],
"command": ["/bin/sh", "-c", e["SCRIPT"]],
"env": json.loads(e["TES_ENV"]),
}],
}, indent=2))
PY
}
submit_role() {
local role="$1" resp id
[[ "$("$HUB" status)" == *"hub running"* ]] || die "hub not running (run: setsid -f $HUB start)"
resp="$(build_task "$role" | curl -fsS --connect-timeout 10 -m 60 -X POST -H 'Content-Type: application/json' \
--data-binary @- "$TES_API/tasks")" || die "submitting $role to $TES_API failed"
id="$(json_get 'd["id"]' <<<"$resp")"
echo "$id" > "$(id_file "$role")"
echo "$role: submitted $id"
}
cancel_role() {
local id; id="$(task_id "$1")"
[[ -n "$id" ]] && curl -fsS -m 30 -X POST "$TES_API/tasks/$id:cancel" >/dev/null && echo "$1: canceled $id" || true
}
roles_arg() { if [[ -z "${1:-}" || "$1" == all ]]; then echo "${ROLES[@]}"; else check_role "$1"; echo "$1"; fi; }
cmd_setup() {
if [[ ! -f "$STATE_DIR/config" ]]; then
local pw dbpw
pw="$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')"
dbpw="$(python3 -c 'import secrets; print(secrets.token_urlsafe(18))')"
printf 'HUB_AUTH=%q\nDB_PASSWORD=%q\n' "omop:$pw" "$dbpw" > "$STATE_DIR/config"
chmod 600 "$STATE_DIR/config"; source "$STATE_DIR/config"
echo "generated hub credentials and DB password in $STATE_DIR/config"
fi
# The credentials may only publish the three reverse ports and reach DB/WebAPI on the hub.
"$HUB" add-user "$HUB_AUTH" \
"^R:127\\.0\\.0\\.1:($HUB_ATLAS_PORT|$HUB_WEBAPI_PORT|$HUB_DB_PORT)\$" \
"^127\\.0\\.0\\.1:($HUB_WEBAPI_PORT|$HUB_DB_PORT)\$"
}
role_state() { local id; id="$(task_id "$1")"; [[ -n "$id" ]] && { task_state "$id" 2>/dev/null || echo UNREACHABLE; } || echo NOT_SUBMITTED; }
is_done() { [[ "$1" =~ ^(NOT_SUBMITTED|COMPLETE|EXECUTOR_ERROR|SYSTEM_ERROR|CANCELED|PREEMPTED)$ ]]; }
tunnel_up() { [[ "$(ss -tln 2>/dev/null)" == *"127.0.0.1:$1 "* ]]; }
webapi_ready() { curl -fs -m 60 "http://127.0.0.1:$HUB_WEBAPI_PORT/WebAPI/info" 2>/dev/null; }
source_ready() { curl -fs -m 60 "http://127.0.0.1:$HUB_WEBAPI_PORT/WebAPI/source/sources" 2>/dev/null | grep -q '"EUNOMIA"'; }
atlas_ready() { curl -fs -m 30 -o /dev/null "http://127.0.0.1:$HUB_ATLAS_PORT/atlas/" 2>/dev/null; }
stack_ready() { webapi_ready >/dev/null && source_ready && atlas_ready; }
cmd_status() {
local r p info
for r in "${ROLES[@]}"; do printf '%-7s %-15s %s\n' "$r" "$(task_id "$r" | grep . || echo -)" "$(role_state "$r")"; done
for p in "$HUB_ATLAS_PORT" "$HUB_WEBAPI_PORT" "$HUB_DB_PORT"; do
tunnel_up "$p" && echo "tunnel up: 127.0.0.1:$p" || echo "not connected: 127.0.0.1:$p"
done
if info="$(webapi_ready)"; then echo "WebAPI ready: $(grep -o '"version":"[^"]*"' <<<"$info" | head -1)"; else echo "WebAPI not ready (its first start takes 10-15 minutes)"; fi
source_ready && echo "data source EUNOMIA registered" || echo "data source EUNOMIA not registered yet"
atlas_ready && echo "Atlas ready: http://localhost:$HUB_ATLAS_PORT/atlas/" || echo "Atlas not ready"
watching && echo "keep-alive loop running (pid $(cat "$WATCH_PID"))" || echo "keep-alive loop not running"
}
# up: submit the roles that aren't running, start the keep-alive loop, then wait until
# Atlas answers and WebAPI has the data source.
cmd_up() {
[[ -n "$HUB_AUTH" ]] || cmd_setup
local r state i p n
lock
for r in "${ROLES[@]}"; do
state="$(role_state "$r")"
if ! is_done "$state"; then echo "$r: task $(task_id "$r") is $state, not resubmitting"
elif [[ $r == db ]] && ! is_done "$(role_state webapi)"; then
# A new database is empty, and WebAPI only migrates on its first start.
unlock; die "the database task ended while WebAPI runs: run $0 restart"
else submit_role "$r"; fi
done
unlock
if ! watching; then
setsid -f "$HERE/omop-tesk-fi.sh" watch >> "$STATE_DIR/watch.log" 2>&1 < /dev/null
echo "keep-alive loop started (log: $STATE_DIR/watch.log)"
fi
echo "waiting for Atlas, WebAPI and the data source (first start 10-15 minutes; Ctrl-C is safe, the tasks keep running)"
for ((i = 0; i < 240; i++)); do
stack_ready && break
if (( i % 30 == 29 )); then
n=0; for p in "$HUB_ATLAS_PORT" "$HUB_WEBAPI_PORT" "$HUB_DB_PORT"; do tunnel_up "$p" && n=$((n + 1)); done
echo "$(date +%H:%M) still waiting: $n of 3 tunnels up, WebAPI $(webapi_ready >/dev/null && echo up || echo starting)"
fi
sleep 10
done
stack_ready || die "not ready after 40 minutes, see: $0 status, $0 logs webapi"
echo "Atlas is up on this VM at 127.0.0.1:$HUB_ATLAS_PORT. On your laptop run:"
echo " ssh -N -L $HUB_ATLAS_PORT:127.0.0.1:$HUB_ATLAS_PORT $(id -un)@$(hostname -f)"
echo "and open http://localhost:$HUB_ATLAS_PORT"
}
# Cancel marks the tasks CANCELED, but their pods keep running on TESK. New credentials and
# a hub restart lock them out. All three roles share the hub user, so this replaces the stack.
lock_out() { rm -f "$STATE_DIR/config"; HUB_AUTH="" DB_PASSWORD=""; cmd_setup; "$HUB" restart; }
# down: stop the loop first (it would resubmit), lock the pods out before cancelling, so that
# an interrupted down never leaves them connected.
cmd_down() {
if watching; then kill "$(cat "$WATCH_PID")" && rm -f "$WATCH_PID" && echo "keep-alive loop stopped"; fi
lock
lock_out
local r; for r in "${ROLES[@]}"; do cancel_role "$r"; done
unlock
}
cmd_logs() {
local id; check_role "${1:-}"; id="$(task_id "$1")"; [[ -n "$id" ]] || die "$1 not submitted"
curl -fsS -m 60 "$TES_API/tasks/$id?view=FULL" | json_get \
'"state: " + d["state"] + "\n" + "\n".join((l.get("stdout") or "") + (l.get("stderr") or "") for t in d.get("logs",[]) for l in t.get("logs",[]))'
}
cmd_selftest() {
local role="${1:-}" resp id state i out
[[ "$role" == db || "$role" == atlas ]] || die "usage: $0 selftest db|atlas"
[[ -n "$HUB_AUTH" && -n "$DB_PASSWORD" ]] || cmd_setup
resp="$(build_task "$role" selftest | curl -fsS --connect-timeout 10 -m 60 -X POST \
-H 'Content-Type: application/json' --data-binary @- "$TES_API/tasks")" || die "submitting to $TES_API failed"
id="$(json_get 'd["id"]' <<<"$resp")"; echo "selftest $role: $id"
for ((i = 0; i < 90; i++)); do
state="$(task_state "$id" 2>/dev/null || echo UNKNOWN)"
case "$state" in COMPLETE|EXECUTOR_ERROR|SYSTEM_ERROR|CANCELED) break ;; esac
sleep 10
done
out="$(curl -fsS -m 60 "$TES_API/tasks/$id?view=FULL" | json_get \
'"state: " + d["state"] + "\n" + "\n".join((l.get("stdout") or "") + (l.get("stderr") or "") for t in d.get("logs",[]) for l in t.get("logs",[]))')"
echo "$out"
[[ "$out" == "state: COMPLETE"* && "$out" == *SELFTEST_OK* ]] || die "selftest $role failed"
}
port_of() { case $1 in db) echo "$HUB_DB_PORT" ;; webapi) echo "$HUB_WEBAPI_PORT" ;; atlas) echo "$HUB_ATLAS_PORT" ;; esac; }
# Replace the whole stack: lock the old pods out, cancel them, submit all roles.
replace_stack() {
local r
lock_out
for r in "${ROLES[@]}"; do cancel_role "$r"; done
for r in "${ROLES[@]}"; do submit_role "$r"; done
}
cmd_watch() {
echo $$ > "$WATCH_PID"
trap 'rm -f "$WATCH_PID"' EXIT
echo "$(date -Is) keeping OMOP tasks alive on $TES_URL (every ${WATCH_INTERVAL}s)"
local r id state
while true; do
lock
for r in "${ROLES[@]}"; do
id="$(task_id "$r")"; state=NONE
[[ -n "$id" ]] && state="$(task_state "$id" 2>/dev/null || echo UNREACHABLE)"
case "$state" in
QUEUED|INITIALIZING|RUNNING|PAUSED|UNREACHABLE|UNKNOWN) ;;
*)
echo "$(date -Is) $r task ${id:-<none>} is $state"
# A new DB starts empty, so WebAPI has to migrate again; and an ended task whose
# pod still holds its port can't be replaced alone. Both need a fresh stack.
if [[ $r == db ]] || tunnel_up "$(port_of "$r")"; then
echo "$(date -Is) replacing the stack"; (replace_stack) || true; break
fi
(submit_role "$r") || true ;;
esac
done
unlock
sleep "$WATCH_INTERVAL"
done
}
case "${1:-}" in
up) cmd_up ;;
down) cmd_down ;;
restart) cmd_down; cmd_up ;;
setup) cmd_setup ;;
submit) lock; for r in $(roles_arg "${2:-}"); do submit_role "$r"; done; unlock ;;
status) cmd_status ;;
logs) cmd_logs "${2:-}" ;;
cancel) for r in $(roles_arg "${2:-}"); do cancel_role "$r"; done ;;
watch) cmd_watch ;;
selftest) cmd_selftest "${2:-}" ;;
print) build_task "${2:-}" ;;
*) sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 1 ;;
esac
__TES_SERVICES_EOF__
# Atlas stack: database task
cat > omop-tesk/pod/db.sh <<'__TES_SERVICES_EOF__'
# CDM database task (image: postgres:16-alpine).
# Loads the Eunomia GiBleed demo CDM into a fresh Postgres, publishes it on the hub and,
# once WebAPI is up, creates the results schema and registers the CDM as a WebAPI source.
export PGDATA=/tmp/pgdata POSTGRES_PASSWORD="$DB_PASSWORD"
export PGHOST=127.0.0.1 PGUSER=postgres PGDATABASE=postgres PGPASSWORD="$DB_PASSWORD"
WEBAPI=http://127.0.0.1:18081/WebAPI
start_tunnel "R:127.0.0.1:${HUB_DB_PORT}:127.0.0.1:5432" "127.0.0.1:18081:127.0.0.1:${HUB_WEBAPI_PORT}"
# The stock entrypoint handles the random UID (nss_wrapper) and runs initdb into /tmp.
docker-entrypoint.sh postgres -c listen_addresses=127.0.0.1 &
PG=$!
until pg_isready -q; do kill -0 $PG 2>/dev/null || { log "postgres failed to start"; exit 1; }; sleep 2; done
if [ "$(psql -Atc "select to_regclass('demo_cdm.person') is not null")" != t ]; then
log "loading Eunomia CDM from $EUNOMIA_URL"
mkdir -p /tmp/cdm && cd /tmp/cdm || exit 1
fetch "$EUNOMIA_URL" eunomia.zip && unzip -q -o eunomia.zip && fetch "$CDM_DDL_URL" ddl.sql \
|| { log "ERROR: download failed"; exit 1; }
# WebAPI's Flyway may already have created "webapi" while this loads.
for s in demo_cdm demo_cdm_results temp webapi; do psql -q -c "create schema if not exists $s"; done
sed 's/@cdmDatabaseSchema/demo_cdm/g' ddl.sql | psql -q -v ON_ERROR_STOP=1 || exit 1
# Not a pipeline: an exit in the loop must end the task, not a subshell.
find . -name '*.csv' ! -path '*__MACOSX*' | sort > /tmp/cdm.files
while read -r f; do
t=$(basename "$f" .csv | tr 'A-Z' 'a-z')
# Eunomia also ships cohort tables, which are not part of the CDM 5.3 DDL.
if [ "$(psql -Atc "select to_regclass('demo_cdm.$t') is not null")" != t ]; then
log "skipping $f: no table demo_cdm.$t in the CDM DDL"; continue
fi
# quoted, because note_nlp has a column named "offset"
cols=$(head -1 "$f" | tr -d '\r' | tr 'A-Z' 'a-z' | sed 's/[^,]*/"&"/g')
psql -q -v ON_ERROR_STOP=1 -c "\\copy demo_cdm.$t ($cols) from '$f' csv header" \
|| { log "ERROR: could not load $t"; exit 1; }
done < /tmp/cdm.files
cd /tmp && rm -rf /tmp/cdm
log "CDM loaded: $(psql -Atc 'select count(*) from demo_cdm.person') persons"
fi
if [ "${SELFTEST:-0}" = 1 ]; then
psql -v ON_ERROR_STOP=1 -c "select relname as table, n_live_tup as rows from pg_stat_user_tables where schemaname='demo_cdm' and n_live_tup > 0 order by 1" \
&& [ "$(psql -Atc 'select count(*) from demo_cdm.person')" -gt 0 ] && { log "SELFTEST_OK"; exit 0; }
log "SELFTEST_FAILED"; exit 1
fi
# Needs WebAPI to have run its migrations (webapi schema) and to serve the results DDL.
register_source() {
[ "$(psql -Atc "select count(*) from webapi.source where source_key='EUNOMIA'" 2>/dev/null)" = 1 ] && return 0
wget -q -O /dev/null "$WEBAPI/info" 2>/dev/null || return 1
[ "$(psql -Atc "select to_regclass('webapi.source_daimon') is not null")" = t ] || return 1
# Without the results schema, cohorts can't be generated: register the source only after
# it exists, so that a failed DDL is retried in the next round.
if [ "$(psql -Atc "select to_regclass('demo_cdm_results.cohort') is not null")" != t ]; then
log "WebAPI is up, creating the results schema"
wget -q -O /tmp/results.sql "$WEBAPI/ddl/results?dialect=postgresql&schema=demo_cdm_results&vocabSchema=demo_cdm&tempSchema=temp&initConceptHierarchy=true" \
&& psql -q -f /tmp/results.sql >/dev/null \
&& [ "$(psql -Atc "select to_regclass('demo_cdm_results.cohort') is not null")" = t ] \
|| { log "WARN: results DDL failed, retrying later"; return 1; }
fi
log "registering the source"
# source_connection is used by WebAPI, which reaches this DB on its own 127.0.0.1:5432.
psql -q -v ON_ERROR_STOP=1 <<SQL || return 1
insert into webapi.source (source_id, source_name, source_key, source_connection, source_dialect)
values (nextval('webapi.source_sequence'), 'Eunomia GiBleed (TESK @ ELIXIR-FI)', 'EUNOMIA',
'jdbc:postgresql://127.0.0.1:5432/postgres?user=postgres&password=${DB_PASSWORD}', 'postgresql');
insert into webapi.source_daimon (source_daimon_id, source_id, daimon_type, table_qualifier, priority)
select nextval('webapi.source_daimon_sequence'), s.source_id, d.t, d.q, d.p
from webapi.source s, (values (0,'demo_cdm',0), (1,'demo_cdm',1), (2,'demo_cdm_results',1), (5,'temp',0)) d(t,q,p)
where s.source_key = 'EUNOMIA';
SQL
wget -q -O /dev/null "$WEBAPI/source/refresh" || true
log "source EUNOMIA registered"
}
log "database ready"
while kill -0 $PG 2>/dev/null; do
register_source || true
sleep 30
done
log "postgres exited"; exit 1
__TES_SERVICES_EOF__
# Atlas stack: WebAPI task
cat > omop-tesk/pod/webapi.sh <<'__TES_SERVICES_EOF__'
# WebAPI task (image: ohdsi/webapi). Reaches the CDM database through the hub on
# 127.0.0.1:5432 and publishes WebAPI (:8080 in the pod) on the hub.
start_tunnel "R:127.0.0.1:${HUB_WEBAPI_PORT}:127.0.0.1:8080" "127.0.0.1:5432:127.0.0.1:${HUB_DB_PORT}"
# chisel accepts locally even when the DB task is down, so ask Postgres itself (SSLRequest).
pg_alive() { timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/5432; printf "\0\0\0\10\4\322\26\57" >&3; head -c1 <&3' 2>/dev/null | grep -q '[NS]'; }
log "waiting for the database task"
until pg_alive; do sleep 10; done
# The DB task creates the webapi schema after loading the CDM; Flyway fills it.
sleep 20
cd /var/lib/ohdsi/webapi || exit 1
# The start command of the WebAPI 2.x images (exploded WAR; java expands only WEB-INF/lib/*,
# the Spring Boot WarLauncher loads the jars itself). WebAPI 3 images have only WebAPI.jar
# and start with "java -jar WebAPI.jar": this script does not support them yet.
log "starting WebAPI (first start runs all Flyway migrations, this takes several minutes)"
# shellcheck disable=SC2086
java $WEBAPI_JAVA_OPTS -cp ".:WebAPI.jar:WEB-INF/lib/*.jar" org.springframework.boot.loader.WarLauncher &
API=$!
# Spring Boot can leave the JVM running after a failed start: give up after STARTUP_TIMEOUT.
i=0
until curl -fs -o /dev/null http://127.0.0.1:8080/WebAPI/info; do
kill -0 $API 2>/dev/null || { log "WebAPI exited during startup"; exit 1; }
i=$((i + 10)); [ $i -ge "${STARTUP_TIMEOUT:-2400}" ] && { log "WebAPI not up after ${i}s"; kill $API; exit 1; }
sleep 10
done
log "WebAPI up: $(curl -fs http://127.0.0.1:8080/WebAPI/info)"
wait $API
log "WebAPI exited"; exit 1
__TES_SERVICES_EOF__
# Atlas stack: Atlas UI task
cat > omop-tesk/pod/atlas.sh <<'__TES_SERVICES_EOF__'
# Atlas task (image: ohdsi/atlas). Serves the Atlas UI on :8080, proxies /WebAPI/ to the
# WebAPI task through the hub, and publishes :8080 on the hub (-> localhost:8080 on the VM).
start_tunnel "R:127.0.0.1:${HUB_ATLAS_PORT}:127.0.0.1:8080" "127.0.0.1:18081:127.0.0.1:${HUB_WEBAPI_PORT}"
# The image's html is read-only for the random UID: render config-local.js into /tmp and
# serve it from there. Relative WebAPI URL = same origin, works on any forwarded port.
D=/tmp/atlas; mkdir -p $D
WEBAPI_URL=/WebAPI/ envsubst < /usr/share/nginx/html/atlas/js/config-local.js > $D/config-local.js
cat > $D/nginx.conf <<EOF
worker_processes 1;
daemon off;
pid $D/nginx.pid;
error_log stderr warn;
events { worker_connections 256; }
http {
include /etc/nginx/mime.types;
access_log off;
absolute_redirect off;
client_body_temp_path $D/body;
proxy_temp_path $D/proxy;
fastcgi_temp_path $D/fastcgi;
uwsgi_temp_path $D/uwsgi;
scgi_temp_path $D/scgi;
gzip on;
gzip_types text/css application/javascript application/json image/svg+xml;
server {
listen 8080;
root /usr/share/nginx/html;
location = / { return 302 /atlas/; }
location = /atlas/js/config-local.js { alias $D/config-local.js; }
location ^~ /WebAPI/ {
proxy_pass http://127.0.0.1:18081;
proxy_http_version 1.1;
proxy_set_header Host \$http_host;
proxy_read_timeout 600s;
}
location ~ ^.*[^/]\$ { try_files \$uri @rewrite; }
location @rewrite { return 302 \$uri/; }
}
}
EOF
nginx -e stderr -c $D/nginx.conf &
WEB=$!
sleep 2
if [ "${SELFTEST:-0}" = 1 ]; then
rc=1
wget -q -O - http://127.0.0.1:8080/atlas/js/config-local.js | grep -n 'webapi_url = ' \
&& wget -q -O /dev/null http://127.0.0.1:8080/atlas/ && { log "SELFTEST_OK"; rc=0; }
kill $WEB; exit $rc
fi
log "Atlas serving on :8080"
wait $WEB
log "nginx exited"; exit 1
__TES_SERVICES_EOF__
# RStudio: image and port
cat > rstudio-tesk/service.conf <<'__TES_SERVICES_EOF__'
# RStudio Server, run with: ../tes-hub/tes-service.sh rstudio-tesk up
NAME=rstudio # hub user, task name
IMAGE=rocker/rstudio:latest
PORT=8787 # rserver port in the pod, and on the VM
RS_LOG_LEVEL=warn # rserver/rsession log level (passed to the pod)
__TES_SERVICES_EOF__
# RStudio: what runs in the task
cat > rstudio-tesk/pod.sh <<'__TES_SERVICES_EOF__'
# Runs inside the TES pod (image rocker/rstudio): random UID, read-only image,
# outbound HTTPS only, no curl. tes-service.sh prepends ../tes-hub/common.sh
# (set -u, log, start_tunnel) and passes service.conf as environment.
# The image's /init (s6) needs root, so start rserver directly as the pod user.
# Its passwd home is "/" (not writable): give R a home under /tmp. Data is lost when the
# task ends (this TESK does not start tasks that request disk_gb).
USER="$(id -un)"; export HOME=/tmp/home USER
RS=/tmp/rs; mkdir -p "$HOME" $RS/data $RS/run $RS/etc $RS/log
# Own config dir (copy of the image's /etc/rstudio) so rserver can log to stderr, which
# TESK returns in the task logs. rsession's stderr ends up in the R console, so it logs
# to a file instead (/tmp/rs/log, readable from RStudio's terminal).
cp -r /etc/rstudio/. $RS/etc/ 2>/dev/null
export RSTUDIO_CONFIG_DIR=$RS/etc
printf '[@rserver]\nlog-level=%s\nlogger-type=stderr\n\n[@rsession]\nlog-level=%s\nlogger-type=file\nlog-dir=%s/log\n' \
"$RS_LOG_LEVEL" "$RS_LOG_LEVEL" $RS > $RS/etc/logging.conf
printf 'provider=sqlite\ndirectory=%s/data\n' $RS > $RS/db.conf
printf 'session-default-working-dir=%s\nsession-default-new-project-dir=%s\n' "$HOME" "$HOME" > $RS/rsession.conf
# rserver starts rsession with HOME taken from passwd ("/", not writable), so the session
# cannot create its state dirs and the browser gets "Unable to connect to service".
# This wrapper points HOME and the XDG dirs to /tmp/home before starting the real rsession.
cat > $RS/rsession.sh <<EOS
#!/bin/sh
export HOME=$HOME USER=$USER RSTUDIO_CONFIG_DIR=$RS/etc
export XDG_DATA_HOME=$HOME/.local/share XDG_CONFIG_HOME=$HOME/.config XDG_CACHE_HOME=$HOME/.cache
exec /usr/lib/rstudio-server/bin/rsession "\$@"
EOS
chmod 755 $RS/rsession.sh
RSERVER_ARGS="--server-daemonize=0 --www-address=127.0.0.1 --www-port=$PORT
--auth-none=1 --server-user=$USER --server-data-dir=$RS/run
--database-config-file=$RS/db.conf --rsession-config-file=$RS/rsession.conf
--rsession-path=$RS/rsession.sh
--secure-cookie-key-file=$RS/cookie.key --server-pid-file=$RS/rserver.pid"
if [ "${SELFTEST:-0}" = 1 ]; then
log "verify-installation (starts a test R session)"
/usr/lib/rstudio-server/bin/rserver $RSERVER_ARGS --verify-installation=1 && { log "SELFTEST_OK"; exit 0; }
exit 1
fi
# Publish the pod's :$PORT as 127.0.0.1:$HUB_PORT on the hub VM.
start_tunnel "R:127.0.0.1:${HUB_PORT}:127.0.0.1:${PORT}"
log "RStudio Server starting on :$PORT as $USER"
/usr/lib/rstudio-server/bin/rserver $RSERVER_ARGS
log "rserver exited ($?)"; exit 1
__TES_SERVICES_EOF__
# make the launchers executable
chmod +x tes-hub/tes-hub.sh tes-hub/tes-service.sh omop-tesk/omop-tesk-fi.sh
# re-enable history expansion, list the installed files
set -H; ls -R ~/eoc_services/{tes-hub,omop-tesk,rstudio-tesk}
Set up the hub¶
# once: download chisel, create the hub key, start the hub on 10.0.2.165:8090
# (the elixir-on.cloud proxy runs on another machine, so the hub listens on the VM address)
./tes-hub/tes-hub.sh setup https://elixir-on.cloud/tes-tunnel/ 10.0.2.165
# check hub, proxy route and all tunnels; must print "proxy route OK: https://elixir-on.cloud/tes-tunnel/"
./tes-hub/tes-hub.sh status
# on the proxy (194.94.4.89), in the server block of elixir-on.cloud
location /tes-tunnel/ {
proxy_pass http://10.0.2.165:8090/; # the hub on eoc-services
proxy_http_version 1.1; # needed for websockets
proxy_set_header Upgrade $http_upgrade; # pass the websocket upgrade
proxy_set_header Connection "upgrade"; # pass the websocket upgrade
proxy_set_header Host $host; # keep the original host name
proxy_read_timeout 1d; # keep idle tunnels open
proxy_send_timeout 1d; # keep idle tunnels open
proxy_buffering off; # stream tunnel traffic unbuffered
}
Start Atlas with OMOP demo data¶
# Atlas uses port 8088 on this VM, because the cloud-registry container holds 127.0.0.1:8080
# (set permanently in ~/.bashrc)
export HUB_ATLAS_PORT=8088
# create credentials, submit DB, WebAPI and Atlas tasks, start the keep-alive loop, wait until ready (first start ~15 min)
./omop-tesk/omop-tesk-fi.sh up
# laptop: forward Atlas, then open http://localhost:8088/atlas/
ssh -N -L 8088:127.0.0.1:8088 ubuntu@eoc-services
Start RStudio Server¶
# submit the RStudio task (rocker/rstudio:latest), start the keep-alive loop, wait for the tunnel (1-3 min)
./tes-hub/tes-service.sh rstudio-tesk up
# laptop: forward RStudio, then open http://localhost:8787
ssh -N -L 8787:127.0.0.1:8787 ubuntu@eoc-services
Add JupyterLab as your own service¶
# service directory next to tes-hub/
mkdir -p jupyter-tesk
# service definition: hub user/task name, image, port
cat > jupyter-tesk/service.conf <<'EOF'
# JupyterLab, run with: ../tes-hub/tes-service.sh jupyter-tesk up
NAME=jupyter # hub user, task name
IMAGE=quay.io/jupyter/base-notebook:latest
PORT=8888 # JupyterLab port in the pod, and on the VM
EOF
# what runs in the task: start JupyterLab without token, open the tunnel, stay in the foreground
cat > jupyter-tesk/pod.sh <<'EOF'
# Runs inside the TES pod (image quay.io/jupyter/base-notebook) as a random UID, so all
# writable state goes to /tmp. No token: the port is only reachable from the hub VM.
export HOME=/tmp/home; mkdir -p "$HOME"; cd "$HOME" || exit 1
jupyter lab --ip=127.0.0.1 --port="$PORT" --no-browser --IdentityProvider.token='' &
APP=$!
if [ "${SELFTEST:-0}" = 1 ]; then
sleep 15
rc=1
python3 -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:$PORT/api').read().decode())" \
&& { log "SELFTEST_OK"; rc=0; }
kill $APP; exit $rc
fi
start_tunnel "R:127.0.0.1:${HUB_PORT}:127.0.0.1:${PORT}"
wait $APP
log "jupyter exited"; exit 1
EOF
# start it
./tes-hub/tes-service.sh jupyter-tesk up
# laptop: forward JupyterLab, then open http://localhost:8888
ssh -N -L 8888:127.0.0.1:8888 ubuntu@eoc-services
Forward the ports¶
# laptop: Atlas, RStudio and JupyterLab in one connection
ssh -N -L 8088:127.0.0.1:8088 -L 8787:127.0.0.1:8787 -L 8888:127.0.0.1:8888 ubuntu@eoc-services
# VS Code Remote-SSH instead: Ports panel -> Forward a Port -> 8088, 8787, 8888
Manage the services¶
# go to the scripts
cd ~/eoc_services
# hub, proxy route and every tunnel
./tes-hub/tes-hub.sh status
# hub users and the ports each may open
./tes-hub/tes-hub.sh users
# Atlas stack: tasks, tunnels, WebAPI and data source
./omop-tesk/omop-tesk-fi.sh status
# Atlas stack: logs of one task (db, webapi or atlas; filled once the task ends)
./omop-tesk/omop-tesk-fi.sh logs webapi
# Atlas stack: replace with a fresh instance
./omop-tesk/omop-tesk-fi.sh restart
# Atlas stack: stop
./omop-tesk/omop-tesk-fi.sh down
# Atlas stack: show the TES request of one task
./omop-tesk/omop-tesk-fi.sh print atlas
# RStudio: task, tunnel, keep-alive loop
./tes-hub/tes-service.sh rstudio-tesk status
# RStudio: task logs (filled once the task ends)
./tes-hub/tes-service.sh rstudio-tesk logs
# RStudio: replace with a fresh instance
./tes-hub/tes-service.sh rstudio-tesk restart
# RStudio: stop
./tes-hub/tes-service.sh rstudio-tesk down
# RStudio: show the TES request
./tes-hub/tes-service.sh rstudio-tesk print
# JupyterLab: status / logs / restart / down
./tes-hub/tes-service.sh jupyter-tesk status
./tes-hub/tes-service.sh jupyter-tesk logs
./tes-hub/tes-service.sh jupyter-tesk restart
./tes-hub/tes-service.sh jupyter-tesk down
# list all tasks on TESK @ ELIXIR-FI
curl -fsS "https://csc-tesk-noauth.rahtiapp.fi/ga4gh/tes/v1/tasks?view=MINIMAL"
After a reboot¶
# start the hub again; running tasks reconnect within ~30 seconds
./tes-hub/tes-hub.sh start
# restart the keep-alive loops of the single-container services
./tes-hub/tes-service.sh rstudio-tesk up
./tes-hub/tes-service.sh jupyter-tesk up
Settings¶
# TES instance: TESK @ ELIXIR-FI (default)
export TES_URL=https://csc-tesk-noauth.rahtiapp.fi
# hub port and chisel version (defaults)
export HUB_LISTEN_PORT=8090 CHISEL_VERSION=1.12.0
# Atlas stack images (defaults)
export DB_IMAGE=postgres:16-alpine WEBAPI_IMAGE=ohdsi/webapi:2.15.2 ATLAS_IMAGE=ohdsi/atlas:2.15.0
# Atlas stack dataset: Eunomia GiBleed (default)
export EUNOMIA_URL=https://raw.githubusercontent.com/OHDSI/EunomiaDatasets/main/datasets/GiBleed/GiBleed_5.3.zip
# WebAPI memory (defaults)
export WEBAPI_RAM_GB=1 WEBAPI_JAVA_OPTS="-Xmx640m -XX:MaxMetaspaceSize=192m -Xss512k -Djava.security.egd=file:///dev/./urandom"
# Atlas stack ports on the VM: Atlas on 8088 (8080 is the cloud-registry), WebAPI and DB defaults
export HUB_ATLAS_PORT=8088 HUB_WEBAPI_PORT=18081 HUB_DB_PORT=15432
# hub settings written by setup
cat tes-hub/.state/config
# the Atlas port is set permanently here
grep HUB_ATLAS_PORT ~/.bashrc
Troubleshooting¶
# hub log: "Login failed" (locked-out old pod, or run the service's restart), "cannot listen" (old pod holds the port)
tail -f tes-hub/.state/hub.log
# failed logins per user today (locked-out pods that still use TESK capacity)
grep "^$(date +%Y/%m/%d).*Login failed" tes-hub/.state/hub.log | grep -o 'user "[a-z]*"' | sort | uniq -c
# restart the hub after a stop or crash
./tes-hub/tes-hub.sh start
# what holds a port on the VM (e.g. 8080 = cloud-registry container)
ss -ltn | grep -E ':(8080|8088|8787|8888|15432|18081)\b'
docker ps --format '{{.Names}}\t{{.Ports}}' | grep 8080
# RStudio shows "Unable to connect to service": short test task, check its output
./tes-hub/tes-service.sh rstudio-tesk selftest
# JupyterLab: short test task
./tes-hub/tes-service.sh jupyter-tesk selftest
# Atlas stack: short test task for the database or Atlas role
./omop-tesk/omop-tesk-fi.sh selftest db
# WebAPI answers through the tunnel (long timeout, it is slow)
curl -fsS -m 60 http://127.0.0.1:18081/WebAPI/info
# Atlas answers on the VM (HUB_ATLAS_PORT: 8088 on this VM, 8080 by default)
curl -fsS -m 30 -o /dev/null -w '%{http_code}\n' "http://127.0.0.1:${HUB_ATLAS_PORT:-8080}/atlas/"
# create and generate a cohort through the WebAPI REST interface; expected: cohort <id> persons: 479
python3 - <<'EOF'
import json, os, time, urllib.request
W = f"http://127.0.0.1:{os.environ.get('HUB_ATLAS_PORT', '8080')}/WebAPI"
def call(method, path, body=None):
req = urllib.request.Request(W + path, method=method, headers={"Content-Type": "application/json"},
data=None if body is None else json.dumps(body).encode())
with urllib.request.urlopen(req, timeout=120) as r:
text = r.read().decode()
return json.loads(text) if text.strip()[:1] in ("[", "{") else text
expression = {
"ConceptSets": [{"id": 0, "name": "GI hemorrhage", "expression": {"items": [
{"concept": {"CONCEPT_ID": 192671}, "includeDescendants": True}]}}],
"PrimaryCriteria": {"CriteriaList": [{"ConditionOccurrence": {"CodesetId": 0}}],
"ObservationWindow": {"PriorDays": 0, "PostDays": 0},
"PrimaryCriteriaLimit": {"Type": "First"}},
"QualifiedLimit": {"Type": "First"}, "ExpressionLimit": {"Type": "First"},
"InclusionRules": [], "CensoringCriteria": [], "CollapseSettings": {"CollapseType": "ERA", "EraPad": 0},
"CensorWindow": {}, "cdmVersionRange": ">=5.0.0"}
cohort = call("POST", "/cohortdefinition/", {"name": "GI bleed, first event (API)",
"expressionType": "SIMPLE_EXPRESSION", "expression": expression})
call("GET", f"/cohortdefinition/{cohort['id']}/generate/EUNOMIA")
while True:
info = call("GET", f"/cohortdefinition/{cohort['id']}/info")
if info and info[0]["status"] == "COMPLETE":
print("cohort", cohort["id"], "persons:", info[0]["personCount"]); break
time.sleep(5)
EOF