011: observability - pg_stat_statements, slow-log, io timing

preload pg_stat_statements (+extension in iwanhae/postgres DBs),
log_min_duration_statement=500ms, track_io_timing=on.

migrations/011-postgres-observability.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# 011-postgres-observability - Turn on the instrumentation that any future
+# tuning depends on (A-tier of the 2026-08 tuning review).
+#
+#   shared_preload_libraries = 'pg_stat_statements'  query stats (restart)
+#   track_io_timing          = on   pg_stat_io timing, cheap on modern Linux
+#   log_min_duration_statement = 500ms   slow queries land in the server log
+#
+# pg_stat_statements extension is created in `iwanhae` (app db) and
+# `postgres` (admin db). Inspect with:
+#   SELECT calls, round(total_exec_time::numeric,1) ms, rows, query
+#     FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 10;
+# up:    write drop-in; restart only if the lib is not yet loaded; extensions
+# down:  drop extensions (loses collected stats), remove drop-in, restart
+set -euo pipefail
+
+PGBIN=/etc/postgresql/18/main
+CONF=$PGBIN/conf.d/011-postgres-observability.conf
+CLUSTER=$(pg_lsclusters --no-header | awk 'NR==1{print $1"/"$2}')
+DBS=(iwanhae postgres)
+
+pg() { runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "$1"; }
+
+lib_loaded() { pg 'SHOW shared_preload_libraries' | grep -q pg_stat_statements; }
+
+write_conf() {
+  install -d -m 755 "$PGBIN/conf.d"
+  cat > "$CONF" <<'EOF'
+# managed by sysmig 011-postgres-observability
+shared_preload_libraries = 'pg_stat_statements'
+track_io_timing = on
+log_min_duration_statement = 500ms
+EOF
+  chown root:postgres "$CONF"
+  chmod 644 "$CONF"
+}
+
+case "${1:-}" in
+  up)
+    write_conf
+    if lib_loaded; then
+      pg_ctlcluster "$CLUSTER" reload
+      echo "    cluster reloaded (pg_stat_statements already loaded)"
+    else
+      pg_ctlcluster "$CLUSTER" restart
+      pg_isready -q
+      echo "    cluster restarted to load pg_stat_statements"
+    fi
+    for db in "${DBS[@]}"; do
+      runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -qc \
+        'CREATE EXTENSION IF NOT EXISTS pg_stat_statements'
+    done
+    lib_loaded || { echo "FATAL: pg_stat_statements not loaded" >&2; exit 1; }
+    [[ $(pg 'SHOW track_io_timing') == on ]] || { echo "FATAL: track_io_timing not on" >&2; exit 1; }
+    [[ $(pg 'SHOW log_min_duration_statement') == '500ms' ]] || { echo "FATAL: log_min_duration_statement wrong" >&2; exit 1; }
+    echo "    extension created in: ${DBS[*]}"
+    ;;
+  status)
+    if [[ $EUID -eq 0 ]]; then
+      q() { runuser -u postgres -- psql -d "$1" -v ON_ERROR_STOP=1 -tAc "$2"; }
+    elif psql -d postgres -tAc 'SELECT 1' >/dev/null 2>&1; then
+      q() { psql -d "$1" -v ON_ERROR_STOP=1 -tAc "$2"; }
+    else
+      echo "? needs root or peer DB access to verify"
+      exit 2
+    fi
+    [[ -f $CONF ]] || { echo "DRIFT drop-in $CONF missing"; exit 1; }
+    if ! q postgres 'SHOW shared_preload_libraries' | grep -q pg_stat_statements; then
+      echo "DRIFT pg_stat_statements not preloaded"; exit 1
+    fi
+    [[ $(q postgres 'SHOW track_io_timing') == on ]] || { echo "DRIFT track_io_timing off"; exit 1; }
+    [[ $(q postgres 'SHOW log_min_duration_statement') == '500ms' ]] || { echo "DRIFT log_min_duration_statement"; exit 1; }
+    for db in "${DBS[@]}"; do
+      [[ $(q "$db" "SELECT count(*) FROM pg_extension WHERE extname='pg_stat_statements'") -ge 1 ]] \
+        || { echo "DRIFT extension missing in $db"; exit 1; }
+    done
+    echo "OK pg_stat_statements + slow-log(500ms) + io timing"
+    ;;
+  down)
+    for db in "${DBS[@]}"; do
+      runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -qc \
+        'DROP EXTENSION IF EXISTS pg_stat_statements' \
+        || { echo "cannot drop extension in $db (in use?) - retry later" >&2; exit 1; }
+    done
+    rm -f "$CONF"
+    if lib_loaded; then
+      pg_ctlcluster "$CLUSTER" restart
+      echo "    extensions dropped, drop-in removed, cluster restarted to unload lib"
+    else
+      echo "    extensions dropped, drop-in removed (lib was not loaded)"
+    fi
+    ;;
+esac