Lab: remember-me, Grafana proxy fix, builder cache dirs, branded QR alerts.

Grafana auth restore + nginx proxy_cookie_path; alert-notifier branded QR; prepare-be-builder-dirs shared ccache/gradle; bump ac-scripts for Docker DLC maintenance.

Co-authored-by: Cursor <cursoragent@cursor.com>

monitoring/artc0/grafana-auth-check.php

@@ -14,6 +14,17 @@ require_once dirname(__DIR__, 4) . '/platform/shared_session.php';
 
 platform_start_session('ac_crash_sess', '/app/androidcast_project');
 
+$composedSrc = dirname(__DIR__, 2) . '/src';
+if (is_file($composedSrc . '/AuthRemember.php')) {
+    require_once $composedSrc . '/Database.php';
+    require_once $composedSrc . '/Rbac.php';
+    require_once $composedSrc . '/Auth.php';
+    require_once $composedSrc . '/AuthRemember.php';
+    require_once $composedSrc . '/AuthEmailSchema.php';
+    require_once $composedSrc . '/AuthFactors.php';
+    AuthRemember::tryRestore();
+}
+
 $user = $_SESSION['user'] ?? null;
 
 if (empty($user['id'])) {

monitoring/cast04/alert-notifier.py

@@ -99,13 +99,25 @@ def shorten(url: str) -> dict:
     }
 
 
-def make_qr_png(url: str) -> bytes:
+def make_qr_png(url: str, slug: str = '') -> bytes:
     """
-    Generate a QR code PNG for *url* using the system qrencode binary.
+    Branded QR PNG: prefer URL shortener /api/v1/qr/{slug}.png (logo overlay),
-    Returns raw PNG bytes.  Raises on failure.
+    fall back to local qrencode.
     """
+    if slug:
+        try:
+            req = urllib.request.Request(
+                f'{SHORTENER_INTERNAL}/api/v1/qr/{slug}.png?size=256',
+                headers={'Host': SHORTENER_HOST_HDR},
+            )
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = resp.read()
+            if len(data) > 100:
+                return data
+        except Exception as exc:
+            log.warning('branded QR fetch failed (%s), falling back to qrencode', exc)
     result = subprocess.run(
-        ['qrencode', '-t', 'PNG', '-o', '-', '-s', '6', '--', url],
+        ['qrencode', '-l', 'H', '-t', 'PNG', '-o', '-', '-s', '6', '--', url],
         capture_output=True,
         check=True,
     )
@@ -287,8 +299,7 @@ class WebhookHandler(http.server.BaseHTTPRequestHandler):
         # ── shorten & QR ────────────────────────────────────────────────────
         try:
             short = shorten(ext_url)
-            # QR image encodes short_url?src=qr — generated locally via qrencode
+            qr_png = make_qr_png(short['qr_url'], short.get('slug', ''))
-            qr_png = make_qr_png(short['qr_url'])
             log.info('shortened %s → %s  qr_url=%s  (%d B)',
                      ext_url, short['short_url'], short['qr_url'], len(qr_png))
         except Exception as exc:

sim/cluster0/lab-seeds/backend/public/api/grafana-auth-check.php

@@ -14,6 +14,17 @@ require_once dirname(__DIR__, 4) . '/platform/shared_session.php';
 
 platform_start_session('ac_crash_sess', '/app/androidcast_project');
 
+$composedSrc = dirname(__DIR__, 2) . '/src';
+if (is_file($composedSrc . '/AuthRemember.php')) {
+    require_once $composedSrc . '/Database.php';
+    require_once $composedSrc . '/Rbac.php';
+    require_once $composedSrc . '/Auth.php';
+    require_once $composedSrc . '/AuthRemember.php';
+    require_once $composedSrc . '/AuthEmailSchema.php';
+    require_once $composedSrc . '/AuthFactors.php';
+    AuthRemember::tryRestore();
+}
+
 $user = $_SESSION['user'] ?? null;
 
 if (empty($user['id'])) {

sim/cluster0/lab-seeds/backend/public/index.php

@@ -165,10 +165,13 @@ if ($route === '/logout') {
 
 if ($route === '/login' && $_SERVER['REQUEST_METHOD'] === 'POST') {
     Auth::captureRedirectFromRequest();
+    AuthRemember::stashPendingFromRequest();
     $user = trim($_POST['username'] ?? '');
     $pass = $_POST['password'] ?? '';
     $result = Auth::login($user, $pass);
     if ($result === true) {
+        $uid = (int) (Auth::user()['id'] ?? 0);
+        AuthRemember::applyPending($uid);
         header('Location: ' . Auth::postLoginRedirect($base . '/'));
         exit;
     }
@@ -191,6 +194,10 @@ if ($route === '/login' && $_SERVER['REQUEST_METHOD'] === 'POST') {
 
 if ($route === '/login') {
     Auth::captureRedirectFromRequest();
+    if (Auth::user()) {
+        header('Location: ' . Auth::postLoginRedirect($base . '/'));
+        exit;
+    }
     if (Auth::pending2faUserId() > 0) {
         header('Location: ' . Auth::authUrl('/two-factor'));
         exit;
@@ -253,6 +260,8 @@ if ($route === '/two-factor' && $_SERVER['REQUEST_METHOD'] === 'POST') {
     }
     $code = trim($_POST['code'] ?? '');
     if (Auth::completeTotpLogin($code)) {
+        $uid = (int) (Auth::user()['id'] ?? 0);
+        AuthRemember::applyPending($uid);
         header('Location: ' . Auth::postLoginRedirect($base . '/'));
         exit;
     }
@@ -290,6 +299,8 @@ if (
     $status = AuthApproval::pollStatus($rawToken);
     if ($status === 'approved') {
         if (Auth::completeTotpLoginByApproval($rawToken)) {
+            $uid = (int) (Auth::user()['id'] ?? 0);
+            AuthRemember::applyPending($uid);
             echo json_encode([
                 'status' => 'approved',
                 'redirect' => Auth::postLoginRedirect($base . '/'),

sim/cluster0/lab-seeds/backend/scripts/prepare-be-builder-dirs.sh

@@ -11,7 +11,9 @@ DOCKER_HOME="${BROADCAST_ROOT}"
 DOWNLOADS="${2:-/var/www/localhost/htdocs/apps/app/androidcast_project/downloads}"
 OTA_ROOT="${3:-/var/www/localhost/htdocs/apps/app/androidcast_project/ota-artifacts}"
 
-mkdir -p "${BUILDS}" "${DOCKER_HOME}/.docker" "${DOWNLOADS}" "${OTA_ROOT}/v0/ota/channel"
+mkdir -p "${BUILDS}" "${BUILDS}/.cache/ccache" "${BUILDS}/.cache/gradle/wrapper" \
+  "${BUILDS}/.cache/gradle/caches" "${DOCKER_HOME}/.docker" "${DOWNLOADS}" "${OTA_ROOT}/v0/ota/channel"
+chmod 1777 "${BUILDS}/.cache/ccache" "${BUILDS}/.cache/gradle" 2>/dev/null || true
 
 # Do not chown -R broadcast root: that resets builds/* to nginx and breaks PHP-FPM (nobody).
 for _d in config bin deploy sql src public; do

sim/cluster0/lab-seeds/backend/src/Auth.php

@@ -34,11 +34,65 @@ final class Auth {
         return $_SESSION['user'];
     }
 
+    private const SESSION_IDLE_SECONDS = 86400;
+
     public static function check(): void {
         if (!self::user()) {
-            header('Location: ' . self::authUrl('/login'));
+            $return = (string) ($_SERVER['REQUEST_URI'] ?? '');
+            $q = $return !== '' ? '?redirect=' . urlencode($return) : '';
+            header('Location: ' . self::authUrl('/login') . $q);
             exit;
         }
+        $last = (int) ($_SESSION['last_activity'] ?? 0);
+        if ($last > 0 && (time() - $last) > self::SESSION_IDLE_SECONDS) {
+            self::logout();
+            $return = (string) ($_SERVER['REQUEST_URI'] ?? '');
+            $q = $return !== '' ? '?redirect=' . urlencode($return) : '';
+            header('Location: ' . self::authUrl('/login') . $q);
+            exit;
+        }
+        self::touchSession();
+    }
+
+    public static function touchSession(): void {
+        if (self::user()) {
+            $_SESSION['last_activity'] = time();
+        }
+    }
+
+    /** Accept only in-project relative URLs (open redirect safe). */
+    public static function safeRedirect(string $url): ?string {
+        $url = trim($url);
+        if ($url === '') {
+            return null;
+        }
+        $project = self::projectBasePath();
+        if (str_starts_with($url, $project . '/') || $url === $project) {
+            return $url;
+        }
+        if (str_starts_with($url, '/') && !str_starts_with($url, '//') && str_starts_with($url, $project)) {
+            return $url;
+        }
+        return null;
+    }
+
+    public static function captureRedirectFromRequest(): void {
+        $from = trim((string) ($_GET['redirect'] ?? $_POST['redirect'] ?? ''));
+        $safe = self::safeRedirect($from);
+        if ($safe !== null) {
+            $_SESSION['post_login_redirect'] = $safe;
+        }
+    }
+
+    public static function peekPostLoginRedirect(string $default): string {
+        $redir = (string) ($_SESSION['post_login_redirect'] ?? '');
+        return self::safeRedirect($redir) ?? $default;
+    }
+
+    public static function postLoginRedirect(string $default): string {
+        $target = self::peekPostLoginRedirect($default);
+        unset($_SESSION['post_login_redirect']);
+        return $target;
     }
 
     public static function can(string $action): bool {
@@ -94,6 +148,7 @@ final class Auth {
             return 'pending_2fa';
         }
         $_SESSION['user'] = Rbac::buildSessionUser($row);
+        self::touchSession();
         AuthAttempts::record('login_ok', $username);
         return true;
     }
@@ -117,6 +172,7 @@ final class Auth {
             return false;
         }
         $_SESSION['user'] = Rbac::buildSessionUser($row);
+        self::touchSession();
         return true;
     }
 
@@ -129,11 +185,68 @@ final class Auth {
     }
 
     public static function clearPending2fa(): void {
-        unset($_SESSION['pending_2fa_user_id'], $_SESSION['pending_2fa_exp']);
+        unset($_SESSION['pending_2fa_user_id'], $_SESSION['pending_2fa_exp'], $_SESSION['pending_2fa_approval_token']);
+    }
+
+    /**
+     * Complete login via a cross-device approval token (no TOTP code required).
+     * The approval token must have been approved by the authenticated mobile user.
+     */
+    public static function completeTotpLoginByApproval(string $rawToken): bool {
+        $uid = self::pending2faUserId();
+        if ($uid <= 0) {
+            return false;
+        }
+        $sessionId = session_id();
+        $consumedUid = AuthApproval::consume($rawToken, $sessionId);
+        if ($consumedUid <= 0 || $consumedUid !== $uid) {
+            return false;
+        }
+        self::clearPending2fa();
+        $stmt = Database::pdo()->prepare('SELECT * FROM users WHERE id = ? LIMIT 1');
+        $stmt->execute([$consumedUid]);
+        $row = $stmt->fetch();
+        if (!$row) {
+            return false;
+        }
+        $_SESSION['user'] = Rbac::buildSessionUser($row);
+        self::touchSession();
+        return true;
+    }
+
+    /** Best-effort client IP (respects X-Forwarded-For from trusted proxy). */
+    public static function clientIp(): string {
+        $fwd = (string) ($_SERVER['HTTP_X_FORWARDED_FOR'] ?? '');
+        if ($fwd !== '') {
+            // Take the first (original client) IP from the chain
+            $first = trim(explode(',', $fwd)[0]);
+            if (filter_var($first, FILTER_VALIDATE_IP)) {
+                return $first;
+            }
+        }
+        return (string) ($_SERVER['REMOTE_ADDR'] ?? '0.0.0.0');
+    }
+
+    /** Origin for outbound links (QR, email) — matches the host the user is browsing. */
+    public static function requestPublicOrigin(): string {
+        $host = trim(explode(',', (string) ($_SERVER['HTTP_X_FORWARDED_HOST'] ?? $_SERVER['HTTP_HOST'] ?? ''))[0]);
+        $host = preg_replace('/:\d+$/', '', $host);
+        if ($host !== '' && !in_array($host, ['127.0.0.1', 'localhost'], true)) {
+            $scheme = 'http';
+            if (function_exists('platform_is_https_request') && platform_is_https_request()) {
+                $scheme = 'https';
+            } elseif (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') {
+                $scheme = 'https';
+            }
+            return $scheme . '://' . $host;
+        }
+        $cfg = rtrim((string) cfg('public_origin', ''), '/');
+        return $cfg !== '' ? $cfg : 'https://apps.f0xx.org';
     }
 
     public static function logout(): void {
         self::clearPending2fa();
+        AuthRemember::clear();
         unset($_SESSION['user']);
     }
 

sim/cluster0/lab-seeds/backend/src/AuthRemember.php

@@ -0,0 +1,127 @@
+<?php
+declare(strict_types=1);
+
+/** Persistent login via HttpOnly cookie backed by auth_tokens (purpose login_magic). */
+final class AuthRemember {
+    private const COOKIE = 'ac_remember';
+    private const TTL_SECONDS = 2592000; // 30 days
+
+    private function __construct() {
+    }
+
+    public static function tryRestore(): void {
+        if (PHP_SAPI === 'cli' || !empty($_SESSION['user'])) {
+            return;
+        }
+        $raw = trim((string) ($_COOKIE[self::COOKIE] ?? ''));
+        if ($raw === '' || !preg_match('/^[a-f0-9]{64}$/', $raw)) {
+            return;
+        }
+        AuthEmailSchema::ensure(Database::pdo());
+        $hash = hash('sha256', $raw);
+        $stmt = Database::pdo()->prepare(
+            "SELECT t.user_id, u.* FROM auth_tokens t
+             INNER JOIN users u ON u.id = t.user_id
+             WHERE t.token_hash = ? AND t.purpose = 'login_magic'
+               AND t.used_at IS NULL AND t.expires_at > ?
+             LIMIT 1"
+        );
+        $now = gmdate('Y-m-d H:i:s');
+        $stmt->execute([$hash, $now]);
+        $row = $stmt->fetch(PDO::FETCH_ASSOC);
+        if (!$row || empty($row['id'])) {
+            self::clearCookie();
+            return;
+        }
+        $status = (string) ($row['status'] ?? 'active');
+        if ($status !== 'active') {
+            self::clearCookie();
+            return;
+        }
+        if (AuthFactors::hasTotp((int) $row['id'])) {
+            // Remember-me skips password but not 2FA.
+            return;
+        }
+        Database::ensureRbacSchema();
+        Rbac::seedMembershipsForUser((int) $row['id'], (string) ($row['role'] ?? 'viewer'));
+        $_SESSION['user'] = Rbac::buildSessionUser($row);
+        Auth::touchSession();
+    }
+
+    public static function issue(int $userId): void {
+        if ($userId <= 0) {
+            return;
+        }
+        AuthEmailSchema::ensure(Database::pdo());
+        $pdo = Database::pdo();
+        $pdo->prepare(
+            "DELETE FROM auth_tokens WHERE user_id = ? AND purpose = 'login_magic' AND used_at IS NULL"
+        )->execute([$userId]);
+        $raw = bin2hex(random_bytes(32));
+        $hash = hash('sha256', $raw);
+        $expires = gmdate('Y-m-d H:i:s', time() + self::TTL_SECONDS);
+        $pdo->prepare(
+            'INSERT INTO auth_tokens (user_id, purpose, token_hash, expires_at, created_at)
+             VALUES (?, ?, ?, ?, ?)'
+        )->execute([$userId, 'login_magic', $hash, $expires, gmdate('Y-m-d H:i:s')]);
+        self::setCookie($raw, self::TTL_SECONDS);
+    }
+
+    public static function clear(): void {
+        $raw = trim((string) ($_COOKIE[self::COOKIE] ?? ''));
+        if ($raw !== '' && preg_match('/^[a-f0-9]{64}$/', $raw)) {
+            AuthEmailSchema::ensure(Database::pdo());
+            $hash = hash('sha256', $raw);
+            Database::pdo()->prepare(
+                "UPDATE auth_tokens SET used_at = ? WHERE token_hash = ? AND purpose = 'login_magic' AND used_at IS NULL"
+            )->execute([gmdate('Y-m-d H:i:s'), $hash]);
+        }
+        self::clearCookie();
+    }
+
+    public static function stashPendingFromRequest(): void {
+        if (!empty($_POST['remember_me'])) {
+            $_SESSION['remember_me_after_login'] = 1;
+        } else {
+            unset($_SESSION['remember_me_after_login']);
+        }
+    }
+
+    public static function applyPending(int $userId): void {
+        if ($userId <= 0 || empty($_SESSION['remember_me_after_login'])) {
+            unset($_SESSION['remember_me_after_login']);
+            return;
+        }
+        unset($_SESSION['remember_me_after_login']);
+        self::issue($userId);
+    }
+
+    private static function cookiePath(): string {
+        $path = trim((string) cfg('session_cookie_path', '/app/androidcast_project'));
+        return $path !== '' ? rtrim($path, '/') : '/';
+    }
+
+    private static function setCookie(string $value, int $maxAge): void {
+        $secure = function_exists('platform_is_https_request') && platform_is_https_request();
+        setcookie(self::COOKIE, $value, [
+            'expires' => time() + $maxAge,
+            'path' => self::cookiePath(),
+            'secure' => $secure,
+            'httponly' => true,
+            'samesite' => 'Lax',
+        ]);
+        $_COOKIE[self::COOKIE] = $value;
+    }
+
+    private static function clearCookie(): void {
+        $secure = function_exists('platform_is_https_request') && platform_is_https_request();
+        setcookie(self::COOKIE, '', [
+            'expires' => time() - 3600,
+            'path' => self::cookiePath(),
+            'secure' => $secure,
+            'httponly' => true,
+            'samesite' => 'Lax',
+        ]);
+        unset($_COOKIE[self::COOKIE]);
+    }
+}

sim/cluster0/lab-seeds/backend/src/bootstrap.php

@@ -46,6 +46,7 @@ require_once __DIR__ . '/Database.php';
 require_once __DIR__ . '/Rbac.php';
 require_once __DIR__ . '/DeviceRepository.php';
 require_once __DIR__ . '/Auth.php';
+require_once __DIR__ . '/AuthRemember.php';
 require_once __DIR__ . '/AuthEmailSchema.php';
 require_once __DIR__ . '/AuthMailer.php';
 require_once __DIR__ . '/AuthRegistration.php';
@@ -78,6 +79,10 @@ require_once __DIR__ . '/AuthApproval.php';
 require_once __DIR__ . '/AuthTwoFactorPage.php';
 require_once __DIR__ . '/AnalyticsHead.php';
 
+if (PHP_SAPI !== 'cli') {
+    AuthRemember::tryRestore();
+}
+
 function cfg(string $key, $default = null) {
     global $config;
     $parts = explode('.', $key);

sim/cluster0/lab-seeds/backend/views/login.php

@@ -53,6 +53,7 @@ $loginRedirect = $loginRedirect ?? '';
     <?php endif; ?>
     <label><span data-i18n="login.username">Username</span><input name="username" autocomplete="username" required></label>
     <label><span data-i18n="login.password">Password</span><input name="password" type="password" autocomplete="current-password" required></label>
+    <label class="build-form-checks"><input type="checkbox" name="remember_me" value="1"> <span data-i18n="login.remember_me">Remember me on this device</span></label>
     <button type="submit" data-i18n="login.submit">Sign in</button>
     <p class="hint"><a href="<?= h($auth) ?>/register" data-i18n="login.register">Create account</a></p>
   </form>

sim/cluster0/nginx/apps-port80.conf

@@ -234,10 +234,12 @@ server {
 	location ~ ^/app/androidcast_project/monitor/(login|user/password|api/user/password) {
 		proxy_pass         http://10.7.16.239:3000;
 		proxy_redirect     off;
+		proxy_cookie_path  / /app/androidcast_project/monitor/;
 		proxy_set_header   Host              $http_host;
 		proxy_set_header   X-Real-IP         $remote_addr;
 		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
 		proxy_set_header   X-Forwarded-Proto $public_scheme;
+		proxy_set_header   X-Forwarded-Host  $http_host;
 		proxy_http_version 1.1;
 	}
 	location /app/androidcast_project/monitor/ {
@@ -246,10 +248,12 @@ server {
 		error_page 401 = @monitor_login_redirect;
 		proxy_pass         http://10.7.16.239:3000;
 		proxy_redirect     off;
+		proxy_cookie_path  / /app/androidcast_project/monitor/;
 		proxy_set_header   Host              $http_host;
 		proxy_set_header   X-Real-IP         $remote_addr;
 		proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
 		proxy_set_header   X-Forwarded-Proto $public_scheme;
+		proxy_set_header   X-Forwarded-Host  $http_host;
 		proxy_set_header   X-WEBAUTH-USER    $grafana_user;
 		proxy_set_header   Authorization     "";
 		proxy_http_version 1.1;

sim/cluster0/scripts/configure-docker-storage.sh

@@ -0,0 +1,83 @@
+#!/bin/sh
+# Docker storage for cast01 builder: dedicated local xvdb + NFS cold backup.
+#
+# /var/lib/docker on /dev/xvdb1 (64G ext4). Do NOT bind-mount NFS for live
+# docker root (vfs + NFS = hung container creates). Optional rsync to
+# /shared/docker for archive/restore only.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+load_cluster_env
+
+DOCKER_BACKUP="${DOCKER_NFS_DIR:-${SHARED_MOUNT}/docker}"
+DOCKER_DST="/var/lib/docker"
+DOCKER_DISK="${DOCKER_DATA_DISK:-/dev/xvdb1}"
+DOCKER_LABEL="${DOCKER_DATA_LABEL:-docker-data}"
+FSTAB="/etc/fstab"
+
+log "configure docker storage (disk ${DOCKER_DISK}, NFS backup ${DOCKER_BACKUP})"
+
+mount | grep -q " ${SHARED_MOUNT} " || die "/shared not mounted"
+mkdir -p "${DOCKER_BACKUP}" "${DOCKER_DST}"
+
+# Remove obsolete NFS bind mount if present.
+if mount | grep -q "${DOCKER_BACKUP} on ${DOCKER_DST}"; then
+  log "removing NFS bind from ${DOCKER_DST}"
+  rc-service docker stop 2>/dev/null || true
+  sleep 2
+  umount -l "${DOCKER_DST}" 2>/dev/null || true
+fi
+sed -i "/\\/shared\\/docker \\/var\\/lib\\/docker/d" "${FSTAB}" 2>/dev/null || true
+
+ensure_docker_disk() {
+  if [ ! -b "${DOCKER_DISK}" ]; then
+    log "WARN ${DOCKER_DISK} missing — using existing ${DOCKER_DST} mount"
+    return 0
+  fi
+  mdev -s 2>/dev/null || true
+  if ! blkid "${DOCKER_DISK}" 2>/dev/null | grep -q ext4; then
+    log "format ${DOCKER_DISK} ext4 label=${DOCKER_LABEL}"
+    rc-service docker stop 2>/dev/null || true
+    sleep 2
+    mkfs.ext4 -L "${DOCKER_LABEL}" -F "${DOCKER_DISK}"
+  fi
+  uuid="$(blkid -s UUID -o value "${DOCKER_DISK}" 2>/dev/null || true)"
+  if [ -n "${uuid}" ]; then
+    sed -i "\|${DOCKER_DST}|d" "${FSTAB}" 2>/dev/null || true
+    if ! grep -q "${uuid}" "${FSTAB}" 2>/dev/null; then
+      echo "UUID=${uuid}  ${DOCKER_DST}  ext4  defaults,noatime,nofail  0  2" >> "${FSTAB}"
+      log "fstab: UUID=${uuid} → ${DOCKER_DST}"
+    fi
+  fi
+  if ! mount | grep -q " on ${DOCKER_DST} "; then
+    log "mount ${DOCKER_DISK} → ${DOCKER_DST}"
+    mount "${DOCKER_DISK}" "${DOCKER_DST}" 2>/dev/null || mount "${DOCKER_DST}" 2>/dev/null || true
+  fi
+}
+
+ensure_docker_disk
+
+if [ "${DOCKER_RESTORE_FROM_NFS:-0}" = "1" ] && [ -d "${DOCKER_BACKUP}/vfs" ]; then
+  log "restore: rsync ${DOCKER_BACKUP} → ${DOCKER_DST}"
+  rc-service docker stop 2>/dev/null || true
+  sleep 2
+  rsync -a "${DOCKER_BACKUP}/" "${DOCKER_DST}/"
+fi
+
+if [ "${DOCKER_BACKUP_TO_NFS:-0}" = "1" ] && [ -d "${DOCKER_DST}/vfs" ]; then
+  log "backup: rsync ${DOCKER_DST} → ${DOCKER_BACKUP}"
+  rsync -a "${DOCKER_DST}/" "${DOCKER_BACKUP}/"
+fi
+
+rc-service docker start 2>/dev/null || rc-service docker restart 2>/dev/null || true
+sleep 2
+if docker info >/dev/null 2>&1; then
+  log "docker OK root=$(docker info 2>/dev/null | awk -F': ' '/Docker Root Dir/{print $2}')"
+  docker system df 2>/dev/null || true
+else
+  log "WARN docker info failed"
+fi
+
+df -h / "${DOCKER_DST}" 2>/dev/null || true
+log "configure-docker-storage_ok $(host_short)"

sim/cluster0/scripts/prune-builder-artifacts.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# Prune obsolete Docker layers and old build artifact trees on builder nodes.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+load_cluster_env
+
+KEEP_BUILDS="${KEEP_BUILDS:-5}"
+BUILDS="${BUILDER_BUILDS_DIR:-/var/www/ac/broadcast/builds}"
+
+log "prune docker build cache + dangling images"
+docker builder prune -af 2>/dev/null || true
+docker image prune -af 2>/dev/null || true
+
+if [ -d "${BUILDS}" ]; then
+  log "prune build dirs under ${BUILDS} (keep newest ${KEEP_BUILDS})"
+  # Numeric build IDs only; sort descending, drop oldest beyond KEEP_BUILDS.
+  ls -1 "${BUILDS}" 2>/dev/null | awk '/^[0-9]+$/' | sort -rn | tail -n +"$((KEEP_BUILDS + 1))" | while read -r id; do
+    log "  remove builds/${id}"
+    rm -rf "${BUILDS}/${id}"
+  done
+fi
+
+docker system df 2>/dev/null || true
+df -h / /shared 2>/dev/null || true
+log "prune-builder-artifacts_ok $(host_short)"