diff --git a/cloud-init/user-data.yaml.tpl b/cloud-init/user-data.yaml.tpl index b4a7318..64591bd 100644 --- a/cloud-init/user-data.yaml.tpl +++ b/cloud-init/user-data.yaml.tpl @@ -53,42 +53,95 @@ write_files: - path: /etc/systemd/system/forgejo-stack.service content: | [Unit] + # systemd owns the container lifecycle. We deliberately do NOT use + # `docker run --restart=unless-stopped`: on COS, docker's auto-restart + # races the persistent-disk mount unit on boot (containers come back up + # before /mnt/disks/forgejo-data is mounted, bind-mounting the empty + # underlying directory instead) AND races the daemon's container-metadata + # restoration (rm -f at boot becomes a no-op, then `docker run` collides + # with the just-restored container). Description=Forgejo + Caddy + Watchtower After=network-online.target docker.service RequiresMountsFor=/mnt/disks/forgejo-data Wants=network-online.target + OnFailure=forgejo-stack-recover.service + StartLimitIntervalSec=600 + StartLimitBurst=5 [Service] Type=oneshot RemainAfterExit=true ExecStartPre=/var/lib/google/forgejo/fetch-secrets.sh - ExecStartPre=/bin/sh -c 'i=0; while [ $i -lt 30 ]; do /usr/bin/docker ps -a --format "{{.Names}}" 2>/dev/null | grep -qE "^(caddy|forgejo|watchtower)$" && break; i=$((i+1)); sleep 1; done' ExecStartPre=-/usr/bin/docker rm -f caddy forgejo watchtower - ExecStartPre=-/usr/bin/docker network create web - ExecStart=/usr/bin/docker run -d --name caddy --network web \ + ExecStartPre=-/usr/bin/docker network rm web + ExecStartPre=/usr/bin/docker network create web + ExecStart=/usr/bin/docker run -d --rm --name caddy --network web \ -p 80:80 -p 443:443 \ -v /mnt/disks/forgejo-data/caddy:/data \ -v /var/lib/forgejo/Caddyfile:/etc/caddy/Caddyfile:ro \ - --restart=unless-stopped \ ${caddy_image} - ExecStart=/usr/bin/docker run -d --name forgejo --network web \ + ExecStart=/usr/bin/docker run -d --rm --name forgejo --network web \ -e FORGEJO__server__DISABLE_SSH=true \ -e FORGEJO__server__ROOT_URL=https://${domain}/ \ -e FORGEJO__service__DISABLE_REGISTRATION=true \ -e FORGEJO__database__DB_TYPE=sqlite3 \ --env-file /run/forgejo-secrets.env \ -v /mnt/disks/forgejo-data/forgejo:/data \ - --restart=unless-stopped \ ${forgejo_image} - ExecStart=/usr/bin/docker run -d --name watchtower \ + ExecStart=/usr/bin/docker run -d --rm --name watchtower \ -v /var/run/docker.sock:/var/run/docker.sock \ - --restart=unless-stopped \ containrrr/watchtower --cleanup --schedule "0 0 4 * * *" - ExecStop=/usr/bin/docker rm -f watchtower forgejo caddy + ExecStop=-/usr/bin/docker rm -f watchtower forgejo caddy + ExecStop=-/usr/bin/docker network rm web [Install] WantedBy=multi-user.target + - path: /etc/systemd/system/forgejo-stack-recover.service + content: | + [Unit] + Description=Restart forgejo-stack after a failure + + [Service] + Type=oneshot + ExecStart=/bin/sh -c 'sleep 60 && /usr/bin/systemctl restart forgejo-stack.service' + + - path: /var/lib/google/forgejo/watchdog.sh + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + # Replaces docker's --restart=unless-stopped, which we can't use here + # (see forgejo-stack.service comment). If any of the three containers + # is missing, restart the whole stack -- systemd is the single owner. + for c in caddy forgejo watchtower; do + if ! /usr/bin/docker ps --format '{{.Names}}' | grep -qx "$c"; then + /usr/bin/logger -t forgejo-watchdog "container '$c' missing, restarting forgejo-stack.service" + exec /usr/bin/systemctl restart forgejo-stack.service + fi + done + + - path: /etc/systemd/system/forgejo-stack-watchdog.service + content: | + [Unit] + Description=Restart forgejo-stack if any container is missing + + [Service] + Type=oneshot + ExecStart=/var/lib/google/forgejo/watchdog.sh + + - path: /etc/systemd/system/forgejo-stack-watchdog.timer + content: | + [Unit] + Description=Check forgejo-stack health every minute + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + + [Install] + WantedBy=timers.target + - path: /var/lib/google/forgejo/backup.sh permissions: '0755' content: | @@ -184,6 +237,7 @@ runcmd: - systemctl daemon-reload - mkdir -p /mnt/disks/forgejo-data/forgejo /mnt/disks/forgejo-data/caddy - systemctl enable --now forgejo-stack.service + - systemctl enable --now forgejo-stack-watchdog.timer - systemctl enable --now forgejo-backup.timer - systemctl enable --now forgejo-reboot.timer - systemctl enable --now forgejo-disk-check.timer