forge/cloud-init/user-data.yaml.tpl

#cloud-config

# Notes on Container-Optimized OS (COS):
#   - /var is mounted noexec, so executable scripts must live under /var/lib/google
#     (one of the exec-allowed writable paths on COS).
#   - Mount units use systemd-escape(1) naming: /mnt/disks/forgejo-data becomes
#     mnt-disks-forgejo\x2ddata.mount. We avoid hardcoding the escaped name in
#     dependencies by using RequiresMountsFor=, which lets systemd resolve it.

write_files:
  - path: /etc/systemd/system/mnt-disks-forgejo\x2ddata.mount
    content: |
      [Unit]
      Description=Mount Forgejo data disk
      Before=docker.service

      [Mount]
      What=/dev/disk/by-id/google-forgejo-data
      Where=/mnt/disks/forgejo-data
      Type=ext4
      Options=defaults,nofail

      [Install]
      WantedBy=multi-user.target

  - path: /var/lib/forgejo/Caddyfile
    content: |
      ${domain} {
        reverse_proxy forgejo:3000
        encode gzip
      }

  - path: /var/lib/google/forgejo/fetch-secrets.sh
    permissions: '0755'
    content: |
      #!/bin/bash
      set -euo pipefail
      TOKEN=$(curl -sf -H "Metadata-Flavor: Google" \
        "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \
        | python3 -c "import sys,json;print(json.load(sys.stdin)['access_token'])")
      fetch() {
        curl -sf -H "Authorization: Bearer $TOKEN" \
          "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$1/versions/latest:access" \
          | python3 -c "import sys,json,base64;print(base64.b64decode(json.load(sys.stdin)['payload']['data']).decode())"
      }
      mkdir -p /run
      umask 077
      {
        echo "FORGEJO__security__SECRET_KEY=$(fetch forgejo-secret-key)"
        echo "FORGEJO__security__INTERNAL_TOKEN=$(fetch forgejo-internal-token)"
      } > /run/forgejo-secrets.env

  - path: /etc/systemd/system/forgejo-stack.service
    content: |
      [Unit]
      # systemd owns the container lifecycle. We deliberately do NOT use
      # `docker run --restart=unless-stopped`: on COS, docker's auto-restart
      # races the persistent-disk mount unit on boot (containers come back up
      # before /mnt/disks/forgejo-data is mounted, bind-mounting the empty
      # underlying directory instead) AND races the daemon's container-metadata
      # restoration (rm -f at boot becomes a no-op, then `docker run` collides
      # with the just-restored container).
      Description=Forgejo + Caddy + Watchtower
      After=network-online.target docker.service
      RequiresMountsFor=/mnt/disks/forgejo-data
      Wants=network-online.target
      OnFailure=forgejo-stack-recover.service
      StartLimitIntervalSec=600
      StartLimitBurst=5

      [Service]
      Type=oneshot
      RemainAfterExit=true
      ExecStartPre=/var/lib/google/forgejo/fetch-secrets.sh
      ExecStartPre=-/usr/bin/docker rm -f caddy forgejo watchtower
      ExecStartPre=-/usr/bin/docker network rm web
      ExecStartPre=/usr/bin/docker network create web
      ExecStart=/usr/bin/docker run -d --rm --name caddy --network web \
        -p 80:80 -p 443:443 \
        -v /mnt/disks/forgejo-data/caddy:/data \
        -v /var/lib/forgejo/Caddyfile:/etc/caddy/Caddyfile:ro \
        ${caddy_image}
      ExecStart=/usr/bin/docker run -d --rm --name forgejo --network web \
        -e FORGEJO__server__DISABLE_SSH=true \
        -e FORGEJO__server__ROOT_URL=https://${domain}/ \
        -e FORGEJO__service__DISABLE_REGISTRATION=true \
        -e FORGEJO__database__DB_TYPE=sqlite3 \
        --env-file /run/forgejo-secrets.env \
        -v /mnt/disks/forgejo-data/forgejo:/data \
        ${forgejo_image}
      ExecStart=/usr/bin/docker run -d --rm --name watchtower \
        -v /var/run/docker.sock:/var/run/docker.sock \
        containrrr/watchtower --cleanup --schedule "0 0 4 * * *"
      ExecStop=-/usr/bin/docker rm -f watchtower forgejo caddy
      ExecStop=-/usr/bin/docker network rm web

      [Install]
      WantedBy=multi-user.target

  - path: /etc/systemd/system/forgejo-stack-recover.service
    content: |
      [Unit]
      Description=Restart forgejo-stack after a failure

      [Service]
      Type=oneshot
      ExecStart=/bin/sh -c 'sleep 60 && /usr/bin/systemctl restart forgejo-stack.service'

  - path: /var/lib/google/forgejo/watchdog.sh
    permissions: '0755'
    content: |
      #!/bin/bash
      set -euo pipefail
      # Replaces docker's --restart=unless-stopped, which we can't use here
      # (see forgejo-stack.service comment). If any of the three containers
      # is missing, restart the whole stack -- systemd is the single owner.
      for c in caddy forgejo watchtower; do
        if ! /usr/bin/docker ps --format '{{.Names}}' | grep -qx "$c"; then
          /usr/bin/logger -t forgejo-watchdog "container '$c' missing, restarting forgejo-stack.service"
          exec /usr/bin/systemctl restart forgejo-stack.service
        fi
      done

  - path: /etc/systemd/system/forgejo-stack-watchdog.service
    content: |
      [Unit]
      Description=Restart forgejo-stack if any container is missing

      [Service]
      Type=oneshot
      ExecStart=/var/lib/google/forgejo/watchdog.sh

  - path: /etc/systemd/system/forgejo-stack-watchdog.timer
    content: |
      [Unit]
      Description=Check forgejo-stack health every minute

      [Timer]
      OnBootSec=2min
      OnUnitActiveSec=1min

      [Install]
      WantedBy=timers.target

  - path: /var/lib/google/forgejo/backup.sh
    permissions: '0755'
    content: |
      #!/bin/bash
      set -euo pipefail
      STAMP=$(date -u +%Y%m%dT%H%M%SZ)
      docker exec forgejo sqlite3 /data/gitea/gitea.db ".backup '/data/gitea/snapshot.db'"
      tar czf /tmp/forgejo-$STAMP.tar.gz -C /mnt/disks/forgejo-data forgejo
      docker run --rm -v /tmp:/tmp google/cloud-sdk:slim \
        gsutil cp /tmp/forgejo-$STAMP.tar.gz gs://${gcs_backup_bucket}/
      rm /tmp/forgejo-$STAMP.tar.gz
      docker exec forgejo rm -f /data/gitea/snapshot.db

  - path: /etc/systemd/system/forgejo-backup.service
    content: |
      [Unit]
      Description=Backup Forgejo to GCS
      After=forgejo-stack.service
      Requires=forgejo-stack.service

      [Service]
      Type=oneshot
      ExecStart=/var/lib/google/forgejo/backup.sh

  - path: /etc/systemd/system/forgejo-backup.timer
    content: |
      [Unit]
      Description=Nightly Forgejo backup

      [Timer]
      OnCalendar=*-*-* 03:30:00
      Persistent=true

      [Install]
      WantedBy=timers.target

  - path: /var/lib/google/forgejo/disk-check.sh
    permissions: '0755'
    content: |
      #!/bin/bash
      set -euo pipefail
      USE=$(df --output=pcent /mnt/disks/forgejo-data | tail -1 | tr -dc '0-9')
      if [ "$USE" -gt 80 ]; then
        logger -t forgejo-disk "DISK_HIGH: /mnt/disks/forgejo-data at $${USE}% used"
      fi

  - path: /etc/systemd/system/forgejo-disk-check.service
    content: |
      [Unit]
      Description=Emit a log line if Forgejo data disk is >80% full

      [Service]
      Type=oneshot
      ExecStart=/var/lib/google/forgejo/disk-check.sh

  - path: /etc/systemd/system/forgejo-disk-check.timer
    content: |
      [Unit]
      Description=Hourly disk-fullness check

      [Timer]
      OnBootSec=5min
      OnUnitActiveSec=1h
      Persistent=true

      [Install]
      WantedBy=timers.target

  - path: /etc/systemd/system/forgejo-reboot.service
    content: |
      [Unit]
      Description=Apply staged COS updates by rebooting

      [Service]
      Type=oneshot
      ExecStart=/sbin/shutdown -r +0

  - path: /etc/systemd/system/forgejo-reboot.timer
    content: |
      [Unit]
      Description=Nightly reboot (lands 30 min after Watchtower so container updates apply first)

      [Timer]
      OnCalendar=*-*-* 04:30:00
      Persistent=true

      [Install]
      WantedBy=timers.target

runcmd:
  - mkdir -p /mnt/disks/forgejo-data
  - if ! blkid /dev/disk/by-id/google-forgejo-data; then mkfs.ext4 -F /dev/disk/by-id/google-forgejo-data; fi
  - systemctl daemon-reload
  - mkdir -p /mnt/disks/forgejo-data/forgejo /mnt/disks/forgejo-data/caddy
  - systemctl enable --now forgejo-stack.service
  - systemctl enable --now forgejo-stack-watchdog.timer
  - systemctl enable --now forgejo-backup.timer
  - systemctl enable --now forgejo-reboot.timer
  - systemctl enable --now forgejo-disk-check.timer