diff options
| -rw-r--r-- | f3s/argocd-apps/services/forgejo.yaml | 28 | ||||
| -rw-r--r-- | f3s/forgejo/Justfile | 44 | ||||
| -rw-r--r-- | f3s/forgejo/README.md | 123 | ||||
| -rw-r--r-- | f3s/forgejo/helm-chart/Chart.yaml | 5 | ||||
| -rw-r--r-- | f3s/forgejo/helm-chart/templates/deployment.yaml | 152 | ||||
| -rw-r--r-- | f3s/forgejo/helm-chart/templates/ingress.yaml | 54 | ||||
| -rw-r--r-- | f3s/forgejo/helm-chart/templates/persistent-volume.yaml | 66 | ||||
| -rw-r--r-- | f3s/forgejo/helm-chart/templates/service.yaml | 36 | ||||
| -rw-r--r-- | frontends/Rexfile | 2 |
9 files changed, 509 insertions, 1 deletions
diff --git a/f3s/argocd-apps/services/forgejo.yaml b/f3s/argocd-apps/services/forgejo.yaml new file mode 100644 index 0000000..e0728ad --- /dev/null +++ b/f3s/argocd-apps/services/forgejo.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: forgejo + namespace: cicd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: http://git-server.cicd.svc.cluster.local/conf.git + targetRevision: master + path: f3s/forgejo/helm-chart + destination: + server: https://kubernetes.default.svc + namespace: services + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=false + retry: + limit: 3 + backoff: + duration: 5s + factor: 2 + maxDuration: 1m diff --git a/f3s/forgejo/Justfile b/f3s/forgejo/Justfile new file mode 100644 index 0000000..ee78a62 --- /dev/null +++ b/f3s/forgejo/Justfile @@ -0,0 +1,44 @@ +NAMESPACE := "services" +APP_NAME := "forgejo" + +status: + @echo "=== Pods ===" + @kubectl get pods -n {{NAMESPACE}} | grep forgejo + @echo "" + @echo "=== Services ===" + @kubectl get svc -n {{NAMESPACE}} forgejo forgejo-ssh + @echo "" + @echo "=== Ingress ===" + @kubectl get ingress -n {{NAMESPACE}} forgejo-ingress forgejo-ingress-lan + @echo "" + @echo "=== PVCs ===" + @kubectl get pvc -n {{NAMESPACE}} | grep forgejo + @echo "" + @echo "=== ArgoCD Status ===" + @kubectl get application {{APP_NAME}} -n cicd -o jsonpath='Sync: {.status.sync.status}, Health: {.status.health.status}' 2>/dev/null && echo "" + +logs lines="100": + kubectl logs -n {{NAMESPACE}} -l app=forgejo --tail={{lines}} -f + +port-forward port="3000": + @echo "Forwarding forgejo to localhost:{{port}}" + kubectl port-forward -n {{NAMESPACE}} svc/forgejo {{port}}:80 + +sync: + @echo "Triggering ArgoCD sync..." + @kubectl annotate application {{APP_NAME}} -n cicd argocd.argoproj.io/refresh=normal --overwrite + @sleep 2 + @kubectl get application {{APP_NAME}} -n cicd -o jsonpath='Sync: {.status.sync.status}, Health: {.status.health.status}' && echo "" + +restart: + @echo "Restarting forgejo..." + kubectl rollout restart -n {{NAMESPACE}} deployment/forgejo + +# One-time after the first successful start. Prompts for the password. +create-admin username="paul" email="paul.buetow@gmail.com": + kubectl exec -n {{NAMESPACE}} -it deploy/forgejo -- \ + forgejo admin user create --admin --username {{username}} --email {{email}} + +# Open a shell as the git user inside the running instance. +shell: + kubectl exec -n {{NAMESPACE}} -it deploy/forgejo -- /bin/sh diff --git a/f3s/forgejo/README.md b/f3s/forgejo/README.md new file mode 100644 index 0000000..ecc35a6 --- /dev/null +++ b/f3s/forgejo/README.md @@ -0,0 +1,123 @@ +# Forgejo + +Self-hosted git forge at `https://code.f3s.buetow.org`, running in the `services` +namespace of the f3s k3s cluster. + +## Relationship to the cgit git-server + +**These two installs are deliberately independent.** Nothing here touches +`f3s/git-server/`: + +| | cgit / git-server | Forgejo | +|---|---|---| +| Web UI | `c-git.f3s.buetow.org` | `code.f3s.buetow.org` | +| Namespace | `cicd` | `services` | +| Repo storage | `/data/nfs/k3svolumes/git-server/repos` (80 bare repos) | `/data/nfs/k3svolumes/forgejo/data` (starts empty) | +| SSH NodePort | 30022 | 30222 | + +ArgoCD keeps reading `conf.git` from the existing git-server +(`http://git-server.cicd.svc.cluster.local/conf.git`). Forgejo has no consumers, +so if it breaks, cluster deploys are unaffected. Repositories are migrated by +hand, one at a time, whenever you feel like it — there is no bulk import. + +## Architecture + +``` +Internet -> relayd (OpenBSD GW, TLS) -> WireGuard -> Traefik -> forgejo svc:80 -> pod:3000 +git+ssh -> NodePort 30222 -----------------------------------> pod:2222 + | + /var/lib/gitea (repos, SQLite) NFS -> ZFS + /etc/gitea (app.ini, secrets) +``` + +- Image `codeberg.org/forgejo/forgejo:16.0.1-rootless` — the rootless variant, so + the whole pod runs as UID/GID 1000 with all capabilities dropped. +- SQLite, not PostgreSQL. Single writer (`replicas: 1` + `Recreate`), and NFSv4.2 + does real byte-range locking, so the classic SQLite-on-NFS corruption mode does + not apply. Revisit if this ever needs to scale out. +- Both volumes carry the standard `.nfs-sentinel` guard so the pod refuses to + start against the local-XFS shadow if a node has NFS unmounted. + +## Initial setup + +### 1. Create the storage directories + +On the current CARP storage master (check with `ifconfig | grep MASTER` on f0/f1): + +```sh +doas mkdir -p /data/nfs/k3svolumes/forgejo/data /data/nfs/k3svolumes/forgejo/config +doas touch /data/nfs/k3svolumes/forgejo/data/.nfs-sentinel \ + /data/nfs/k3svolumes/forgejo/config/.nfs-sentinel +doas chown -R 1000:1000 /data/nfs/k3svolumes/forgejo +doas chmod -R 0750 /data/nfs/k3svolumes/forgejo +``` + +The PVs use `type: Directory`, so the pod will not schedule until these exist. + +### 2. Publish the hostname + +`code.f3s.buetow.org` must be added to `@f3s_hosts` in `frontends/Rexfile`. That +one array drives the DNS zone, the relayd routing rule, the ACME certificate and +the gogios monitoring checks. + +Deploy order matters — relayd loads `tls keypair code.f3s.buetow.org` at startup, +so the certificate has to exist first: + +```sh +cd frontends +rex -H blowfish.buetow.org:2 nsd httpd acme acme_invoke relayd +rex -H fishfinger.buetow.org:2 nsd httpd acme acme_invoke relayd +``` + +`acme.sh` copies the `foo.zone` cert as a placeholder for any host that has none +yet, so relayd will still start on the first pass; the real certificate arrives +on the same run. Deploying one gateway at a time avoids restarting both public +frontends simultaneously. + +### 3. Deploy + +```sh +kubectl apply -f ../argocd-apps/services/forgejo.yaml +``` + +Or just push — ArgoCD picks it up automatically. + +### 4. Create the admin user + +The web installer is locked (`INSTALL_LOCK=true`) and registration is disabled, +because this instance is reachable from the public internet. Create the first +account from the CLI: + +```sh +just create-admin +``` + +## Repository URLs + +```sh +# HTTPS +git clone https://code.f3s.buetow.org/<user>/<repo>.git + +# SSH (NodePort; LAN only unless you forward it) +git clone ssh://git@r0.lan.buetow.org:30222/<user>/<repo>.git +``` + +## Operations + +```sh +just status # pods, services, ingress, PVCs, ArgoCD sync +just logs # follow logs +just restart # rollout restart +just shell # shell inside the pod +just port-forward # reach the UI on localhost:3000 +``` + +## Backup + +Covered by the ZFS snapshots and zrepl replication of `/data/nfs`. The SQLite +database and `app.ini` both live on that volume. To restore: roll back the ZFS +snapshot and restart the deployment. + +Note that `/etc/gitea/app.ini` holds `SECRET_KEY` and `INTERNAL_TOKEN`, generated +on first start. Restoring the data volume without the matching config volume +invalidates sessions and stored credentials. diff --git a/f3s/forgejo/helm-chart/Chart.yaml b/f3s/forgejo/helm-chart/Chart.yaml new file mode 100644 index 0000000..d0bf564 --- /dev/null +++ b/f3s/forgejo/helm-chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: forgejo +description: A Helm chart for deploying Forgejo, the self-hosted git forge at code.f3s.buetow.org. +version: 0.1.0 +appVersion: "16.0.1" diff --git a/f3s/forgejo/helm-chart/templates/deployment.yaml b/f3s/forgejo/helm-chart/templates/deployment.yaml new file mode 100644 index 0000000..132fed4 --- /dev/null +++ b/f3s/forgejo/helm-chart/templates/deployment.yaml @@ -0,0 +1,152 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forgejo + namespace: services + labels: + app: forgejo +spec: + replicas: 1 + # Recreate so the old pod fully terminates before the new one starts — + # avoids NFS-lock races on the hostPath-backed PVC during rolling updates. + # This also matters for SQLite: exactly one process may hold the database. + strategy: + type: Recreate + selector: + matchLabels: + app: forgejo + template: + metadata: + labels: + app: forgejo + spec: + securityContext: + # The -rootless image runs entirely as the unprivileged git user (1000). + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + initContainers: + - name: nfs-check-data + image: busybox:stable + command: + - sh + - -c + - | + test -f /mnt/.nfs-sentinel || ( + echo "ERROR: NFS sentinel missing at /mnt/.nfs-sentinel" + echo "refusing to start; node likely has NFS unmounted" + echo "pod would otherwise bind-mount the local-XFS shadow" + exit 1 + ) + volumeMounts: + - name: forgejo-data + mountPath: /mnt + readOnly: true + - name: nfs-check-config + image: busybox:stable + command: + - sh + - -c + - | + test -f /mnt/.nfs-sentinel || ( + echo "ERROR: NFS sentinel missing at /mnt/.nfs-sentinel" + echo "refusing to start; node likely has NFS unmounted" + echo "pod would otherwise bind-mount the local-XFS shadow" + exit 1 + ) + volumeMounts: + - name: forgejo-config + mountPath: /mnt + readOnly: true + + containers: + - name: forgejo + image: codeberg.org/forgejo/forgejo:16.0.1-rootless + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http + protocol: TCP + - containerPort: 2222 + name: ssh + protocol: TCP + env: + # SQLite rather than a separate PostgreSQL pod: this is a single-writer + # instance (replicas 1 + Recreate), and NFSv4.2 does real byte-range + # locking, so the usual SQLite-on-NFS corruption mode does not apply. + # Revisit if this ever needs more than one replica. + - name: FORGEJO__database__DB_TYPE + value: "sqlite3" + - name: FORGEJO__database__PATH + value: "/var/lib/gitea/data/forgejo.db" + + # Public identity. ROOT_URL must match what relayd terminates TLS for, + # otherwise Forgejo generates clone URLs and redirects on the wrong host. + - name: FORGEJO__server__DOMAIN + value: "code.f3s.buetow.org" + - name: FORGEJO__server__ROOT_URL + value: "https://code.f3s.buetow.org/" + - name: FORGEJO__server__HTTP_PORT + value: "3000" + + # Built-in SSH server. SSH_LISTEN_PORT is what the container binds; + # SSH_PORT is what Forgejo advertises in clone URLs, i.e. the NodePort + # users actually reach. git-server already owns 30022, so this is 30222. + - name: FORGEJO__server__START_SSH_SERVER + value: "true" + - name: FORGEJO__server__SSH_LISTEN_PORT + value: "2222" + - name: FORGEJO__server__SSH_DOMAIN + value: "code.f3s.buetow.org" + - name: FORGEJO__server__SSH_PORT + value: "30222" + + # This instance is reachable from the public internet through relayd. + # Lock the installer (otherwise the first visitor gets the setup wizard) + # and keep signups closed; create the admin with the CLI, see README. + - name: FORGEJO__security__INSTALL_LOCK + value: "true" + - name: FORGEJO__service__DISABLE_REGISTRATION + value: "true" + + # Catches stale NFS file handles (ESTALE) after an NFS server restart, + # which a plain HTTP probe would not notice until a request touched disk. + livenessProbe: + exec: + command: ["test", "-f", "/var/lib/gitea/.nfs-sentinel"] + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /api/healthz + port: 3000 + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 5 + volumeMounts: + - name: forgejo-data + mountPath: /var/lib/gitea + - name: forgejo-config + mountPath: /etc/gitea + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1000m + memory: 1Gi + + volumes: + - name: forgejo-data + persistentVolumeClaim: + claimName: forgejo-data-pvc + - name: forgejo-config + persistentVolumeClaim: + claimName: forgejo-config-pvc diff --git a/f3s/forgejo/helm-chart/templates/ingress.yaml b/f3s/forgejo/helm-chart/templates/ingress.yaml new file mode 100644 index 0000000..4916098 --- /dev/null +++ b/f3s/forgejo/helm-chart/templates/ingress.yaml @@ -0,0 +1,54 @@ +# Forgejo web UI ingress. +# +# code.f3s.buetow.org must also be listed in @f3s_hosts in frontends/Rexfile -- +# that array drives the DNS zone, the relayd routing rule and the ACME cert. +# Adding it here alone is not enough to make the name resolve or serve TLS. +# +# cgit stays where it is, at c-git.f3s.buetow.org; the two are unrelated. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: forgejo-ingress + namespace: services + annotations: + spec.ingressClassName: traefik + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + rules: + - host: code.f3s.buetow.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: forgejo + port: + number: 80 +--- +# LAN ingress. *.f3s.lan resolves to the storage VIP via Pi-hole, and Traefik +# terminates TLS here with the shared f3s-lan-tls cert. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: forgejo-ingress-lan + namespace: services + annotations: + spec.ingressClassName: traefik + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure +spec: + tls: + - hosts: + - code.f3s.lan.buetow.org + secretName: f3s-lan-tls + rules: + - host: code.f3s.lan.buetow.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: forgejo + port: + number: 80 diff --git a/f3s/forgejo/helm-chart/templates/persistent-volume.yaml b/f3s/forgejo/helm-chart/templates/persistent-volume.yaml new file mode 100644 index 0000000..40c8788 --- /dev/null +++ b/f3s/forgejo/helm-chart/templates/persistent-volume.yaml @@ -0,0 +1,66 @@ +# Forgejo storage. Deliberately separate from the git-server/cgit volume: +# Forgejo owns its own repositories under /data/nfs/k3svolumes/forgejo/data and +# never touches the 80 bare repos cgit serves out of +# /data/nfs/k3svolumes/git-server/repos. +# +# Both directories must exist and contain a .nfs-sentinel file before the pod +# starts -- see the initContainers in deployment.yaml and the README. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: forgejo-data-pv +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data/nfs/k3svolumes/forgejo/data + type: Directory +--- +# app.ini lives here, and with it the SECRET_KEY and INTERNAL_TOKEN that Forgejo +# generates on first start. Losing this volume invalidates existing sessions and +# any stored credentials, so it is kept on NFS (ZFS-snapshotted) rather than in +# an emptyDir. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: forgejo-config-pv +spec: + capacity: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data/nfs/k3svolumes/forgejo/config + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: forgejo-data-pvc + namespace: services +spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: forgejo-config-pvc + namespace: services +spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/f3s/forgejo/helm-chart/templates/service.yaml b/f3s/forgejo/helm-chart/templates/service.yaml new file mode 100644 index 0000000..7c3bfe7 --- /dev/null +++ b/f3s/forgejo/helm-chart/templates/service.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: forgejo + namespace: services + labels: + app: forgejo +spec: + selector: + app: forgejo + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 3000 + type: ClusterIP +--- +# SSH for git clone/push. NodePort 30222 -- git-server/cgit already owns 30022, +# and the two installs are intentionally independent. +apiVersion: v1 +kind: Service +metadata: + name: forgejo-ssh + namespace: services + labels: + app: forgejo +spec: + selector: + app: forgejo + ports: + - name: ssh + protocol: TCP + port: 2222 + targetPort: 2222 + nodePort: 30222 + type: NodePort diff --git a/frontends/Rexfile b/frontends/Rexfile index 57f8141..3e72227 100644 --- a/frontends/Rexfile +++ b/frontends/Rexfile @@ -85,7 +85,7 @@ our $secrets = sub { read_file './secrets/' . shift }; # k3s cluster running on FreeBSD in my LAN our @f3s_hosts = - qw/f3s.buetow.org ychat.f3s.buetow.org player.f3s.buetow.org xplayer.f3s.buetow.org pihole.f3s.buetow.org jellyfin.f3s.buetow.org navidrome.f3s.buetow.org git.f3s.buetow.org c-git.f3s.buetow.org immich.f3s.buetow.org argocd.f3s.buetow.org keybr.f3s.buetow.org anki.f3s.buetow.org bag.f3s.buetow.org flux.f3s.buetow.org audiobookshelf.f3s.buetow.org garage.f3s.buetow.org grafana.f3s.buetow.org radicale.f3s.buetow.org syncthing.f3s.buetow.org koreader.f3s.buetow.org filebrowser.f3s.buetow.org webdav.f3s.buetow.org pkgrepo.f3s.buetow.org goprecords.f3s.buetow.org ipv6test.f3s.buetow.org ipv4.ipv6test.f3s.buetow.org ipv6.ipv6test.f3s.buetow.org/; + qw/f3s.buetow.org ychat.f3s.buetow.org player.f3s.buetow.org xplayer.f3s.buetow.org pihole.f3s.buetow.org jellyfin.f3s.buetow.org navidrome.f3s.buetow.org git.f3s.buetow.org c-git.f3s.buetow.org code.f3s.buetow.org immich.f3s.buetow.org argocd.f3s.buetow.org keybr.f3s.buetow.org anki.f3s.buetow.org bag.f3s.buetow.org flux.f3s.buetow.org audiobookshelf.f3s.buetow.org garage.f3s.buetow.org grafana.f3s.buetow.org radicale.f3s.buetow.org syncthing.f3s.buetow.org koreader.f3s.buetow.org filebrowser.f3s.buetow.org webdav.f3s.buetow.org pkgrepo.f3s.buetow.org goprecords.f3s.buetow.org ipv6test.f3s.buetow.org ipv4.ipv6test.f3s.buetow.org ipv6.ipv6test.f3s.buetow.org/; # optionally, only enable manually for temp time, as no password protection yet # push @f3s_hosts, 'registry.f3s.buetow.org'; |