summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-05-18 20:17:42 +0300
committerPaul Buetow <paul@buetow.org>2026-05-18 20:17:42 +0300
commit4356fde60cd64b48bee83385c78f37dd13e2659d (patch)
treef2bedf8e213a6be7edf99060433fde61d8af4684
parentf84f1d930ca2b3e2fdd0cf162310fd2d22f913c0 (diff)
Add mage E2E target and convert nightly.yml to manual-only trigger
Replaces the automated cron-based LLM e2e trigger with a local Mage target. `mage E2E` builds the runner and executes all scenarios against a running server. The Woodpecker pipeline now requires an explicit cron event (which is not configured), preventing accidental CI runs while keeping the secret wiring in place for future manual re-triggers via the Woodpecker UI. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
-rw-r--r--.woodpecker/nightly.yml34
-rw-r--r--magefile.go49
2 files changed, 68 insertions, 15 deletions
diff --git a/.woodpecker/nightly.yml b/.woodpecker/nightly.yml
index 3f3fb24..dd12fc1 100644
--- a/.woodpecker/nightly.yml
+++ b/.woodpecker/nightly.yml
@@ -1,12 +1,18 @@
-# Nightly LLM e2e pipeline for codeberg.org/snonux/player
+# LLM e2e pipeline for codeberg.org/snonux/player
#
-# Trigger: Woodpecker cron schedule 0 2 * * * (02:00 UTC daily, main only).
-# Manual dispatch is also allowed for ad-hoc runs.
+# Trigger: Manual only — do NOT configure a Woodpecker cron schedule for this
+# pipeline. The intended way to run LLM e2e tests is locally via:
#
-# Purpose: LLM-assisted end-to-end test harness (currently a placeholder).
-# The real test suite is a future task. This pipeline wires in the correct
-# structure — secret injection, branch guard, cost logging — so that the
-# first real LLM e2e tests can be added without restructuring the YAML.
+# mage E2E
+#
+# The Mage target (magefile.go at the project root) builds the runner,
+# sets the required env vars, and invokes `node dist/index.js` against
+# a locally running Player server.
+#
+# This pipeline file exists so the secret is available when the
+# pipeline is re-triggered manually via the Woodpecker UI. It will
+# never fire automatically: the `event: cron` filter only matches a
+# Woodpecker cron job, and no cron is configured for this project.
#
# Secrets:
# ANTHROPIC_API_KEY — injected from a Woodpecker protected project secret.
@@ -20,7 +26,7 @@
# A future monitoring step can fail the job if cost exceeds a threshold.
---
-# ── Nightly LLM e2e (main only, cron 0 2 * * *) ───────────────────────────────
+# ── LLM e2e (manual trigger via Woodpecker UI or mage E2E locally) ─────────────
steps:
- name: llm-e2e
@@ -35,8 +41,6 @@ steps:
- from_secret: ANTHROPIC_API_KEY
commands:
# Belt-and-suspenders branch guard: skip if not running on main.
- # The cron trigger is already scoped to main; this check guards against
- # accidental manual triggers on feature branches exposing the secret.
- |
if [ "$CI_COMMIT_BRANCH" != "main" ]; then
echo "Skipping LLM e2e: not on main branch (branch=$CI_COMMIT_BRANCH)"
@@ -48,10 +52,11 @@ steps:
- 'echo "LLM e2e harness: placeholder (no tests implemented yet)"'
- exit 0
- # Run only on pushes to main (manual dispatch) and via the cron trigger.
- # The cron event is automatically scoped to main by Woodpecker's scheduler.
+ # Only fire when explicitly triggered via a Woodpecker cron event.
+ # No cron is configured for this project, so this pipeline never runs
+ # automatically. Re-trigger it manually via the Woodpecker UI when needed.
when:
- - event: [push, cron]
+ - event: cron
branch: main
# ── Cost logging step ──────────────────────────────────────────────────────────
@@ -70,7 +75,6 @@ steps:
# (e.g., $1.00/run — see docs/ci-design.md § 8).
- 'echo "Cost: $0.00 (placeholder -- no LLM calls yet)"'
- # Run whenever llm-e2e ran — same branch/event filter applies.
when:
- - event: [push, cron]
+ - event: cron
branch: main
diff --git a/magefile.go b/magefile.go
new file mode 100644
index 0000000..e3ce281
--- /dev/null
+++ b/magefile.go
@@ -0,0 +1,49 @@
+//go:build mage
+
+// Package main provides Mage targets for the Player project.
+// Run `mage -l` to list available targets.
+// Prerequisites: Node.js 18+, npm, and a running Player server for E2E tests.
+package main
+
+import (
+ "os"
+ "os/exec"
+)
+
+// runnerDir is the path to the LLM e2e runner, relative to the project root.
+const runnerDir = "player-server/test/e2e-llm/runner"
+
+// E2E builds the LLM e2e runner and runs all scenarios against a live server.
+//
+// The Player server must already be running before invoking this target.
+// Start it from player-server/ with:
+//
+// MEDIA_ROOT=./testmedia SECURE_COOKIES=false DB_PATH=/tmp/player-e2e-llm.db ./player
+//
+// Override the target URL with PLAYER_URL (default: http://localhost:8080).
+// Set ANTHROPIC_API_KEY to enable the Haiku screenshot oracle (Layer 5).
+func E2E() error {
+ if err := npmRun("build"); err != nil {
+ return err
+ }
+ return nodeRun("dist/index.js")
+}
+
+// npmRun runs an npm script in runnerDir, inheriting stdout/stderr and env.
+func npmRun(script string) error {
+ cmd := exec.Command("npm", "run", script)
+ cmd.Dir = runnerDir
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ return cmd.Run()
+}
+
+// nodeRun executes a Node.js script in runnerDir, inheriting stdout/stderr and env.
+func nodeRun(script string) error {
+ cmd := exec.Command("node", script)
+ cmd.Dir = runnerDir
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ cmd.Env = os.Environ()
+ return cmd.Run()
+}