diff options
5 files changed, 419 insertions, 0 deletions
diff --git a/player-server/test/e2e-llm/fixtures/mock-rss-server.js b/player-server/test/e2e-llm/fixtures/mock-rss-server.js new file mode 100644 index 0000000..8d9446c --- /dev/null +++ b/player-server/test/e2e-llm/fixtures/mock-rss-server.js @@ -0,0 +1,149 @@ +#!/usr/bin/env node +/** + * mock-rss-server.js — minimal Node.js HTTP server that serves a valid podcast + * RSS 2.0 feed at http://localhost:8888/feed.xml. + * + * Used by scenario S02 to provide a controllable RSS endpoint so the e2e tests + * do not depend on external network access or a live podcast feed. + * + * Usage: + * node mock-rss-server.js # listens on port 8888 (default) + * PORT=9999 node mock-rss-server.js + * + * The server serves: + * GET /feed.xml — podcast RSS feed with two episodes + * GET /episode1.mp3 — tiny fake audio blob (not real audio, just non-empty) + * GET /episode2.mp3 — tiny fake audio blob (not real audio, just non-empty) + * + * All other paths return 404. + */ + +"use strict"; + +const http = require("http"); + +const PORT = parseInt(process.env.PORT || "8888", 10); +const BASE_URL = `http://localhost:${PORT}`; + +// A minimal valid RSS 2.0 podcast feed with two episodes. +// Duration values use HH:MM:SS format as accepted by most podcast parsers. +function buildFeed() { + return `<?xml version="1.0" encoding="UTF-8"?> +<rss version="2.0" + xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" + xmlns:content="http://purl.org/rss/modules/content/"> + <channel> + <title>Test Podcast — E2E Fixture</title> + <link>${BASE_URL}</link> + <description>A mock podcast feed used by the Player e2e-llm test suite.</description> + <language>en-us</language> + <itunes:author>Player E2E Test Harness</itunes:author> + <itunes:image href="${BASE_URL}/cover.jpg"/> + <itunes:explicit>false</itunes:explicit> + + <item> + <title>Episode 1 — Introduction</title> + <link>${BASE_URL}/episode1.mp3</link> + <guid isPermaLink="false">e2e-fixture-episode-1</guid> + <description>First episode of the mock podcast.</description> + <pubDate>Mon, 01 Jan 2024 10:00:00 +0000</pubDate> + <enclosure url="${BASE_URL}/episode1.mp3" length="1024" type="audio/mpeg"/> + <itunes:duration>00:01:30</itunes:duration> + <itunes:explicit>false</itunes:explicit> + </item> + + <item> + <title>Episode 2 — The Follow-Up</title> + <link>${BASE_URL}/episode2.mp3</link> + <guid isPermaLink="false">e2e-fixture-episode-2</guid> + <description>Second episode of the mock podcast.</description> + <pubDate>Tue, 02 Jan 2024 10:00:00 +0000</pubDate> + <enclosure url="${BASE_URL}/episode2.mp3" length="2048" type="audio/mpeg"/> + <itunes:duration>00:03:00</itunes:duration> + <itunes:explicit>false</itunes:explicit> + </item> + </channel> +</rss>`; +} + +// A tiny non-empty blob returned for episode download requests. +// The Player server probes the file with ffprobe after download; since this is +// not real audio the probe will fail, but the download step itself will succeed +// (HTTP 200 with Content-Type audio/mpeg). The scenario only asserts the HTTP +// status of the download trigger, not the resulting ffprobe output. +const FAKE_AUDIO = Buffer.from( + "ID3\x03\x00\x00\x00\x00\x00\x00" + "fake-mp3-content-for-e2e-testing", + "ascii", +); + +// Route the incoming request to the appropriate response. +function handleRequest(req, res) { + const url = req.url.split("?")[0]; // strip query string + + if (url === "/feed.xml") { + // Serve the RSS feed with appropriate headers. + const body = buildFeed(); + res.writeHead(200, { + "Content-Type": "application/rss+xml; charset=utf-8", + "Content-Length": Buffer.byteLength(body, "utf8"), + "Cache-Control": "no-cache", + }); + res.end(body); + return; + } + + if (url === "/episode1.mp3" || url === "/episode2.mp3") { + // Serve a fake audio blob so the Player download endpoint can retrieve it. + res.writeHead(200, { + "Content-Type": "audio/mpeg", + "Content-Length": FAKE_AUDIO.length, + "Accept-Ranges": "bytes", + }); + res.end(FAKE_AUDIO); + return; + } + + if (url === "/cover.jpg") { + // Serve a 1×1 white JPEG as a minimal cover image. + const TINY_JPEG = Buffer.from( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8U" + + "HRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgN" + + "DRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" + + "MjL/wAARCAABAAEDASIAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAA" + + "AAAAAAAAAAAAAP/EABQBAQAAAAAAAAAAAAAAAAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA" + + "/9oADAMBAAIRAxEAPwCwABmX/9k=", + "base64", + ); + res.writeHead(200, { + "Content-Type": "image/jpeg", + "Content-Length": TINY_JPEG.length, + }); + res.end(TINY_JPEG); + return; + } + + // Any other path → 404. + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found\n"); +} + +// Start the server and print a ready message so the harness can wait for it. +const server = http.createServer(handleRequest); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`mock-rss-server: listening on ${BASE_URL}`); + console.log(`mock-rss-server: feed available at ${BASE_URL}/feed.xml`); +}); + +// Graceful shutdown on SIGINT / SIGTERM so the harness can stop the server +// cleanly after the scenario completes. +function shutdown(signal) { + console.log(`mock-rss-server: received ${signal}, shutting down`); + server.close(() => { + console.log("mock-rss-server: closed"); + process.exit(0); + }); +} + +process.on("SIGINT", () => shutdown("SIGINT")); +process.on("SIGTERM", () => shutdown("SIGTERM")); diff --git a/player-server/test/e2e-llm/scenarios/S01-bootstrap.md b/player-server/test/e2e-llm/scenarios/S01-bootstrap.md new file mode 100644 index 0000000..3d2380d --- /dev/null +++ b/player-server/test/e2e-llm/scenarios/S01-bootstrap.md @@ -0,0 +1,46 @@ +--- +id: S01 +title: "Bootstrap → first user" +tags: [auth, web, bootstrap] +preconditions: + server_state: fresh # harness must start the server against an empty temp DB + fixtures: [] +assertions: + - db: "SELECT id FROM users WHERE is_admin=1" # at least one admin row + - db: "SELECT id FROM users" # at least one user row + - url_contains: /login.html +skip: false +--- + +1. Confirm the server is reachable: `GET /healthz` must return HTTP 200. + +2. Navigate to the root URL (`/`). Confirm the browser is redirected to + `/bootstrap.html` (the BootstrapRedirect middleware redirects unauthenticated + requests when the database is empty). + +3. Confirm the bootstrap form is visible: look for a `<form>` that contains + inputs for "username" and "password". + +4. Fill in the username field with `admin` and the password field with + `TestPassw0rd!`. Fill in the confirm-password field (if present) with the + same value. + +5. Submit the form by clicking the submit button (or pressing Enter). The + browser should be redirected to `/login.html` after a successful bootstrap. + +6. Confirm the browser URL now contains `/login.html`. + +7. Fill in the login form with username `admin` and password `TestPassw0rd!` + and submit it. + +8. Confirm the browser lands on the home page (`/` or `/index.html`) and that + an authenticated session cookie named `session` is present in the browser. + +9. Confirm the admin panel link is accessible: navigate to the web UI and + verify that an element indicating admin access (e.g. a settings or admin + menu item) is visible. + +10. Via the API, call `GET /api/v1/admin/users` with the session cookie. Confirm + the response is HTTP 200 and contains exactly one user object whose + `is_admin` field is `true` (SQLite stores this as 1) and whose `username` + is `admin`. diff --git a/player-server/test/e2e-llm/scenarios/S02-podcast.md b/player-server/test/e2e-llm/scenarios/S02-podcast.md new file mode 100644 index 0000000..29e9bc9 --- /dev/null +++ b/player-server/test/e2e-llm/scenarios/S02-podcast.md @@ -0,0 +1,72 @@ +--- +id: S02 +title: "Podcast subscribe → download → mark complete" +tags: [podcast, api, progress] +preconditions: + server_state: running # server running with an existing admin account + fixtures: + - mock-rss-server # start the mock RSS server before running steps +assertions: + - db: "SELECT id FROM podcast_feeds WHERE feed_url='http://localhost:8888/feed.xml'" + - db: "SELECT id FROM podcast_episodes WHERE feed_id=(SELECT id FROM podcast_feeds WHERE feed_url='http://localhost:8888/feed.xml')" + - db: "SELECT episode_id FROM podcast_status WHERE is_completed=1" +skip: false +--- + +# Setup note +Before running this scenario, start the mock RSS server fixture: + +```sh +node player-server/test/e2e-llm/fixtures/mock-rss-server.js & +# Server listens at http://localhost:8888/feed.xml +``` + +Stop it after the scenario completes with `kill %1` (or equivalent). + +--- + +1. Authenticate as an admin user: call `POST /api/v1/auth/login` with body + `{"username": "admin", "password": "TestPassw0rd!"}`. Save the `session` + cookie returned in the response for all subsequent requests. + +2. Subscribe to the mock podcast feed: call `POST /api/v1/podcasts` with body + `{"feed_url": "http://localhost:8888/feed.xml", "set_name": "Test Podcast"}`. + Confirm the response is HTTP 200 and the returned JSON object contains a + non-zero `id` field. Save `feed_id` from the response. + +3. Confirm `GET /api/v1/podcasts` returns HTTP 200 and the list contains an + entry whose `feed_url` is `http://localhost:8888/feed.xml`. + +4. Retrieve the list of episodes for the new feed: call + `GET /api/v1/podcasts/{feed_id}/episodes`. Confirm the response is HTTP 200 + and contains at least one episode object. Save the `id` of the first episode + as `episode_id`. + +5. Confirm the first episode has a non-empty `title` field and a non-null + `episode_url` field. + +6. Download the episode: call + `POST /api/v1/podcasts/episodes/{episode_id}/download`. + Confirm the response is HTTP 200 and the returned JSON is a media object with + a non-zero `id` field and a non-empty `file_name` field. Save `id` as + `media_id`. + +7. Confirm the episode now appears in the list returned by + `GET /api/v1/podcasts/{feed_id}/episodes` with `is_downloaded: true`. + +8. Record playback progress for the downloaded media: call + `POST /api/v1/progress` with body + `{"media_id": <media_id from step 6>, "position_seconds": 120.0}`. + Confirm the response is HTTP 200. + +9. Confirm the episode appears in the in-progress list: call + `GET /api/v1/in-progress` and verify the list contains a media item whose + `id` matches the media from step 6. + +10. Mark the episode as complete: call + `POST /api/v1/podcasts/episodes/{episode_id}/complete`. Confirm the response + is HTTP 204 (No Content). + +11. Retrieve the episode list again: + `GET /api/v1/podcasts/{feed_id}/episodes`. Confirm the target episode now + has `is_completed: true` in its status fields. diff --git a/player-server/test/e2e-llm/scenarios/S03-upload.md b/player-server/test/e2e-llm/scenarios/S03-upload.md new file mode 100644 index 0000000..3961e4a --- /dev/null +++ b/player-server/test/e2e-llm/scenarios/S03-upload.md @@ -0,0 +1,76 @@ +--- +id: S03 +title: "Upload via API → verify on web" +tags: [upload, api, web, visual] +preconditions: + server_state: running # server running with an existing admin account + fixtures: [] +assertions: + - db: "SELECT id FROM media WHERE file_name='test-audio-e2e.mp3'" + - selector_visible: ".media-card" + - status_code: "GET /api/v1/media 200" +skip: false +--- + +# Visual check note +Step 10 triggers a Haiku screenshot oracle when `LLM_E2E_SCREENSHOTS=true`. +Set that env var and provide `ANTHROPIC_API_KEY` to enable the visual assertion. +If the env var is not set, the step is skipped and the selector assertion in the +YAML front-matter is used instead. + +--- + +1. Authenticate as an admin user: call `POST /api/v1/auth/login` with body + `{"username": "admin", "password": "TestPassw0rd!"}`. Save the `session` + cookie for subsequent requests. + +2. Create an API token for the upload step (Bearer auth): call + `POST /api/v1/auth/tokens` with body + `{"name": "e2e-upload-test", "expires_in_days": 1}` and the session cookie. + Save the `token` value from the response as `BEARER_TOKEN`. + +3. List the available sets: call `GET /api/v1/sets`. Confirm the response is + HTTP 200. Save the `id` of the first set (e.g. the set named `musicvideos` + or any non-podcast set) as `set_id`. + +4. Prepare a small test audio file to upload. Use the fixture file at + `player-server/testmedia/podcast/` or create a minimal 1-second silent MP3 + named `test-audio-e2e.mp3` using `ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 1 -q:a 9 -acodec libmp3lame /tmp/test-audio-e2e.mp3` + (requires ffmpeg on PATH). Confirm the file exists at `/tmp/test-audio-e2e.mp3`. + +5. Upload the file to the set: call + `POST /api/v1/sets/{set_id}/upload` as a `multipart/form-data` request with: + - `Authorization: Bearer {BEARER_TOKEN}` header + - form field `file` containing the contents of `/tmp/test-audio-e2e.mp3` + with filename `test-audio-e2e.mp3`. + Confirm the response is HTTP 200 and the returned JSON contains a non-zero + `id` field. Save `media_id` from the response. + +6. Confirm the media is in the database: call + `GET /api/v1/media/{media_id}` with the session cookie. Confirm the response + is HTTP 200, the `file_name` field is `test-audio-e2e.mp3`, and the + `type` field is `audio`. + +7. Trigger a media rescan so the server reindexes the upload: + call `POST /api/v1/admin/rescan` with the session cookie. Confirm the + response is HTTP 200. + +8. Wait for the scan to complete: poll `GET /api/v1/admin/scan-progress` until + the response indicates the scan is done (e.g. `"scanning": false` or an + empty progress object). Poll up to 30 seconds with 2-second intervals. + +9. Open the web UI in a Playwright browser context: navigate to + `{PLAYER_URL}/index.html` and authenticate by injecting the session cookie. + Navigate to the set page that contains the uploaded file (use the set + browsing UI or navigate directly to the URL for the set). + +10. Confirm the uploaded file's media card is visible in the media grid. + Look for a `.media-card` element (or equivalent) whose title or filename + contains `test-audio-e2e`. + **Visual check (Layer 5):** Take a screenshot of the media grid and ask + Claude Haiku: "Is there a media card visible in the grid? Answer yes or no + and give a one-sentence reason." This step requires `LLM_E2E_SCREENSHOTS=true`. + +11. Clean up: delete the test media item by calling + `DELETE /api/v1/media/{media_id}` with the session cookie. Confirm the + response is HTTP 200. diff --git a/player-server/test/e2e-llm/scenarios/S04-share-link.md b/player-server/test/e2e-llm/scenarios/S04-share-link.md new file mode 100644 index 0000000..37ce8d5 --- /dev/null +++ b/player-server/test/e2e-llm/scenarios/S04-share-link.md @@ -0,0 +1,76 @@ +--- +id: S04 +title: "Share-link round-trip" +tags: [share, auth, web, visual] +preconditions: + server_state: running # server running with admin account and at least one media item + fixtures: [] +assertions: + - db: "SELECT token FROM shares WHERE token IS NOT NULL" + - url_contains: /login.html + - status_code: "GET /api/v1/media 200" +skip: false +--- + +# Visual check note +Steps 8 and 11 trigger the Haiku screenshot oracle when +`LLM_E2E_SCREENSHOTS=true`. Set that env var and provide `ANTHROPIC_API_KEY` +to enable visual assertions. Without it, the selector assertions in the YAML +front-matter are used instead. + +--- + +1. Authenticate as an admin user: call `POST /api/v1/auth/login` with body + `{"username": "admin", "password": "TestPassw0rd!"}`. Save the `session` + cookie for subsequent authenticated requests. + +2. Find a media item to share: call `GET /api/v1/media?limit=1` with the + session cookie. Confirm the response is HTTP 200 and contains at least one + media object. Save the `id` of the first item as `media_id`. + +3. Create a share link for that media item: call + `POST /api/v1/media/{media_id}/shares` with the session cookie. Confirm the + response is HTTP 200. Save the `token` field from the response as + `share_token`. The share URL is `{PLAYER_URL}/s/{share_token}`. + +4. Confirm the share is recorded: call + `GET /api/v1/media/{media_id}/shares` with the session cookie. Confirm the + response contains a share entry whose `token` matches `share_token`. + +5. Open a fresh, unauthenticated Playwright browser context (no cookies). In + this context, navigate to the share URL `{PLAYER_URL}/s/{share_token}`. + Confirm the response is HTTP 200 and the browser does NOT redirect to + `/login.html` (the share page is publicly accessible). + +6. Confirm the share page renders the media player: look for a `<audio>` or + `<video>` element, or a play button element (e.g. a button with aria-label + "Play" or a class like `.play-btn`). The element must be present and visible. + +7. Confirm the share page contains the correct media title by verifying that a + heading or text element on the page matches the `file_name` or `title` + of the shared media item (obtained in step 2). + +8. **Visual check (Layer 5):** Take a screenshot of the share page and ask + Claude Haiku: "Is there a media player visible on this page? Answer yes or + no and give a one-sentence reason." This step requires `LLM_E2E_SCREENSHOTS=true`. + +9. From the same unauthenticated browser context, navigate to the root URL + `{PLAYER_URL}/`. Confirm the browser is redirected to `/login.html`. + The share token must not grant general access to the application. + +10. Confirm the root URL returns HTTP 401 or redirects to `/login.html` for + unauthenticated requests: call `GET {PLAYER_URL}/api/v1/media` without any + session cookie. Confirm the response is HTTP 401. + +11. **Visual check (Layer 5):** Take a screenshot after the redirect to + `/login.html` and ask Claude Haiku: "Does this screenshot show a login + form? Answer yes or no and give a one-sentence reason." This step requires + `LLM_E2E_SCREENSHOTS=true`. + +12. Revoke the share link: from an authenticated context (using the admin + session cookie), call `DELETE /api/v1/shares/{share_token}`. Confirm the + response is HTTP 200. + +13. Confirm the share is no longer accessible: in the unauthenticated context, + navigate to `{PLAYER_URL}/s/{share_token}` again. Confirm the response is + HTTP 404 or HTTP 410 Gone (the token has been revoked). |