Parallelize template rendering jobs

7 files changed, 268 insertions, 58 deletions

diff --git a/README.md b/README.md
index 74b17a9..31482e6 100644
--- a/README.md
+++ b/README.md
@@ -180,9 +180,9 @@ effective paths, skipped existing files, staging output directory, and tarball
 decisions. If `--quiet` and `--verbose` are repeated or combined, the last output
 flag wins.
 
-ImageMagick photo processing runs in parallel. The default is `IMAGE_JOBS=3`.
-Set `IMAGE_JOBS` in the config, or pass `--image-jobs N`, to tune the number of
-concurrent ImageMagick jobs.
+ImageMagick photo processing and per-photo HTML template rendering run in
+parallel. The default is `IMAGE_JOBS=3`. Set `IMAGE_JOBS` in the config, or pass
+`--image-jobs N`, to tune the number of concurrent image and template jobs.
 
 Each ImageMagick command is bounded by `IMAGEMAGICK_TIMEOUT=60` seconds, and
 tarball creation is bounded by `TAR_TIMEOUT=120` seconds. Set either config
diff --git a/bin/photoalbum b/bin/photoalbum
index 6b686a1..323477d 100755
--- a/bin/photoalbum
+++ b/bin/photoalbum
@@ -1045,7 +1045,7 @@ scalephotos() {
     fi
 }
 
-wait_for_image_job_pid() {
+wait_for_parallel_job_pid() {
     local -r pid="$1"; shift
     local -i status=0
 
@@ -1057,7 +1057,7 @@ wait_for_image_job_pid() {
     return "$status"
 }
 
-image_job_pid_is_running() {
+parallel_job_pid_is_running() {
     local -r pid="$1"; shift
     local running_pid
 
@@ -1070,10 +1070,10 @@ image_job_pid_is_running() {
     return 1
 }
 
-reap_finished_image_jobs() {
-    local -r image_job_pids_name="$1"; shift
+reap_finished_parallel_jobs() {
+    local -r job_pids_name="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
+    local -n job_pids_ref="$job_pids_name"
     local -n failed_ref="$1"; shift
     local -i reaped_status="$1"; shift
     local -i has_reaped_status=1
@@ -1084,13 +1084,13 @@ reap_finished_image_jobs() {
 
     mapfile -t running_pids < <(jobs -rp)
 
-    for pid in "${image_job_pids_ref[@]}"; do
-        if image_job_pid_is_running "$pid" "${running_pids[@]}"; then
+    for pid in "${job_pids_ref[@]}"; do
+        if parallel_job_pid_is_running "$pid" "${running_pids[@]}"; then
             remaining_pids+=("$pid")
             continue
         fi
 
-        if wait_for_image_job_pid "$pid"; then
+        if wait_for_parallel_job_pid "$pid"; then
             status=0
         else
             status=$?
@@ -1105,50 +1105,84 @@ reap_finished_image_jobs() {
         fi
     done
 
-    image_job_pids_ref=("${remaining_pids[@]}")
+    job_pids_ref=("${remaining_pids[@]}")
 }
 
-wait_for_next_image_job() {
-    local -r image_job_pids_name="$1"; shift
+wait_for_next_parallel_job() {
+    local -r job_pids_name="$1"; shift
     local -r failed_name="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
+    local -n job_pids_ref="$job_pids_name"
     local -i status=0
 
     set +e
-    wait -n "${image_job_pids_ref[@]}"
+    wait -n "${job_pids_ref[@]}"
     status=$?
     set -e
 
-    reap_finished_image_jobs "$image_job_pids_name" "$failed_name" "$status"
+    reap_finished_parallel_jobs "$job_pids_name" "$failed_name" "$status"
 }
 
-wait_for_image_job_slot() {
-    local -r image_job_pids_name="$1"; shift
+wait_for_parallel_job_slot() {
+    local -r job_pids_name="$1"; shift
     local -r failed_name="$1"; shift
+    local -r max_jobs="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
-    local -r max_jobs="${IMAGE_JOBS:-3}"
+    local -n job_pids_ref="$job_pids_name"
 
-    while (( ${#image_job_pids_ref[@]} >= max_jobs )); do
-        wait_for_next_image_job "$image_job_pids_name" "$failed_name"
+    while (( ${#job_pids_ref[@]} >= max_jobs )); do
+        wait_for_next_parallel_job "$job_pids_name" "$failed_name"
     done
 }
 
-wait_for_image_jobs() {
+wait_for_parallel_jobs() {
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$1"; shift
+    local -n job_pids_ref="$1"; shift
     local -n failed_ref="$1"; shift
 
     : "$failed_ref"
-    while (( ${#image_job_pids_ref[@]} > 0 )); do
-        if ! wait_for_image_job_pid "${image_job_pids_ref[0]}"; then
+    while (( ${#job_pids_ref[@]} > 0 )); do
+        if ! wait_for_parallel_job_pid "${job_pids_ref[0]}"; then
             failed_ref=1
         fi
-        image_job_pids_ref=("${image_job_pids_ref[@]:1}")
+        job_pids_ref=("${job_pids_ref[@]:1}")
     done
 }
 
+wait_for_image_job_slot() {
+    local -r image_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_job_slot \
+        "$image_job_pids_name" \
+        "$failed_name" \
+        "${IMAGE_JOBS:-3}"
+}
+
+wait_for_image_jobs() {
+    local -r image_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_jobs "$image_job_pids_name" "$failed_name"
+}
+
+wait_for_template_render_job_slot() {
+    local -r render_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_job_slot \
+        "$render_job_pids_name" \
+        "$failed_name" \
+        "${IMAGE_JOBS:-3}"
+}
+
+wait_for_template_render_jobs() {
+    local -r render_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_jobs "$render_job_pids_name" "$failed_name"
+}
+
 scale_photo() {
     local -r photo="$1"; shift
     local destphoto
@@ -1770,7 +1804,7 @@ queue_album_view_render_job() {
     # shellcheck disable=SC2178
     local -n render_failed_ref="$1"; shift
 
-    wait_for_image_job_slot render_job_pids_ref render_failed_ref
+    wait_for_template_render_job_slot render_job_pids_ref render_failed_ref
     render_photo_view_and_details \
         "$photos_dir" \
         "$blurs_dir" \
@@ -1789,7 +1823,7 @@ wait_for_album_view_render_jobs() {
     # shellcheck disable=SC2178
     local -n render_failed_ref="$1"; shift
 
-    wait_for_image_jobs render_job_pids_ref render_failed_ref
+    wait_for_template_render_jobs render_job_pids_ref render_failed_ref
     if (( render_failed_ref != 0 )); then
         return 1
     fi
diff --git a/src/lib/album.source.sh b/src/lib/album.source.sh
index 886bd81..0efff6e 100644
--- a/src/lib/album.source.sh
+++ b/src/lib/album.source.sh
@@ -512,7 +512,7 @@ queue_album_view_render_job() {
     # shellcheck disable=SC2178
     local -n render_failed_ref="$1"; shift
 
-    wait_for_image_job_slot render_job_pids_ref render_failed_ref
+    wait_for_template_render_job_slot render_job_pids_ref render_failed_ref
     render_photo_view_and_details \
         "$photos_dir" \
         "$blurs_dir" \
@@ -531,7 +531,7 @@ wait_for_album_view_render_jobs() {
     # shellcheck disable=SC2178
     local -n render_failed_ref="$1"; shift
 
-    wait_for_image_jobs render_job_pids_ref render_failed_ref
+    wait_for_template_render_jobs render_job_pids_ref render_failed_ref
     if (( render_failed_ref != 0 )); then
         return 1
     fi
diff --git a/src/lib/image.source.sh b/src/lib/image.source.sh
index ccdca34..fef9345 100644
--- a/src/lib/image.source.sh
+++ b/src/lib/image.source.sh
@@ -80,7 +80,7 @@ scalephotos() {
     fi
 }
 
-wait_for_image_job_pid() {
+wait_for_parallel_job_pid() {
     local -r pid="$1"; shift
     local -i status=0
 
@@ -92,7 +92,7 @@ wait_for_image_job_pid() {
     return "$status"
 }
 
-image_job_pid_is_running() {
+parallel_job_pid_is_running() {
     local -r pid="$1"; shift
     local running_pid
 
@@ -105,10 +105,10 @@ image_job_pid_is_running() {
     return 1
 }
 
-reap_finished_image_jobs() {
-    local -r image_job_pids_name="$1"; shift
+reap_finished_parallel_jobs() {
+    local -r job_pids_name="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
+    local -n job_pids_ref="$job_pids_name"
     local -n failed_ref="$1"; shift
     local -i reaped_status="$1"; shift
     local -i has_reaped_status=1
@@ -119,13 +119,13 @@ reap_finished_image_jobs() {
 
     mapfile -t running_pids < <(jobs -rp)
 
-    for pid in "${image_job_pids_ref[@]}"; do
-        if image_job_pid_is_running "$pid" "${running_pids[@]}"; then
+    for pid in "${job_pids_ref[@]}"; do
+        if parallel_job_pid_is_running "$pid" "${running_pids[@]}"; then
             remaining_pids+=("$pid")
             continue
         fi
 
-        if wait_for_image_job_pid "$pid"; then
+        if wait_for_parallel_job_pid "$pid"; then
             status=0
         else
             status=$?
@@ -140,50 +140,84 @@ reap_finished_image_jobs() {
         fi
     done
 
-    image_job_pids_ref=("${remaining_pids[@]}")
+    job_pids_ref=("${remaining_pids[@]}")
 }
 
-wait_for_next_image_job() {
-    local -r image_job_pids_name="$1"; shift
+wait_for_next_parallel_job() {
+    local -r job_pids_name="$1"; shift
     local -r failed_name="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
+    local -n job_pids_ref="$job_pids_name"
     local -i status=0
 
     set +e
-    wait -n "${image_job_pids_ref[@]}"
+    wait -n "${job_pids_ref[@]}"
     status=$?
     set -e
 
-    reap_finished_image_jobs "$image_job_pids_name" "$failed_name" "$status"
+    reap_finished_parallel_jobs "$job_pids_name" "$failed_name" "$status"
 }
 
-wait_for_image_job_slot() {
-    local -r image_job_pids_name="$1"; shift
+wait_for_parallel_job_slot() {
+    local -r job_pids_name="$1"; shift
     local -r failed_name="$1"; shift
+    local -r max_jobs="$1"; shift
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$image_job_pids_name"
-    local -r max_jobs="${IMAGE_JOBS:-3}"
+    local -n job_pids_ref="$job_pids_name"
 
-    while (( ${#image_job_pids_ref[@]} >= max_jobs )); do
-        wait_for_next_image_job "$image_job_pids_name" "$failed_name"
+    while (( ${#job_pids_ref[@]} >= max_jobs )); do
+        wait_for_next_parallel_job "$job_pids_name" "$failed_name"
     done
 }
 
-wait_for_image_jobs() {
+wait_for_parallel_jobs() {
     # shellcheck disable=SC2178
-    local -n image_job_pids_ref="$1"; shift
+    local -n job_pids_ref="$1"; shift
     local -n failed_ref="$1"; shift
 
     : "$failed_ref"
-    while (( ${#image_job_pids_ref[@]} > 0 )); do
-        if ! wait_for_image_job_pid "${image_job_pids_ref[0]}"; then
+    while (( ${#job_pids_ref[@]} > 0 )); do
+        if ! wait_for_parallel_job_pid "${job_pids_ref[0]}"; then
             failed_ref=1
         fi
-        image_job_pids_ref=("${image_job_pids_ref[@]:1}")
+        job_pids_ref=("${job_pids_ref[@]:1}")
     done
 }
 
+wait_for_image_job_slot() {
+    local -r image_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_job_slot \
+        "$image_job_pids_name" \
+        "$failed_name" \
+        "${IMAGE_JOBS:-3}"
+}
+
+wait_for_image_jobs() {
+    local -r image_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_jobs "$image_job_pids_name" "$failed_name"
+}
+
+wait_for_template_render_job_slot() {
+    local -r render_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_job_slot \
+        "$render_job_pids_name" \
+        "$failed_name" \
+        "${IMAGE_JOBS:-3}"
+}
+
+wait_for_template_render_jobs() {
+    local -r render_job_pids_name="$1"; shift
+    local -r failed_name="$1"; shift
+
+    wait_for_parallel_jobs "$render_job_pids_name" "$failed_name"
+}
+
 scale_photo() {
     local -r photo="$1"; shift
     local destphoto
diff --git a/src/photoalbum.default.conf b/src/photoalbum.default.conf
index 6fad282..b6f23a2 100644
--- a/src/photoalbum.default.conf
+++ b/src/photoalbum.default.conf
@@ -7,7 +7,7 @@ THUMBHEIGHT=300
 HEIGHT=1200
 # Max previews per page.
 MAXPREVIEWS=40
-# Parallel ImageMagick jobs for photo, thumbnail, and blur processing.
+# Parallel jobs for image processing and HTML template rendering.
 IMAGE_JOBS=3
 # Timeout in seconds for each ImageMagick command.
 IMAGEMAGICK_TIMEOUT=60
diff --git a/tests/cli.sh b/tests/cli.sh
index 77d7cce..3a38526 100755
--- a/tests/cli.sh
+++ b/tests/cli.sh
@@ -1072,6 +1072,68 @@ test_generate_image_jobs_limits_parallel_identify() {
     test::teardown
 }
 
+test_generate_image_jobs_limits_parallel_template_rendering() {
+    local active_file
+    local config_file
+    local fake_bin
+    local lock_file
+    local log_file
+    local max_file
+    local max_seen
+    local render_count
+    local template_dir
+
+    test::setup
+    fake_bin="$TEST_TMPDIR/bin"
+    config_file="$TEST_TMPDIR/photoalbum.conf"
+    template_dir="$TEST_TMPDIR/templates"
+    lock_file="$TEST_TMPDIR/template.lock"
+    active_file="$TEST_TMPDIR/template.active"
+    max_file="$TEST_TMPDIR/template.max"
+    log_file="$TEST_TMPDIR/template.log"
+
+    test::install_fake_imagemagick "$fake_bin"
+    test::install_parallel_template_spy \
+        "$template_dir" \
+        "$lock_file" \
+        "$active_file" \
+        "$max_file" \
+        "$log_file"
+    mkdir -p "$TEST_TMPDIR/incoming"
+    printf 'fake image\n' > "$TEST_TMPDIR/incoming/01.jpg"
+    printf 'fake image\n' > "$TEST_TMPDIR/incoming/02.jpg"
+    printf 'fake image\n' > "$TEST_TMPDIR/incoming/03.jpg"
+    printf 'fake image\n' > "$TEST_TMPDIR/incoming/04.jpg"
+    test::write_album_config \
+        "$config_file" "$TEST_TMPDIR/incoming" "$TEST_TMPDIR/dist" \
+        'Parallel template album' 40
+
+    (
+        cd "$TEST_TMPDIR"
+        PATH="$fake_bin:$PATH" "$TEST_PHOTOALBUM" \
+            --image-jobs 2 \
+            --template "$template_dir" \
+            --generate
+    )
+
+    max_seen=$(<"$max_file")
+    if (( max_seen != 2 )); then
+        echo 'FAIL: expected max parallel template render jobs to be 2' >&2
+        echo "max_seen=$max_seen" >&2
+        cat "$log_file" >&2
+        exit 1
+    fi
+
+    render_count=$(grep -c '^start ' "$log_file")
+    if (( render_count != 8 )); then
+        echo 'FAIL: expected view and details templates for each image' >&2
+        echo "render_count=$render_count" >&2
+        cat "$log_file" >&2
+        exit 1
+    fi
+    test::teardown
+}
+
 test_repeated_output_flags_use_last_value() {
     local config_file
     local fake_bin
@@ -3561,6 +3623,9 @@ main() {
         '--generate --image-jobs limits ImageMagick identify parallelism' \
         test_generate_image_jobs_limits_parallel_identify
     test::run_case \
+        '--generate --image-jobs limits template rendering parallelism' \
+        test_generate_image_jobs_limits_parallel_template_rendering
+    test::run_case \
         'repeated output flags use last value' \
         test_repeated_output_flags_use_last_value
     test::run_case \
diff --git a/tests/helpers.sh b/tests/helpers.sh
index bb52b67..7c0a104 100755
--- a/tests/helpers.sh
+++ b/tests/helpers.sh
@@ -385,6 +385,83 @@ MAGICK
     cp "$bin_dir/magick" "$bin_dir/convert"
 }
 
+test::write_parallel_template_spy_file() {
+    local -r template_path="$1"; shift
+    local -r template_label="$1"; shift
+    local -r lock_file="$1"; shift
+    local -r active_file="$1"; shift
+    local -r max_file="$1"; shift
+    local -r log_file="$1"; shift
+
+    {
+        printf 'template_label=%q\n' "$template_label"
+        printf 'lock_file=%q\n' "$lock_file"
+        printf 'active_file=%q\n' "$active_file"
+        printf 'max_file=%q\n' "$max_file"
+        printf 'log_file=%q\n' "$log_file"
+        cat <<'TEMPLATE'
+
+(
+    flock 9
+    active=0
+    if [ -f "$active_file" ]; then
+        active=$(<"$active_file")
+    fi
+    active=$(( active + 1 ))
+    printf '%s\n' "$active" > "$active_file"
+
+    max=0
+    if [ -f "$max_file" ]; then
+        max=$(<"$max_file")
+    fi
+    if (( active > max )); then
+        printf '%s\n' "$active" > "$max_file"
+    fi
+
+    printf 'start %s %s %s\n' \
+        "$active" "$template_label" "$render_photo_html" >> "$log_file"
+) 9>"$lock_file"
+
+sleep 0.1
+printf '<div class="%s">%s</div>\n' "$template_label" "$render_photo_html"
+
+(
+    flock 9
+    active=$(<"$active_file")
+    active=$(( active - 1 ))
+    printf '%s\n' "$active" > "$active_file"
+    printf 'finish %s %s %s\n' \
+        "$active" "$template_label" "$render_photo_html" >> "$log_file"
+) 9>"$lock_file"
+TEMPLATE
+    } > "$template_path"
+}
+
+test::install_parallel_template_spy() {
+    local -r template_dir="$1"; shift
+    local -r lock_file="$1"; shift
+    local -r active_file="$1"; shift
+    local -r max_file="$1"; shift
+    local -r log_file="$1"; shift
+
+    mkdir -p "$template_dir"
+    cp "$TEST_REPO_ROOT"/share/templates/default/*.tmpl "$template_dir/"
+    test::write_parallel_template_spy_file \
+        "$template_dir/view.tmpl" \
+        view \
+        "$lock_file" \
+        "$active_file" \
+        "$max_file" \
+        "$log_file"
+    test::write_parallel_template_spy_file \
+        "$template_dir/details.tmpl" \
+        details \
+        "$lock_file" \
+        "$active_file" \
+        "$max_file" \
+        "$log_file"
+}
+
 test::install_failing_imagemagick() {
     local -r bin_dir="$1"; shift