summaryrefslogtreecommitdiff
path: root/src/lib
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-06-24 11:00:33 +0300
committerPaul Buetow <paul@buetow.org>2026-06-24 11:00:33 +0300
commit928f28d0c51ca47f9958fda4c4bb7c8b0c571e34 (patch)
tree5d9d7a80aeb845598d1bdac928ed5a1ed2e9540f /src/lib
parent6697431d3855d3030b35c2ce232c9ea0e9282ba7 (diff)
Encapsulate job pool behind a single handle (job_pool_*)
Replace the four-parallel-nameref job-pool API (pids/statuses/labels/failed, each needing its own `shellcheck disable=SC2034` at every call site) with a single pool handle: a name prefix whose four backing variables (${pool}_pids/_statuses/_labels/_failed) are derived on demand by the helpers. Bash can't nest indexed arrays in an associative array, so a prefixed-handle with declare -g backing vars is the simplest pure-nameref encoding (no eval). New public API: job_pool_init <pool> job_pool_submit <pool> <label> <cmd...> job_pool_wait <pool> # returns 1 if any job failed Migrate all callers (scalephotos, create_all_photo_derivatives, render_album_pages, render_view_redirects, render_filter_pages) to the handle. queue_preview_page_render_job / queue_album_view_render_job / _album_record_view_photo / _stats_enqueue_filter_album now take one pool arg instead of four names. Drop the dead wrappers and unused parameterization: wait_for_image_job_slot, wait_for_template_render_job_slot and their _jobs variants only ever passed IMAGE_JOBS, so the max_jobs parameter is gone and throttling is fixed at IMAGE_JOBS inside the pool. Also removes wait_for_album_view_render_jobs (now just job_pool_wait). Throttling (max IMAGE_JOBS concurrent), failure detection and failed-job propagation are unchanged; the parallel-throttling and failure-logging tests pass unmodified. SC2034 disable-comments across the touched files drop 29->7. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Diffstat (limited to 'src/lib')
-rw-r--r--src/lib/album-render.source.sh145
-rw-r--r--src/lib/image-pipeline.source.sh35
-rw-r--r--src/lib/image.source.sh33
-rw-r--r--src/lib/job-pool.source.sh257
-rw-r--r--src/lib/stats-filter-album.source.sh47
5 files changed, 186 insertions, 331 deletions
diff --git a/src/lib/album-render.source.sh b/src/lib/album-render.source.sh
index 3024923..d438cef 100644
--- a/src/lib/album-render.source.sh
+++ b/src/lib/album-render.source.sh
@@ -132,13 +132,13 @@ render_full_preview_page() {
fi
}
-# Enqueue one complete preview page as a background render job, throttled to
-# IMAGE_JOBS via the shared template render job pool. Mirrors
-# queue_album_view_render_job: wait for a free slot, background the whole-page
-# assembly, then track its pid/label so a failed job flips render_failed and
-# makes generation fail loudly. The page's photos are passed as trailing
-# positional args; the background subshell forks a private copy of them, so the
-# caller is free to reuse its per-page accumulator for the next page.
+# Enqueue one complete preview page as a background render job in the given job
+# pool (throttled to IMAGE_JOBS). Mirrors queue_album_view_render_job: job_pool_submit
+# waits for a free slot, backgrounds the whole-page assembly, and tracks its
+# pid/label so a failed job is reported by job_pool_wait and makes generation
+# fail loudly. The page's photos are passed as trailing positional args; the
+# background subshell forks a private copy of them, so the caller is free to
+# reuse its per-page accumulator for the next page.
queue_preview_page_render_job() {
local -r photos_dir="$1"; shift
local -r html_dir="$1"; shift
@@ -151,24 +151,11 @@ queue_preview_page_render_job() {
local -r header_bar="$1"; shift
local -r prev_page="$1"; shift
local -r next_page="$1"; shift
- # shellcheck disable=SC2178
- local -n render_job_pids_ref="$1"; shift
- # Passed by name to wait_for_template_render_job_slot.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_statuses_ref="$1"; shift
- # Passed by name to wait_for_template_render_job_slot and assigned below.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_labels_ref="$1"; shift
- # shellcheck disable=SC2178
- local -n render_failed_ref="$1"; shift
+ local -r pool="$1"; shift
# Remaining positional args ("$@") are this page's photos in order.
- wait_for_template_render_job_slot \
- render_job_pids_ref \
- render_job_statuses_ref \
- render_job_labels_ref \
- render_failed_ref
- render_full_preview_page \
+ job_pool_submit "$pool" "template render job for preview $page_name" \
+ render_full_preview_page \
"$photos_dir" \
"$html_dir" \
"$thumbs_dir" \
@@ -180,9 +167,7 @@ queue_preview_page_render_job() {
"$header_bar" \
"$prev_page" \
"$next_page" \
- "$@" &
- render_job_pids_ref+=("$!")
- render_job_labels_ref["$!"]="template render job for preview $page_name"
+ "$@"
}
# Build a whole thumbnail-grid buffer by walking a list of photos and grouping
@@ -592,37 +577,26 @@ render_view_redirects() {
local -n redirect_last_views_ref="$1"; shift
local max_page
local page
- # Render job pool, throttled to IMAGE_JOBS by the job-pool helpers.
- # shellcheck disable=SC2034
- local -a render_job_pids=()
- # shellcheck disable=SC2034
- local -A render_job_statuses=()
- # shellcheck disable=SC2034
- local -A render_job_labels=()
- local -i render_failed=0
if (( ${#view_pages_ref[@]} == 0 )); then
return
fi
+ # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle
+ # "render_jobs". job_pool_wait returns 1 if any render job failed.
+ job_pool_init render_jobs
+
max_page=${view_pages_ref[$(( ${#view_pages_ref[@]} - 1 ))]}
for page in "${view_pages_ref[@]}"; do
- wait_for_template_render_job_slot \
- render_job_pids render_job_statuses render_job_labels render_failed
- render_page_view_redirects \
+ job_pool_submit render_jobs \
+ "template render job for redirect page $page" \
+ render_page_view_redirects \
"$html_dir" "$page" "${redirect_last_views_ref[$page]}" \
- "$max_page" &
- render_job_pids+=("$!")
- # shellcheck disable=SC2034
- render_job_labels["$!"]="template render job for redirect page $page"
+ "$max_page"
done
- wait_for_template_render_jobs \
- render_job_pids render_job_statuses render_job_labels render_failed
- if (( render_failed != 0 )); then
- return 1
- fi
+ job_pool_wait render_jobs
}
render_album_index_redirect() {
@@ -686,23 +660,10 @@ queue_album_view_render_job() {
local -r page_num="$1"; shift
local -r preview_num="$1"; shift
local -r photo="$1"; shift
- # shellcheck disable=SC2178
- local -n render_job_pids_ref="$1"; shift
- # Passed by name to wait_for_template_render_job_slot.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_statuses_ref="$1"; shift
- # Passed by name to wait_for_template_render_job_slot and assigned below.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_labels_ref="$1"; shift
- # shellcheck disable=SC2178
- local -n render_failed_ref="$1"; shift
-
- wait_for_template_render_job_slot \
- render_job_pids_ref \
- render_job_statuses_ref \
- render_job_labels_ref \
- render_failed_ref
- render_photo_view_and_details \
+ local -r pool="$1"; shift
+
+ job_pool_submit "$pool" "template render job for photo $photo" \
+ render_photo_view_and_details \
"$photos_dir" \
"$blurs_dir" \
"$html_dir" \
@@ -710,31 +671,7 @@ queue_album_view_render_job() {
"$tarball_name" \
"$page_num" \
"$preview_num" \
- "$photo" &
- render_job_pids_ref+=("$!")
- render_job_labels_ref["$!"]="template render job for photo $photo"
-}
-
-wait_for_album_view_render_jobs() {
- # shellcheck disable=SC2178
- local -n render_job_pids_ref="$1"; shift
- # Passed by name to wait_for_template_render_jobs.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_statuses_ref="$1"; shift
- # Passed by name to wait_for_template_render_jobs.
- # shellcheck disable=SC2034,SC2178
- local -n render_job_labels_ref="$1"; shift
- # shellcheck disable=SC2178
- local -n render_failed_ref="$1"; shift
-
- wait_for_template_render_jobs \
- render_job_pids_ref \
- render_job_statuses_ref \
- render_job_labels_ref \
- render_failed_ref
- if (( render_failed_ref != 0 )); then
- return 1
- fi
+ "$photo"
}
# Group the album's photos into pages of at most MAXPREVIEWS, in their final
@@ -783,17 +720,14 @@ _album_record_view_photo() {
local -ri page_num="$1"; shift
local -ri preview_num="$1"; shift
local -r photo="$1"; shift
- local -r pids_name="$1"; shift
- local -r statuses_name="$1"; shift
- local -r labels_name="$1"; shift
- local -r failed_name="$1"; shift
+ local -r pool="$1"; shift
local -r view_pages_name="$1"; shift
local -r last_views_name="$1"; shift
queue_album_view_render_job \
"$photos_dir" "$blurs_dir" "$html_dir" "$backhref" "$tarball_name" \
"$page_num" "$preview_num" "$photo" \
- "$pids_name" "$statuses_name" "$labels_name" "$failed_name"
+ "$pool"
record_rendered_view_page "$view_pages_name" "$last_views_name" \
"$page_num" "$preview_num"
# Read later through the album_view_page_for_photo accessor (e.g. by the
@@ -819,15 +753,6 @@ render_album_pages() {
local prev_name=''
local record
local -i preview_num
- # Passed by name to the render-pool helpers below.
- # shellcheck disable=SC2034
- local -i render_failed=0
- # shellcheck disable=SC2034
- local -a render_job_pids=()
- # shellcheck disable=SC2034
- local -A render_job_labels=()
- # shellcheck disable=SC2034
- local -A render_job_statuses=()
# Passed by name to record_rendered_view_page and render_view_redirects.
# shellcheck disable=SC2034
local -A rendered_last_views=()
@@ -836,6 +761,10 @@ render_album_pages() {
local -a page_photos=()
local -a page_records=()
+ # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle
+ # "render_jobs". job_pool_wait returns 1 if any render job failed.
+ job_pool_init render_jobs
+
# Rebuild the photo -> view-page map for this album from scratch so a
# re-generate (or a smaller incoming set) does not keep stale entries. The
# per-photo entries are written in _album_record_view_photo, which shellcheck
@@ -867,8 +796,7 @@ render_album_pages() {
_album_record_view_photo \
"$photos_dir" "$blurs_dir" "$html_dir" "$backhref" \
"$tarball_name" "$page_num" "$preview_num" "$photo" \
- render_job_pids render_job_statuses render_job_labels \
- render_failed rendered_view_pages rendered_last_views
+ render_jobs rendered_view_pages rendered_last_views
done
# A page has a "next" link unless it is the last record.
@@ -880,18 +808,13 @@ render_album_pages() {
"$photos_dir" "$html_dir" "$thumbs_dir" "$blurs_dir" "$backhref" \
"$tarball_name" "$name" "$page_num" "$header_bar" \
"$prev_name" "$next_name" \
- render_job_pids render_job_statuses render_job_labels \
- render_failed \
+ render_jobs \
"${page_photos[@]}"
prev_name="$name"
done
- if ! wait_for_album_view_render_jobs \
- render_job_pids \
- render_job_statuses \
- render_job_labels \
- render_failed; then
+ if ! job_pool_wait render_jobs; then
return 1
fi
render_view_redirects "$html_dir" rendered_view_pages rendered_last_views
diff --git a/src/lib/image-pipeline.source.sh b/src/lib/image-pipeline.source.sh
index 876501b..934c3b5 100644
--- a/src/lib/image-pipeline.source.sh
+++ b/src/lib/image-pipeline.source.sh
@@ -38,41 +38,22 @@ create_all_photo_derivatives() {
local -r photos_dir="$1"; shift
local -r thumbs_dir="$1"; shift
local -r blurs_dir="$1"; shift
- local -i failed=0
- local -a image_job_pids=()
- # Passed by name to wait_for_image_job_slot and wait_for_image_jobs.
- # shellcheck disable=SC2034
- local -A image_job_labels=()
- # Passed by name to wait_for_image_job_slot and wait_for_image_jobs.
- # shellcheck disable=SC2034
- local -A image_job_statuses=()
local photo
+ # Throttled background job pool (max IMAGE_JOBS concurrent), addressed by the
+ # single handle "image_jobs". job_pool_wait returns 1 if any job failed.
+ job_pool_init image_jobs
+
while IFS= read -r photo; do
- wait_for_image_job_slot \
- image_job_pids \
- image_job_statuses \
- image_job_labels \
- failed
- create_photo_derivatives "$photos_dir" "$thumbs_dir" "$blurs_dir" \
- "$photo" &
- image_job_pids+=("$!")
- # Read through a nameref in the job-pool helpers.
- # shellcheck disable=SC2034
- image_job_labels["$!"]="image derivative job for photo $photo"
+ job_pool_submit image_jobs "image derivative job for photo $photo" \
+ create_photo_derivatives "$photos_dir" "$thumbs_dir" "$blurs_dir" \
+ "$photo"
done < <(
find "$DIST_DIR/$photos_dir" -maxdepth 1 -type f -printf '%f\n' \
| sort
)
- wait_for_image_jobs \
- image_job_pids \
- image_job_statuses \
- image_job_labels \
- failed
- if (( failed != 0 )); then
- return 1
- fi
+ job_pool_wait image_jobs
}
prepare_generation_photo_assets() {
diff --git a/src/lib/image.source.sh b/src/lib/image.source.sh
index 2581228..32d1362 100644
--- a/src/lib/image.source.sh
+++ b/src/lib/image.source.sh
@@ -102,37 +102,18 @@ warn_unsupported_incoming_files() {
}
scalephotos() {
- local -i failed=0
- local -a image_job_pids=()
- # Passed by name to wait_for_image_job_slot and wait_for_image_jobs.
- # shellcheck disable=SC2034
- local -A image_job_labels=()
- # Passed by name to wait_for_image_job_slot and wait_for_image_jobs.
- # shellcheck disable=SC2034
- local -A image_job_statuses=()
local photo
+ # Throttled background job pool (max IMAGE_JOBS concurrent), addressed by the
+ # single handle "image_jobs". job_pool_wait returns 1 if any job failed.
+ job_pool_init image_jobs
+
while IFS= read -r photo; do
- wait_for_image_job_slot \
- image_job_pids \
- image_job_statuses \
- image_job_labels \
- failed
- scale_photo "$photo" &
- image_job_pids+=("$!")
- # Read through a nameref in the job-pool helpers.
- # shellcheck disable=SC2034
- image_job_labels["$!"]="image job for photo $photo"
+ job_pool_submit image_jobs "image job for photo $photo" \
+ scale_photo "$photo"
done < <(incoming_image_files)
- wait_for_image_jobs \
- image_job_pids \
- image_job_statuses \
- image_job_labels \
- failed
- if (( failed != 0 )); then
- return 1
- fi
+ job_pool_wait image_jobs
}
scale_photo() {
diff --git a/src/lib/job-pool.source.sh b/src/lib/job-pool.source.sh
index a50a6e8..d3a3a9c 100644
--- a/src/lib/job-pool.source.sh
+++ b/src/lib/job-pool.source.sh
@@ -1,15 +1,61 @@
-wait_for_parallel_job_pid() {
+# Background-job pool, throttled to a fixed number of concurrent children.
+#
+# A "pool" is encapsulated behind a single handle: a name prefix string. The
+# helpers derive the pool's four backing variables from that prefix on demand:
+#
+# ${pool}_pids indexed array - pids of jobs still being tracked
+# ${pool}_statuses assoc array - pid -> exit status, for jobs already
+# reaped by `wait -n` but not yet finished
+# ${pool}_labels assoc array - pid -> human label, for failure messages
+# ${pool}_failed integer - 1 once any job has exited non-zero
+#
+# Encapsulating the four parallel arrays behind one prefix means callers declare
+# and pass a single handle instead of four namerefs (each previously needing its
+# own `shellcheck disable=SC2034`). Bash cannot nest indexed arrays inside an
+# associative array, so a literal single-variable struct is impossible; a name
+# prefix with `declare -g` backing variables is the simplest encoding that keeps
+# all four pieces together under one name while staying pure-nameref (no eval).
+#
+# Public API:
+# job_pool_init <pool> - create/reset the four backing variables
+# job_pool_submit <pool> <label> <cmd...>
+# - block until a slot is free (< IMAGE_JOBS
+# running), then background <cmd...> and
+# track it under <label>
+# job_pool_wait <pool> - wait for all remaining jobs; return 1 if
+# any job (now or earlier) failed
+#
+# Throttling is fixed at IMAGE_JOBS (the configured max concurrent jobs); the
+# previous per-call max_jobs parameterization was never varied, so it is gone.
+
+# Create or reset a pool's backing variables. `declare -g` makes them globals so
+# the derived namerefs in the helpers can see them regardless of the calling
+# function's scope; re-running it clears any stale state from a prior pool of the
+# same name (e.g. a re-generate).
+job_pool_init() {
+ local -r pool="$1"; shift
+
+ declare -ga "${pool}_pids=()"
+ declare -gA "${pool}_statuses=()"
+ declare -gA "${pool}_labels=()"
+ declare -gi "${pool}_failed=0"
+}
+
+# Reap a single finished pid: record its exit status. If `wait -n` already
+# observed this pid (its status is cached in ${pool}_statuses), reuse that;
+# otherwise `wait` on it directly. The status is returned through status_ref.
+_job_pool_reap_pid() {
+ local -r pool="$1"; shift
local -r pid="$1"; shift
- local -r job_statuses_name="$1"; shift
local -n status_ref="$1"; shift
# shellcheck disable=SC2178
- local -n job_statuses_ref="$job_statuses_name"
+ local -n statuses_ref="${pool}_statuses"
local -i wait_status=0
- if [ -n "${job_statuses_ref[$pid]+x}" ]; then
+ if [ -n "${statuses_ref[$pid]+x}" ]; then
# shellcheck disable=SC2034
- status_ref="${job_statuses_ref[$pid]}"
- unset "job_statuses_ref[$pid]"
+ status_ref="${statuses_ref[$pid]}"
+ unset "statuses_ref[$pid]"
return
fi
@@ -22,7 +68,8 @@ wait_for_parallel_job_pid() {
status_ref="$wait_status"
}
-parallel_job_pid_is_running() {
+# True if pid is among the still-running pids passed as the remaining args.
+_job_pool_pid_is_running() {
local -r pid="$1"; shift
local running_pid
@@ -35,42 +82,32 @@ parallel_job_pid_is_running() {
return 1
}
-log_parallel_job_failure() {
+# Finalise one reaped pid: on non-zero status flip the pool's failed flag and
+# log the job's label, then drop the label entry.
+_job_pool_finish_pid() {
+ local -r pool="$1"; shift
local -r pid="$1"; shift
local -r status="$1"; shift
- local -r job_labels_name="$1"; shift
# shellcheck disable=SC2178
- local -n job_labels_ref="$job_labels_name"
+ local -n labels_ref="${pool}_labels"
+ local -n failed_ref="${pool}_failed"
local label
- label="${job_labels_ref[$pid]:-pid $pid}"
- printf 'ERROR: parallel job failed (%s): %s\n' "$status" "$label" >&2
-}
-
-finish_parallel_job() {
- local -r pid="$1"; shift
- local -r status="$1"; shift
- local -r job_labels_name="$1"; shift
- local -n failed_target_ref="$1"; shift
- # shellcheck disable=SC2178
- local -n job_labels_ref="$job_labels_name"
-
if (( status != 0 )); then
- # shellcheck disable=SC2034
- failed_target_ref=1
- log_parallel_job_failure "$pid" "$status" "$job_labels_name"
+ failed_ref=1
+ label="${labels_ref[$pid]:-pid $pid}"
+ printf 'ERROR: parallel job failed (%s): %s\n' "$status" "$label" >&2
fi
- unset "job_labels_ref[$pid]"
+ unset "labels_ref[$pid]"
}
-reap_finished_parallel_jobs() {
- local -r job_pids_name="$1"; shift
- local -r job_statuses_name="$1"; shift
- local -r job_labels_name="$1"; shift
+# Sweep the pool's tracked pids: any that are no longer running get reaped and
+# finished; still-running ones are kept for the next sweep.
+_job_pool_reap_finished() {
+ local -r pool="$1"; shift
# shellcheck disable=SC2178
- local -n job_pids_ref="$job_pids_name"
- local -n failed_ref="$1"; shift
+ local -n pids_ref="${pool}_pids"
local -i status=0
local -a remaining_pids=()
local -a running_pids=()
@@ -78,138 +115,90 @@ reap_finished_parallel_jobs() {
mapfile -t running_pids < <(jobs -rp)
- for pid in "${job_pids_ref[@]}"; do
- if parallel_job_pid_is_running "$pid" "${running_pids[@]}"; then
+ for pid in "${pids_ref[@]}"; do
+ if _job_pool_pid_is_running "$pid" "${running_pids[@]}"; then
remaining_pids+=("$pid")
continue
fi
- wait_for_parallel_job_pid "$pid" "$job_statuses_name" status
- finish_parallel_job "$pid" "$status" "$job_labels_name" failed_ref
+ _job_pool_reap_pid "$pool" "$pid" status
+ _job_pool_finish_pid "$pool" "$pid" "$status"
done
- job_pids_ref=("${remaining_pids[@]}")
+ pids_ref=("${remaining_pids[@]}")
}
-wait_for_next_parallel_job() {
- local -r job_pids_name="$1"; shift
- local -r job_statuses_name="$1"; shift
- local -r job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
+# Block until at least one tracked job finishes, then reap every job that is now
+# done. `wait -n` returns the first child to exit; its status is cached so the
+# subsequent sweep can finish it with the rest.
+_job_pool_wait_for_next() {
+ local -r pool="$1"; shift
# shellcheck disable=SC2178
- local -n job_pids_ref="$job_pids_name"
+ local -n pids_ref="${pool}_pids"
# shellcheck disable=SC2178
- local -n job_statuses_ref="$job_statuses_name"
+ local -n statuses_ref="${pool}_statuses"
local completed_pid=''
local -i status=0
set +e
- wait -n -p completed_pid "${job_pids_ref[@]}"
+ wait -n -p completed_pid "${pids_ref[@]}"
status=$?
set -e
if [ -n "$completed_pid" ]; then
- job_statuses_ref["$completed_pid"]="$status"
+ statuses_ref["$completed_pid"]="$status"
fi
- reap_finished_parallel_jobs \
- "$job_pids_name" \
- "$job_statuses_name" \
- "$job_labels_name" \
- "$failed_name"
+ _job_pool_reap_finished "$pool"
}
-wait_for_parallel_job_slot() {
- local -r job_pids_name="$1"; shift
- local -r job_statuses_name="$1"; shift
- local -r job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
- local -r max_jobs="$1"; shift
+# Block until the pool has a free slot (fewer than IMAGE_JOBS jobs tracked),
+# reaping finished jobs while it waits. Throttles concurrency to IMAGE_JOBS.
+_job_pool_wait_for_slot() {
+ local -r pool="$1"; shift
# shellcheck disable=SC2178
- local -n job_pids_ref="$job_pids_name"
-
- while (( ${#job_pids_ref[@]} >= max_jobs )); do
- wait_for_next_parallel_job \
- "$job_pids_name" \
- "$job_statuses_name" \
- "$job_labels_name" \
- "$failed_name"
+ local -n pids_ref="${pool}_pids"
+
+ while (( ${#pids_ref[@]} >= IMAGE_JOBS )); do
+ _job_pool_wait_for_next "$pool"
done
}
-wait_for_parallel_jobs() {
+# Wait for a free slot, then background <cmd...> and track it under <label> so a
+# later failure can be reported against a meaningful name. The whole command
+# (and its arguments) runs in the child; the caller is free to reuse its own
+# variables for the next submission once this returns.
+job_pool_submit() {
+ local -r pool="$1"; shift
+ local -r label="$1"; shift
# shellcheck disable=SC2178
- local -n job_pids_ref="$1"; shift
- local -r job_statuses_name="$1"; shift
- local -r job_labels_name="$1"; shift
- local -n failed_ref="$1"; shift
- local -i status=0
-
- : "$failed_ref"
- while (( ${#job_pids_ref[@]} > 0 )); do
- wait_for_parallel_job_pid \
- "${job_pids_ref[0]}" \
- "$job_statuses_name" \
- status
- finish_parallel_job \
- "${job_pids_ref[0]}" \
- "$status" \
- "$job_labels_name" \
- failed_ref
- job_pids_ref=("${job_pids_ref[@]:1}")
- done
-}
+ local -n pids_ref="${pool}_pids"
+ # shellcheck disable=SC2178
+ local -n labels_ref="${pool}_labels"
-wait_for_image_job_slot() {
- local -r image_job_pids_name="$1"; shift
- local -r image_job_statuses_name="$1"; shift
- local -r image_job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
-
- wait_for_parallel_job_slot \
- "$image_job_pids_name" \
- "$image_job_statuses_name" \
- "$image_job_labels_name" \
- "$failed_name" \
- "$IMAGE_JOBS"
+ _job_pool_wait_for_slot "$pool"
+ "$@" &
+ pids_ref+=("$!")
+ labels_ref["$!"]="$label"
}
-wait_for_image_jobs() {
- local -r image_job_pids_name="$1"; shift
- local -r image_job_statuses_name="$1"; shift
- local -r image_job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
-
- wait_for_parallel_jobs \
- "$image_job_pids_name" \
- "$image_job_statuses_name" \
- "$image_job_labels_name" \
- "$failed_name"
-}
+# Wait for all remaining jobs to finish, finishing each in turn so a failed
+# child flips the pool's failed flag. Returns 1 if any job in the pool's
+# lifetime failed, 0 otherwise.
+job_pool_wait() {
+ local -r pool="$1"; shift
+ # shellcheck disable=SC2178
+ local -n pids_ref="${pool}_pids"
+ local -n failed_ref="${pool}_failed"
+ local -i status=0
-wait_for_template_render_job_slot() {
- local -r render_job_pids_name="$1"; shift
- local -r render_job_statuses_name="$1"; shift
- local -r render_job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
-
- wait_for_parallel_job_slot \
- "$render_job_pids_name" \
- "$render_job_statuses_name" \
- "$render_job_labels_name" \
- "$failed_name" \
- "$IMAGE_JOBS"
-}
+ while (( ${#pids_ref[@]} > 0 )); do
+ _job_pool_reap_pid "$pool" "${pids_ref[0]}" status
+ _job_pool_finish_pid "$pool" "${pids_ref[0]}" "$status"
+ pids_ref=("${pids_ref[@]:1}")
+ done
-wait_for_template_render_jobs() {
- local -r render_job_pids_name="$1"; shift
- local -r render_job_statuses_name="$1"; shift
- local -r render_job_labels_name="$1"; shift
- local -r failed_name="$1"; shift
-
- wait_for_parallel_jobs \
- "$render_job_pids_name" \
- "$render_job_statuses_name" \
- "$render_job_labels_name" \
- "$failed_name"
+ if (( failed_ref != 0 )); then
+ return 1
+ fi
}
diff --git a/src/lib/stats-filter-album.source.sh b/src/lib/stats-filter-album.source.sh
index c1c1e1c..c475006 100644
--- a/src/lib/stats-filter-album.source.sh
+++ b/src/lib/stats-filter-album.source.sh
@@ -196,14 +196,7 @@ END
# background render so parallelism follows the configured IMAGE_JOBS.
_stats_enqueue_filter_album() {
local -r pagebase="$1"; shift
- local -r pids_name="$1"; shift
- local -r statuses_name="$1"; shift
- local -r labels_name="$1"; shift
- local -r failed_name="$1"; shift
- # shellcheck disable=SC2178
- local -n pids_ref="$pids_name"
- # shellcheck disable=SC2178
- local -n labels_ref="$labels_name"
+ local -r pool="$1"; shift
local photo
local -a photo_list=()
local -i i n
@@ -213,20 +206,14 @@ _stats_enqueue_filter_album() {
done <<< "${STATS_FILTER_PHOTOS[$pagebase]}"
n=${#photo_list[@]}
- wait_for_template_render_job_slot \
- "$pids_name" "$statuses_name" "$labels_name" "$failed_name"
- _stats_render_filter_gallery "$pagebase" &
- pids_ref+=("$!")
- labels_ref["$!"]="filter gallery $pagebase"
+ job_pool_submit "$pool" "filter gallery $pagebase" \
+ _stats_render_filter_gallery "$pagebase"
for (( i = 1; i <= n; i++ )); do
- wait_for_template_render_job_slot \
- "$pids_name" "$statuses_name" "$labels_name" "$failed_name"
- _stats_render_filter_view_page \
+ job_pool_submit "$pool" "filter view $pagebase/$i" \
+ _stats_render_filter_view_page \
"$pagebase" "${photo_list[i - 1]}" "$i" \
- "$(( i == 1 ? n : i - 1 ))" "$(( i == n ? 1 : i + 1 ))" &
- pids_ref+=("$!")
- labels_ref["$!"]="filter view $pagebase/$i"
+ "$(( i == 1 ? n : i - 1 ))" "$(( i == n ? 1 : i + 1 ))"
done
}
@@ -241,24 +228,18 @@ _stats_enqueue_filter_album() {
# collect_photo_exif_stats first to fill STATS_FILTER_PHOTOS.
render_filter_pages() {
local pagebase
- # Render job pool, throttled to IMAGE_JOBS by the job-pool helpers.
- local -a render_job_pids=()
- # shellcheck disable=SC2034
- local -A render_job_statuses=()
- # shellcheck disable=SC2034
- local -A render_job_labels=()
- local -i render_failed=0
if (( ${#STATS_FILTER_PHOTOS[@]} == 0 )); then
return
fi
+
+ # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle
+ # "render_jobs". job_pool_wait returns 1 if any render job failed.
+ job_pool_init render_jobs
+
while IFS= read -r pagebase; do
- _stats_enqueue_filter_album "$pagebase" \
- render_job_pids render_job_statuses render_job_labels render_failed
+ _stats_enqueue_filter_album "$pagebase" render_jobs
done < <(printf '%s\n' "${!STATS_FILTER_PHOTOS[@]}" | LC_ALL=C sort)
- wait_for_template_render_jobs \
- render_job_pids render_job_statuses render_job_labels render_failed
- if (( render_failed != 0 )); then
- return 1
- fi
+
+ job_pool_wait render_jobs
}