diff options
| author | Paul Buetow <paul@buetow.org> | 2026-06-24 11:00:33 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2026-06-24 11:00:33 +0300 |
| commit | 928f28d0c51ca47f9958fda4c4bb7c8b0c571e34 (patch) | |
| tree | 5d9d7a80aeb845598d1bdac928ed5a1ed2e9540f /src | |
| parent | 6697431d3855d3030b35c2ce232c9ea0e9282ba7 (diff) | |
Encapsulate job pool behind a single handle (job_pool_*)
Replace the four-parallel-nameref job-pool API (pids/statuses/labels/failed,
each needing its own `shellcheck disable=SC2034` at every call site) with a
single pool handle: a name prefix whose four backing variables
(${pool}_pids/_statuses/_labels/_failed) are derived on demand by the helpers.
Bash can't nest indexed arrays in an associative array, so a prefixed-handle
with declare -g backing vars is the simplest pure-nameref encoding (no eval).
New public API:
job_pool_init <pool>
job_pool_submit <pool> <label> <cmd...>
job_pool_wait <pool> # returns 1 if any job failed
Migrate all callers (scalephotos, create_all_photo_derivatives,
render_album_pages, render_view_redirects, render_filter_pages) to the handle.
queue_preview_page_render_job / queue_album_view_render_job /
_album_record_view_photo / _stats_enqueue_filter_album now take one pool arg
instead of four names.
Drop the dead wrappers and unused parameterization: wait_for_image_job_slot,
wait_for_template_render_job_slot and their _jobs variants only ever passed
IMAGE_JOBS, so the max_jobs parameter is gone and throttling is fixed at
IMAGE_JOBS inside the pool. Also removes wait_for_album_view_render_jobs (now
just job_pool_wait).
Throttling (max IMAGE_JOBS concurrent), failure detection and failed-job
propagation are unchanged; the parallel-throttling and failure-logging tests
pass unmodified. SC2034 disable-comments across the touched files drop 29->7.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/album-render.source.sh | 145 | ||||
| -rw-r--r-- | src/lib/image-pipeline.source.sh | 35 | ||||
| -rw-r--r-- | src/lib/image.source.sh | 33 | ||||
| -rw-r--r-- | src/lib/job-pool.source.sh | 257 | ||||
| -rw-r--r-- | src/lib/stats-filter-album.source.sh | 47 |
5 files changed, 186 insertions, 331 deletions
diff --git a/src/lib/album-render.source.sh b/src/lib/album-render.source.sh index 3024923..d438cef 100644 --- a/src/lib/album-render.source.sh +++ b/src/lib/album-render.source.sh @@ -132,13 +132,13 @@ render_full_preview_page() { fi } -# Enqueue one complete preview page as a background render job, throttled to -# IMAGE_JOBS via the shared template render job pool. Mirrors -# queue_album_view_render_job: wait for a free slot, background the whole-page -# assembly, then track its pid/label so a failed job flips render_failed and -# makes generation fail loudly. The page's photos are passed as trailing -# positional args; the background subshell forks a private copy of them, so the -# caller is free to reuse its per-page accumulator for the next page. +# Enqueue one complete preview page as a background render job in the given job +# pool (throttled to IMAGE_JOBS). Mirrors queue_album_view_render_job: job_pool_submit +# waits for a free slot, backgrounds the whole-page assembly, and tracks its +# pid/label so a failed job is reported by job_pool_wait and makes generation +# fail loudly. The page's photos are passed as trailing positional args; the +# background subshell forks a private copy of them, so the caller is free to +# reuse its per-page accumulator for the next page. queue_preview_page_render_job() { local -r photos_dir="$1"; shift local -r html_dir="$1"; shift @@ -151,24 +151,11 @@ queue_preview_page_render_job() { local -r header_bar="$1"; shift local -r prev_page="$1"; shift local -r next_page="$1"; shift - # shellcheck disable=SC2178 - local -n render_job_pids_ref="$1"; shift - # Passed by name to wait_for_template_render_job_slot. - # shellcheck disable=SC2034,SC2178 - local -n render_job_statuses_ref="$1"; shift - # Passed by name to wait_for_template_render_job_slot and assigned below. - # shellcheck disable=SC2034,SC2178 - local -n render_job_labels_ref="$1"; shift - # shellcheck disable=SC2178 - local -n render_failed_ref="$1"; shift + local -r pool="$1"; shift # Remaining positional args ("$@") are this page's photos in order. - wait_for_template_render_job_slot \ - render_job_pids_ref \ - render_job_statuses_ref \ - render_job_labels_ref \ - render_failed_ref - render_full_preview_page \ + job_pool_submit "$pool" "template render job for preview $page_name" \ + render_full_preview_page \ "$photos_dir" \ "$html_dir" \ "$thumbs_dir" \ @@ -180,9 +167,7 @@ queue_preview_page_render_job() { "$header_bar" \ "$prev_page" \ "$next_page" \ - "$@" & - render_job_pids_ref+=("$!") - render_job_labels_ref["$!"]="template render job for preview $page_name" + "$@" } # Build a whole thumbnail-grid buffer by walking a list of photos and grouping @@ -592,37 +577,26 @@ render_view_redirects() { local -n redirect_last_views_ref="$1"; shift local max_page local page - # Render job pool, throttled to IMAGE_JOBS by the job-pool helpers. - # shellcheck disable=SC2034 - local -a render_job_pids=() - # shellcheck disable=SC2034 - local -A render_job_statuses=() - # shellcheck disable=SC2034 - local -A render_job_labels=() - local -i render_failed=0 if (( ${#view_pages_ref[@]} == 0 )); then return fi + # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle + # "render_jobs". job_pool_wait returns 1 if any render job failed. + job_pool_init render_jobs + max_page=${view_pages_ref[$(( ${#view_pages_ref[@]} - 1 ))]} for page in "${view_pages_ref[@]}"; do - wait_for_template_render_job_slot \ - render_job_pids render_job_statuses render_job_labels render_failed - render_page_view_redirects \ + job_pool_submit render_jobs \ + "template render job for redirect page $page" \ + render_page_view_redirects \ "$html_dir" "$page" "${redirect_last_views_ref[$page]}" \ - "$max_page" & - render_job_pids+=("$!") - # shellcheck disable=SC2034 - render_job_labels["$!"]="template render job for redirect page $page" + "$max_page" done - wait_for_template_render_jobs \ - render_job_pids render_job_statuses render_job_labels render_failed - if (( render_failed != 0 )); then - return 1 - fi + job_pool_wait render_jobs } render_album_index_redirect() { @@ -686,23 +660,10 @@ queue_album_view_render_job() { local -r page_num="$1"; shift local -r preview_num="$1"; shift local -r photo="$1"; shift - # shellcheck disable=SC2178 - local -n render_job_pids_ref="$1"; shift - # Passed by name to wait_for_template_render_job_slot. - # shellcheck disable=SC2034,SC2178 - local -n render_job_statuses_ref="$1"; shift - # Passed by name to wait_for_template_render_job_slot and assigned below. - # shellcheck disable=SC2034,SC2178 - local -n render_job_labels_ref="$1"; shift - # shellcheck disable=SC2178 - local -n render_failed_ref="$1"; shift - - wait_for_template_render_job_slot \ - render_job_pids_ref \ - render_job_statuses_ref \ - render_job_labels_ref \ - render_failed_ref - render_photo_view_and_details \ + local -r pool="$1"; shift + + job_pool_submit "$pool" "template render job for photo $photo" \ + render_photo_view_and_details \ "$photos_dir" \ "$blurs_dir" \ "$html_dir" \ @@ -710,31 +671,7 @@ queue_album_view_render_job() { "$tarball_name" \ "$page_num" \ "$preview_num" \ - "$photo" & - render_job_pids_ref+=("$!") - render_job_labels_ref["$!"]="template render job for photo $photo" -} - -wait_for_album_view_render_jobs() { - # shellcheck disable=SC2178 - local -n render_job_pids_ref="$1"; shift - # Passed by name to wait_for_template_render_jobs. - # shellcheck disable=SC2034,SC2178 - local -n render_job_statuses_ref="$1"; shift - # Passed by name to wait_for_template_render_jobs. - # shellcheck disable=SC2034,SC2178 - local -n render_job_labels_ref="$1"; shift - # shellcheck disable=SC2178 - local -n render_failed_ref="$1"; shift - - wait_for_template_render_jobs \ - render_job_pids_ref \ - render_job_statuses_ref \ - render_job_labels_ref \ - render_failed_ref - if (( render_failed_ref != 0 )); then - return 1 - fi + "$photo" } # Group the album's photos into pages of at most MAXPREVIEWS, in their final @@ -783,17 +720,14 @@ _album_record_view_photo() { local -ri page_num="$1"; shift local -ri preview_num="$1"; shift local -r photo="$1"; shift - local -r pids_name="$1"; shift - local -r statuses_name="$1"; shift - local -r labels_name="$1"; shift - local -r failed_name="$1"; shift + local -r pool="$1"; shift local -r view_pages_name="$1"; shift local -r last_views_name="$1"; shift queue_album_view_render_job \ "$photos_dir" "$blurs_dir" "$html_dir" "$backhref" "$tarball_name" \ "$page_num" "$preview_num" "$photo" \ - "$pids_name" "$statuses_name" "$labels_name" "$failed_name" + "$pool" record_rendered_view_page "$view_pages_name" "$last_views_name" \ "$page_num" "$preview_num" # Read later through the album_view_page_for_photo accessor (e.g. by the @@ -819,15 +753,6 @@ render_album_pages() { local prev_name='' local record local -i preview_num - # Passed by name to the render-pool helpers below. - # shellcheck disable=SC2034 - local -i render_failed=0 - # shellcheck disable=SC2034 - local -a render_job_pids=() - # shellcheck disable=SC2034 - local -A render_job_labels=() - # shellcheck disable=SC2034 - local -A render_job_statuses=() # Passed by name to record_rendered_view_page and render_view_redirects. # shellcheck disable=SC2034 local -A rendered_last_views=() @@ -836,6 +761,10 @@ render_album_pages() { local -a page_photos=() local -a page_records=() + # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle + # "render_jobs". job_pool_wait returns 1 if any render job failed. + job_pool_init render_jobs + # Rebuild the photo -> view-page map for this album from scratch so a # re-generate (or a smaller incoming set) does not keep stale entries. The # per-photo entries are written in _album_record_view_photo, which shellcheck @@ -867,8 +796,7 @@ render_album_pages() { _album_record_view_photo \ "$photos_dir" "$blurs_dir" "$html_dir" "$backhref" \ "$tarball_name" "$page_num" "$preview_num" "$photo" \ - render_job_pids render_job_statuses render_job_labels \ - render_failed rendered_view_pages rendered_last_views + render_jobs rendered_view_pages rendered_last_views done # A page has a "next" link unless it is the last record. @@ -880,18 +808,13 @@ render_album_pages() { "$photos_dir" "$html_dir" "$thumbs_dir" "$blurs_dir" "$backhref" \ "$tarball_name" "$name" "$page_num" "$header_bar" \ "$prev_name" "$next_name" \ - render_job_pids render_job_statuses render_job_labels \ - render_failed \ + render_jobs \ "${page_photos[@]}" prev_name="$name" done - if ! wait_for_album_view_render_jobs \ - render_job_pids \ - render_job_statuses \ - render_job_labels \ - render_failed; then + if ! job_pool_wait render_jobs; then return 1 fi render_view_redirects "$html_dir" rendered_view_pages rendered_last_views diff --git a/src/lib/image-pipeline.source.sh b/src/lib/image-pipeline.source.sh index 876501b..934c3b5 100644 --- a/src/lib/image-pipeline.source.sh +++ b/src/lib/image-pipeline.source.sh @@ -38,41 +38,22 @@ create_all_photo_derivatives() { local -r photos_dir="$1"; shift local -r thumbs_dir="$1"; shift local -r blurs_dir="$1"; shift - local -i failed=0 - local -a image_job_pids=() - # Passed by name to wait_for_image_job_slot and wait_for_image_jobs. - # shellcheck disable=SC2034 - local -A image_job_labels=() - # Passed by name to wait_for_image_job_slot and wait_for_image_jobs. - # shellcheck disable=SC2034 - local -A image_job_statuses=() local photo + # Throttled background job pool (max IMAGE_JOBS concurrent), addressed by the + # single handle "image_jobs". job_pool_wait returns 1 if any job failed. + job_pool_init image_jobs + while IFS= read -r photo; do - wait_for_image_job_slot \ - image_job_pids \ - image_job_statuses \ - image_job_labels \ - failed - create_photo_derivatives "$photos_dir" "$thumbs_dir" "$blurs_dir" \ - "$photo" & - image_job_pids+=("$!") - # Read through a nameref in the job-pool helpers. - # shellcheck disable=SC2034 - image_job_labels["$!"]="image derivative job for photo $photo" + job_pool_submit image_jobs "image derivative job for photo $photo" \ + create_photo_derivatives "$photos_dir" "$thumbs_dir" "$blurs_dir" \ + "$photo" done < <( find "$DIST_DIR/$photos_dir" -maxdepth 1 -type f -printf '%f\n' \ | sort ) - wait_for_image_jobs \ - image_job_pids \ - image_job_statuses \ - image_job_labels \ - failed - if (( failed != 0 )); then - return 1 - fi + job_pool_wait image_jobs } prepare_generation_photo_assets() { diff --git a/src/lib/image.source.sh b/src/lib/image.source.sh index 2581228..32d1362 100644 --- a/src/lib/image.source.sh +++ b/src/lib/image.source.sh @@ -102,37 +102,18 @@ warn_unsupported_incoming_files() { } scalephotos() { - local -i failed=0 - local -a image_job_pids=() - # Passed by name to wait_for_image_job_slot and wait_for_image_jobs. - # shellcheck disable=SC2034 - local -A image_job_labels=() - # Passed by name to wait_for_image_job_slot and wait_for_image_jobs. - # shellcheck disable=SC2034 - local -A image_job_statuses=() local photo + # Throttled background job pool (max IMAGE_JOBS concurrent), addressed by the + # single handle "image_jobs". job_pool_wait returns 1 if any job failed. + job_pool_init image_jobs + while IFS= read -r photo; do - wait_for_image_job_slot \ - image_job_pids \ - image_job_statuses \ - image_job_labels \ - failed - scale_photo "$photo" & - image_job_pids+=("$!") - # Read through a nameref in the job-pool helpers. - # shellcheck disable=SC2034 - image_job_labels["$!"]="image job for photo $photo" + job_pool_submit image_jobs "image job for photo $photo" \ + scale_photo "$photo" done < <(incoming_image_files) - wait_for_image_jobs \ - image_job_pids \ - image_job_statuses \ - image_job_labels \ - failed - if (( failed != 0 )); then - return 1 - fi + job_pool_wait image_jobs } scale_photo() { diff --git a/src/lib/job-pool.source.sh b/src/lib/job-pool.source.sh index a50a6e8..d3a3a9c 100644 --- a/src/lib/job-pool.source.sh +++ b/src/lib/job-pool.source.sh @@ -1,15 +1,61 @@ -wait_for_parallel_job_pid() { +# Background-job pool, throttled to a fixed number of concurrent children. +# +# A "pool" is encapsulated behind a single handle: a name prefix string. The +# helpers derive the pool's four backing variables from that prefix on demand: +# +# ${pool}_pids indexed array - pids of jobs still being tracked +# ${pool}_statuses assoc array - pid -> exit status, for jobs already +# reaped by `wait -n` but not yet finished +# ${pool}_labels assoc array - pid -> human label, for failure messages +# ${pool}_failed integer - 1 once any job has exited non-zero +# +# Encapsulating the four parallel arrays behind one prefix means callers declare +# and pass a single handle instead of four namerefs (each previously needing its +# own `shellcheck disable=SC2034`). Bash cannot nest indexed arrays inside an +# associative array, so a literal single-variable struct is impossible; a name +# prefix with `declare -g` backing variables is the simplest encoding that keeps +# all four pieces together under one name while staying pure-nameref (no eval). +# +# Public API: +# job_pool_init <pool> - create/reset the four backing variables +# job_pool_submit <pool> <label> <cmd...> +# - block until a slot is free (< IMAGE_JOBS +# running), then background <cmd...> and +# track it under <label> +# job_pool_wait <pool> - wait for all remaining jobs; return 1 if +# any job (now or earlier) failed +# +# Throttling is fixed at IMAGE_JOBS (the configured max concurrent jobs); the +# previous per-call max_jobs parameterization was never varied, so it is gone. + +# Create or reset a pool's backing variables. `declare -g` makes them globals so +# the derived namerefs in the helpers can see them regardless of the calling +# function's scope; re-running it clears any stale state from a prior pool of the +# same name (e.g. a re-generate). +job_pool_init() { + local -r pool="$1"; shift + + declare -ga "${pool}_pids=()" + declare -gA "${pool}_statuses=()" + declare -gA "${pool}_labels=()" + declare -gi "${pool}_failed=0" +} + +# Reap a single finished pid: record its exit status. If `wait -n` already +# observed this pid (its status is cached in ${pool}_statuses), reuse that; +# otherwise `wait` on it directly. The status is returned through status_ref. +_job_pool_reap_pid() { + local -r pool="$1"; shift local -r pid="$1"; shift - local -r job_statuses_name="$1"; shift local -n status_ref="$1"; shift # shellcheck disable=SC2178 - local -n job_statuses_ref="$job_statuses_name" + local -n statuses_ref="${pool}_statuses" local -i wait_status=0 - if [ -n "${job_statuses_ref[$pid]+x}" ]; then + if [ -n "${statuses_ref[$pid]+x}" ]; then # shellcheck disable=SC2034 - status_ref="${job_statuses_ref[$pid]}" - unset "job_statuses_ref[$pid]" + status_ref="${statuses_ref[$pid]}" + unset "statuses_ref[$pid]" return fi @@ -22,7 +68,8 @@ wait_for_parallel_job_pid() { status_ref="$wait_status" } -parallel_job_pid_is_running() { +# True if pid is among the still-running pids passed as the remaining args. +_job_pool_pid_is_running() { local -r pid="$1"; shift local running_pid @@ -35,42 +82,32 @@ parallel_job_pid_is_running() { return 1 } -log_parallel_job_failure() { +# Finalise one reaped pid: on non-zero status flip the pool's failed flag and +# log the job's label, then drop the label entry. +_job_pool_finish_pid() { + local -r pool="$1"; shift local -r pid="$1"; shift local -r status="$1"; shift - local -r job_labels_name="$1"; shift # shellcheck disable=SC2178 - local -n job_labels_ref="$job_labels_name" + local -n labels_ref="${pool}_labels" + local -n failed_ref="${pool}_failed" local label - label="${job_labels_ref[$pid]:-pid $pid}" - printf 'ERROR: parallel job failed (%s): %s\n' "$status" "$label" >&2 -} - -finish_parallel_job() { - local -r pid="$1"; shift - local -r status="$1"; shift - local -r job_labels_name="$1"; shift - local -n failed_target_ref="$1"; shift - # shellcheck disable=SC2178 - local -n job_labels_ref="$job_labels_name" - if (( status != 0 )); then - # shellcheck disable=SC2034 - failed_target_ref=1 - log_parallel_job_failure "$pid" "$status" "$job_labels_name" + failed_ref=1 + label="${labels_ref[$pid]:-pid $pid}" + printf 'ERROR: parallel job failed (%s): %s\n' "$status" "$label" >&2 fi - unset "job_labels_ref[$pid]" + unset "labels_ref[$pid]" } -reap_finished_parallel_jobs() { - local -r job_pids_name="$1"; shift - local -r job_statuses_name="$1"; shift - local -r job_labels_name="$1"; shift +# Sweep the pool's tracked pids: any that are no longer running get reaped and +# finished; still-running ones are kept for the next sweep. +_job_pool_reap_finished() { + local -r pool="$1"; shift # shellcheck disable=SC2178 - local -n job_pids_ref="$job_pids_name" - local -n failed_ref="$1"; shift + local -n pids_ref="${pool}_pids" local -i status=0 local -a remaining_pids=() local -a running_pids=() @@ -78,138 +115,90 @@ reap_finished_parallel_jobs() { mapfile -t running_pids < <(jobs -rp) - for pid in "${job_pids_ref[@]}"; do - if parallel_job_pid_is_running "$pid" "${running_pids[@]}"; then + for pid in "${pids_ref[@]}"; do + if _job_pool_pid_is_running "$pid" "${running_pids[@]}"; then remaining_pids+=("$pid") continue fi - wait_for_parallel_job_pid "$pid" "$job_statuses_name" status - finish_parallel_job "$pid" "$status" "$job_labels_name" failed_ref + _job_pool_reap_pid "$pool" "$pid" status + _job_pool_finish_pid "$pool" "$pid" "$status" done - job_pids_ref=("${remaining_pids[@]}") + pids_ref=("${remaining_pids[@]}") } -wait_for_next_parallel_job() { - local -r job_pids_name="$1"; shift - local -r job_statuses_name="$1"; shift - local -r job_labels_name="$1"; shift - local -r failed_name="$1"; shift +# Block until at least one tracked job finishes, then reap every job that is now +# done. `wait -n` returns the first child to exit; its status is cached so the +# subsequent sweep can finish it with the rest. +_job_pool_wait_for_next() { + local -r pool="$1"; shift # shellcheck disable=SC2178 - local -n job_pids_ref="$job_pids_name" + local -n pids_ref="${pool}_pids" # shellcheck disable=SC2178 - local -n job_statuses_ref="$job_statuses_name" + local -n statuses_ref="${pool}_statuses" local completed_pid='' local -i status=0 set +e - wait -n -p completed_pid "${job_pids_ref[@]}" + wait -n -p completed_pid "${pids_ref[@]}" status=$? set -e if [ -n "$completed_pid" ]; then - job_statuses_ref["$completed_pid"]="$status" + statuses_ref["$completed_pid"]="$status" fi - reap_finished_parallel_jobs \ - "$job_pids_name" \ - "$job_statuses_name" \ - "$job_labels_name" \ - "$failed_name" + _job_pool_reap_finished "$pool" } -wait_for_parallel_job_slot() { - local -r job_pids_name="$1"; shift - local -r job_statuses_name="$1"; shift - local -r job_labels_name="$1"; shift - local -r failed_name="$1"; shift - local -r max_jobs="$1"; shift +# Block until the pool has a free slot (fewer than IMAGE_JOBS jobs tracked), +# reaping finished jobs while it waits. Throttles concurrency to IMAGE_JOBS. +_job_pool_wait_for_slot() { + local -r pool="$1"; shift # shellcheck disable=SC2178 - local -n job_pids_ref="$job_pids_name" - - while (( ${#job_pids_ref[@]} >= max_jobs )); do - wait_for_next_parallel_job \ - "$job_pids_name" \ - "$job_statuses_name" \ - "$job_labels_name" \ - "$failed_name" + local -n pids_ref="${pool}_pids" + + while (( ${#pids_ref[@]} >= IMAGE_JOBS )); do + _job_pool_wait_for_next "$pool" done } -wait_for_parallel_jobs() { +# Wait for a free slot, then background <cmd...> and track it under <label> so a +# later failure can be reported against a meaningful name. The whole command +# (and its arguments) runs in the child; the caller is free to reuse its own +# variables for the next submission once this returns. +job_pool_submit() { + local -r pool="$1"; shift + local -r label="$1"; shift # shellcheck disable=SC2178 - local -n job_pids_ref="$1"; shift - local -r job_statuses_name="$1"; shift - local -r job_labels_name="$1"; shift - local -n failed_ref="$1"; shift - local -i status=0 - - : "$failed_ref" - while (( ${#job_pids_ref[@]} > 0 )); do - wait_for_parallel_job_pid \ - "${job_pids_ref[0]}" \ - "$job_statuses_name" \ - status - finish_parallel_job \ - "${job_pids_ref[0]}" \ - "$status" \ - "$job_labels_name" \ - failed_ref - job_pids_ref=("${job_pids_ref[@]:1}") - done -} + local -n pids_ref="${pool}_pids" + # shellcheck disable=SC2178 + local -n labels_ref="${pool}_labels" -wait_for_image_job_slot() { - local -r image_job_pids_name="$1"; shift - local -r image_job_statuses_name="$1"; shift - local -r image_job_labels_name="$1"; shift - local -r failed_name="$1"; shift - - wait_for_parallel_job_slot \ - "$image_job_pids_name" \ - "$image_job_statuses_name" \ - "$image_job_labels_name" \ - "$failed_name" \ - "$IMAGE_JOBS" + _job_pool_wait_for_slot "$pool" + "$@" & + pids_ref+=("$!") + labels_ref["$!"]="$label" } -wait_for_image_jobs() { - local -r image_job_pids_name="$1"; shift - local -r image_job_statuses_name="$1"; shift - local -r image_job_labels_name="$1"; shift - local -r failed_name="$1"; shift - - wait_for_parallel_jobs \ - "$image_job_pids_name" \ - "$image_job_statuses_name" \ - "$image_job_labels_name" \ - "$failed_name" -} +# Wait for all remaining jobs to finish, finishing each in turn so a failed +# child flips the pool's failed flag. Returns 1 if any job in the pool's +# lifetime failed, 0 otherwise. +job_pool_wait() { + local -r pool="$1"; shift + # shellcheck disable=SC2178 + local -n pids_ref="${pool}_pids" + local -n failed_ref="${pool}_failed" + local -i status=0 -wait_for_template_render_job_slot() { - local -r render_job_pids_name="$1"; shift - local -r render_job_statuses_name="$1"; shift - local -r render_job_labels_name="$1"; shift - local -r failed_name="$1"; shift - - wait_for_parallel_job_slot \ - "$render_job_pids_name" \ - "$render_job_statuses_name" \ - "$render_job_labels_name" \ - "$failed_name" \ - "$IMAGE_JOBS" -} + while (( ${#pids_ref[@]} > 0 )); do + _job_pool_reap_pid "$pool" "${pids_ref[0]}" status + _job_pool_finish_pid "$pool" "${pids_ref[0]}" "$status" + pids_ref=("${pids_ref[@]:1}") + done -wait_for_template_render_jobs() { - local -r render_job_pids_name="$1"; shift - local -r render_job_statuses_name="$1"; shift - local -r render_job_labels_name="$1"; shift - local -r failed_name="$1"; shift - - wait_for_parallel_jobs \ - "$render_job_pids_name" \ - "$render_job_statuses_name" \ - "$render_job_labels_name" \ - "$failed_name" + if (( failed_ref != 0 )); then + return 1 + fi } diff --git a/src/lib/stats-filter-album.source.sh b/src/lib/stats-filter-album.source.sh index c1c1e1c..c475006 100644 --- a/src/lib/stats-filter-album.source.sh +++ b/src/lib/stats-filter-album.source.sh @@ -196,14 +196,7 @@ END # background render so parallelism follows the configured IMAGE_JOBS. _stats_enqueue_filter_album() { local -r pagebase="$1"; shift - local -r pids_name="$1"; shift - local -r statuses_name="$1"; shift - local -r labels_name="$1"; shift - local -r failed_name="$1"; shift - # shellcheck disable=SC2178 - local -n pids_ref="$pids_name" - # shellcheck disable=SC2178 - local -n labels_ref="$labels_name" + local -r pool="$1"; shift local photo local -a photo_list=() local -i i n @@ -213,20 +206,14 @@ _stats_enqueue_filter_album() { done <<< "${STATS_FILTER_PHOTOS[$pagebase]}" n=${#photo_list[@]} - wait_for_template_render_job_slot \ - "$pids_name" "$statuses_name" "$labels_name" "$failed_name" - _stats_render_filter_gallery "$pagebase" & - pids_ref+=("$!") - labels_ref["$!"]="filter gallery $pagebase" + job_pool_submit "$pool" "filter gallery $pagebase" \ + _stats_render_filter_gallery "$pagebase" for (( i = 1; i <= n; i++ )); do - wait_for_template_render_job_slot \ - "$pids_name" "$statuses_name" "$labels_name" "$failed_name" - _stats_render_filter_view_page \ + job_pool_submit "$pool" "filter view $pagebase/$i" \ + _stats_render_filter_view_page \ "$pagebase" "${photo_list[i - 1]}" "$i" \ - "$(( i == 1 ? n : i - 1 ))" "$(( i == n ? 1 : i + 1 ))" & - pids_ref+=("$!") - labels_ref["$!"]="filter view $pagebase/$i" + "$(( i == 1 ? n : i - 1 ))" "$(( i == n ? 1 : i + 1 ))" done } @@ -241,24 +228,18 @@ _stats_enqueue_filter_album() { # collect_photo_exif_stats first to fill STATS_FILTER_PHOTOS. render_filter_pages() { local pagebase - # Render job pool, throttled to IMAGE_JOBS by the job-pool helpers. - local -a render_job_pids=() - # shellcheck disable=SC2034 - local -A render_job_statuses=() - # shellcheck disable=SC2034 - local -A render_job_labels=() - local -i render_failed=0 if (( ${#STATS_FILTER_PHOTOS[@]} == 0 )); then return fi + + # Render job pool (max IMAGE_JOBS concurrent), addressed by the single handle + # "render_jobs". job_pool_wait returns 1 if any render job failed. + job_pool_init render_jobs + while IFS= read -r pagebase; do - _stats_enqueue_filter_album "$pagebase" \ - render_job_pids render_job_statuses render_job_labels render_failed + _stats_enqueue_filter_album "$pagebase" render_jobs done < <(printf '%s\n' "${!STATS_FILTER_PHOTOS[@]}" | LC_ALL=C sort) - wait_for_template_render_jobs \ - render_job_pids render_job_statuses render_job_labels render_failed - if (( render_failed != 0 )); then - return 1 - fi + + job_pool_wait render_jobs } |
