diff options
| -rw-r--r-- | README.md | 7 | ||||
| -rwxr-xr-x | bin/shuriken | 55 | ||||
| -rw-r--r-- | docs/configuration.md | 19 | ||||
| -rw-r--r-- | docs/generation.md | 2 | ||||
| -rw-r--r-- | src/lib/album-photo-select.source.sh | 55 | ||||
| -rwxr-xr-x | tests/cli.sh | 98 |
6 files changed, 162 insertions, 74 deletions
@@ -123,9 +123,10 @@ Feature toggles at a glance: repeatable. * **Chronological order** (`CHRONOLOGICAL_ORDER=no`, the default): set to `yes` (or pass `--chronological`) to order the main album's photos by EXIF - date taken instead of filename/shuffle order, falling back to source mtime - for photos with no usable EXIF date. Takes precedence over `SHUFFLE` when - both are enabled. + date taken instead of filename/shuffle order (years before 2001 are + treated as an implausible camera clock-reset default, not a real date), + falling back to filename order for photos with no usable EXIF date. Takes + precedence over `SHUFFLE` when both are enabled. ## Documentation diff --git a/bin/shuriken b/bin/shuriken index fd7eda1..627de13 100755 --- a/bin/shuriken +++ b/bin/shuriken @@ -3540,7 +3540,8 @@ build_preview_thumbnail() { # Main album display order, in precedence order (task 8v0): # 1. CHRONOLOGICAL_ORDER=yes -> chronological_photo_files (EXIF date taken, -# ascending, falling back to mtime for photos without one). +# ascending, falling back to filename order for photos without a usable +# one). # 2. otherwise -> the historical maybe_shuffle path (seeded/random SHUFFLE, or # plain filename sort when SHUFFLE=no). # CHRONOLOGICAL_ORDER therefore takes precedence over SHUFFLE when both are @@ -3564,6 +3565,16 @@ album_photo_files() { | maybe_shuffle } +# Digital cameras were not commercially available before the 1990s, and a +# long tail of camera bodies -- including the Fujifilm X100V that motivated +# this constant -- silently default their clock to "2000-01-01" (or similar) +# once its battery dies, then stamp every EXIF timestamp with that bogus date +# instead of omitting it. An EXIF year below this is therefore treated as +# unusable rather than real, so a whole clock-reset camera roll does not sort +# to the very front of an otherwise correctly-dated album (see +# chronological_sort_key_for_photo). +readonly CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR=2001 + # Build the sort key chronological_photo_files uses to order one photo: a # tab-separated "<group>\t<time>\t<photo>" line consumed by a plain lexicographic # sort (see chronological_photo_files). EXIF reads always target the INCOMING_DIR @@ -3571,28 +3582,27 @@ album_photo_files() { # resized DIST_DIR copy, so ordering and tooltip/details agree about a photo's # taken time and both share the same identify cache entry. # -# group 0 when photo_date_taken (album-metadata.source.sh) found a real EXIF -# date, 1 otherwise. Group 0 always sorts before group 1, so photos -# with a genuine timestamp are never displaced by an approximate -# fallback for photos that lack one. +# group 0 when photo_date_taken (album-metadata.source.sh) found a real, +# plausible (CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR or later) EXIF date, 1 +# otherwise. Group 0 always sorts before group 1, so photos with a +# genuine timestamp are never displaced by an approximate fallback. # time the EXIF date normalized from "YYYY:MM:DD HH:MM:SS" to a 14-digit # "YYYYMMDDHHMMSS" string (colons/space just stripped -- the calendar # substrings are untouched, so this stays safe even though the EXIF # string is not `date -d`-parseable, see docs/stats-exif-audit.md) for -# group 0, or the INCOMING_DIR file's mtime (compat.source.sh $STAT, -# zero-padded so it sorts lexicographically) for group 1. Fixed width -# within each group keeps a plain sort numerically correct. -# photo final tiebreaker so photos sharing a timestamp (e.g. burst shots) or -# missing both an EXIF date and a readable mtime still sort in a -# stable, reproducible order across regenerations of the same -# incoming set. +# group 0, or the photo's own filename for group 1 (see the fallback +# comment below for why filename, not mtime). Fixed width within +# group 0 keeps a plain sort numerically correct. +# photo final tiebreaker so photos sharing a timestamp (e.g. burst shots) +# still sort in a stable, reproducible order across regenerations of +# the same incoming set. chronological_sort_key_for_photo() { local -r photo="$1"; shift local date_time - local mtime date_time=$(photo_date_taken "$photo" "$INCOMING_DIR/$photo") - if [[ "$date_time" =~ ^([0-9]{4}):([0-9]{2}):([0-9]{2})\ ([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; then + if [[ "$date_time" =~ ^([0-9]{4}):([0-9]{2}):([0-9]{2})\ ([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]] \ + && ((10#${BASH_REMATCH[1]} >= CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR)); then printf '0\t%s%s%s%s%s%s\t%s\n' \ "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" \ "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${BASH_REMATCH[6]}" \ @@ -3600,14 +3610,15 @@ chronological_sort_key_for_photo() { return fi - # No usable EXIF date-taken (missing tag or a malformed value): fall back to - # the source file's mtime so ordering still reflects "roughly when this - # photo appeared" rather than an arbitrary readdir order, and stays fully - # deterministic across runs. A missing/unreadable source file (should not - # happen; INCOMING_DIR is validated before generation) reads as mtime 0 so - # this never aborts the render. - mtime=$("$STAT" -c '%Y' "$INCOMING_DIR/$photo" 2>/dev/null) || mtime=0 - printf '1\t%020d\t%s\n' "$mtime" "$photo" + # No usable EXIF date-taken (tag missing, malformed, or an implausible + # clock-reset default): fall back to plain filename order rather than + # source mtime. mtime looked like a reasonable fallback but is not a + # reliable proxy for capture order in practice -- copying/rsyncing an + # incoming directory commonly touches every file's mtime to the transfer + # time, unrelated to when it was actually shot -- whereas sequential + # camera filenames (e.g. Fujifilm's DSCFnnnn shutter-count naming) track + # real shooting order even across a clock reset. + printf '1\t%s\t%s\n' "$photo" "$photo" } # Chronological ordering for CHRONOLOGICAL_ORDER=yes: every photo in diff --git a/docs/configuration.md b/docs/configuration.md index c2261c0..171d59d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -18,7 +18,7 @@ values for the current run. | `IMAGE_JOBS` | `3` | Parallel jobs for image processing and HTML template rendering. Positive integer. | | `IMAGEMAGICK_TIMEOUT` | `60` | Per-ImageMagick-command timeout in seconds. Positive integer. | | `TAR_TIMEOUT` | `120` | Tarball creation timeout in seconds. Positive integer. | -| `CHRONOLOGICAL_ORDER` | `no` | Order the main album's photos chronologically by EXIF date taken (ascending), falling back to source mtime when a photo has no usable EXIF date. `yes`/`no`. Takes precedence over `SHUFFLE` when both are set. See "Photo ordering" below. | +| `CHRONOLOGICAL_ORDER` | `no` | Order the main album's photos chronologically by EXIF date taken (ascending; years before 2001 are treated as an implausible camera clock-reset default, not a real date), falling back to filename order when a photo has no usable EXIF date. `yes`/`no`. Takes precedence over `SHUFFLE` when both are set. See "Photo ordering" below. | | `SHUFFLE` | `no` | Randomly shuffle all previews. `yes`/`no`. Ignored when `CHRONOLOGICAL_ORDER=yes`. | | `SPLASH_PAGE` | `yes` | Generate a splash landing page at `index.html`. `yes`/`no`. | | `DETAILS_PAGE` | `yes` | Generate each photo's `*-details.html` page (and its "Details" link). `yes`/`no`. See "Details pages" below. | @@ -74,11 +74,18 @@ for reproducibility). Set `CHRONOLOGICAL_ORDER=yes` (or pass `--chronological`) to instead order them chronologically by EXIF date taken (ascending), reusing the same `DateTimeOriginal` -> `DateTimeDigitized` -> `DateTime` tag fallback chain, and the same per-photo EXIF cache, as the "Taken:" tooltip field and the -details page. A photo with none of those three tags falls back to its source -file's modification time, so ordering is always fully deterministic and never -crashes on EXIF-less photos (screenshots, downloaded images, ...); photos with -a real EXIF date always sort before mtime-fallback photos, so an approximate -fallback never displaces a genuine timestamp. +details page. An EXIF date is only trusted from year 2001 onward: many camera +bodies default their clock to a `2000-01-01`-ish date once the battery dies +and stamp every timestamp with that bogus value instead of omitting it, so an +older year is treated the same as a missing tag rather than sorting a whole +clock-reset camera roll to the front of the album. A photo with no usable EXIF +date falls back to plain filename order (not modification time -- copying or +rsyncing an incoming directory commonly rewrites every file's mtime to the +transfer time, unrelated to capture order), so ordering is always fully +deterministic and never crashes on EXIF-less photos (screenshots, downloaded +images, ...); photos with a real, plausible EXIF date always sort before +filename-fallback photos, so an approximate fallback never displaces a +genuine timestamp. **`CHRONOLOGICAL_ORDER` takes precedence over `SHUFFLE`** when both are set to `yes`: a chronological album is meant to read as a timeline, so an enabled diff --git a/docs/generation.md b/docs/generation.md index 3941800..2d88d20 100644 --- a/docs/generation.md +++ b/docs/generation.md @@ -92,7 +92,7 @@ seed and inputs to produce the same HTML. `CHRONOLOGICAL_ORDER=yes` (see "Photo ordering" in [configuration.md](configuration.md)) is always deterministic regardless of -`RANDOM_SEED`: it orders by each photo's EXIF date taken (with a source-mtime +`RANDOM_SEED`: it orders by each photo's EXIF date taken (with a filename fallback), so it needs no seed to repeat, and it takes precedence over `SHUFFLE` when both are enabled. diff --git a/src/lib/album-photo-select.source.sh b/src/lib/album-photo-select.source.sh index 283e17b..cae4f9f 100644 --- a/src/lib/album-photo-select.source.sh +++ b/src/lib/album-photo-select.source.sh @@ -12,7 +12,8 @@ # Main album display order, in precedence order (task 8v0): # 1. CHRONOLOGICAL_ORDER=yes -> chronological_photo_files (EXIF date taken, -# ascending, falling back to mtime for photos without one). +# ascending, falling back to filename order for photos without a usable +# one). # 2. otherwise -> the historical maybe_shuffle path (seeded/random SHUFFLE, or # plain filename sort when SHUFFLE=no). # CHRONOLOGICAL_ORDER therefore takes precedence over SHUFFLE when both are @@ -36,6 +37,16 @@ album_photo_files() { | maybe_shuffle } +# Digital cameras were not commercially available before the 1990s, and a +# long tail of camera bodies -- including the Fujifilm X100V that motivated +# this constant -- silently default their clock to "2000-01-01" (or similar) +# once its battery dies, then stamp every EXIF timestamp with that bogus date +# instead of omitting it. An EXIF year below this is therefore treated as +# unusable rather than real, so a whole clock-reset camera roll does not sort +# to the very front of an otherwise correctly-dated album (see +# chronological_sort_key_for_photo). +readonly CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR=2001 + # Build the sort key chronological_photo_files uses to order one photo: a # tab-separated "<group>\t<time>\t<photo>" line consumed by a plain lexicographic # sort (see chronological_photo_files). EXIF reads always target the INCOMING_DIR @@ -43,28 +54,27 @@ album_photo_files() { # resized DIST_DIR copy, so ordering and tooltip/details agree about a photo's # taken time and both share the same identify cache entry. # -# group 0 when photo_date_taken (album-metadata.source.sh) found a real EXIF -# date, 1 otherwise. Group 0 always sorts before group 1, so photos -# with a genuine timestamp are never displaced by an approximate -# fallback for photos that lack one. +# group 0 when photo_date_taken (album-metadata.source.sh) found a real, +# plausible (CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR or later) EXIF date, 1 +# otherwise. Group 0 always sorts before group 1, so photos with a +# genuine timestamp are never displaced by an approximate fallback. # time the EXIF date normalized from "YYYY:MM:DD HH:MM:SS" to a 14-digit # "YYYYMMDDHHMMSS" string (colons/space just stripped -- the calendar # substrings are untouched, so this stays safe even though the EXIF # string is not `date -d`-parseable, see docs/stats-exif-audit.md) for -# group 0, or the INCOMING_DIR file's mtime (compat.source.sh $STAT, -# zero-padded so it sorts lexicographically) for group 1. Fixed width -# within each group keeps a plain sort numerically correct. -# photo final tiebreaker so photos sharing a timestamp (e.g. burst shots) or -# missing both an EXIF date and a readable mtime still sort in a -# stable, reproducible order across regenerations of the same -# incoming set. +# group 0, or the photo's own filename for group 1 (see the fallback +# comment below for why filename, not mtime). Fixed width within +# group 0 keeps a plain sort numerically correct. +# photo final tiebreaker so photos sharing a timestamp (e.g. burst shots) +# still sort in a stable, reproducible order across regenerations of +# the same incoming set. chronological_sort_key_for_photo() { local -r photo="$1"; shift local date_time - local mtime date_time=$(photo_date_taken "$photo" "$INCOMING_DIR/$photo") - if [[ "$date_time" =~ ^([0-9]{4}):([0-9]{2}):([0-9]{2})\ ([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; then + if [[ "$date_time" =~ ^([0-9]{4}):([0-9]{2}):([0-9]{2})\ ([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]] \ + && ((10#${BASH_REMATCH[1]} >= CHRONOLOGICAL_MIN_PLAUSIBLE_YEAR)); then printf '0\t%s%s%s%s%s%s\t%s\n' \ "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" \ "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${BASH_REMATCH[6]}" \ @@ -72,14 +82,15 @@ chronological_sort_key_for_photo() { return fi - # No usable EXIF date-taken (missing tag or a malformed value): fall back to - # the source file's mtime so ordering still reflects "roughly when this - # photo appeared" rather than an arbitrary readdir order, and stays fully - # deterministic across runs. A missing/unreadable source file (should not - # happen; INCOMING_DIR is validated before generation) reads as mtime 0 so - # this never aborts the render. - mtime=$("$STAT" -c '%Y' "$INCOMING_DIR/$photo" 2>/dev/null) || mtime=0 - printf '1\t%020d\t%s\n' "$mtime" "$photo" + # No usable EXIF date-taken (tag missing, malformed, or an implausible + # clock-reset default): fall back to plain filename order rather than + # source mtime. mtime looked like a reasonable fallback but is not a + # reliable proxy for capture order in practice -- copying/rsyncing an + # incoming directory commonly touches every file's mtime to the transfer + # time, unrelated to when it was actually shot -- whereas sequential + # camera filenames (e.g. Fujifilm's DSCFnnnn shutter-count naming) track + # real shooting order even across a clock reset. + printf '1\t%s\t%s\n' "$photo" "$photo" } # Chronological ordering for CHRONOLOGICAL_ORDER=yes: every photo in diff --git a/tests/cli.sh b/tests/cli.sh index 0618163..a7b82b4 100755 --- a/tests/cli.sh +++ b/tests/cli.sh @@ -7279,13 +7279,14 @@ test_stats_collect_reads_cached_identify_output() { test::teardown } -# task 8v0: chronological_photo_files (album-photo-select.source.sh) orders -# photos by EXIF date taken, ascending, falling back to source mtime (and then -# filename) for photos with no usable EXIF date. Exercises the function -# directly (not a full --generate) for precise control over dates/mtimes, the -# same pattern test_stats_collect_reads_cached_identify_output above uses for -# the EXIF cache layer. -test_chronological_photo_files_sorts_by_exif_date_with_mtime_fallback() { +# task 8v0 (and its bugfix once irregular.ninja/so.war.das surfaced a real +# camera clock-reset roll): chronological_photo_files +# (album-photo-select.source.sh) orders photos by EXIF date taken, ascending, +# falling back to filename order for photos with no usable EXIF date. +# Exercises the function directly (not a full --generate) for precise control +# over dates/mtimes, the same pattern test_stats_collect_reads_cached_identify_output +# above uses for the EXIF cache layer. +test_chronological_photo_files_sorts_by_exif_date_with_filename_fallback() { local incoming_dir local dist_dir local cache_dir @@ -7306,12 +7307,14 @@ test_chronological_photo_files_sorts_by_exif_date_with_mtime_fallback() { done # a.jpg/d.jpg have no DateTimeOriginal/Digitized/DateTime tag at all (only - # an unrelated Make tag), so they must fall back to mtime. d.jpg's mtime is - # deliberately EARLIER than a.jpg's, and both are deliberately out of - # filename order, to prove the fallback is a real mtime sort rather than an - # accidental filename sort. - touch -d '2020-01-01 00:00:00' "$incoming_dir/d.jpg" - touch -d '2020-01-02 00:00:00' "$incoming_dir/a.jpg" + # an unrelated Make tag), so they must fall back to filename order. Their + # mtimes are deliberately set OUT of filename order (d.jpg newer than + # a.jpg) to prove the fallback is a real filename sort rather than an + # accidental mtime sort (mtime is not trustworthy: bulk-copying/rsyncing an + # incoming directory commonly rewrites every file's mtime to the transfer + # time regardless of capture order). + touch -d '2020-01-02 00:00:00' "$incoming_dir/d.jpg" + touch -d '2020-01-01 00:00:00' "$incoming_dir/a.jpg" # b.jpg/c.jpg have a real EXIF date, deliberately reversed relative to their # filenames (c.jpg is EARLIER than b.jpg) so a filename-sort bug would not @@ -7338,11 +7341,63 @@ test_chronological_photo_files_sorts_by_exif_date_with_mtime_fallback() { mapfile -t ordered < <(chronological_photo_files photos) - # Real EXIF dates (c.jpg 2021, b.jpg 2023) sort before the mtime fallback - # group (d.jpg 2020-01-01, a.jpg 2020-01-02): a genuine timestamp is never - # displaced by an approximate fallback, even though the fallback group's - # own dates are chronologically earlier. - test "${ordered[*]}" = 'c.jpg b.jpg d.jpg a.jpg' + # Real EXIF dates (c.jpg 2021, b.jpg 2023) sort before the filename + # fallback group (a.jpg, d.jpg): a genuine timestamp is never displaced by + # an approximate fallback. Within the fallback group, a.jpg sorts before + # d.jpg by filename despite its mtime being earlier -- proving mtime plays + # no part in the fallback order. + test "${ordered[*]}" = 'c.jpg b.jpg a.jpg d.jpg' + + test::teardown +} + +# Regression test for the real-world bug found via irregular.ninja/so.war.das: +# a Fujifilm X100V whose battery died reset its clock to 2000-01-01, and every +# EXIF timestamp on that roll (DateTimeOriginal/Digitized/DateTime, all +# identical) was stamped with that bogus date instead of omitting it. Trusting +# it at face value put an entire clock-reset roll at the very FRONT of an +# album ahead of hundreds of correctly (2020s-)dated photos -- the opposite of +# what CHRONOLOGICAL_ORDER promises. old.jpg's implausible year-2000 EXIF date +# must therefore be treated the same as a missing tag (filename fallback, group +# 1), not as a real group-0 timestamp. +test_chronological_photo_files_treats_implausible_exif_year_as_camera_clock_reset() { + local incoming_dir + local dist_dir + local cache_dir + local name + local -a ordered=() + + test::setup + test::source_shuriken_lib + + incoming_dir="$TEST_TMPDIR/incoming" + dist_dir="$TEST_TMPDIR/dist" + cache_dir="$TEST_TMPDIR/cache/exif" + mkdir -p "$incoming_dir" "$dist_dir/photos" "$cache_dir" + + for name in old.jpg real.jpg; do + printf 'fake\n' > "$incoming_dir/$name" + printf 'fake\n' > "$dist_dir/photos/$name" + done + + { + photo_cache_signature 'old.jpg' "$incoming_dir/old.jpg" + printf ' exif:DateTimeOriginal: 2000:01:01 05:50:52\n' + } > "$cache_dir/old.jpg.txt" + { + photo_cache_signature 'real.jpg' "$incoming_dir/real.jpg" + printf ' exif:DateTimeOriginal: 2026:06:07 13:19:23\n' + } > "$cache_dir/real.jpg.txt" + + export INCOMING_DIR="$incoming_dir" + export DIST_DIR="$dist_dir" + + mapfile -t ordered < <(chronological_photo_files photos) + + # real.jpg (a plausible 2026 date, group 0) sorts before old.jpg (an + # implausible 2000 clock-reset date, demoted to the group-1 filename + # fallback), even though 2000 is numerically earlier than 2026. + test "${ordered[*]}" = 'real.jpg old.jpg' test::teardown } @@ -7970,8 +8025,11 @@ main() { 'stats collect reads cached identify output' \ test_stats_collect_reads_cached_identify_output test::run_case \ - 'chronological_photo_files sorts by EXIF date with mtime fallback' \ - test_chronological_photo_files_sorts_by_exif_date_with_mtime_fallback + 'chronological_photo_files sorts by EXIF date with filename fallback' \ + test_chronological_photo_files_sorts_by_exif_date_with_filename_fallback + test::run_case \ + 'chronological_photo_files treats implausible EXIF year as camera clock reset' \ + test_chronological_photo_files_treats_implausible_exif_year_as_camera_clock_reset test::run_case \ 'album/stats decoupling boundary (pn0)' \ test_album_stats_decoupling_boundary |