From 48e4ffe00e473629445ba52e93186dfa8f93c121 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 3 Jul 2022 09:17:56 +0300 Subject: add cache to speed up atom feed generation --- lib/atomfeed.source.sh | 47 ++++++++++++++++++++++++++++++++++++++++------- lib/gemfeed.source.sh | 4 +--- 2 files changed, 41 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/atomfeed.source.sh b/lib/atomfeed.source.sh index 716ef5c..391842a 100644 --- a/lib/atomfeed.source.sh +++ b/lib/atomfeed.source.sh @@ -32,19 +32,51 @@ META cat "$meta_file" } -# Retrieve the core content as XHTML of the blog post. -atomfeed::content () { +atomfeed::_from_cache () { local -r gmi_file_path="$1"; shift - log VERBOSE "Retrieving feed content from $gmi_file_path" + local -r cache_file_path="$1"; shift + + if [ ! -f "${cache_file_path}.info" ]; then + # No cache there. + return 1 + elif ! diff "${cache_file_path}.info" <(ls -l "$gmi_file_path"); then + # Need to refresh the cache. + return 1 + fi + + log VERBOSE "Retrieving feed content for $gmi_file_path from $cache_file_path" + cat "$cache_file_path" +} + +atomfeed::_make_cache () { + local -r gmi_file_path="$1"; shift + local -r cache_file_path="$1"; shift + + log VERBOSE "Making feed content cache from $gmi_file_path" + + local -r cache_file_dir="$(dirname "$cache_file_path")" + if [ ! -d "$cache_file_dir" ]; then + mkdir -p "$cache_file_dir" + fi # sed: Remove all before the first header # sed: Make HTML links absolute, Atom relative URLs feature seems a mess # across different Atom clients. html::fromgmi < <($SED '/Go back to the main site/d' "$gmi_file_path") | - $SED " - s|href=\"\./|href=\"https://$DOMAIN/gemfeed/|g; - s|src=\"\./|src=\"https://$DOMAIN/gemfeed/|g; - " + $SED "s|href=\"\./|href=\"https://$DOMAIN/gemfeed/|g; + s|src=\"\./|src=\"https://$DOMAIN/gemfeed/|g;" | + tee "$cache_file_path" + + ls -l "$gmi_file_path" > "${cache_file_path}.info" +} + +# Retrieve the core content as XHTML of the blog post. +atomfeed::content () { + local -r gmi_file_path="$1"; shift + local -r cache_file_path="${gmi_file_path/gemtext/cache}.atomcache" + + atomfeed::_from_cache "$gmi_file_path" "$cache_file_path" || + atomfeed::_make_cache "$gmi_file_path" "$cache_file_path" } # Generate an atom.xml feed file. @@ -53,6 +85,7 @@ atomfeed::generate () { local -r atom_file="$gemfeed_dir/atom.xml" local -r now=$($DATE --iso-8601=seconds) log INFO "Generating Atom feed to $atom_file" + log INFO 'This may takes a while with an empty cache....' assert::not_empty now "$now" diff --git a/lib/gemfeed.source.sh b/lib/gemfeed.source.sh index 1b2ee5c..0d9f2f9 100644 --- a/lib/gemfeed.source.sh +++ b/lib/gemfeed.source.sh @@ -3,9 +3,7 @@ gemfeed::get_posts () { local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" local -r gmi_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}-.*\.gmi$' - ls "$gemfeed_dir" | - $GREP -E "$gmi_pattern" | - sort -r + ls "$gemfeed_dir" | $GREP -E "$gmi_pattern" | sort -r } # Add the links from gemfeed/index.gmi to the main index site. -- cgit v1.2.3 From 0064164a5e5e3435cf2a7fb484be9761fc974691 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 30 Jul 2022 11:55:38 +0100 Subject: rewording and actually add filter code --- lib/generate.source.sh | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/generate.source.sh b/lib/generate.source.sh index b2dafbc..f72f84b 100644 --- a/lib/generate.source.sh +++ b/lib/generate.source.sh @@ -124,6 +124,11 @@ generate::fromgmi () { # Add content while read -r src; do + # User can specify a content filter + if test ! -z "$CONTENT_FILTER" && ! $GREP -q "$CONTENT_FILTER" <<< "$src"; then + continue + fi + num_gmi_files=$(( num_gmi_files + 1 )) log INFO "Generating output formats from $src" for format in "$@"; do -- cgit v1.2.3 From bff59497009cfc9b30530be4e6583fde9208c373 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 30 Jul 2022 12:00:05 +0100 Subject: fix HTML inline processing for list elements --- lib/html.source.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/html.source.sh b/lib/html.source.sh index c303d2e..ac604c9 100644 --- a/lib/html.source.sh +++ b/lib/html.source.sh @@ -98,7 +98,8 @@ html::fromgmi () { while IFS='' read -r line; do if [[ "$is_list" == yes ]]; then if [[ "$line" == '* '* ]]; then - echo "
  • $(html::encode "${line/\* /}")
    • " + echo "
  • $(html::encode "${line/\* /}")
    • " | + html::process_inline else is_list=no echo "" @@ -110,7 +111,7 @@ html::fromgmi () { echo "
    " is_plain=no else - html::encode "$line" + html::encode "$line" | html::process_inline fi continue fi -- cgit v1.2.3 From 34c7954565eadf7949d13e04a86c19be017ef17b Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 16 Aug 2022 07:56:41 +0100 Subject: fix for macOS --- lib/gemfeed.source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gemfeed.source.sh b/lib/gemfeed.source.sh index 0d9f2f9..aee70d3 100644 --- a/lib/gemfeed.source.sh +++ b/lib/gemfeed.source.sh @@ -24,7 +24,7 @@ gemfeed::updatemainindex () { gemfeed::_get_word_count () { local -r gmi_file="$1"; shift - sed '/^```/,/^```/d' "$gmi_file" | wc -w | cut -d' ' -f1 + $SED '/^```/,/^```/d' "$gmi_file" | wc -w } # Generate a index.gmi in the ./gemfeed subdir. -- cgit v1.2.3 From 072e1eef2e56f28a4b9c0540e5bd31a8002592d1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 28 Aug 2022 10:17:08 +0100 Subject: then there are no notes and no feeds dont process them --- lib/atomfeed.source.sh | 4 ++++ lib/gemfeed.source.sh | 7 ++++++- lib/generate.source.sh | 4 ++++ lib/notes.source.sh | 5 +++++ 4 files changed, 19 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/atomfeed.source.sh b/lib/atomfeed.source.sh index 391842a..c5e492f 100644 --- a/lib/atomfeed.source.sh +++ b/lib/atomfeed.source.sh @@ -82,6 +82,10 @@ atomfeed::content () { # Generate an atom.xml feed file. atomfeed::generate () { local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" + if [ ! -d "$gemfeed_dir" ]; then + return + fi + local -r atom_file="$gemfeed_dir/atom.xml" local -r now=$($DATE --iso-8601=seconds) log INFO "Generating Atom feed to $atom_file" diff --git a/lib/gemfeed.source.sh b/lib/gemfeed.source.sh index aee70d3..8eaeb95 100644 --- a/lib/gemfeed.source.sh +++ b/lib/gemfeed.source.sh @@ -24,12 +24,17 @@ gemfeed::updatemainindex () { gemfeed::_get_word_count () { local -r gmi_file="$1"; shift - $SED '/^```/,/^```/d' "$gmi_file" | wc -w + sed '/^```/,/^```/d' "$gmi_file" | wc -w | cut -d' ' -f1 } # Generate a index.gmi in the ./gemfeed subdir. gemfeed::generate () { local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" + if [ ! -d "$gemfeed_dir" ]; then + log INFO "Capsule without Gemfeed" + return + fi + log INFO "Generating Gemfeed index for $gemfeed_dir" cat < "$gemfeed_dir/index.gmi.tmp" diff --git a/lib/generate.source.sh b/lib/generate.source.sh index f72f84b..5392163 100644 --- a/lib/generate.source.sh +++ b/lib/generate.source.sh @@ -63,6 +63,10 @@ generate::convert_gmi_atom_to_html_atom () { return fi + if [ ! -f "$CONTENT_BASE_DIR/gemtext/gemfeed/atom.xml" ]; then + return + fi + log INFO 'Converting Gemtext Atom feed to HTML Atom feed' $SED 's|.gmi|.html|g; s|gemini://|https://|g' \ diff --git a/lib/notes.source.sh b/lib/notes.source.sh index b82631d..0f5251d 100644 --- a/lib/notes.source.sh +++ b/lib/notes.source.sh @@ -11,6 +11,11 @@ notes::_get_notes () { # Generate a index.gmi in the ./notes subdir. notes::generate () { local -r notes_dir="$CONTENT_BASE_DIR/gemtext/notes" + if [ ! -d "$notes_dir" ]; then + log INFO "Capsule without Notes section" + return + fi + log INFO "Generating Notes index for $notes_dir" cat < "$notes_dir/index.gmi.tmp" -- cgit v1.2.3 From e9044cd87daabcb83522a68732e0b96189baf854 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sat, 19 Nov 2022 10:52:17 +0000 Subject: add DRAFT feature --- lib/atomfeed.source.sh | 1 + lib/gemfeed.source.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/atomfeed.source.sh b/lib/atomfeed.source.sh index c5e492f..65afa5a 100644 --- a/lib/atomfeed.source.sh +++ b/lib/atomfeed.source.sh @@ -88,6 +88,7 @@ atomfeed::generate () { local -r atom_file="$gemfeed_dir/atom.xml" local -r now=$($DATE --iso-8601=seconds) + log INFO "Generating Atom feed to $atom_file" log INFO 'This may takes a while with an empty cache....' diff --git a/lib/gemfeed.source.sh b/lib/gemfeed.source.sh index 8eaeb95..6cb9a5a 100644 --- a/lib/gemfeed.source.sh +++ b/lib/gemfeed.source.sh @@ -3,7 +3,7 @@ gemfeed::get_posts () { local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" local -r gmi_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}-.*\.gmi$' - ls "$gemfeed_dir" | $GREP -E "$gmi_pattern" | sort -r + ls "$gemfeed_dir" | $GREP -f -v DRAFT- | $GREP -E "$gmi_pattern" | sort -r } # Add the links from gemfeed/index.gmi to the main index site. -- cgit v1.2.3 From 4bf2c17e3f647ebf004b2a1ca5e6eb682e4ad966 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 20 Nov 2022 10:22:00 +0000 Subject: add --draft switch --- lib/generate.source.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/generate.source.sh b/lib/generate.source.sh index 5392163..9a9e90b 100644 --- a/lib/generate.source.sh +++ b/lib/generate.source.sh @@ -185,3 +185,16 @@ generate::fromgmi () { done log INFO "You may want to commit all changes to version control!" } + +# Only generate draft posts +generate::draft () { + if [ ! -z "$CONTENT_FILTER" ]; then + log ERROR "ERROR, you can't set a content filter manually in draft mode" + exit 2 + fi + CONTENT_FILTER=DRAFT- + generate::fromgmi $@ + + log INFO 'For HTML preview, open in your browser:' + find $CONTENT_BASE_DIR/html -name DRAFT-\*.html +} -- cgit v1.2.3 From 4c4f379ea616eeec320ec27776c739fadf70d2da Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Sun, 20 Nov 2022 10:30:13 +0000 Subject: dont format inline in plain block --- lib/html.source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/html.source.sh b/lib/html.source.sh index ac604c9..03bbb45 100644 --- a/lib/html.source.sh +++ b/lib/html.source.sh @@ -111,7 +111,7 @@ html::fromgmi () { echo "
    " is_plain=no else - html::encode "$line" | html::process_inline + html::encode "$line" fi continue fi -- cgit v1.2.3