From 2a3fa5b8c749e631d2d7ed1bb01300f023c4b9ea Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 25 May 2021 22:10:09 +0100 Subject: rename packages to lib --- lib/assert.source.sh | 59 ++++++++++++++++++ lib/atomfeed.source.sh | 128 ++++++++++++++++++++++++++++++++++++++ lib/gemfeed.source.sh | 54 +++++++++++++++++ lib/generate.source.sh | 161 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/git.source.sh | 50 +++++++++++++++ lib/html.source.sh | 162 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/log.source.sh | 30 +++++++++ lib/md.source.sh | 63 +++++++++++++++++++ 8 files changed, 707 insertions(+) create mode 100644 lib/assert.source.sh create mode 100644 lib/atomfeed.source.sh create mode 100644 lib/gemfeed.source.sh create mode 100644 lib/generate.source.sh create mode 100644 lib/git.source.sh create mode 100644 lib/html.source.sh create mode 100644 lib/log.source.sh create mode 100644 lib/md.source.sh (limited to 'lib') diff --git a/lib/assert.source.sh b/lib/assert.source.sh new file mode 100644 index 0000000..d53a728 --- /dev/null +++ b/lib/assert.source.sh @@ -0,0 +1,59 @@ +# Unit test for whether 2 given strings equal. +assert::equals () { + local -r result="$1"; shift + local -r expected="$1"; shift + local -r callee=${FUNCNAME[1]} + + if [[ "$result" != "$expected" ]]; then + cat < "$atom_file.tmp" + + + $now + $DOMAIN feed + $SUBTITLE + + + gemini://$DOMAIN/ +ATOMHEADER + + while read -r gmi_file; do + # Load cached meta information about the post. + source <(atomfeed::meta "$gemfeed_dir/$gmi_file") + + # Get HTML content for the feed + local content="$(atomfeed::content "$gemfeed_dir/$gmi_file")" + + assert::not_empty meta_title "$meta_title" + assert::not_empty meta_date "$meta_date" + assert::not_empty meta_author "$meta_author" + assert::not_empty meta_email "$meta_email" + assert::not_empty meta_summary "$meta_summary" + assert::not_empty content "$content" + + cat <> "$atom_file.tmp" + + $meta_title + + gemini://$DOMAIN/gemfeed/$gmi_file + $meta_date + + $meta_author + $meta_email + + $meta_summary + +
+ $content +
+
+
+ATOMENTRY + done < <(gemfeed::get_posts | head -n $ATOM_MAX_ENTRIES) + + cat <> "$atom_file.tmp" +
+ATOMFOOTER + + # Delete the 3rd line of the atom feeds (global feed update timestamp) + if ! diff -u <($SED 3d "$atom_file") <($SED 3d "$atom_file.tmp"); then + log INFO 'Feed got something new!' + mv "$atom_file.tmp" "$atom_file" + git::add gemtext "$atom_file" + else + log INFO 'Nothing really new in the feed' + rm "$atom_file.tmp" + fi +} diff --git a/lib/gemfeed.source.sh b/lib/gemfeed.source.sh new file mode 100644 index 0000000..c842bb1 --- /dev/null +++ b/lib/gemfeed.source.sh @@ -0,0 +1,54 @@ +# Filter out blog posts from other files in the gemfeed dir. +gemfeed::get_posts () { + local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" + local -r gmi_pattern='^[0-9]{4}-[0-9]{2}-[0-9]{2}-.*\.gmi$' + local -r draft_pattern='\.draft\.gmi$' + + ls "$gemfeed_dir" | + $GREP -E "$gmi_pattern" | + $GREP -E -v "$draft_pattern" | + sort -r +} + +# Add the links from gemfeed/index.gmi to the main index site. +gemfeed::updatemainindex () { + local -r index_gmi="$CONTENT_BASE_DIR/gemtext/index.gmi" + local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" + + log VERBOSE "Updating $index_gmi with posts from $gemfeed_dir" + + # Remove old gemfeeds from main index + $SED '/^=> .\/gemfeed\/[0-9].* - .*/d;' "$index_gmi" > "$index_gmi.tmp" + # Add current gemfeeds to main index + $SED -n '/^=> / { s| ./| ./gemfeed/|; p; }' "$gemfeed_dir/index.gmi" >> "$index_gmi.tmp" + + mv "$index_gmi.tmp" "$index_gmi" + git::add gemtext "$index_gmi" +} + +# Generate a index.gmi in the ./gemfeed subdir. +gemfeed::generate () { + local -r gemfeed_dir="$CONTENT_BASE_DIR/gemtext/gemfeed" + log INFO "Generating Gemfeed index for $gemfeed_dir" + +cat < "$gemfeed_dir/index.gmi.tmp" +# $DOMAIN's Gemfeed + +## $SUBTITLE + +GEMFEED + + gemfeed::get_posts | while read -r gmi_file; do + # Extract first heading as post title. + local title=$($SED -n '/^# / { s/# //; p; q; }' "$gemfeed_dir/$gmi_file" | tr '"' "'") + # Extract the date from the file name. + local filename_date=$(basename "$gemfeed_dir/$gmi_file" | cut -d- -f1,2,3) + + echo "=> ./$gmi_file $filename_date - $title" >> "$gemfeed_dir/index.gmi.tmp" + done + + mv "$gemfeed_dir/index.gmi.tmp" "$gemfeed_dir/index.gmi" + git::add gemtext "$gemfeed_dir/index.gmi" + + gemfeed::updatemainindex +} diff --git a/lib/generate.source.sh b/lib/generate.source.sh new file mode 100644 index 0000000..6f042eb --- /dev/null +++ b/lib/generate.source.sh @@ -0,0 +1,161 @@ +# Generate a HTML or Markdown link from given Gemtext link. +generate::make_link () { + local -r what="$1"; shift + local -r line="${1/=> }"; shift + local link + local descr + + while read -r token; do + if [ -z "$link" ]; then + link="$token" + elif [ -z "$descr" ]; then + descr="$token" + else + descr="$descr $token" + fi + done < <(echo "$line" | tr ' ' '\n') + + if $GREP -E -q "$IMAGE_PATTERN" <<< "$link"; then + if [[ "$what" == md ]]; then + md::make_img "$link" "$descr" + else + html::make_img "$link" "$(html::encode "$descr")" + fi + return + fi + + if [[ "$what" == md ]]; then + md::make_link "$link" "$descr" + else + html::make_link "$link" "$(html::encode "$descr")" + fi +} + +# Add other docs (e.g. images, videos) from Gemtext to output format. +generate::fromgmi_add_docs () { + local -r src="$1"; shift + local -r format="$1"; shift + local -r dest=${src/gemtext/$format} + local -r dest_dir=$(dirname "$dest") + + if [[ ! -d "$dest_dir" ]]; then + mkdir -p "$dest_dir" + fi + cp "$src" "$dest" + git::add "$format" "$dest" +} + +# Remove docs from output format which aren't present in Gemtext anymore. +generate::fromgmi_cleanup_docs () { + local -r src="$1"; shift + local -r format="$1"; shift + local dest=${src/.$format/.gmi} + dest=${dest/$format/gemtext} + + if [[ ! -f "$dest" ]]; then + git::rm "$format" "$src" + fi +} + +# Convert the Gemtext Atom feed to a HTML Atom feed. +generate::convert_gmi_atom_to_html_atom () { + local -r format="$1"; shift + if [[ "$format" != html ]]; then + return + fi + + log INFO 'Converting Gemtext Atom feed to HTML Atom feed' + + $SED 's|.gmi|.html|g; s|gemini://|https://|g' \ + < $CONTENT_BASE_DIR/gemtext/gemfeed/atom.xml \ + > $CONTENT_BASE_DIR/html/gemfeed/atom.xml + + git::add "$format" "$CONTENT_BASE_DIR/html/gemfeed/atom.xml" +} + +# Internal helper function for generate::fromgmi +generate::_fromgmi () { + local -r src="$1"; shift + local -r format="$1"; shift + local dest=${src/gemtext/$format} + dest=${dest/.gmi/.$format} + local dest_dir=$(dirname "$dest") + + if [[ ! -d "$dest_dir" ]]; then + mkdir -p "$dest_dir" + fi + + if [[ "$format" == html ]]; then + cat "$HTML_HEADER" > "$dest.tmp" + html::fromgmi < "$src" >> "$dest.tmp" + cat "$HTML_FOOTER" >> "$dest.tmp" + + elif [[ "$format" == md ]]; then + md::fromgmi < "$src" >> "$dest.tmp" + fi + + local title=$($SED -n '/^# / { s/# //; p; q; }' "$src" | tr '"' "'") + if [[ -z "$title" ]]; then + title=$SUBTITLE + fi + $SED -i "s|%%TITLE%%|$title|g" "$dest.tmp" + mv "$dest.tmp" "$dest" + + git::add "$format" "$dest" +} + +# Generate a given output format from a Gemtext file. +generate::fromgmi () { + local -i num_gmi_files=0 + local -i num_doc_files=0 + + log INFO "Generating $* from Gemtext" + + while read -r src; do + num_gmi_files=$(( num_gmi_files + 1 )) + for format in "$@"; do + generate::_fromgmi "$src" "$format" + done + done < <(find "$CONTENT_BASE_DIR/gemtext" -type f -name \*.gmi) + + log INFO "Converted $num_gmi_files Gemtext files" + + # Add non-.gmi files to html dir. + log VERBOSE "Adding other docs to $*" + + while read -r src; do + num_doc_files=$(( num_doc_files + 1 )) + for format in "$@"; do + generate::fromgmi_add_docs "$src" "$format" + done + done < <(find "$CONTENT_BASE_DIR/gemtext" -type f | $GREP -E -v '(\.git.*|\.gmi|atom.xml|\.tmp)$') + + log INFO "Added $num_doc_files other documents to each of $*" + + # Add atom feed for HTML + for format in "$@"; do + generate::convert_gmi_atom_to_html_atom "$format" + done + + # Remove obsolete files from ./html/. + # Note: The _config.yml is the config file for GitHub pages (md format). + # Anoter note: The CNAME file is required by GitHub pages as well for custom domains. + for format in "$@"; do + find "$CONTENT_BASE_DIR/$format" -type f | + $GREP -E -v '(\.git.*|_config.yml|CNAME)$'| + while read -r src; do + generate::fromgmi_cleanup_docs "$src" "$format" + done + done + + if [[ -z "$GIT_COMMIT_MESSAGE" ]]; then + GIT_COMMIT_MESSAGE='Publishing new version' + fi + git::commit gemtext "$GIT_COMMIT_MESSAGE" + git::commit meta "$GIT_COMMIT_MESSAGE" + + for format in "$@"; do + git::commit "$format" "$GIT_COMMIT_MESSAGE" + log INFO "$format can be found in $CONTENT_BASE_DIR/$format now" + done +} diff --git a/lib/git.source.sh b/lib/git.source.sh new file mode 100644 index 0000000..c502b00 --- /dev/null +++ b/lib/git.source.sh @@ -0,0 +1,50 @@ +# Add a static content file to git +git::add () { + if [[ "$USE_GIT" != yes ]]; then + return + fi + + local -r content_dir="$CONTENT_BASE_DIR/$1"; shift + local file="$1"; shift + file=${file/$content_dir/.\/} + + cd "$content_dir" &>/dev/null + git add "$file" + cd - &>/dev/null +} + +# Remove a static content file from git +git::rm () { + if [[ "$USE_GIT" != yes ]]; then + return + fi + + local -r content_dir="$CONTENT_BASE_DIR/$1"; shift + local file="$1"; shift + file=${file/$content_dir/.\/} + + cd "$content_dir" &>/dev/null + git rm "$file" + cd - &>/dev/null +} + +# Commit all changes +git::commit () { + if [[ "$USE_GIT" != yes ]]; then + return + fi + + local -r content_dir="$CONTENT_BASE_DIR/$1"; shift + local -r message="$1"; shift + + cd "$content_dir" &>/dev/null + set +e + git commit -a -m "$message" + if [[ "$GIT_PUSH" == yes ]]; then + log INFO "Invoking git pull/push in $content_dir" + git pull + git push + fi + set -e + cd - &>/dev/null +} diff --git a/lib/html.source.sh b/lib/html.source.sh new file mode 100644 index 0000000..6049c2f --- /dev/null +++ b/lib/html.source.sh @@ -0,0 +1,162 @@ +# Convert special characters to their HTML codes +html::encode () { + $SED ' + s|\&|\&|g; + s|<|\<|g; + s|>|\>|g; + ' <<< "$@" +} + +# Make a HTML paragraph. +html::make_paragraph () { + local -r text="$1"; shift + + if [[ -n "$text" ]]; then + echo "

$(html::encode "$text")

" + fi +} + +# Make a HTML header. +html::make_heading () { + local -r text=$($SED -E 's/^#+ //' <<< "$1"); shift + local -r level="$1"; shift + echo "$(html::encode "$text")" +} + +# Make a HTML quotation +html::make_quote () { + local -r quote="${1/> }" + echo "

$(html::encode "$quote")

" +} + +# Make a HTML image +html::make_img () { + local link="$1"; shift + local descr="$1"; shift + + if [ -z "$descr" ]; then + echo -n "" + else + echo -n "$descr:" + echo -n "\"$descr\"" + fi + + echo "
" +} + +# Make a HTML hyperlink +html::make_link () { + local link="$1"; shift + local descr="$1"; shift + + if ! $GREP -F -q '://' <<< "$link"; then + link=${link/.gmi/.html} + fi + + if [[ -z "$descr" ]]; then + descr="$link" + fi + + echo "$descr
" +} + +# Convert Gemtext to HTML +html::fromgmi () { + local is_list=no + local is_plain=no + + while IFS='' read -r line; do + if [[ "$is_list" == yes ]]; then + if [[ "$line" == '* '* ]]; then + echo "
  • $(html::encode "${line/\* /}")
  • " + else + is_list=no + echo "" + fi + continue + + elif [[ "$is_plain" == yes ]]; then + if [[ "$line" == '```'* ]]; then + echo "" + is_plain=no + else + html::encode "$line" + fi + continue + fi + + case "$line" in + '* '*) + is_list=yes + echo "
      " + echo "
    • ${line/\* /}
    • " + ;; + '```'*) + is_plain=yes + echo "
      "
      +                ;;
      +            '# '*)
      +                html::make_heading "$line" 1
      +                ;;
      +            '## '*)
      +                html::make_heading "$line" 2
      +                ;;
      +            '### '*)
      +                html::make_heading "$line" 3
      +                ;;
      +            '> '*)
      +                html::make_quote "$line"
      +                ;;
      +            '=> '*)
      +                generate::make_link html "$line"
      +                ;;
      +            *)
      +                html::make_paragraph "$line"
      +                ;;
      +        esac
      +    done
      +}
      +
      +# Test HTML package.
      +html::test () {
      +    local line='Hello world! This is a paragraph.'
      +    assert::equals "$(html::make_paragraph "$line")" '

      Hello world! This is a paragraph.

      ' + + line='' + assert::equals "$(html::make_paragraph "$line")" '' + + line='Foo &<>& Bar!' + assert::equals "$(html::make_paragraph "$line")" '

      Foo &<>& Bar!

      ' + + line='# Header 1' + assert::equals "$(html::make_heading "$line" 1)" '

      Header 1

      ' + + line='## Header 2' + assert::equals "$(html::make_heading "$line" 2)" '

      Header 2

      ' + + line='### Header 3' + assert::equals "$(html::make_heading "$line" 3)" '

      Header 3

      ' + + line='> This is a quote' + assert::equals "$(html::make_quote "$line")" '

      This is a quote

      ' + + line='=> https://example.org' + assert::equals "$(generate::make_link html "$line")" \ + 'https://example.org
      ' + + line='=> index.html' + assert::equals "$(generate::make_link html "$line")" \ + 'index.html
      ' + + line='=> http://example.org Description of the link' + assert::equals "$(generate::make_link html "$line")" \ + 'Description of the link
      ' + + line='=> http://example.org/image.png' + assert::equals "$(generate::make_link html "$line")" \ + '
      ' + + line='=> http://example.org/image.png Image description' + assert::equals "$(generate::make_link html "$line")" \ + 'Image description:Image description
      ' +} diff --git a/lib/log.source.sh b/lib/log.source.sh new file mode 100644 index 0000000..56c6587 --- /dev/null +++ b/lib/log.source.sh @@ -0,0 +1,30 @@ +# Log a message. +log () { + local -r level="$1"; shift + local message + + for message in "$@"; do + echo "$message" + done | log::_pipe "$level" +} + +# Log a stream through a pipe. +log::pipe () { + log::_pipe "$1" +} + +# Internal log implementation. +log::_pipe () { + local -r level="$1"; shift + + if [[ "$level" == VERBOSE && -z "$LOG_VERBOSE" ]]; then + return + fi + + local -r callee=${FUNCNAME[2]} + local -r stamp=$($DATE +%Y%m%d-%H%M%S) + + while read -r line; do + echo "$level|$stamp|$callee|$line" >&2 + done +} diff --git a/lib/md.source.sh b/lib/md.source.sh new file mode 100644 index 0000000..e7bfae1 --- /dev/null +++ b/lib/md.source.sh @@ -0,0 +1,63 @@ +# Make a Markdown image. +md::make_img () { + local link="$1"; shift + local descr="$1"; shift + + if [ -z "$descr" ]; then + echo "[![$link]($link)]($link) " + else + echo "[![$descr]($link \"$descr\")]($link) " + fi +} + +# Make a Markdown hyperlink. +md::make_link () { + local link="$1"; shift + local descr="$1"; shift + + if ! $GREP -F -q '://' <<< "$link"; then + link=${link/.gmi/.md} + fi + if [[ -z "$descr" ]]; then + descr="$link" + fi + + echo "[$descr]($link) " +} + +# Convert Gemtext to Markdown. +md::fromgmi () { + while IFS='' read -r line; do + case "$line" in + '=> '*) + generate::make_link md "$line" + ;; + *) + echo "$line" + ;; + esac + done +} + +# Test the Markdown package. +md::test () { + local line='=> https://example.org' + assert::equals "$(generate::make_link md "$line")" \ + '[https://example.org](https://example.org) ' + + line='=> index.md' + assert::equals "$(generate::make_link md "$line")" \ + '[index.md](index.md) ' + + line='=> http://example.org Description of the link' + assert::equals "$(generate::make_link md "$line")" \ + '[Description of the link](http://example.org) ' + + line='=> http://example.org/image.png' + assert::equals "$(generate::make_link md "$line")" \ + '[![http://example.org/image.png](http://example.org/image.png)](http://example.org/image.png) ' + + line='=> http://example.org/image.png Image description' + assert::equals "$(generate::make_link md "$line")" \ + '[![Image description](http://example.org/image.png "Image description")](http://example.org/image.png) ' +} -- cgit v1.2.3