diff options
24 files changed, 1240 insertions, 0 deletions
diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b47fb27 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +NAME=staticfarm-apache-handlers +all: version documentation build +build: +install: + test ! -d $(DESTDIR)/usr/share/staticfarm/apache/handlers/StaticFarm && mkdir -p $(DESTDIR)/usr/share/staticfarm/apache/handlers/StaticFarm || exit 0 + cp -R ./src/StaticFarm/* $(DESTDIR)/usr/share/staticfarm/apache/handlers/StaticFarm +deinstall: + test -d $(DESTDIR)/usr/share/staticfarm/apache/handlers && rm -r $(DESTDIR)/usr/share/staticfarm/apache/handlers || exit 0 +clean: +# Parses the version out of the Debian changelog +version: + cut -d' ' -f2 debian/changelog | head -n 1 | sed 's/(//;s/)//' > .version +# Builds the documentation into a manpage +documentation: + pod2man --release="$(NAME) $$(cat .version)" \ + --center="User Commands" ./docs/$(NAME).pod > ./docs/$(NAME).1 + pod2text ./docs/$(NAME).pod > ./docs/$(NAME).txt + cp ./docs/$(NAME).pod ./README.pod +# Build a debian package (don't sign it, modify the arguments if you want to sign it) +deb: + dpkg-buildpackage -uc -us +dch: + dch -i +release: dch version deb + git commit -a -m 'New release' + bash -c "git tag $$(cat .version)" + git push --tags + git push origin master +clean-top: + rm ../$(NAME)_*.tar.gz + rm ../$(NAME)_*.dsc + rm ../$(NAME)_*.changes + rm ../$(NAME)_*.deb diff --git a/README.pod b/README.pod new file mode 100644 index 0000000..cc338da --- /dev/null +++ b/README.pod @@ -0,0 +1,7 @@ +=head1 NAME + +staticfarm-apache-handlers + +=head1 WTF? + +This just includes a few mod_perl2 handlers to include into an Apache2. diff --git a/debian/README b/debian/README new file mode 100644 index 0000000..f23cc95 --- /dev/null +++ b/debian/README @@ -0,0 +1,7 @@ +The Debian Package staticfarm-apache-handlers +---------------------------- + +This is just a humble go project. Not sure if everything meets the debian +policy though. Alt least the resulting debian package passes a pedantic lintian + + -- Paul Buetow <paul@buetow.org> Wed, 02 Jan 2013 15:23:53 +0200 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..0c61418 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +staticfarm-apache-handlers (1.1.3) unitix-60; urgency=low + + * Re-import + + -- Paul Buetow <paul@buetow.org> Fri, 19 Apr 2013 14:44:21 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +8 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..361a332 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: staticfarm-apache-handlers +Section: utils +Priority: optional +Maintainer: Paul Buetow <paul@buetow.org> +Build-Depends: +Standards-Version: 3.9.2 +Homepage: http://github.com/rantanplan/staticfarm-apache-handlers +Vcs-Git: https://github.com/rantanplan/staticfarm-apache-handlers +Vcs-Browser: https://github.com/rantanplan/staticfarm-apache-handlers + +Package: staticfarm-apache-handlers +Architecture: all +Depends: libapache2-mod-perl2,libjson-perl,libfile-mimeinfo-perl +Description: Apache Handlers for a Static Content Farm + This are just mod_perl2 Handlers for a Static Content Farm. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..479f27a --- /dev/null +++ b/debian/copyright @@ -0,0 +1,30 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: staticfarm-apache-handlers +Source: https://github.com/rantanplan/static-content-farm + +Files: * +Copyright: 2012 Paul Buetow <paul@buetow.org> +License: GPL-3.0+ + +Files: debian/* +Copyright: 2012 Paul Buetow <paul@buetow.org> +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". + + diff --git a/debian/files b/debian/files new file mode 100644 index 0000000..5c74678 --- /dev/null +++ b/debian/files @@ -0,0 +1 @@ +staticfarm-apache-handlers_1.1.3_all.deb utils optional diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..b760bee --- /dev/null +++ b/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +%: + dh $@ diff --git a/debian/staticfarm-apache-handlers.debhelper.log b/debian/staticfarm-apache-handlers.debhelper.log new file mode 100644 index 0000000..545a50f --- /dev/null +++ b/debian/staticfarm-apache-handlers.debhelper.log @@ -0,0 +1,45 @@ +dh_auto_configure +dh_auto_build +dh_auto_test +dh_prep +dh_installdirs +dh_auto_install +dh_install +dh_installdocs +dh_installchangelogs +dh_installexamples +dh_installman +dh_installcatalogs +dh_installcron +dh_installdebconf +dh_installemacsen +dh_installifupdown +dh_installinfo +dh_pysupport +dh_installinit +dh_installmenu +dh_installmime +dh_installmodules +dh_installlogcheck +dh_installlogrotate +dh_installpam +dh_installppp +dh_installudev +dh_installwm +dh_installxfonts +dh_installgsettings +dh_bugfiles +dh_ucf +dh_lintian +dh_gconf +dh_icons +dh_perl +dh_usrlocal +dh_link +dh_compress +dh_fixperms +dh_installdeb +dh_gencontrol +dh_md5sums +dh_builddeb +dh_builddeb diff --git a/debian/staticfarm-apache-handlers.manpages b/debian/staticfarm-apache-handlers.manpages new file mode 100644 index 0000000..bcf67d4 --- /dev/null +++ b/debian/staticfarm-apache-handlers.manpages @@ -0,0 +1 @@ +docs/staticfarm-apache-handlers.1 diff --git a/debian/staticfarm-apache-handlers.substvars b/debian/staticfarm-apache-handlers.substvars new file mode 100644 index 0000000..abd3ebe --- /dev/null +++ b/debian/staticfarm-apache-handlers.substvars @@ -0,0 +1 @@ +misc:Depends= diff --git a/debian/staticfarm-apache-handlers/DEBIAN/control b/debian/staticfarm-apache-handlers/DEBIAN/control new file mode 100644 index 0000000..25b2f7f --- /dev/null +++ b/debian/staticfarm-apache-handlers/DEBIAN/control @@ -0,0 +1,11 @@ +Package: staticfarm-apache-handlers +Version: 1.1.3 +Architecture: all +Maintainer: Paul Buetow <paul@buetow.org> +Installed-Size: 62 +Depends: libapache2-mod-perl2, libjson-perl, libfile-mimeinfo-perl +Section: utils +Priority: optional +Homepage: http://github.com/rantanplan/staticfarm-apache-handlers +Description: Apache Handlers for a Static Content Farm + This are just mod_perl2 Handlers for a Static Content Farm. diff --git a/debian/staticfarm-apache-handlers/DEBIAN/md5sums b/debian/staticfarm-apache-handlers/DEBIAN/md5sums new file mode 100644 index 0000000..e34bcbe --- /dev/null +++ b/debian/staticfarm-apache-handlers/DEBIAN/md5sums @@ -0,0 +1,5 @@ +5998f5eb5bab0e50b625d884d2b31929 usr/share/doc/staticfarm-apache-handlers/changelog.gz +e375479dfe40b9fe67881f51b49bfdb4 usr/share/doc/staticfarm-apache-handlers/copyright +0b8797a0c3782136204d6e18a7e80812 usr/share/man/man1/staticfarm-apache-handlers.1.gz +ecacc92d2225456d311f4e1ab8ce64b2 usr/share/staticfarm/apache/handlers/StaticFarm/API.pm +db8852dd8765940e8e8eceac0313d49d usr/share/staticfarm/apache/handlers/StaticFarm/CacheControl.pm diff --git a/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/changelog.gz b/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/changelog.gz Binary files differnew file mode 100644 index 0000000..e5dbe83 --- /dev/null +++ b/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/changelog.gz diff --git a/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/copyright b/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/copyright new file mode 100644 index 0000000..479f27a --- /dev/null +++ b/debian/staticfarm-apache-handlers/usr/share/doc/staticfarm-apache-handlers/copyright @@ -0,0 +1,30 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: staticfarm-apache-handlers +Source: https://github.com/rantanplan/static-content-farm + +Files: * +Copyright: 2012 Paul Buetow <paul@buetow.org> +License: GPL-3.0+ + +Files: debian/* +Copyright: 2012 Paul Buetow <paul@buetow.org> +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". + + diff --git a/debian/staticfarm-apache-handlers/usr/share/man/man1/staticfarm-apache-handlers.1.gz b/debian/staticfarm-apache-handlers/usr/share/man/man1/staticfarm-apache-handlers.1.gz Binary files differnew file mode 100644 index 0000000..02ce36e --- /dev/null +++ b/debian/staticfarm-apache-handlers/usr/share/man/man1/staticfarm-apache-handlers.1.gz diff --git a/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/API.pm b/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/API.pm new file mode 100644 index 0000000..50e50ca --- /dev/null +++ b/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/API.pm @@ -0,0 +1,248 @@ +package StaticFarm::API; + +use strict; +use warnings; + +use Apache2::Const -compile => qw(:common); +use Apache2::RequestIO (); +use Apache2::RequestRec (); +use Apache2::ServerUtil; + +use ExtUtils::Command; + +use File::Path qw(remove_tree); +use JSON; +use POSIX qw(strftime ctime); + +use constant IOBUFSIZE => 8192; + +my $CONTENT_DIR = $ENV{API_CONTENT_DIR}; + +# ... now setup some serious stuff!! +my $URI_PREFIX = '/-api'; + +sub handler { + my $r = shift; + $r->content_type('application/json'); + + my $method = $r->method(); + + my $d = { + method => $method, + uri => $r->uri(), + args => $r->args(), + out => { message => "" }, + }; + + ($d->{path}) = $r->uri() =~ /^$URI_PREFIX(.*)/; + $d->{fullpath} = "$CONTENT_DIR$d->{path}"; + + my %params = map { + s/\.\.//g; + my ($k, $v) = split '=', $_; + $v //= ''; + $k => $v; + } split '&', $r->args(); + + $d->{params} = \%params; + + if ($method eq 'GET') { + handler_get($r, $d); + + } elsif ($method eq 'DELETE') { + handler_delete($r, $d); + + } elsif ($method eq 'POST') { + handler_post($r, $d); + + } elsif ($method eq 'PUT') { + handler_put($r, $d); + + } else { + handler_unknown($r, $d); + } + + return Apache2::Const::DONE; +} + +sub data_out { + my ($r, $d, $status, $message) = @_; + my $p = $d->{params}; + + $d->{out}{message} = $message if defined $message; + $d->{out}{fullpath} = $d->{fullpath}; + + $status //= 200; + $d->{out}{status} = $status; + $r->status($status); + + if (exists $p->{debug} and $p->{debug} == 1) { + for (grep !/^out$/, keys %$d) { + $d->{out}{debug}{$_} = $d->{$_}; + } + } + + print JSON->new->allow_nonref->pretty->encode($d->{out}); +} + +sub my_time { + return strftime("%Y-%m-%d %H:%M:%S", localtime(shift)); +} + +sub path_stat { + my $f = shift; + + my @stat = stat($f); + + my %data = ( + size => -s $f, + hardlinks => $stat[3], + uid => $stat[4], + gid => $stat[5], + last_access => my_time($stat[8]), + last_modified => my_time($stat[9]), + last_status_change => my_time($stat[10]), + blocks => $stat[12], + ascii => (-T $f eq '' ? 0 : 1), + is_directory => (-d $f eq '' ? 0 : 1), + is_symlink => (-l $f eq '' ? 0 : 1), + is_file => (-f $f eq '' ? 0 : 1), + ); + + return \%data; +} + +sub path_ls { + my $f = shift; + + return [ map { s#.*/##; $_ } glob("$f/*") ]; +} + +sub path_exists { + my ($r, $d) = @_; + + unless ( -e $d->{fullpath}) { + data_out($r, $d, 404, "No such file or directory: $d->{fullpath}"); + return 0; + } + + return 1; +} + +sub path_writable { + my ($r, $d) = @_; + + if (-e $d->{fullpath} && ! -w $d->{fullpath}) { + data_out($r, $d, 403, "Error: $d->{fullpath} permission denied."); + return 0; + } + + return 1; +} + +sub path_write { + my ($r, $d, $content) = @_; + + return unless path_writable($r, $d); + + if (defined $content) { + if ( -f $d->{fullpath} or ! -e $d->{fullpath} ) { + open my $fh, '>', $d->{fullpath} or do { + $d->{out}{message} = "Error: $d->{fullpath} $!"; + data_out($r, $d, 500); + return; + }; + + print $fh $content; + close $fh; + + $d->{out}{message} = "Wrote file successfully."; + } else { + data_out($r, $d, 500, "Can't put or post content like that. Destination may be a directory."); + return; + } + + } else { + system("/usr/bin/touch \"$d->{fullpath}\""); + $d->{out}{message} = "Touched file or directory successfully."; + } + + $d->{out}{stat} = path_stat($d->{fullpath}); + data_out($r, $d); +} + + +sub handler_get { + my ($r, $d) = @_; + my $p = $d->{params}; + + return unless path_exists($r, $d); + + $d->{out}{stat} = path_stat($d->{fullpath}); + $d->{out}{content} = path_ls($d->{fullpath}) + if -d $d->{fullpath} and exists $p->{ls} and $p->{ls} == 1; + + data_out($r, $d, 200); +} + +sub handler_delete { + my ($r, $d) = @_; + my $p = $d->{params}; + + return unless path_exists($r, $d); + + $d->{out}{stat} = path_stat($d->{fullpath}); + + if ( -d $d->{fullpath} ) { + if (exists $p->{iamsure} and $p->{iamsure} == 1) { + my $err; + remove_tree($d->{fullpath}, { error => \$err }); + unless (@$err) { + data_out($r, $d, 200, 'Directory deleted successfully.'); + } else { + data_out($r, $d, 500, "Directory could not deleted completely. Run ls to see what's left over."); + } + } else { + data_out($r, $d, 403, 'Not removing directory recursively. If you want to do this set iamsure=1.'); + } + + } elsif ( -f $d->{fullpath} ) { + if (unlink($d->{fullpath})) { + data_out($r, $d, 200, 'File deleted successfully.'); + } else { + data_out($r, $d, 500, "Error deleting: $d->{fullpath} $!."); + } + } +} + + +sub handler_post { + my ($r, $d) = @_; + my ($content, $buffer, $len) = ('', '', IOBUFSIZE); + + $content .= $buffer while $r->read($buffer, $len); + path_write($r, $d, $content); +} + +sub handler_put { + my ($r, $d) = @_; + my $p = $d->{params}; + + my $content = do { + if (exists $p->{content}) { + $p->{content}; + } else { + undef; + } + }; + + path_write($r, $d, $content); +} + +sub handler_unknown { + my ($r, $d) = @_; + + data_out($r, $d, 501, "Method $d->{method} is not implemented."); +} + +1; diff --git a/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/CacheControl.pm b/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/CacheControl.pm new file mode 100644 index 0000000..1525cd2 --- /dev/null +++ b/debian/staticfarm-apache-handlers/usr/share/staticfarm/apache/handlers/StaticFarm/CacheControl.pm @@ -0,0 +1,195 @@ +# (C) 2013 Paul Buetow + +package StaticFarm::CacheControl; + +use strict; +use warnings; + +use Apache2::Const -compile => qw(HTTP_OK HTTP_NO_CONTENT HTTP_NOT_FOUND); +use Apache2::Log; +use Apache2::RequestIO; +use Apache2::RequestRec; +use Apache2::Response; +use Apache2::ServerUtil; +use APR::Table; + +use File::Basename; +use File::Copy qw(move); +use File::MimeInfo; +use File::Path qw(make_path); +use LWP::Simple qw($ua getstore); + +my $FETCH_FALLBACK_ENABLE = $ENV{CACHECONTROL_FETCH_FALLBACK_ENABLE}; +my $FETCH_FALLBACK_HOSTSDIR = $ENV{CACHECONTROL_FETCH_FALLBACK_HOSTSDIR}; +my $FETCH_MW_HA_HOST = $ENV{CACHECONTROL_FETCH_MW_HA_HOST}; +my $FETCH_PROTO = $ENV{CACHECONTROL_FETCH_PROTO}; +my $FETCH_REASK_AFTER = $ENV{CACHECONTROL_FETCH_REASK_AFTER}; +my $FETCH_TIMEOUT = $ENV{CACHECONTROL_FETCH_TIMEOUT}; +my $FETCH_MAX_LIMIT = $ENV{CACHECONTROL_FETCH_MAX_LIMIT}; +my $FETCH_MAX_INTERVAL = $ENV{CACHECONTROL_FETCH_MAX_INTERVAL}; +my $VERBOSE = $ENV{CACHECONTROL_WARN_VERBOSE}; + +# ... now setup some serious stuff!! +my $SERVER_ROOT = Apache2::ServerUtil::server_root(); +my $DOCUMENT_ROOT = "$SERVER_ROOT/htdocs"; +my $RUN_DIR = "$SERVER_ROOT/run"; +my $STATIC_ROOT = "$DOCUMENT_ROOT/static"; +my $DOT_RE = qr/\.\./; +my $QRY_RE = qr/\?.*/; +my $IGNORE_RE = qr/favicon.ico/; + +# TMP_DIR is in DOCUMENT_ROOT due FS performance issue (must be on same partition) +my $TMP_DIR = "$RUN_DIR/cachetmp"; + +my %NOT_FOUND; +my $FETCH_MAX_COUNTER = 0; +my $FETCH_MAX_TIME = 0; + +sub my_warn { + my $msg = shift; + + Apache2::ServerRec::warn("CacheControl: $msg"); +} + +sub my_response { + my ($r, $what, $msg) = @_; + + $r->custom_response($what, "<body><html>$msg</html></body>"); + + return $what; +} + +sub my_getstore { + my ($url, $tmp_file) = @_; + + my_warn("Fetching $url -> $tmp_file with timeout $FETCH_TIMEOUT") if $VERBOSE == 1; + + $ua->timeout($FETCH_TIMEOUT); + my $http_code = getstore($url, $tmp_file); + + if ($http_code >= 301) { + unlink $tmp_file if -f $tmp_file; + + my_warn("Document $url not fetchable (HTTP status is $http_code)"); + } + + return $http_code; +} + +sub handler { + my $r = shift; + + return fetch_file($r); +} + +sub get_fallback_mw_hosts { + opendir my $dh, $FETCH_FALLBACK_HOSTSDIR or return (); + + my @fallbacks; + while (my $d = readdir($dh)) { + next if $d =~ /^\./; + push @fallbacks, $d; + } + close $dh; + + return @fallbacks; +} +sub fetch_file { + my $r = shift; + + unless (-e $STATIC_ROOT) { + my_warn("Static root $STATIC_ROOT does not exist."); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + } + + my $request_uri = $ENV{REQUEST_URI}; + $request_uri =~ s/$DOT_RE//g; + $request_uri =~ s/$QRY_RE//; + + my $mw_url = "$FETCH_PROTO://$FETCH_MW_HA_HOST/static/$ENV{SERVER_NAME}"; + my $file = "$STATIC_ROOT/$ENV{SERVER_NAME}$request_uri"; + my $basename = basename($file); + my $tmp_file = "$TMP_DIR/$basename"; + + if ($request_uri =~ $IGNORE_RE) { + my_warn("Ignoring $file, don't try to fetch from MW"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + } + + $r->uri($request_uri); + + unless (-e $TMP_DIR) { + my_warn("Creating directory $TMP_DIR") if $VERBOSE == 1; + make_path($TMP_DIR); + } + + my $now = time(); + # Prevent DOS attacks against the middleware server + if (++$FETCH_MAX_COUNTER > $FETCH_MAX_LIMIT) { + if ($now - $FETCH_MAX_TIME > $FETCH_MAX_INTERVAL) { + $FETCH_MAX_COUNTER = 1; + $FETCH_MAX_TIME= $now; + } else { + my_warn("Don't try to fetch $request_uri from mw, because in FETCH_MAX_INTERVAL=$FETCH_MAX_INTERVAL seconds we had already $FETCH_MAX_COUNTER tries but FETCH_MAX_LIMIT=$FETCH_MAX_LIMIT seconds"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + } + } + + if ($FETCH_REASK_AFTER != 0 && exists $NOT_FOUND{$request_uri}) { + my $last_access = $now - $NOT_FOUND{$request_uri}; + if ($last_access < $FETCH_REASK_AFTER) { + my_warn("Don't try to fetch $request_uri from mw, because you can ask for this file only 1 time within FETCH_REASK_AFTER=$FETCH_REASK_AFTER seconds"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + } else { + delete $NOT_FOUND{$request_uri}; + } + } + + my $url = "$FETCH_PROTO://$FETCH_MW_HA_HOST/static/$ENV{SERVER_NAME}/$request_uri"; + my $http_code = my_getstore($url, $tmp_file); + + if ($http_code >= 500 && $FETCH_FALLBACK_ENABLE == 1) { + # The staticmw ha address (FETCH_MW_HA_HOST) is not reachable or broken, try fallback MW hosts + for (get_fallback_mw_hosts()) { + $url = "$FETCH_PROTO://$_/static/$ENV{SERVER_NAME}/$request_uri"; + $http_code = my_getstore($url, $tmp_file); + last if $http_code < 400; + } + } + + if ($http_code >= 301) { + $NOT_FOUND{$request_uri} = time() if $FETCH_REASK_AFTER != 0; + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + + } else { + my $dirname = dirname($file); + + unless (-d $dirname) { + my_warn("Creating directory $dirname") if $VERBOSE == 1; + make_path($dirname); + } + + my_warn("Moving $tmp_file -> $file") if $VERBOSE == 1; + + unless (move($tmp_file, $file)) { + my_warn("Could not move file $tmp_file -> $file: $!"); + return Apache2::Const::HTTP_NO_CONTENT; + } + + open my $fh, $file or do { + my_warn("Could not open file $file: $!"); + return Apache2::Const::HTTP_NO_CONTENT; + }; + + $r->content_type(mimetype($file)); + print while <$fh>; + close $fh; + + return Apache2::Const::OK; + } +} + +1; diff --git a/docs/staticfarm-apache-handlers.1 b/docs/staticfarm-apache-handlers.1 new file mode 100644 index 0000000..1ef0ebd --- /dev/null +++ b/docs/staticfarm-apache-handlers.1 @@ -0,0 +1,136 @@ +.\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.16) +.\" +.\" Standard preamble: +.\" ======================================================================== +.de Sp \" Vertical space (when we can't use .PP) +.if t .sp .5v +.if n .sp +.. +.de Vb \" Begin verbatim text +.ft CW +.nf +.ne \\$1 +.. +.de Ve \" End verbatim text +.ft R +.fi +.. +.\" Set up some character translations and predefined strings. \*(-- will +.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left +.\" double quote, and \*(R" will give a right double quote. \*(C+ will +.\" give a nicer C++. Capital omega is used to do unbreakable dashes and +.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, +.\" nothing in troff, for use with C<>. +.tr \(*W- +.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' +.ie n \{\ +. ds -- \(*W- +. ds PI pi +. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch +. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch +. ds L" "" +. ds R" "" +. ds C` "" +. ds C' "" +'br\} +.el\{\ +. ds -- \|\(em\| +. ds PI \(*p +. ds L" `` +. ds R" '' +'br\} +.\" +.\" Escape single quotes in literal strings from groff's Unicode transform. +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" +.\" If the F register is turned on, we'll generate index entries on stderr for +.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index +.\" entries marked with X<> in POD. Of course, you'll have to process the +.\" output yourself in some meaningful fashion. +.ie \nF \{\ +. de IX +. tm Index:\\$1\t\\n%\t"\\$2" +.. +. nr % 0 +. rr F +.\} +.el \{\ +. de IX +.. +.\} +.\" +.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). +.\" Fear. Run. Save yourself. No user-serviceable parts. +. \" fudge factors for nroff and troff +.if n \{\ +. ds #H 0 +. ds #V .8m +. ds #F .3m +. ds #[ \f1 +. ds #] \fP +.\} +.if t \{\ +. ds #H ((1u-(\\\\n(.fu%2u))*.13m) +. ds #V .6m +. ds #F 0 +. ds #[ \& +. ds #] \& +.\} +. \" simple accents for nroff and troff +.if n \{\ +. ds ' \& +. ds ` \& +. ds ^ \& +. ds , \& +. ds ~ ~ +. ds / +.\} +.if t \{\ +. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" +. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' +. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' +. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' +. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' +. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' +.\} +. \" troff and (daisy-wheel) nroff accents +.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' +.ds 8 \h'\*(#H'\(*b\h'-\*(#H' +.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] +.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' +.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' +.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] +.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] +.ds ae a\h'-(\w'a'u*4/10)'e +.ds Ae A\h'-(\w'A'u*4/10)'E +. \" corrections for vroff +.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' +.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' +. \" for low resolution devices (crt and lpr) +.if \n(.H>23 .if \n(.V>19 \ +\{\ +. ds : e +. ds 8 ss +. ds o a +. ds d- d\h'-1'\(ga +. ds D- D\h'-1'\(hy +. ds th \o'bp' +. ds Th \o'LP' +. ds ae ae +. ds Ae AE +.\} +.rm #[ #] #H #V #F C +.\" ======================================================================== +.\" +.IX Title "STATICFARM-APACHE-HANDLERS 1" +.TH STATICFARM-APACHE-HANDLERS 1 "2015-01-02" "staticfarm-apache-handlers 1.1.3" "User Commands" +.\" For nroff, turn off justification. Always turn off hyphenation; it makes +.\" way too many mistakes in technical documents. +.if n .ad l +.nh +.SH "NAME" +staticfarm\-apache\-handlers +.SH "WTF?" +.IX Header "WTF?" +This just includes a few mod_perl2 handlers to include into an Apache2. diff --git a/docs/staticfarm-apache-handlers.pod b/docs/staticfarm-apache-handlers.pod new file mode 100644 index 0000000..cc338da --- /dev/null +++ b/docs/staticfarm-apache-handlers.pod @@ -0,0 +1,7 @@ +=head1 NAME + +staticfarm-apache-handlers + +=head1 WTF? + +This just includes a few mod_perl2 handlers to include into an Apache2. diff --git a/docs/staticfarm-apache-handlers.txt b/docs/staticfarm-apache-handlers.txt new file mode 100644 index 0000000..b2295d3 --- /dev/null +++ b/docs/staticfarm-apache-handlers.txt @@ -0,0 +1,6 @@ +NAME + staticfarm-apache-handlers + +WTF? + This just includes a few mod_perl2 handlers to include into an Apache2. + diff --git a/src/StaticFarm/API.pm b/src/StaticFarm/API.pm new file mode 100644 index 0000000..50e50ca --- /dev/null +++ b/src/StaticFarm/API.pm @@ -0,0 +1,248 @@ +package StaticFarm::API; + +use strict; +use warnings; + +use Apache2::Const -compile => qw(:common); +use Apache2::RequestIO (); +use Apache2::RequestRec (); +use Apache2::ServerUtil; + +use ExtUtils::Command; + +use File::Path qw(remove_tree); +use JSON; +use POSIX qw(strftime ctime); + +use constant IOBUFSIZE => 8192; + +my $CONTENT_DIR = $ENV{API_CONTENT_DIR}; + +# ... now setup some serious stuff!! +my $URI_PREFIX = '/-api'; + +sub handler { + my $r = shift; + $r->content_type('application/json'); + + my $method = $r->method(); + + my $d = { + method => $method, + uri => $r->uri(), + args => $r->args(), + out => { message => "" }, + }; + + ($d->{path}) = $r->uri() =~ /^$URI_PREFIX(.*)/; + $d->{fullpath} = "$CONTENT_DIR$d->{path}"; + + my %params = map { + s/\.\.//g; + my ($k, $v) = split '=', $_; + $v //= ''; + $k => $v; + } split '&', $r->args(); + + $d->{params} = \%params; + + if ($method eq 'GET') { + handler_get($r, $d); + + } elsif ($method eq 'DELETE') { + handler_delete($r, $d); + + } elsif ($method eq 'POST') { + handler_post($r, $d); + + } elsif ($method eq 'PUT') { + handler_put($r, $d); + + } else { + handler_unknown($r, $d); + } + + return Apache2::Const::DONE; +} + +sub data_out { + my ($r, $d, $status, $message) = @_; + my $p = $d->{params}; + + $d->{out}{message} = $message if defined $message; + $d->{out}{fullpath} = $d->{fullpath}; + + $status //= 200; + $d->{out}{status} = $status; + $r->status($status); + + if (exists $p->{debug} and $p->{debug} == 1) { + for (grep !/^out$/, keys %$d) { + $d->{out}{debug}{$_} = $d->{$_}; + } + } + + print JSON->new->allow_nonref->pretty->encode($d->{out}); +} + +sub my_time { + return strftime("%Y-%m-%d %H:%M:%S", localtime(shift)); +} + +sub path_stat { + my $f = shift; + + my @stat = stat($f); + + my %data = ( + size => -s $f, + hardlinks => $stat[3], + uid => $stat[4], + gid => $stat[5], + last_access => my_time($stat[8]), + last_modified => my_time($stat[9]), + last_status_change => my_time($stat[10]), + blocks => $stat[12], + ascii => (-T $f eq '' ? 0 : 1), + is_directory => (-d $f eq '' ? 0 : 1), + is_symlink => (-l $f eq '' ? 0 : 1), + is_file => (-f $f eq '' ? 0 : 1), + ); + + return \%data; +} + +sub path_ls { + my $f = shift; + + return [ map { s#.*/##; $_ } glob("$f/*") ]; +} + +sub path_exists { + my ($r, $d) = @_; + + unless ( -e $d->{fullpath}) { + data_out($r, $d, 404, "No such file or directory: $d->{fullpath}"); + return 0; + } + + return 1; +} + +sub path_writable { + my ($r, $d) = @_; + + if (-e $d->{fullpath} && ! -w $d->{fullpath}) { + data_out($r, $d, 403, "Error: $d->{fullpath} permission denied."); + return 0; + } + + return 1; +} + +sub path_write { + my ($r, $d, $content) = @_; + + return unless path_writable($r, $d); + + if (defined $content) { + if ( -f $d->{fullpath} or ! -e $d->{fullpath} ) { + open my $fh, '>', $d->{fullpath} or do { + $d->{out}{message} = "Error: $d->{fullpath} $!"; + data_out($r, $d, 500); + return; + }; + + print $fh $content; + close $fh; + + $d->{out}{message} = "Wrote file successfully."; + } else { + data_out($r, $d, 500, "Can't put or post content like that. Destination may be a directory."); + return; + } + + } else { + system("/usr/bin/touch \"$d->{fullpath}\""); + $d->{out}{message} = "Touched file or directory successfully."; + } + + $d->{out}{stat} = path_stat($d->{fullpath}); + data_out($r, $d); +} + + +sub handler_get { + my ($r, $d) = @_; + my $p = $d->{params}; + + return unless path_exists($r, $d); + + $d->{out}{stat} = path_stat($d->{fullpath}); + $d->{out}{content} = path_ls($d->{fullpath}) + if -d $d->{fullpath} and exists $p->{ls} and $p->{ls} == 1; + + data_out($r, $d, 200); +} + +sub handler_delete { + my ($r, $d) = @_; + my $p = $d->{params}; + + return unless path_exists($r, $d); + + $d->{out}{stat} = path_stat($d->{fullpath}); + + if ( -d $d->{fullpath} ) { + if (exists $p->{iamsure} and $p->{iamsure} == 1) { + my $err; + remove_tree($d->{fullpath}, { error => \$err }); + unless (@$err) { + data_out($r, $d, 200, 'Directory deleted successfully.'); + } else { + data_out($r, $d, 500, "Directory could not deleted completely. Run ls to see what's left over."); + } + } else { + data_out($r, $d, 403, 'Not removing directory recursively. If you want to do this set iamsure=1.'); + } + + } elsif ( -f $d->{fullpath} ) { + if (unlink($d->{fullpath})) { + data_out($r, $d, 200, 'File deleted successfully.'); + } else { + data_out($r, $d, 500, "Error deleting: $d->{fullpath} $!."); + } + } +} + + +sub handler_post { + my ($r, $d) = @_; + my ($content, $buffer, $len) = ('', '', IOBUFSIZE); + + $content .= $buffer while $r->read($buffer, $len); + path_write($r, $d, $content); +} + +sub handler_put { + my ($r, $d) = @_; + my $p = $d->{params}; + + my $content = do { + if (exists $p->{content}) { + $p->{content}; + } else { + undef; + } + }; + + path_write($r, $d, $content); +} + +sub handler_unknown { + my ($r, $d) = @_; + + data_out($r, $d, 501, "Method $d->{method} is not implemented."); +} + +1; diff --git a/src/StaticFarm/CacheControl.pm b/src/StaticFarm/CacheControl.pm new file mode 100644 index 0000000..1525cd2 --- /dev/null +++ b/src/StaticFarm/CacheControl.pm @@ -0,0 +1,195 @@ +# (C) 2013 Paul Buetow + +package StaticFarm::CacheControl; + +use strict; +use warnings; + +use Apache2::Const -compile => qw(HTTP_OK HTTP_NO_CONTENT HTTP_NOT_FOUND); +use Apache2::Log; +use Apache2::RequestIO; +use Apache2::RequestRec; +use Apache2::Response; +use Apache2::ServerUtil; +use APR::Table; + +use File::Basename; +use File::Copy qw(move); +use File::MimeInfo; +use File::Path qw(make_path); +use LWP::Simple qw($ua getstore); + +my $FETCH_FALLBACK_ENABLE = $ENV{CACHECONTROL_FETCH_FALLBACK_ENABLE}; +my $FETCH_FALLBACK_HOSTSDIR = $ENV{CACHECONTROL_FETCH_FALLBACK_HOSTSDIR}; +my $FETCH_MW_HA_HOST = $ENV{CACHECONTROL_FETCH_MW_HA_HOST}; +my $FETCH_PROTO = $ENV{CACHECONTROL_FETCH_PROTO}; +my $FETCH_REASK_AFTER = $ENV{CACHECONTROL_FETCH_REASK_AFTER}; +my $FETCH_TIMEOUT = $ENV{CACHECONTROL_FETCH_TIMEOUT}; +my $FETCH_MAX_LIMIT = $ENV{CACHECONTROL_FETCH_MAX_LIMIT}; +my $FETCH_MAX_INTERVAL = $ENV{CACHECONTROL_FETCH_MAX_INTERVAL}; +my $VERBOSE = $ENV{CACHECONTROL_WARN_VERBOSE}; + +# ... now setup some serious stuff!! +my $SERVER_ROOT = Apache2::ServerUtil::server_root(); +my $DOCUMENT_ROOT = "$SERVER_ROOT/htdocs"; +my $RUN_DIR = "$SERVER_ROOT/run"; +my $STATIC_ROOT = "$DOCUMENT_ROOT/static"; +my $DOT_RE = qr/\.\./; +my $QRY_RE = qr/\?.*/; +my $IGNORE_RE = qr/favicon.ico/; + +# TMP_DIR is in DOCUMENT_ROOT due FS performance issue (must be on same partition) +my $TMP_DIR = "$RUN_DIR/cachetmp"; + +my %NOT_FOUND; +my $FETCH_MAX_COUNTER = 0; +my $FETCH_MAX_TIME = 0; + +sub my_warn { + my $msg = shift; + + Apache2::ServerRec::warn("CacheControl: $msg"); +} + +sub my_response { + my ($r, $what, $msg) = @_; + + $r->custom_response($what, "<body><html>$msg</html></body>"); + + return $what; +} + +sub my_getstore { + my ($url, $tmp_file) = @_; + + my_warn("Fetching $url -> $tmp_file with timeout $FETCH_TIMEOUT") if $VERBOSE == 1; + + $ua->timeout($FETCH_TIMEOUT); + my $http_code = getstore($url, $tmp_file); + + if ($http_code >= 301) { + unlink $tmp_file if -f $tmp_file; + + my_warn("Document $url not fetchable (HTTP status is $http_code)"); + } + + return $http_code; +} + +sub handler { + my $r = shift; + + return fetch_file($r); +} + +sub get_fallback_mw_hosts { + opendir my $dh, $FETCH_FALLBACK_HOSTSDIR or return (); + + my @fallbacks; + while (my $d = readdir($dh)) { + next if $d =~ /^\./; + push @fallbacks, $d; + } + close $dh; + + return @fallbacks; +} +sub fetch_file { + my $r = shift; + + unless (-e $STATIC_ROOT) { + my_warn("Static root $STATIC_ROOT does not exist."); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + } + + my $request_uri = $ENV{REQUEST_URI}; + $request_uri =~ s/$DOT_RE//g; + $request_uri =~ s/$QRY_RE//; + + my $mw_url = "$FETCH_PROTO://$FETCH_MW_HA_HOST/static/$ENV{SERVER_NAME}"; + my $file = "$STATIC_ROOT/$ENV{SERVER_NAME}$request_uri"; + my $basename = basename($file); + my $tmp_file = "$TMP_DIR/$basename"; + + if ($request_uri =~ $IGNORE_RE) { + my_warn("Ignoring $file, don't try to fetch from MW"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + } + + $r->uri($request_uri); + + unless (-e $TMP_DIR) { + my_warn("Creating directory $TMP_DIR") if $VERBOSE == 1; + make_path($TMP_DIR); + } + + my $now = time(); + # Prevent DOS attacks against the middleware server + if (++$FETCH_MAX_COUNTER > $FETCH_MAX_LIMIT) { + if ($now - $FETCH_MAX_TIME > $FETCH_MAX_INTERVAL) { + $FETCH_MAX_COUNTER = 1; + $FETCH_MAX_TIME= $now; + } else { + my_warn("Don't try to fetch $request_uri from mw, because in FETCH_MAX_INTERVAL=$FETCH_MAX_INTERVAL seconds we had already $FETCH_MAX_COUNTER tries but FETCH_MAX_LIMIT=$FETCH_MAX_LIMIT seconds"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + } + } + + if ($FETCH_REASK_AFTER != 0 && exists $NOT_FOUND{$request_uri}) { + my $last_access = $now - $NOT_FOUND{$request_uri}; + if ($last_access < $FETCH_REASK_AFTER) { + my_warn("Don't try to fetch $request_uri from mw, because you can ask for this file only 1 time within FETCH_REASK_AFTER=$FETCH_REASK_AFTER seconds"); + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + } else { + delete $NOT_FOUND{$request_uri}; + } + } + + my $url = "$FETCH_PROTO://$FETCH_MW_HA_HOST/static/$ENV{SERVER_NAME}/$request_uri"; + my $http_code = my_getstore($url, $tmp_file); + + if ($http_code >= 500 && $FETCH_FALLBACK_ENABLE == 1) { + # The staticmw ha address (FETCH_MW_HA_HOST) is not reachable or broken, try fallback MW hosts + for (get_fallback_mw_hosts()) { + $url = "$FETCH_PROTO://$_/static/$ENV{SERVER_NAME}/$request_uri"; + $http_code = my_getstore($url, $tmp_file); + last if $http_code < 400; + } + } + + if ($http_code >= 301) { + $NOT_FOUND{$request_uri} = time() if $FETCH_REASK_AFTER != 0; + return my_response($r, Apache2::Const::HTTP_NOT_FOUND, "File not found!"); + #return Apache2::Const::HTTP_NOT_FOUND; + + } else { + my $dirname = dirname($file); + + unless (-d $dirname) { + my_warn("Creating directory $dirname") if $VERBOSE == 1; + make_path($dirname); + } + + my_warn("Moving $tmp_file -> $file") if $VERBOSE == 1; + + unless (move($tmp_file, $file)) { + my_warn("Could not move file $tmp_file -> $file: $!"); + return Apache2::Const::HTTP_NO_CONTENT; + } + + open my $fh, $file or do { + my_warn("Could not open file $file: $!"); + return Apache2::Const::HTTP_NO_CONTENT; + }; + + $r->content_type(mimetype($file)); + print while <$fh>; + close $fh; + + return Apache2::Const::OK; + } +} + +1; |