summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAssaf Gordon <assafgordon@gmail.com>2016-10-21 22:02:44 (GMT)
committerAssaf Gordon <assafgordon@gmail.com>2016-10-21 22:04:56 (GMT)
commit6c218bd2f2508935b9327c1504ca6cb278ebb432 (patch)
treefb8f0e275746e2d6e8778a61c91efa89c1dc7436
parent204ea5fc70be942d178c21a692f13c447fee3080 (diff)
downloadbin_scripts-6c218bd2f2508935b9327c1504ca6cb278ebb432.zip
bin_scripts-6c218bd2f2508935b9327c1504ca6cb278ebb432.tar.gz
bin_scripts-6c218bd2f2508935b9327c1504ca6cb278ebb432.tar.bz2
run-with-log: new scriptv0.8
Runs a program, captures its stdout/err into a log file with unique timestamp, and email the log upon error. Example: run-with-log -e me@example.org -p pipeline- -- \ pipeline.sh --param1 --param3 [...] See 'run-with-log -h' for more details.
-rw-r--r--.gitignore1
-rw-r--r--Makefile.am1
-rw-r--r--README1
-rw-r--r--configure.ac1
-rwxr-xr-xscripts/run-with-log.sh285
5 files changed, 289 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
index b44e3f5..9ad7f94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,7 @@ scripts/ppsx
scripts/pss
scripts/psx
scripts/rsx
+scripts/run-with-log
scripts/sort-header
scripts/sum_file_sizes
scripts/sumcol
diff --git a/Makefile.am b/Makefile.am
index ebe2b8f..5e3a26b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -30,6 +30,7 @@ bin_SCRIPTS = \
scripts/nfs_iostat \
scripts/ppsx \
scripts/pss \
+ scripts/run-with-log \
scripts/sort-header \
scripts/sum_file_sizes \
scripts/sumcol \
diff --git a/README b/README
index 33718d7..c85a947 100644
--- a/README
+++ b/README
@@ -23,6 +23,7 @@ Available scripts
* ppsx - copy user+hostname+fullpath of file/dir to clipboard.
* psx - copy fullpath of file/dir to clipboard.
* rsx - copy rsync-compatible URL of file/dir to clipboard.
+* run-with-log - run a program, log stdout/err to file, email log on errors.
* sort-header - wrapper for GNU sort, with header line support.
* sum_file_sizes - sum the size of files.
* sumcol - sum the values in a column of input file.
diff --git a/configure.ac b/configure.ac
index ceada01..2bba18c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -56,6 +56,7 @@ AC_CONFIG_FILES( [
scripts/xtime:scripts/xtime.sh
scripts/xxcat:scripts/xxcat.sh
scripts/create-ssha-passwd:scripts/create-ssha-passwd.py
+ scripts/run-with-log:scripts/run-with-log.sh
] )
AC_OUTPUT()
diff --git a/scripts/run-with-log.sh b/scripts/run-with-log.sh
new file mode 100755
index 0000000..3720106
--- /dev/null
+++ b/scripts/run-with-log.sh
@@ -0,0 +1,285 @@
+#!/bin/sh
+
+# @autogenerated_warning@
+# @autogenerated_timestamp@
+# @PACKAGE@ @VERSION@
+# @PACKAGE_URL@
+
+COPYRIGHT="
+Copyright (C) 2016 A. Gordon (assafgordon@gmail.com)
+License: GPLv3+
+"
+
+
+## Runs a program, saving STDOUT/STDERR into a log, and optionally
+## emailing it on errors.
+
+## TODO: future improvements:
+## 1. Detect existing log files (e.g. with date collosion) and
+## add a unique sequence to the filename.
+## 2. Don't require "-p PREFIX", use "${COMMAND}-" as default.
+
+set -u
+
+
+die()
+{
+ ## Die BEFORE starting the program (e.g. bad parameters)
+ BASE=$(basename "$0")
+ echo "$BASE: error: $*" >&2
+ exit 1
+}
+
+die_with_log()
+{
+ ## Runtime error, try to send some notification...
+ BASE=$(basename "$0")
+ echo "$BASE: error: $*" >&2
+ if test -n "$email" ; then
+ echo "$BASE: error: $*" \
+ | mail -s "run-with-log: FATAL RUNTIME ERROR" "$email"
+ fi
+ exit 1
+}
+
+check_no_newlines()
+{
+ # Typical Usage:
+ # check_no_newlines "$VAR" || die "newlines not allowed in \$VAR"
+
+ # Ensure the variable contains a single line.
+ # More than one line is invalid (and will not be detected by grep below).
+ # zero lines means the variable does not contain any newlines.
+ _lc=$(printf "%s" "$1" | LC_ALL=C wc -l) \
+ || die "failed to count lines on '$1'"
+ test "$_lc" -eq 0 && return 0
+ return 1
+}
+
+valid_email()
+{
+ # Typical usage:
+ # valid_email "$VAR" || die "invalid email: '$VAR'"
+
+ check_no_newlines "$1" || return 1
+
+ # Validate a subset of all possible valid email addresses,
+ # (but a useful enough subset).
+ # Emails without hostnames are OK (local unix users).
+ # Emails with non-FQDN hostnames are OK (local machines).
+ # Some invalid email forms will pass, but they can be used
+ # safely as command-line parameters for the mail program,
+ # which will detect and reject them.
+ printf "%s" "$1" \
+ | LC_ALL=C grep -Eq '^[A-Za-z0-9][-A-Za-z0-9_\.\+]*(@[-A-Za-z0-9_\.]+)?$' \
+ && return 0
+
+ # Default to not valid
+ return 1
+}
+
+
+show_help_and_exit()
+{
+ BASE=$(basename "$0")
+ echo "
+$BASE - Runs a program, saving STDOUT/STDERR to a file,
+ and opptionally emailing the log.
+
+$COPYRIGHT
+Version: @VERSION@
+See: @PACKAGE_URL@
+
+Usage: $BASE [OPTIONS] -- COMMAND [ARGS]
+
+OPTIONS:
+ -h - This help screen.
+ -e EMAIL - Email log to this address (default: $USER).
+ -A - Always email logs (default: only on COMMAND failure)
+ -L FILE - Write log to FILE (disables auto filename generation).
+ -p PREFIX - Write log to PREFIX with '-{DATE}.log' suffix appended.
+ -L and -p are mutually exclusive.
+ -n NAME - Use NAME in email subject line, instead of COMMAND.
+
+
+Example:
+
+ $BASE -e me@example.org -p pipeline- -- \\
+ pipeline.sh --param1 --param3 [...]
+
+The above command runs 'pipeline.sh' and saves STDOUT/STDERR to
+pipeline-{DATE}.log.gz (in the current directory).
+If pipeline.sh exit with non-zero exit code, sends an email
+with the log attached to 'me@example.org'.
+The subject of the email will be 'pipeline.sh - {DATE} - ERROR'.
+
+
+"
+ exit 0
+}
+
+
+##
+## Script starts here
+##
+
+## parse parameterse
+show_help=
+email=
+always_send_email=
+logfile_prefix=
+logfile_fixed=
+name=
+while getopts he:AL:n:p: param
+do
+ case $param in
+ A) always_send_email=y
+ ;;
+ e) email="$OPTARG"
+ valid_email "$email" || die "invalid email: '$email'"
+ ;;
+ L) logfile_fixed="$OPTARG"
+ ;;
+ p) logfile_prefix="$OPTARG"
+ ;;
+ h) show_help=y
+ ;;
+ n) name="$OPTARG"
+ ;;
+ ?) die "unknown option. See -h for help."
+ esac
+done
+[ -n "$show_help" ] && show_help_and_exit;
+
+shift $((OPTIND-1))
+
+test -n "$logfile_fixed" && test -n "$logfile_prefix" \
+ && die "-L and -p are mutually-exclusive. See -h for help."
+test -z "$logfile_fixed" && test -z "$logfile_prefix" \
+ && die "missing -L or -p. See -h for help."
+
+test "$#" -gt 0 || die "missing COMMAND to run. See -h for help."
+COMMAND=$(basename "$1") || die "failed to get basename of '$1'"
+test -z "$name" \
+ && name="$COMMAND" \
+ || name="$name ($COMMAND)"
+
+# If no email specified with -e, send to the current user.
+# This of course requires that the MTA is properly
+# configured on the host.
+if test -z "$email" ; then
+ # FIXME: 'set -u' is defined: will this even work?
+ test -z "$USER" && die "no email specified (-e), and \$USER is empty."
+ email="$USER"
+fi
+
+##
+## Determine log filename
+##
+LOG=
+BEGDATE=$(date +%F-%H%M%S) || die_with_log "failed to get current date"
+
+if test -n "$logfile_fixed" ; then
+ LOG="$logfile_fixed"
+elif test -n "$logfile_prefix" ; then
+ LOG="${logfile_prefix}${BEGDATE}.log"
+fi
+touch "$LOG" || die_with_log "failed to touch log file '$LOG'"
+
+## Get hostname
+hostname=$(hostname) || die "failed to get hostname"
+## 'realpath' is easier, but can't assume it is installed.
+LOG_DISPLAY_NAME=$(
+ cd $(dirname "$LOG") ;
+ b=$(basename "$LOG")
+ echo "$PWD/$b" ) \
+ || die "failed to get directory name for '$LOG'"
+
+##
+## Run the program
+##
+## NOTES:
+## 1. there's a race here: if another process modifies '$LOG'
+## to prevent writing to it, after the 'touch' above succeeded).
+## 2. If there's a system error (e.g. fork/exec fails),
+## the error will be printed to our STDERR, not the log.
+## (and $rc will be >= 126). FIXME: log this errors as well?
+"$@" 1>"$LOG" 2>&1
+
+rc=$?
+
+ENDDATE=$(date +%F-%H%M%S) || die_with_log "failed to get current date"
+
+gzip -f "$LOG" || die_with_log "failed to gzip log file '$LOG'"
+LOG="$LOG.gz"
+test -e "$LOG" || die_with_log "compressed log '$LOG' not found after gzip"
+LOG_DISPLAY_NAME="$LOG_DISPLAY_NAME.gz"
+
+##
+## Send report
+##
+
+msg_status=
+msg_first_line=
+send_email=
+if test "$rc" -eq 0 ; then
+ test -n "$always_send_email" && send_email=y
+ msg_status="OK"
+ msg_first_line="Command '$COMMAND' completed successfully."
+elif test "$rc" -lt 126 ; then
+ send_email=y
+ msg_status="ERROR"
+ msg_first_line="Command '$COMMAND' FAILED - returned error/exit code $rc"
+else
+ send_email=y
+ msg_status="FATAL ERROR"
+ msg_first_line="Command '$COMMAND' FAILED - shell returned error code '$rc'"
+fi
+
+
+
+## Try to grab the last 10 lines from the log.
+## Trim excessive information and invalid characters.
+## Ignore any failures.
+last_log_lines=$(gzip -dc < "$LOG" \
+ | tail -n10 \
+ | cut -b1-80 \
+ | LC_ALL=C tr -dc '[:print:][:space:]' \
+ | sed 's/^/ /')
+##
+## Build email body and subject
+##
+msg_body="Hello,
+
+$msg_first_line
+
+Command line:
+ $*
+
+Start: $BEGDATE
+End: $ENDDATE
+
+Host:
+ $hostname
+
+PWD:
+ $PWD
+
+Log:
+ $LOG_DISPLAY_NAME
+
+Last lines from log:
+$last_log_lines
+
+Log attached.
+
+"
+
+msg_subject="$name - $BEGDATE - $msg_status"
+
+if test -n "$send_email" ; then
+ echo "$msg_body" | mail -s "$msg_subject" -a "$LOG" "$email" \
+ || die_with_log "fatal error: failed to send email"
+fi
+
+exit 0