summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorA. Gordon <assafgordon@gmail.com>2014-11-05 04:33:01 (GMT)
committerA. Gordon <assafgordon@gmail.com>2014-11-05 04:33:01 (GMT)
commit1d9c3c2c99d464c08735fda9d043b89e129dd672 (patch)
treec62d29eb179973651d6d3a0d7ddeb63d42c7e2c3
parent93b656f958ab10e453d464bde0b57daf44be1bdc (diff)
downloadgnu-sv-gitweb-1d9c3c2c99d464c08735fda9d043b89e129dd672.zip
gnu-sv-gitweb-1d9c3c2c99d464c08735fda9d043b89e129dd672.tar.gz
gnu-sv-gitweb-1d9c3c2c99d464c08735fda9d043b89e129dd672.tar.bz2
web-server: add update-scripts
-rw-r--r--web-server/README-webserver.md51
-rw-r--r--web-server/rsyslog-conf/10-savannah-updates.conf1
-rwxr-xr-xweb-server/update-scripts/new.py96
-rw-r--r--web-server/update-scripts/nohup.out1
-rwxr-xr-xweb-server/update-scripts/show-updates.sh24
-rwxr-xr-xweb-server/update-scripts/update-git-watch.sh44
-rwxr-xr-xweb-server/update-scripts/update-git.sh87
7 files changed, 303 insertions, 1 deletions
diff --git a/web-server/README-webserver.md b/web-server/README-webserver.md
index 3f59fe8..a53ee87 100644
--- a/web-server/README-webserver.md
+++ b/web-server/README-webserver.md
@@ -66,7 +66,6 @@ The update scripts on this server can simply create new project when they see on
# NOTE: this will take some time...
./update-scripts/create-gnu-sv-www-repo.sh
-
# Test project updates:
# Update all repositories:
@@ -75,6 +74,56 @@ The update scripts on this server can simply create new project when they see on
# Or update only recently changed repositories:
update-recent-gnu-sv-www-repo.sh
+## Update Mechanism
+
+1. The CGI script 'new.py' is exposed by 'gnu.org' apache configuration.
+ calling it with the project name,type,vcs will create a marker file
+ in `/usr/local/to-update-git` or `/usr/local/to-update` .
+2. A cron job calls `update-git.sh` script running as `wwwgit` user.
+ The script reads the marker files, and does a `git pull` for each project.
+3. Both of the above scripts write messages to syslog with the
+ `savannah-update` tag.
+4. rsyslog is configured to write these messages to a log file in
+ `/var/www/log/savannah-updates.log`.
+5. A CGI script `/var/www/log/show-updates.sh` is exposed in `nongnu.org`
+ and shows the tail of the update log.
+
+Configuration steps:
+
+ # Create directory for marker files to update projects
+ # The 'new.py' script writes files in these directories
+ sudo chown www-data:www-data /usr/local/to-update/
+ sudo chmod 0777 /usr/local/to-update/
+ sudo chown www-data:www-data /usr/local/to-update-git/
+ sudo chmod 0777 /usr/local/to-update-git/
+
+ # Create a directory for 'new.py'
+ sudo mkdir /var/www/new-savannah-project
+ sudo cp ./update-scripts/new.py /var/www/new-savannah-project
+ sudo chmod a+x /var/www/new-savannah-project
+
+ # Create a directory for the recently-updated list
+ # 1. The apache configuration in 'nongnu.org' adds an alias to this CGI
+ # script: http://nongnu.housegordon.org/gsv-updates/ .
+ # 2. The rsyslog configuration
+ sudo mkdir /var/www/log/
+ sudo cp ./update-scripts/show-updates.sh
+ # Make the log readable by www-data user (the apache server).
+ # TODO: set this properly in rsyslog?
+ sudo touch /var/www/log/savannah-updates.log
+ sudo a+r /var/www/log/savannah-updates.log
+
+ # Add the rsyslogd configuration file
+ sudo cp ./rsyslog-conf/10-savannah-updates.conf /etc/rsyslog.d/
+
+To trigger an update, run (from `fencepost.gnu.org`):
+
+ curl 'http://gnu.housegordon.org/new-savannah-project/new.py?type=gnu&project=autoconf&vcs=git'
+
+To view the recent updates, visit
+ <http://nongnu.housegordon.org/gsv-updates/>
+
+
## Scripts to update the server after every update
diff --git a/web-server/rsyslog-conf/10-savannah-updates.conf b/web-server/rsyslog-conf/10-savannah-updates.conf
new file mode 100644
index 0000000..eb450cd
--- /dev/null
+++ b/web-server/rsyslog-conf/10-savannah-updates.conf
@@ -0,0 +1 @@
+:syslogtag, isequal, "savannah-update:" /var/www/log/savannah-updates.log
diff --git a/web-server/update-scripts/new.py b/web-server/update-scripts/new.py
new file mode 100755
index 0000000..cb5fed2
--- /dev/null
+++ b/web-server/update-scripts/new.py
@@ -0,0 +1,96 @@
+import os,sys
+import string
+from mod_python import apache, util
+from syslog import *
+import stat
+import re
+
+# baughj, 2007.07.06 - add logging, usage of subprocess
+# baughj, 2007.07.16 - make sure to chgrp/chmod checked out directory so that
+# it can be updated by wwwcvs user later - why doesn't everything (cron job,
+# etc) run as one user?
+# add new 'translation' type for translation team webspace, see RT #348523.
+# ward, 2008-08-18
+
+# Stripped out actual cvs update, this script now just requests an update.
+# See /usr/local/bin/update-cvs.sh for the actual cvs update code (and that
+# script is run from cron)
+# ward, 2012-09-26
+
+# 1. Add support for GIT repositories.
+# 2. Return proper HTTP error code with informative messages.
+# 3. request_update takes additional parameter (vcs).
+# 4. no special treatment for project 'www'.
+# Assaf Gordon, 2014-11-04
+
+CHECKOUT = "/var/www/savannah-checkouts/"
+LOGFILE = "/var/log/wwwcvs/new-savannah-project.log"
+
+CVS_UPDATE_DIR="/usr/local/to-update"
+GIT_UPDATE_DIR="/usr/local/to-update-git"
+
+def touch(fname, times=None):
+ with file(fname, 'a'):
+ os.utime(fname, times)
+ os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | \
+ stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
+
+def request_update (vcs, type, project):
+ if vcs == "git":
+ destdir = GIT_UPDATE_DIR
+ else:
+ destdir = CVS_UPDATE_DIR
+
+ marker = os.path.join( destdir, type + "___" + project )
+ touch(marker)
+ syslog(LOG_INFO, "Update requested for project '%s' type '%s' vcs '%s'" \
+ % (project, type, vcs))
+ return
+
+def bad_request_error(req,msg):
+ text = 'new.py: error: ' + str(msg) + "\n"
+ syslog(LOG_ERR, text)
+ req.status = 400
+ req.write(text)
+ # NOTE: "OK" means apache will send our text and status to the client.
+ # The status will be HTTP 400 - Bad request.
+ # When using "apache.HTTP_BAD_REQUEST" - the Apache boiler-plate text
+ # will be returned to the user, without our error message.
+ return apache.OK
+
+def handler (req):
+ openlog('savannah-update', 0, LOG_LOCAL6)
+ form = util.FieldStorage (req)
+
+ type = None
+ project = None
+ vcs = "cvs"
+ if form.has_key('type'):
+ type = form['type']
+ if form.has_key('project'):
+ project = form['project']
+ if form.has_key('vcs'):
+ vcs = form['vcs']
+
+ # Input validation
+ if type is None:
+ return bad_request_error(req,'missing type parameter')
+ if project is None:
+ return bad_request_error(req,'missing project parameter')
+
+ allowed_types = [ "gnu", "non-gnu", "www", "translations" ]
+ if not (type in allowed_types):
+ return bad_request_error(req,"invalid type '%s'" % (type))
+
+ allowed_vcses = [ "cvs", "git" ]
+ if not (vcs in allowed_vcses):
+ return bad_request_error(req,"invalid vcs '%s'" % (vcs))
+
+ if not re.match('^[A-Za-z0-9][A-Za-z0-9_\-]+$', project):
+ return bad_request_error(req,"invalid project name '%s'" % (project))
+
+ req.content_type = 'text/html'
+ req.send_http_header ()
+ request_update(vcs,type,project)
+ req.write("project '%s' marked for update\n" % (project))
+ return apache.OK
diff --git a/web-server/update-scripts/nohup.out b/web-server/update-scripts/nohup.out
new file mode 100644
index 0000000..f0f24a9
--- /dev/null
+++ b/web-server/update-scripts/nohup.out
@@ -0,0 +1 @@
+sudo: sh./update-git-watch.sh: command not found
diff --git a/web-server/update-scripts/show-updates.sh b/web-server/update-scripts/show-updates.sh
new file mode 100755
index 0000000..4beb306
--- /dev/null
+++ b/web-server/update-scripts/show-updates.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+##
+## CGI script to show the last few lines from
+## the syslog savannah-updates log file.
+##
+## The 'new.py' CGI script and the 'update-git.sh' both
+## write to SYSLOG with tag 'savannah-update'.
+##
+## Rsyslog is configured to write these tags to the file
+## /var/www/log/savannah-updates.log
+##
+## This script will send the end of the log file to the user.
+
+LOGFILE=/var/www/log/savannah-updates.log
+
+echo "Content-type: text/plain"
+echo ""
+
+echo "Server Time: $(date)"
+echo
+tail -n 1000 "$LOGFILE" \
+ | tac \
+ | sed 's/gnumockup savannah-update: //'
diff --git a/web-server/update-scripts/update-git-watch.sh b/web-server/update-scripts/update-git-watch.sh
new file mode 100755
index 0000000..2e1adec
--- /dev/null
+++ b/web-server/update-scripts/update-git-watch.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# Assaf Gordon, 2014-11-04 - git adaptation, logging to syslog
+
+# The syslog identity: for easier debugging, should
+# match the same identity as in 'new.py'.
+SYSLOG_TAG="savannah-update"
+
+MARKER_GIT_DIR="/usr/local/to-update-git/"
+
+RUNUSER="wwwgit"
+
+log()
+{
+ BASE=$(basename -- "$0")
+ DATE=$(date +%F-%H%M%S)
+ logger -t "$SYSLOG_TAG" "$@"
+ echo "$DATE: $@" >&2
+}
+
+die()
+{
+ BASE=$(basename -- "$0")
+ echo "$BASE: error: $@" >&2
+ logger -t "$SYSLOG_TAG" "$BASE: error: $@"
+ exit 1
+}
+
+
+UPDATE_SCRIPT=$(dirname -- "$0")/update-git.sh
+test -e "$UPDATE_SCRIPT" \
+ || die "update script not found (expecting '$UPDATE_SCRIPT')"
+
+test "$(whoami)" = "$RUNUSER" \
+ || die "Script must be run as user: $RUNUSER"
+
+while true ;
+do
+ inotifywait -q "$MARKER_GIT_DIR" \
+ || die "FATAL: inotifywait failed! updates are stopped."
+
+ "$UPDATE_SCRIPT" \
+ || log "FATAL: last update attempt failed."
+done
diff --git a/web-server/update-scripts/update-git.sh b/web-server/update-scripts/update-git.sh
new file mode 100755
index 0000000..03d8ad9
--- /dev/null
+++ b/web-server/update-scripts/update-git.sh
@@ -0,0 +1,87 @@
+#!/bin/sh
+
+##
+## Handles GIT updates.
+## Based on 'update-cvs.sh' file.
+##
+# This script looks at
+# /usr/local/to-update-git/
+# where the
+# /var/www/new-savannah-project/new.py
+# script creates empty files if there is work to do.
+#
+# Ward Vandewege, 2012-09-26
+#
+# Assaf Gordon, 2014-11-04 - git adaptation, logging to syslog
+
+# The syslog identity: for easier debugging, should
+# match the same identity as in 'new.py'.
+SYSLOG_TAG="savannah-update"
+
+MARKER_DIR="/usr/local/to-update-git/"
+
+RUNUSER="wwwgit"
+
+GIT_REPOS="/var/lib/savannah-checkouts/"
+
+die()
+{
+ BASE=$(basename -- "$0")
+ echo "$BASE: error: $@" >&2
+ logger -t "$SYSLOG_TAG" "$BASE: error: $@"
+ exit 1
+}
+
+log()
+{
+ BASE=$(basename -- "$0")
+ DATE=$(date +%F-%H%M%S)
+ logger -t "$SYSLOG_TAG" "$@"
+ echo "$DATE: $@" >&2
+}
+
+test "$(whoami)" = "$RUNUSER" \
+ || die "Script must be run as user: $RUNUSER"
+
+
+## Proecss all marker files in the marker directory.
+## The CGI script 'new.py' will create files such as:
+## gnu___coreutils
+## non-gnu___lwip
+for f in $(find "$MARKER_DIR" -type f -name '*___*') ;
+do
+ marker=$(basename -- "$f")
+ type=${marker%___*}
+ project=${marker#*___}
+
+ # Validate project type
+ if ! echo "$type" | grep -qE '^(gnu|non-gnu|www|translations)$' ; then
+ log "skipping invalid marker type '$marker'"
+ rm -f -- "$f"
+ continue
+ fi
+
+ # Validate project name
+ if ! echo "$project" | grep -qE '[A-Za-z0-9][A-Za-z0-9_\-]+$' ; then
+ log "skipping invalid marker project name '$marker'"
+ rm -f -- "$f"
+ continue
+ fi
+
+ # TODO: if the project doesn't exist - CREATE IT ?
+ if ! test -d "$GIT_REPOS/$project" ; then
+ log "skipping non-existing project '$marker'"
+ rm -f -- "$f"
+ continue
+ fi
+
+ log "project $project (type $type) - starting update"
+ MSG=$( cd "$GIT_REPOS/$project" ; git pull --quiet 2>&1 )
+ if test $? -eq 0 ; then
+ REV=$( cd "$GIT_REPOS/$project" ; git rev-parse HEAD)
+ log "project $project (type $type) - update complete (commit: $REV)"
+ else
+ log "failed to update '$GIT_REPOS/$project': $MSG"
+ fi
+ rm -f -- "$f"
+done