source: trunk/debathena/scripts/build-server/autodebathenify @ 24866

#!/bin/bash

# This script is intended to be run from a cron job by a local account
# on a build machine--either debuild on debuild.mit.edu or builder on
# linux-build-10.mit.edu.  The account should have:
#
# * A keytab installed in $HOME/keytab containing a keytab for
#   daemon/HOSTNAME, which should have write access to the build
#   directory and apt repository.
#
# * A GPG secret key capable of signing packages for the apt
#   repository.
#
# * A file named $HOME/autodebathenify.config specifying shell
#   variable assignments for:
#   - error_addr: An email address to send errors to
#   - scripts_dir: Directory containing Debathena scripts (da, etc.)
#   - build_dir: Build directory location, up through "third".
#   - packages: A list of packages to keep up to date
#   It should also specify an environment variable assignment for
#   DEBATHENA_APT.

# Send all output to a logfile.
mkdir -p "$HOME/logs"
logfile=$HOME/logs/$(date '+%Y-%m-%d-%H').log
exec </dev/null >>$logfile 2>&1

# Read configuration; if it's not complete, do nothing.
config=$HOME/autodebathenify.config
. $HOME/autodebathenify.config
if [ ! -n "$error_addr" -o ! -n "$scripts_dir" -o ! -n "$build_dir" -o \
     ! -n "$packages" ]; then
  echo "Incomplete $config; doing nothing."
  exit
fi

suppress=$HOME/autodebathenify.suppress
if [ -e "$suppress" ]; then
  if [ $(($(date +%s) - $(stat -c %Y "$suppress"))) -ge 86400 ]; then
    /usr/sbin/sendmail -t << EOM
From: Autodebathenify <$error_addr>
To: Autodebathenify <$error_addr>
Subject: WARNING: Previous autodebathenify failure has not been addressed

Autodebathenify previously experienced a failure on $(stat -c %y "$suppress")

Please correct this failure immediately.

Autodebathenify will stop running until $suppress is removed.
EOM
  fi
  echo "$suppress exists; not running."
  exit
fi

# Send an error report to $error_addr and suppress future runs until a
# human has checked things out.  This function does not remove the run
# file and does not exit the script.
error() {
  touch "$suppress"
  /usr/sbin/sendmail -t << EOM
From: Autodebathenify <$error_addr>
To: Autodebathenify <$error_addr>
Subject: Autodebathenify failure: $1

Autodebathenify has experienced a failure.

Please see $logfile on $(hostname) for more details.

Autodebathenify will stop running until $suppress is removed.
EOM
}

# Similar to error, but do not suppress future runs.
error_transient() {
  /usr/sbin/sendmail -t << EOM
From: Autodebathenify <$error_addr>
To: Autodebathenify <$error_addr>
Subject: Autodebathenify failure: $1

Autodebathenify has experienced a transient failure.

Autodebathenify will continue running; investigation is not required
unless this failure repeats itself.  If that happens, see $logfile on
$(hostname) for more details.
EOM
}

# Exit handler.  If a subsidiary command exited with status 3, we
# treat it as a transient failure; otherwise we treat it as a failure
# requiring human investigation.  In either case, remove the run file.
on_failure() {
  if [ 3 -eq "$?" ]; then
    error_transient "$state"
  else
    error "$state"
  fi
  rm -f "$runfile"
}

# Make sure runs of this script do not overlap.
runfile=$HOME/autodebathenify.running
if [ -e "$runfile" ]; then
  # A previous invocation appears to still be running.
  pid=$(cat "$runfile")
  if ! kill -0 "$pid" 2>/dev/null; then
    # Okay, it's not really running.  Perhaps the machine has
    # rebooted.  Wait for manual intervention before running again.
    error "Stale runfile"
  else
    echo "$runfile exists; not running."
  fi
  exit
fi
echo $$ > $runfile

trap on_failure EXIT
set -e

# Acquire credentials.
state="acquiring credentials"
export KRB5CCNAME=$HOME/autodebathenify.ccache
kinit -k -t "$HOME/keytab" daemon/$(hostname --fqdn)
aklog

export PATH=${scripts_dir}:$PATH
. "$scripts_dir"/debian-versions.sh

state="changing to build dir"
cd "$build_dir"
for pkg in $packages; do
  state="changing to $pkg in build dir"
  cd "$pkg"
  for code in $DEBIAN_CODES; do
    state="updating $pkg for $code"
    echo "--- Updating $pkg for $code (amd64) ---"
    ./debathenify-"$pkg" "$code"-amd64 -A source binary upload
    echo "--- Updating $pkg for $code (i386) ---"
    ./debathenify-"$pkg" "$code"-i386 source binary upload
  done
  cd ..
done

state="cleaning up"
kdestroy
rm -f "$runfile"
trap '' EXIT