[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] djm: "documented jenkins maintenance", script to support logged node maintenance
Holger Levsen (@holger)
gitlab at salsa.debian.org
Sun Apr 2 15:53:49 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
291bc540 by Holger Levsen at 2023-04-02T16:53:21+02:00
djm: "documented jenkins maintenance", script to support logged node maintenance
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- + bin/djm
Changes:
=====================================
bin/djm
=====================================
@@ -0,0 +1,184 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+#
+# djm - documented jenkins maintenance
+#
+# Copyright 2023 Holger Levsen <holger at layer-acht.org>
+# released under the GPLv2
+#
+
+set -e
+set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
+
+#
+# define environment and parse parameters
+#
+
+LOGFILE=~/.djm.log
+TARGET=$1
+ACTION=$2
+REASON=$3
+DRY_RUN=$4
+FAIL_REASON=""
+
+if [ -z "$TARGET" ] ; then
+ FAIL_REASON="${FAIL_REASON}Undefined target.\n"
+else case $TARGET in
+ all) TARGET="$(./nodes/list_nodes )"
+ ;;
+ *) TARGET="$(./nodes/list_nodes | egrep $TARGET || true)"
+ ;;
+ esac
+fi
+
+if [ -z "$TARGET" ] ; then
+ FAIL_REASON="${FAIL_REASON}Undefined target.\n"
+fi
+
+case $ACTION in
+ r|reboot) ACTION=reboot ;;
+ p|powercycle) ACTION=powercycle ;;
+ a|autoremove) ACTION=autoremove ;;
+ b|bash) ACTION=bash ;;
+ *) FAIL_REASON="${FAIL_REASON}Undefined action.\n"
+ ;;
+esac
+
+case $REASON in
+ ku) REASON="kernel upgrade" ;;
+ np) REASON="no ping" ;;
+ rm) REASON="regular maintenance" ;;
+ *) FAIL_REASON="${FAIL_REASON}Undefined reason.\n"
+ ;;
+esac
+
+if [ -z "$DRY_RUN" ] ; then
+ DRY_MODE=false
+else
+ DRY_MODE=true
+fi
+
+#
+# display help text in case of problems (or no params given)
+#
+
+if [ -n "$FAIL_REASON" ] ; then
+ echo
+ echo problem parsing parameters, djm needs at least three:
+ echo
+ echo "TARGET=\$1 'all' or grepable (jenkins, amd64, ionos, osuosl3) from ./nodes/list_nodes"
+ echo "ACTION=\$2 r_eboot, _p_owercycle, _a_utoremove, _b_bash (for manual maintenance)"
+ echo "REASON=\$3 ku=kernel update, np=no pings, rm=regular maintenance - TODO: define more"
+ echo "DRY_RUN=\$4 optional to force dry-run mode"
+ echo
+ echo params given:
+ echo
+ echo "\$1 = $1"
+ echo "\$2 = $2"
+ echo "\$3 = $3"
+ echo "\$4 = $4"
+ echo
+ echo parsing problems:
+ echo
+ echo -e "$FAIL_REASON"
+ echo
+ exit 1
+fi
+
+#
+# main
+#
+
+if $DRY_MODE ; then
+ echo "###"
+ echo "###"
+ echo "### running in dry-run mode, not doing anything for real."
+ echo "###"
+ echo "###"
+else
+ echo
+ echo Please confirm you want to do this:
+ echo
+fi
+
+for NODE in $TARGET ; do
+ SHORTNODE=$(echo $NODE | cut -d '.' -f1)
+ echo $(date), $SHORTNODE, $ACTION, $REASON
+done
+
+if $DRY_MODE ; then
+ echo
+ echo "# dry-run end."
+ exit 0
+else
+ echo
+ echo -n "confirm: y/N: "
+ read CONFIRM
+ echo
+ case $CONFIRM in
+ y|Y) : ;;
+ *) echo "Ok, aborting" ; exit 0 ;;
+ esac
+fi
+
+BG=""
+get_arch_color() {
+ case "$1" in
+ *amd64*) BG=lightgreen ;;
+ *i386*) BG=lightblue ;;
+ *arm64*) BG=orange ;;
+ *armhf*) BG=lightyellow ;;
+ *jenkins.debian.*) BG=yellow ;;
+ *) BG=white ;;
+ esac
+}
+
+LOGDATE=$(date)
+for NODE in $TARGET ; do
+ SHORTNODE=$(echo $NODE | cut -d '.' -f1)
+ echo $(date), $SHORTNODE, $ACTION, $REASON
+ get_arch_color $NODE
+ #
+ # action
+ #
+ case $ACTION in
+ reboot) ssh $NODE "sudo reboot" || true
+ TMPFILE=$(mktemp --tmpdir=/tmp djm-XXXXXXXX)
+ cat > $TMPFILE << EOF
+#!/bin/bash
+for i in $(seq 1 42 | xargs echo) ; do
+ echo -n .
+ sleep 1
+done
+while ! ssh $NODE 'uptime ; echo press enter ; read a' ; do
+ for i in $(seq 1 30 | xargs echo) ; do
+ echo -n "."
+ sleep 1
+ done
+done
+rm $TMPFILE
+EOF
+ echo xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE"
+ xterm -T "$SHORTNODE / reboot ($TMPFILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE" &
+ echo
+ ;;
+ powercycle) echo FIXME, not yet implemented.
+ exit 1
+ ;;
+ autoremove) xterm -T "$SHORTNODE / autoremove" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove ; sudo apt-get clean'" &
+ ;;
+ bash) xterm -T "$SHORTNODE / bash" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
+ ;;
+ *) echo unsupported action, sorry, please improve.
+ exit 1
+ ;;
+ esac
+
+ #
+ # log
+ #
+ echo "$LOGDATE, $SHORTNODE, $ACTION, $REASON" >> $LOGFILE
+done
+
+echo the end
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/291bc540db81ccdb6a5d973fbbac2ab76927f104
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/291bc540db81ccdb6a5d973fbbac2ab76927f104
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230402/5c99475d/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list