[Qa-jenkins-scm] [jenkins.debian.net] 04/04: add LEDE build script and job
Holger Levsen
holger at layer-acht.org
Sat Jul 23 14:45:43 UTC 2016
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 270066ccae1e1f99e838e6da9895e8567239a42e
Author: Alexander Couzens <lynxis at fe80.eu>
Date: Sat Jul 23 02:16:26 2016 +0200
add LEDE build script and job
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
bin/jenkins_node_wrapper.sh | 2 +
bin/reproducible_common.sh | 1 +
bin/reproducible_html_rsync_remote_results.sh | 2 +-
bin/reproducible_lede.sh | 233 +++++++++++++++++++++
bin/templates/project_links.mustache | 1 +
.../sites-available/jenkins.debian.net.conf | 5 +
job-cfg/reproducible.yaml | 3 +
update_jdn.sh | 2 +-
8 files changed, 247 insertions(+), 2 deletions(-)
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index 1457e0d..480abe6 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -118,6 +118,8 @@ elif [[ "$*" =~ ^reproducible_coreboot ]] ; then
exec /srv/jenkins/bin/reproducible_coreboot.sh ; croak "Exec failed";
elif [[ "$*" =~ ^reproducible_openwrt ]] ; then
exec /srv/jenkins/bin/reproducible_openwrt.sh ; croak "Exec failed";
+elif [[ "$*" =~ ^reproducible_lede ]] ; then
+ exec /srv/jenkins/bin/reproducible_lede.sh ; croak "Exec failed";
elif [[ "$*" =~ ^reproducible_netbsd ]] ; then
exec /srv/jenkins/bin/reproducible_netbsd.sh ; croak "Exec failed";
elif [[ "$*" =~ ^reproducible_freebsd ]] ; then
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 307329c..7f23960 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -297,6 +297,7 @@ write_page_header() {
write_page " <a href=\"https://tests.reproducible-builds.org/freebsd/\">FreeBSD</a> /"
write_page " <a href=\"https://tests.reproducible-builds.org/netbsd/\">NetBSD</a> /"
write_page " <a href=\"https://tests.reproducible-builds.org/openwrt/\">OpenWrt</a>"
+ write_page " <a href=\"https://tests.reproducible-builds.org/lede/\">LEDE</a>"
write_page " </li></ul></li>"
write_page " </ul>"
# end project links
diff --git a/bin/reproducible_html_rsync_remote_results.sh b/bin/reproducible_html_rsync_remote_results.sh
index 0828be9..d459f58 100755
--- a/bin/reproducible_html_rsync_remote_results.sh
+++ b/bin/reproducible_html_rsync_remote_results.sh
@@ -12,7 +12,7 @@ common_init "$@"
# that's all
rsync_remote_results() {
- for PROJECT in coreboot openwrt netbsd ; do
+ for PROJECT in coreboot lede openwrt netbsd ; do
echo "$(date -u) - Starting to rsync results for '$PROJECT'."
local RESULTS=$(mktemp --tmpdir=$TEMPDIR -d reproducible-rsync-XXXXXXXXX)
# copy the new results from build node to webserver node
diff --git a/bin/reproducible_lede.sh b/bin/reproducible_lede.sh
new file mode 100755
index 0000000..712b488
--- /dev/null
+++ b/bin/reproducible_lede.sh
@@ -0,0 +1,233 @@
+#!/bin/bash
+
+# Copyright 2014-2015 Holger Levsen <holger at layer-acht.org>
+# © 2015 Reiner Herrmann <reiner at reiner-h.de>
+# 2016 Alexander Couzens <lynxis at fe80.eu>
+# released under the GPLv=2
+
+OPENWRT_GIT_REPO=https://git.lede-project.org/source.git
+OPENWRT_GIT_BRANCH=master
+DEBUG=true
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+. /srv/jenkins/bin/reproducible_openwrt_common.sh
+set -e
+
+#
+# main
+#
+TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-lede-build-XXXXXXXX) # used to build on tmpfs
+TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-lede-results-XXXXXXXX) # accessable in schroots, used to compare results
+BANNER_HTML=$(mktemp --tmpdir=$TMPDIR)
+DATE=$(date -u +'%Y-%m-%d')
+START=$(date +'%s')
+trap cleanup_tmpdirs INT TERM EXIT
+
+cd $TMPBUILDDIR
+echo "============================================================================="
+echo "$(date -u) - Cloning LEDE git repository."
+echo "============================================================================="
+git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO lede
+cd lede
+OPENWRT="$(git log -1)"
+OPENWRT_VERSION=$(git describe --always)
+echo "This is lede $OPENWRT_VERSION."
+echo
+git log -1
+
+# External feeds disabled for now as they break building (especially with CONFIG_ALL=y).
+#echo "============================================================================="
+#echo "$(date -u) - Updating package feeds."
+#echo "============================================================================="
+#./scripts/feeds update -a
+#./scripts/feeds install -a
+
+build_two_times lede ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n"
+
+# for now we only build one architecture until it's at most reproducible
+#build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n"
+#build_two_times ramips_rt288x_RTN15 "CONFIG_TARGET_ramips=y\nCONFIG_TARGET_ramips_rt288x=y\nCONFIG_TARGET_ramips_rt288x_RTN15=y\n"
+
+#
+# create html about toolchain used
+#
+echo "============================================================================="
+echo "$(date -u) - Creating Documentation HTML"
+echo "============================================================================="
+TOOLCHAIN_HTML=$(mktemp --tmpdir=$TMPDIR)
+echo "<table><tr><th>Target toolchains built</th></tr>" > $TOOLCHAIN_HTML
+for i in $(ls -1d staging_dir/toolchain*|cut -d "-" -f2-|xargs echo) ; do
+ echo " <tr><td><code>$i</code></td></tr>" >> $TOOLCHAIN_HTML
+done
+echo "</table>" >> $TOOLCHAIN_HTML
+echo "<table><tr><th>Contents of <code>build_dir/host/</code></th></tr>" >> $TOOLCHAIN_HTML
+for i in $(ls -1 build_dir/host/) ; do
+ echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML
+done
+echo "</table>" >> $TOOLCHAIN_HTML
+echo "<table><tr><th>Downloaded software</th></tr>" >> $TOOLCHAIN_HTML
+for i in $(ls -1 dl/) ; do
+ echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML
+done
+echo "</table>" >> $TOOLCHAIN_HTML
+echo "<table><tr><th>Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)</th><th>installed version</th></tr>" >> $TOOLCHAIN_HTML
+for i in gcc binutils bzip2 flex python perl make findutils grep diffutils unzip gawk util-linux zlib1g-dev libc6-dev git subversion ; do
+ echo " <tr><td>$i</td><td>" >> $TOOLCHAIN_HTML
+ dpkg -s $i|grep '^Version'|cut -d " " -f2 >> $TOOLCHAIN_HTML
+ echo " </td></tr>" >> $TOOLCHAIN_HTML
+done
+echo "</table>" >> $TOOLCHAIN_HTML
+
+# clean up builddir to save space on tmpfs
+rm -rf $TMPBUILDDIR/lede
+
+# run diffoscope on the results
+# (this needs refactoring rather badly)
+TIMEOUT="30m"
+DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
+echo "============================================================================="
+echo "$(date -u) - Running $DIFFOSCOPE on LEDE images and packages."
+echo "============================================================================="
+DBD_HTML=$(mktemp --tmpdir=$TMPDIR)
+DBD_GOOD_PKGS_HTML=$(mktemp --tmpdir=$TMPDIR)
+DBD_BAD_PKGS_HTML=$(mktemp --tmpdir=$TMPDIR)
+# run diffoscope on the images
+GOOD_IMAGES=0
+ALL_IMAGES=0
+SIZE=""
+create_results_dirs lede
+cd $TMPDIR/b1
+tree .
+for i in * ; do
+ cd $i
+
+ # search images in both paths to find non-existing ones
+ IMGS1=$(find * -type f -name "*.bin" -o -name "*.squashfs" | sort -u )
+ pushd $TMPDIR/b2/$i
+ IMGS2=$(find * -type f -name "*.bin" -o -name "*.squashfs" | sort -u )
+ popd
+
+ echo " <table><tr><th>Images for <code>$i</code></th></tr>" >> $DBD_HTML
+ for j in $(printf "$IMGS1\n$IMGS2" | sort -u ) ; do
+ let ALL_IMAGES+=1
+ if [ ! -f $TMPDIR/b1/$i/$j -o ! -f $TMPDIR/b2/$i/$j ] ; then
+ echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $j (${SIZE}) failed to build.</td></tr>" >> $DBD_HTML
+ rm -f $BASE/lede/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
+ continue
+ fi
+ call_diffoscope $i $j
+ get_filesize $j
+ if [ -f $TMPDIR/$i/$j.html ] ; then
+ mkdir -p $BASE/lede/dbd/$i
+ mv $TMPDIR/$i/$j.html $BASE/lede/dbd/$i/$j.html
+ echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> (${SIZE}) is unreproducible.</td></tr>" >> $DBD_HTML
+ else
+ SHASUM=$(sha256sum $j|cut -d " " -f1)
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> $DBD_HTML
+ let GOOD_IMAGES+=1
+ rm -f $BASE/lede/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
+ fi
+ done
+ cd ..
+ echo " </table>" >> $DBD_HTML
+done
+GOOD_PERCENT_IMAGES=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc)
+# run diffoscope on the packages
+GOOD_PACKAGES=0
+ALL_PACKAGES=0
+create_results_dirs
+cd $TMPDIR/b1
+for i in * ; do
+ cd $i
+
+ # search packages in both paths to find non-existing ones
+ PKGS1=$(find * -type f -name "*.ipk" | sort -u )
+ pushd $TMPDIR/b2/$i
+ PKGS2=$(find * -type f -name "*.ipk" | sort -u )
+ popd
+
+ for j in $(printf "$PKGS1\n$PKGS2" | sort -u ) ; do
+ let ALL_PACKAGES+=1
+ if [ ! -f $TMPDIR/b1/$i/$j -o ! -f $TMPDIR/b2/$i/$j ] ; then
+ echo " <tr><td><img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> $j (${SIZE}) failed to build.</td></tr>" >> $DBD_BAD_PKGS_HTML
+ rm -f $BASE/lede/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
+ continue
+ fi
+ call_diffoscope $i $j
+ get_filesize $j
+ if [ -f $TMPDIR/$i/$j.html ] ; then
+ mkdir -p $BASE/lede/dbd/$i/$(dirname $j)
+ mv $TMPDIR/$i/$j.html $BASE/lede/dbd/$i/$j.html
+ echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> $DBD_BAD_PKGS_HTML
+ else
+ SHASUM=$(sha256sum $j|cut -d " " -f1)
+ echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> $DBD_GOOD_PKGS_HTML
+ let GOOD_PACKAGES+=1
+ rm -f $BASE/lede/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
+ fi
+ done
+ cd ..
+done
+echo " <table><tr><th>Unreproducible and otherwise broken packages</th></tr>" >> $DBD_HTML
+cat $DBD_BAD_PKGS_HTML >> $DBD_HTML
+echo " </table>" >> $DBD_HTML
+echo " <table><tr><th>Reproducible packages</th></tr>" >> $DBD_HTML
+cat $DBD_GOOD_PKGS_HTML >> $DBD_HTML
+echo " </table>" >> $DBD_HTML
+GOOD_PERCENT_PACKAGES=$(echo "scale=1 ; ($GOOD_PACKAGES*100/$ALL_PACKAGES)" | bc)
+# are we there yet?
+if [ "$GOOD_PERCENT_IMAGES" = "100.0" ] || [ "$GOOD_PERCENT_PACKAGES" = "100.0" ]; then
+ MAGIC_SIGN="!"
+else
+ MAGIC_SIGN="?"
+fi
+
+#
+# finally create the webpage
+#
+cd $TMPDIR ; mkdir lede
+PAGE=lede/lede.html
+cat > $PAGE <<- EOF
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width">
+ <title>Reproducible LEDE ?</title>
+ <link rel='stylesheet' id='kamikaze-style-css' href='cascade.css?ver=4.0' type='text/css' media='all'>
+ </head>
+ <body>
+ <div id="header">
+ <p><center>
+ <code>
+EOF
+cat $BANNER_HTML >> $PAGE
+write_page " </code></center></p>"
+write_page " </div><div id=\"main-content\">"
+write_page " <h1>LEDE - <em>reproducible</em> wireless freedom$MAGIC_SIGN</h1>"
+write_page_intro LEDE
+write_page " <p>$GOOD_IMAGES ($GOOD_PERCENT_IMAGES%) out of $ALL_IMAGES built images and $GOOD_PACKAGES ($GOOD_PERCENT_PACKAGES%) out of $ALL_PACKAGES built packages were reproducible in our test setup."
+write_page " These tests were last run on $DATE for version ${OPENWRT_VERSION} using ${DIFFOSCOPE}.</p>"
+write_variation_table LEDE
+cat $DBD_HTML >> $PAGE
+write_page " <table><tr><th>git commit built</th></tr><tr><td><code>"
+echo -n "$OPENWRT" >> $PAGE
+write_page " </code></td></tr></table>"
+cat $TOOLCHAIN_HTML >> $PAGE
+write_page " </div>"
+write_page_footer LEDE
+publish_page
+rm -f $DBD_HTML $DBD_GOOD_PKGS_HTML $DBD_BAD_PKGS_HTML $TOOLCHAIN_HTML $BANNER_HTML
+
+# the end
+calculate_build_duration
+print_out_duration
+irc_message reproducible-builds "$REPRODUCIBLE_URL/lede/ has been updated. ($GOOD_PERCENT_IMAGES% images and $GOOD_PERCENT_PACKAGES% packages reproducible)"
+echo "============================================================================="
+
+# remove everything, we don't need it anymore...
+cleanup_tmpdirs
+trap - INT TERM EXIT
diff --git a/bin/templates/project_links.mustache b/bin/templates/project_links.mustache
index e5cbf07..1e12592 100644
--- a/bin/templates/project_links.mustache
+++ b/bin/templates/project_links.mustache
@@ -13,5 +13,6 @@
<a href="https://tests.reproducible-builds.org/freebsd/">FreeBSD</a> /
<a href="https://tests.reproducible-builds.org/netbsd/">NetBSD</a> /
<a href="https://tests.reproducible-builds.org/openwrt/">OpenWrt</a>
+ <a href="https://tests.reproducible-builds.org/lede/">LEDE</a>
</li></ul></li>
</ul>
diff --git a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
index 8390ba5..a10f998 100644
--- a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
+++ b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
@@ -297,6 +297,11 @@ Use common-debian-service-https-redirect reproducible-builds.org
RewriteCond %{REQUEST_URI} ^/openwrt(/|)$
RewriteRule ^/(.*) /openwrt/openwrt.html [R,L]
+ # redirect /lede/ to lede/lede.html
+ # note: the missing slash in the RewriteRule is wanted to avoid a double slash
+ RewriteCond %{REQUEST_URI} ^/lede(/|)$
+ RewriteRule ^/(.*) /lede/lede.html [R,L]
+
# redirect /netbsd/ to netbsd/netbsd.html
# note: the missing slash in the RewriteRule is wanted to avoid a double slash
RewriteCond %{REQUEST_URI} ^/netbsd(/|)$
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 6c0e8bb..251a052 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -640,6 +640,9 @@
- 'netbsd':
my_description: 'Rebuild NetBSD (for some archs) twice (with variations) and investigate regarding reproducibility.'
my_timed: '42 23 * * 3'
+ - 'lede':
+ my_description: 'Rebuild LEDE (some targets, some packages) twice (with variations) and investigate regarding reproducibility.'
+ my_timed: '42 23 * * 4'
my_trigger: 'reproducible_html_rsync_remote_results'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_hname: 'profitbricks3'
diff --git a/update_jdn.sh b/update_jdn.sh
index f24f9b6..33e4b38 100755
--- a/update_jdn.sh
+++ b/update_jdn.sh
@@ -255,7 +255,7 @@ if [ -f /etc/debian_version ] ; then
profitbricks-build4-amd64|profitbricks-build5-amd64|profitbricks-build6-i386) DEBS="$DEBS ntpdate" ;;
*) ;;
esac
- # needed to run coreboot/openwrt/netbsd/fedora/fdroid jobs
+ # needed to run coreboot/openwrt/lede/netbsd/fedora/fdroid jobs
case $HOSTNAME in
profitbricks-build3-amd64|profitbricks-build4-amd64) DEBS="$DEBS
bison
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list