[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: reproducible Debian real-world: improve output

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
18cd2324 by Holger Levsen at 2019-02-21T14:41:14Z
reproducible Debian real-world: improve output

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
41fcbb67 by Holger Levsen at 2019-02-21T20:23:00Z
reproducible Debian real-world: refactor for huge speed gain

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
395e8ad7 by Holger Levsen at 2019-02-21T20:24:29Z
reproducible Debian real-world: plan rebuilder(s)

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- bin/reproducible_compare_Debian_sha1sums.sh


Changes:

=====================================
bin/reproducible_compare_Debian_sha1sums.sh
=====================================
@@ -25,6 +25,13 @@ set -e
 # - store date when a package was last reproduced... (and constantly do that...)
 # - throw away results (if none has been|which have not) signed with a tests.r-b.o key
 # - json files from buildinfo.d.n are never re-downloaded
+# - rebuilder:
+#   - run on osuoslXXX
+#   - loop randomly through unreproducible packages first, then reproducible ones. do one attempt only.
+#   - one job run tests one package.
+#   - use same debuild options possible? or try all sane options?
+#   - submit .buildinfo file to b.d.n and then fetch the json again.
+#   - debootstrap stretch and upgrade from there?
 
 RELEASE=buster
 MODE="$1"
@@ -53,7 +60,7 @@ case "$MODE" in
 	random)		SORT="sort -R";;
 	reverse)	SORT="sort -r" ;;
 	forward)	SORT="sort" ;;
-	*)		SORT="sort" ; MODE="results" ;;
+	*)		SORT="sort" ; MODE="results" ; RESULTS=$(mktemp --tmpdir=$TMPDIR sha1-results-XXXXXXX) ; find $SHA1DIR -name "*REPRODUCIBLE.buster" > $RESULTS
 esac
 packages="$(grep ^Package: $PACKAGES| awk '{print $2}' | $SORT | xargs echo)"
 
@@ -79,6 +86,7 @@ cleanup_all() {
 		echo
 		echo "$(du -sch $SHA1DIR)"
 		echo
+		rm $RESULTS
 	fi
 	rm $log $PACKAGES
 }
@@ -89,6 +97,25 @@ rm -f $SHA1DIR/*.lock	# this is a tiny bit hackish, but also an elegant way to g
 			# (locks are held for 30s only anyway and there is an 3/60000th chance of a race condition only anyway)
 
 for package in $packages ; do
+	if [ "$MODE" = "results" ] ; then
+		result=$(grep "/${package}_" $RESULTS || true)
+		if [ -n "$result" ] ; then
+			if $(echo $result | grep -q UNREPRODUCIBLE) ; then
+				package_file=$(echo $result | sed 's#\.deb\.UNREPRODUCIBLE\.buster$#.deb#' )
+				count=1
+				SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+				package_file=$(basename $package_file)
+				echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
+			else
+				package_file=$(echo $result | sed 's#\.deb\.REPRODUCIBLE\.buster$#.deb#' )
+				count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
+				SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
+				package_file=$(basename $package_file)
+				echo "$(date -u) - REPRODUCIBLE: $package_file ($SHA1SUM_PKG) - reproduced $count times."
+			fi
+			continue
+		fi
+	fi
 	LOCK="$SHA1DIR/${package}.lock"
 	if [ -e $LOCK ] ; then
 		echo "$(date -u) - skipping locked package $package"
@@ -104,6 +131,7 @@ for package in $packages ; do
 	mkdir -p $pool_dir
 	cd $pool_dir
 	if [ "$MODE" = "results" ] ; then
+		# this code block can be removed once all packages with existing results have been processed once...
 		if [ -e ${package_file}.REPRODUCIBLE.$RELEASE ] ; then
 			count=$(cat ${package_file}.REPRODUCIBLE.$RELEASE)
 			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
@@ -113,7 +141,6 @@ for package in $packages ; do
 			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
 			echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
 		elif [ -e ${package_file}.json ] ; then
-			# this code block can be removed once all packages with existing results have been processed once...
 			count=$(fmt ${package_file}.json | grep -c '\.buildinfo' || true)
 			SHA1SUM_PKG="$(cat ${package_file}.sha1output | awk '{print $1}' )"
 			if [ "${count}" -ge 2 ]; then
@@ -150,7 +177,7 @@ for package in $packages ; do
 			echo "$(date -u) - UNREPRODUCIBLE: $package_file ($SHA1SUM_PKG) only on ftp.debian.org."
 		fi
 	else
-		echo "$(date -u) - not updating .buildinfo files for known ${package_file}"
+		echo "$(date -u) - not updating data about ${package_file}"
 	fi
 	rm -f $LOCK
 done | tee $log



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/a97aad1b9486ce0e4aa3da5920ca5be604b20dea...395e8ad70e51f884260a04477bea2392c9f64fb2

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/a97aad1b9486ce0e4aa3da5920ca5be604b20dea...395e8ad70e51f884260a04477bea2392c9f64fb2
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190221/ab85e570/attachment-0001.html>