[Collab-qa-commits] r528 - svnbuildstat/trunk/script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Fri Nov 30 10:18:05 UTC 2007
Author: goneri-guest
Date: 2007-11-30 10:18:04 +0000 (Fri, 30 Nov 2007)
New Revision: 528
Modified:
svnbuildstat/trunk/script/svnbuildstat_agent.pl
Log:
-fix some serious problem with the purge after the build
-remove the code related to the QA checks
-upload the .deb files too
Modified: svnbuildstat/trunk/script/svnbuildstat_agent.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_agent.pl 2007-11-29 20:31:56 UTC (rev 527)
+++ svnbuildstat/trunk/script/svnbuildstat_agent.pl 2007-11-30 10:18:04 UTC (rev 528)
@@ -2,7 +2,7 @@
use strict;
-use lib ".";
+use lib "/home/sites/svnbuildstat.debian.net/svnbuildstat/lib";
use Data::Dumper;
use Net::FTP;
@@ -12,6 +12,7 @@
use LWP::Simple;
use POSIX ":sys_wait_h";
use File::Basename;
+use File::stat;
###########################################################################
###########################################################################
@@ -21,24 +22,34 @@
my $RELEASE = "0.0.1";
my $hostname = hostname();
my $distro = "sid";
-my $maxjobs = $config->agent_maxjobs;
chomp (my $arch = `dpkg-architecture -qDEB_HOST_ARCH`);
my %job;
-die unless $maxjobs =~ /^\d+$/;
-
-if ($config->agent_vardir !~ /\/.+/) {
- die "vardir not correclty set in the [agent] section";
-}
+#my $maxjobs = $config->agent_maxjobs;
+my $maxjobs = 1;
my $workplace = $config->agent_vardir."/workplace";
my $pbuilderplace = $config->agent_vardir."/pbuilder";
my $tarballsplace = $config->agent_vardir."/tarballs";
-my $buildarea = $config->agent_vardir."/build-area";
-my $reportarea = $config->agent_vardir."/report";
+my $buildarea = $config->agent_vardir."/report";
my $tmpdir = $config->agent_vardir."/tmp";
my $ccachedir = $config->agent_vardir."/ccache";
my $aptcachedir = $config->agent_vardir."/aptcache";
+my $debmirror = $config->agent_debmirror;
+my $buildadminaddr = $config->agent_buildadminaddr;
+my $ftphost = $config->agent_ftphost;
+my $ftplogin = $config->agent_ftplogin;
+my $ftppassword = $config->agent_ftppassword;
+my $serveruri = "http://localhost:3000/packages/tobuildv3";
+my $pbuildertgz = "$pbuilderplace/$distro.tar.gz";
+
+# XXX
+if ($config->agent_vardir !~ /\/.+/) {
+ die "vardir not correclty set in the [agent] section";
+}
+
+die unless $maxjobs =~ /^\d+$/;
+
$ENV{LC_ALL} = 'C';
$ENV{LANG} = 'C';
@@ -48,12 +59,10 @@
# Purge
if (-d $workplace) {`rm -r $workplace`;}
if (-d $buildarea) {`rm -r $buildarea`;}
-if (-d $reportarea) {`rm -r $reportarea`;}
if (-d $tmpdir) {`rm -r $tmpdir`;}
# Create the directorys
die if (!-d $workplace && !mkdir $workplace);
die if (!-d $buildarea && !mkdir $buildarea);
-die if (!-d $reportarea && !mkdir $reportarea);
die if (!-d $pbuilderplace && !mkdir $pbuilderplace);
die if (!-d $tarballsplace && !mkdir $tarballsplace);
die if (!-d $tmpdir && !mkdir $tmpdir);
@@ -87,184 +96,199 @@
sub howxsnext {
my $p;
+ my $report = {};
- my $dsc = LWP::Simple::get("http://svnbuildstat.debian.net/packages/tobuildv2/".$arch);
- if (!$dsc) {
- print "Failed to contact the serveur to get a package to build\n";
+ my $serverOrder = LWP::Simple::get($serveruri.'/'.$arch);
+ return unless $serverOrder;
+ foreach (split $/, $serverOrder) {
+ chomp;
+ print "->".$_."\n";
+
+ $report->{$1} = $2 if (/(.*)=(.*)/);
}
- $dsc;
+
+ return unless $report->{dsc};
+ return unless $report->{package_id};
+ $report;
}
-sub build {
- my $dsc = shift;
+sub upgradeChroot {
- my $threadworkplace = $workplace .'/'. $$;
- my $threadreportarea = $reportarea .'/'. $$;
- my $threadbuildarea = $buildarea .'/'. $$;
- my $threadtmpdir = $tmpdir .'/'. $$;
-
- die if (!-d $threadworkplace && !mkdir $threadworkplace);
- die if (!-d $threadreportarea && !mkdir $threadreportarea);
- die if (!-d $threadbuildarea && !mkdir $threadbuildarea);
- die if (!-d $threadtmpdir && !mkdir $threadtmpdir);
-
- my %report;
- $report{'arch'} = $arch;
- $report{'buildadminaddr'} = $config->agent_buildadminaddr;
- $report{hostname} = $hostname;
- $report{distro} = $distro;
-
- $report{'agent_release'} = $RELEASE;
- $report{'linda_release'} = `dpkg-query -W -f='\${Version}' linda`;
- $report{'lintian_release'} = `dpkg-query -W -f='\${Version}' lintian`;
- $report{'pbuilder_release'} = `dpkg-query -W -f='\${Version}' pbuilder`;
- $report{'piuparts_release'} = `dpkg-query -W -f='\${Version}' piuparts`;
-###########################################################################
-###########################################################################
-###########################################################################
-
- my $pbuildertgz = "$pbuilderplace/$distro.tar.gz";
- if (-f $pbuildertgz) {
- my @s = stat($pbuildertgz);
- if ($s[9] + 3600*24*7 < time) {
- `mv $pbuildertgz $pbuildertgz.bak`
+# print "updateChroot\n";
+ my $pdebuildparam = "--debootstrap debootstrap --mirror ".$debmirror." --buildplace $pbuilderplace --othermirror 'deb ".$debmirror." $distro main contrib non-free' --distribution $distro --basetgz $pbuildertgz";
+ if (-s $pbuildertgz) {
+ my $sb = stat($pbuildertgz);
+ if (time - $sb->ctime > 3600 * 24) {
+ # I recreate the chroot every 3600 hours
+ `mv $pbuildertgz $pbuildertgz.save`;
+ }
}
- }
- my $pdebuildparam = "--debootstrap debootstrap --mirror ".$config->agent_debmirror." --buildplace $pbuilderplace --othermirror 'deb ".$config->agent_debmirror." $distro main contrib non-free' --distribution $distro --basetgz $pbuildertgz";
- `/usr/sbin/pbuilder update $pdebuildparam >$threadreportarea/pbuilder-update.log 2>&1`;
- if (($? >> 8)!=0){
- `/usr/sbin/pbuilder create $pdebuildparam >$threadreportarea/pbuilder-create.log 1>&2`;
- if (($? >> 8)!=0) {
- print "Failed to create pbuilder image\n";
- `mv $pbuildertgz.bak $pbuildertgz`if -f "$pbuildertgz.bak"; # restore the previous tarball
- exit 1
+ if (! -s $pbuildertgz) {
+ # TODO redirect he log on another place
+ `/usr/sbin/pbuilder create $pdebuildparam >/tmp/pbuilder-create.log 1>&2`;
+ if (($? >> 8)!=0) {
+ print "Failed to create pbuilder image\n";
+ if (-f "$pbuildertgz.save") {
+ # I restore the previous chroot
+ `cp $pbuildertgz.save $pbuildertgz`;
+ }
+ } else {
+ # I save the created chroot
+ `cp $pbuildertgz $pbuildertgz.save`;
+ }
}
- }
- my $tarballdir;
- foreach (`cd $threadworkplace; dget -qx $dsc`) {
- $tarballdir = $1 if /dpkg-source:\sextracting\s\S+\sin\s(\S+)/;
- }
- print "tarballdir: $tarballdir\n";
-die "FUCK" unless $tarballdir;
- if (!open CTRL, "<$threadworkplace/$tarballdir/debian/control") {
- return;
- } else {
+}
- foreach (<CTRL>) {
- if (/^Source:\s+(\S+)/) {
- $report{'source'} = $1;
- last;
- }
- }
- close CTRL;
- }
+sub sendReport {
- if (!open REV, "<$threadworkplace/$tarballdir/debian/rev") {
- return;
- } else {
- $report{'svn_rev'} = <REV>;
- print "rev: ".$report{'svn_rev'}."\n";
- close REV;
- }
+ my ($report, $threadbuildarea) = @_;
- foreach (`cd $threadworkplace/$tarballdir; dpkg-parsechangelog 2>&1`) {
- if (/Source: (.+)/) {
- $report{'source'} = $1;
- } elsif (/Version: (.+?)(-.+|$)/) {
- $report{'release'} = $1;
- }
- }
+ $report->{'arch'} = $arch;
+ $report->{'buildadminaddr'} = $buildadminaddr;
+ $report->{hostname} = $hostname;
+ $report->{distro} = $distro;
- $report{'stamp_build-start'} = time;
+ $report->{'agent_release'} = $RELEASE;
+ $report->{'pbuilder_release'} = `dpkg-query -W -f='\${Version}' pbuilder`;
+ my $tmp = basename($report->{dsc});
+ $tmp =~ s/\.dsc//;
+ my $prefix = $tmp."_".$arch."_$hostname";
+ my $logfile = $prefix.'.log';
+ my $infofile = $prefix.'.info';
- my $dsc_localpath = $threadworkplace.'/'.basename($dsc);
- `pbuilder --build --buildresult $threadbuildarea --configfile $ccachedir/ccache.cfg --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz $dsc_localpath >$threadreportarea/build.log.tmp 2>&1`;
- $report{'stamp_build-end'} = time;
-
-# To avoid breakage with tar on the service side, I do some clean up in the logs
- open BUILDLOGTMP, "<$threadreportarea/build.log.tmp" or die;
- open BUILDLOG, ">$threadreportarea/build.log" or die;
+# To avoid strange breakage I do some clean up in the log file
+ open BUILDLOGTMP, "<$threadbuildarea/build.log.tmp" or die;
+ open BUILDLOG, ">$threadbuildarea/".$logfile or die;
foreach (<BUILDLOGTMP>) {
s/[[:cntrl:]]//g;
print BUILDLOG $_."\n";
}
-
close BUILDLOGTMP;
close BUILDLOG;
- unlink "$threadreportarea/build.log.tmp";
+ unlink "$threadbuildarea/build.log.tmp";
- my $debfiles;
- $debfiles .= $_." " foreach (<$threadbuildarea/*.{,u}deb>);
- print "debfiles: ".$debfiles." in ".$threadbuildarea."\n";
- if ($debfiles) {
- $report{build} = "ok";
- `cd $threadbuildarea ; lintian --allow-root $debfiles *.dsc > $threadreportarea/lintian.log`;
- `cd $threadbuildarea ; linda --show-tag $debfiles *.dsc 2>/dev/null > $threadreportarea/linda.log`;
- print "\npiupart\n";
- `cd $threadbuildarea ; /usr/sbin/piuparts $debfiles | grep -v DEBUG > $threadreportarea/piuparts.log`;
- $report{piuparts} = (($? >> 8)==0)?'ok':'nok';
- } else {
- $report{build} = "nok";
+# Prepare and send the report
+ my $ftp = Net::FTP->new($ftphost, Debug => 0) or die "Cannot connect
+ to ".$ftphost.": $@";
+ $ftp->login($ftplogin,$ftppassword) or die "Cannot login ", $ftp->message;
+ $ftp->mkdir($report->{package_id}); # Do not die since mkdir fails if the
+ # directory exist
+ $ftp->cwd($report->{package_id}) or die "Can't cwd ", $ftp->message;
+
+ $report->{build} = "nok";
+ foreach (<$threadbuildarea/*.{,u}deb>) {
+ $report->{build} = "ok"; # There is .deb file => the build is ok... I guess
+ $ftp->put($_) or warn "Failed to send the info file\n";
}
-# Prepare and send the report
- open BUILDREPORT,">".$threadreportarea."/info";
- foreach (sort keys %report) {
- chomp $report{$_};
+ open BUILDREPORT,">".$threadbuildarea."/".$infofile;
+ foreach (sort keys %$report) {
+ chomp $report->{$_};
- print BUILDREPORT $_."=".$report{$_}."\n";
+ print BUILDREPORT $_."=".$report->{$_}."\n";
}
close BUILDREPORT;
-
- my $ftp = Net::FTP->new($config->agent_ftphost, Debug => 0) or die "Cannot connect
- to ".$config->agent_ftphost.": $@";
- my $reporttarball = "$report{'source'}_$report{'svn_rev'}_".$arch."_$hostname.tar";
- $ftp->login($config->agent_ftplogin,$config->agent_ftppassword) or die "Cannot login ", $ftp->message;
- `cd $threadtmpdir ; cp -r $threadreportarea report ; tar cf $reporttarball report 2>&1`;
- $ftp->put($threadtmpdir."/".$reporttarball) or warn "Failed to send the report\n";
+
+ $ftp->put($threadbuildarea."/".$infofile) or warn "Failed to send the info file\n";
$ftp->close or warn "Failed to close the ftp connection to the server\n";
}
-#$SIG{CHLD} = 'IGNORE';
-while (sleep 1) {
- if (isFull($workplace)||isFull($reportarea)||isFull($buildarea)) {
+sub build {
+ my $report = shift;
+
+ print "a\n";
+ print Dumper($report);
+ return unless $report;
+ die unless -d $workplace;
+ die unless -d $buildarea;
+ die unless -d $tmpdir;
+
+ my $threadworkplace = $workplace .'/'. $$;
+ my $threadbuildarea = $buildarea .'/'. $$;
+
+ die if (!-d $threadworkplace && !mkdir $threadworkplace);
+ die if (!-d $threadbuildarea && !mkdir $threadbuildarea);
+
+###########################################################################
+###########################################################################
+###########################################################################
+
+# if (-f $pbuildertgz) {
+# my $sb = stat($pbuildertgz);
+# if ($sb->ctime + 3600*24*7 < time) {
+# `mv $pbuildertgz $pbuildertgz.bak`
+# }
+# }
+
+ print "dsc: ".$report->{dsc}."\n";
+
+ my $tarballdir;
+ my $cmd = "cd $threadworkplace; dget -qx ".$report->{dsc};
+ foreach (`$cmd`) {
+ $tarballdir = $1 if /dpkg-source:\sextracting\s\S+\sin\s(\S+)/;
+ }
+ print "tarballdir:$threadworkplace; $tarballdir\n";
+#die "FUCK" unless $tarballdir;
+ return unless $tarballdir;
+
+ $report->{'stamp_build-start'} = time;
+ my $dsc_localpath = $threadworkplace.'/'.basename($report->{dsc});
+ `cd $threadworkplace/$tarballdir; pbuilder --build --buildresult $threadbuildarea --configfile $ccachedir/ccache.cfg --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz $dsc_localpath >$threadbuildarea/build.log.tmp 2>&1`;
+ $report->{'stamp_build-end'} = time;
+
+ sendReport($report, $threadbuildarea, $threadbuildarea);
+
+ `rm -rf $threadworkplace $threadbuildarea $threadbuildarea`;
+
+ 1;
+}
+
+while (sleep 1) { # TODO replace by a more important valious
+
+ if (isFull($workplace)||isFull($buildarea)) {
`rm -f $tarballsplace/* $aptcachedir/*`;
print "disk is full\n";
- if (%job&&(isFull($workplace)||isFull($reportarea)||isFull($buildarea))) {
+ if (%job&&(isFull($workplace)||isFull($buildarea))) {
print "Emergency clean up\n";
kill 9, $_ foreach (keys %job);
}
}
+ upgradeChroot();
+
foreach my $pid (keys %job) {
- if (time > $job{$pid} + 3600*3) {
+ if (time > $job{$pid} + 3600*4) {
print "kill\n";
kill 9, $pid; # timeout
sleep 5;
}
if (! -d "/proc/".$pid) {
-# `rm -rf $workplace/$pid $reportarea/$pid $buildarea/$pid`;
+ `rm -rf $workplace/$pid $buildarea/$pid`;
delete ($job{$pid});
}
waitpid($pid,WNOHANG);
}
if (keys %job < $maxjobs) {
- my $pid = fork;
- if ($pid) {
- $job{$pid} = time;
+ my $newpid = fork;
+ if ($newpid) {
+ $job{$newpid} = time;
} else {
- my $dsc = howxsnext ();
- build($dsc) if $dsc;
+ my $report = howxsnext ();
+ if (!exists($report->{dsc})) {
+ print "Can't get a package to rebuild\n";
+ } else {
+ build($report);
+ }
print "done\n";
- last;
+ exit;
}
}
More information about the Collab-qa-commits
mailing list