[Collab-qa-commits] r505 - in svnbuildstat/trunk: . lib/SvnBuildStat lib/SvnBuildStat/Common lib/SvnBuildStat/WWW/Controller script
goneri-guest at alioth.debian.org
goneri-guest at alioth.debian.org
Sun Nov 11 16:33:23 UTC 2007
Author: goneri-guest
Date: 2007-11-11 16:33:23 +0000 (Sun, 11 Nov 2007)
New Revision: 505
Added:
svnbuildstat/trunk/script/svnbuildstat_update-db-git.pl
Modified:
svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
svnbuildstat/trunk/lib/SvnBuildStat/Common/Svn.pm
svnbuildstat/trunk/lib/SvnBuildStat/WWW/Controller/Packages.pm
svnbuildstat/trunk/script/svnbuildstat_update-db-svn.pl
svnbuildstat/trunk/svnbuildstat_www.yml
Log:
* clean up of svnbuildstat_update-db-svn.pl.
* move the debian/control parser in SvnBuildStat::Common
* do not use Thread::Pool::Simple for the moment, it creates hardly
understandable error.
* initinal import of svnbuildstat_update-db-git.pl
Modified: svnbuildstat/trunk/lib/SvnBuildStat/Common/Svn.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Common/Svn.pm 2007-11-10 14:11:07 UTC (rev 504)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Common/Svn.pm 2007-11-11 16:33:23 UTC (rev 505)
@@ -1,5 +1,13 @@
-use SvnBuildStat::Svn;
+package SvnBuildStat::Common::Svn;
+use strict;
+use warnings;
+
+require Exporter;
+
+our @ISA = "Exporter";
+our @EXPORT_OK = qw(getRev);
+
sub getRev {
my $uri = shift;
Modified: svnbuildstat/trunk/lib/SvnBuildStat/Common.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/Common.pm 2007-11-10 14:11:07 UTC (rev 504)
+++ svnbuildstat/trunk/lib/SvnBuildStat/Common.pm 2007-11-11 16:33:23 UTC (rev 505)
@@ -1,5 +1,13 @@
-use SvnBuildStat::Common;
+package SvnBuildStat::Common;
+use strict;
+use warnings;
+
+require Exporter;
+
+our @ISA = "Exporter";
+our @EXPORT_OK = qw(mkTarballFromPackage testUrl mkRootdirectoryFromPackage getDataFromDebianFtp parseControl);
+
sub mkTarballFromPackage {
my $package = shift;
@@ -74,5 +82,37 @@
return { tarballuri => $tarballuri, isindebian => $isindebian, isnative => $isnative };
}
+sub parseControl {
+ my @control = @_;
+ my $ret = { maintainers => []};
+
+ foreach (@control) {
+ $ret->{packagesrc} = $1 if /^Source:\ *(.*)/;
+ if (/^(Maintainer|Uploaders):\ *(.*)/) {
+ my $tmp = $2;
+ foreach (split /,/, $tmp) {
+ if (/(.+)<(.+)>/) {
+ my $name = $1;
+ my $email = $2;
+ $name =~ s/^\ *//;
+ $name =~ s/\ $//;
+ push @{$ret->{maintainers}}, {email=>$email, name => $name};
+ }
+
+ }
+ } elsif (/^Architecture: (.*)/) {
+ my $arch = $1;
+ $ret->{i386} = 't' if $arch =~ /(any|all|i386)/;
+ $ret->{powerpc} = 't' if $arch =~ /(any|all|powerpc)/;
+ $ret->{sparc} = 't' if $arch =~ /(any|all|sparc)/;
+ $ret->{amd64} = 't' if $arch =~ /(any|all|amd64)/;
+ }
+ }
+
+ return unless (exists ($ret->{packagesrc}) && $ret->{packagesrc});
+
+ $ret;
+}
+
1;
Modified: svnbuildstat/trunk/lib/SvnBuildStat/WWW/Controller/Packages.pm
===================================================================
--- svnbuildstat/trunk/lib/SvnBuildStat/WWW/Controller/Packages.pm 2007-11-10 14:11:07 UTC (rev 504)
+++ svnbuildstat/trunk/lib/SvnBuildStat/WWW/Controller/Packages.pm 2007-11-11 16:33:23 UTC (rev 505)
@@ -100,8 +100,8 @@
$c->stash->{tarballuri} = $package->tarballuri;
$c->stash->{svndebrelease} = $package->svndebrelease;
$c->stash->{iswatchfilebroken} = $package->iswatchfilebroken;
- $c->stash->{uri} = $package->uri;
- $c->stash->{weburi} = $package->uri;
+ $c->stash->{vcsuri} = $package->vcsuri;
+ $c->stash->{weburi} = $package->vcsuri;
$c->stash->{weburi} =~ s!svn://svn.debian.org/svn!http://svn.debian.org/wsvn!;
$c->stash->{isuptodate} = $package->isuptodate;
$c->stash->{upstreamrelease} = $package->upstreamrelease;
@@ -179,6 +179,7 @@
"ppc64|sh|armeb|m32r|hurd-i386|kfreebsd-gnu)";
+
if (!(defined ($arch) && $arch && $arch =~ /$archs/)) {
$c->response->body("missing or invalid arch"); # Seems to create an error
return;
@@ -194,8 +195,13 @@
my $svndebrelease = $package->realsvndebrelease;
$svndebrelease =~ s/^\d+://; # remove the EPOCH
- # TODO: put the repository URL in the cfg
- $c->response->body("http://nana.rulezlan.org/debian/".$package->name."_".$svndebrelease.".dsc");
+ if ($package->isinmypool) {
+ # TODO: put the repository URL in the cfg
+ $c->response->body("http://nana.rulezlan.org/debian/".$package->repository_id->team_id->shortname."/".$package->name."/".$package->name."_".$svndebrelease.".dsc");
+ } else {
+ $c->response->body($package->dscuri);
+ }
+
}
=head1 AUTHOR
Added: svnbuildstat/trunk/script/svnbuildstat_update-db-git.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db-git.pl (rev 0)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db-git.pl 2007-11-11 16:33:23 UTC (rev 505)
@@ -0,0 +1,108 @@
+#!/usr/bin/perl -w
+use File::Temp qw/ tempfile /;
+
+use strict;
+
+use LWP::UserAgent;
+use Data::Dumper;
+use File::Basename;
+use Time::Local 'timelocal_nocheck';
+
+use lib '/home/sites/svnbuildstat.debian.net/svnbuildstat/lib';
+use SvnBuildStat::Schema;
+use SvnBuildStat::Config;
+use Logger::Syslog;
+#use SvnBuildStat::Common::Svn;
+
+my $config;
+my $schema;
+
+sub createTarballUrlFromTarballlayout {
+ my ($repository, $package) = @_;
+
+ return unless $$repository->tarballlayout;
+ my $packagename = $$package->name;
+ my $tarball = SvnBuildStat::Common::mkTarballFromPackage($package);
+ my $tarballuri = $$repository->tarballlayout;
+ $tarballuri =~ s/\@TARBALL@/$tarball/;
+ $tarballuri =~ s/\@PACKAGE@/$packagename/;
+# print "\n--->".$tarballuri."\n";
+
+ return $tarballuri;
+}
+
+sub importRepository {
+ my $repository = shift;
+
+
+ my $tarballonrepository;
+ print "Repository: ".$$repository->name."\n";
+ if (!$$repository->uri =~ /ssh:\/\/([\w\.-]+)(\/.+)$/) {
+ info ("can't parse ".$$repository->name." uri");
+ return;
+ }
+# my $repohost = $1; harcoded for the moment
+ my $reporoot = $2;
+
+ my $lastchange = `ssh goneri-guest\@alioth.debian.org GIT_DIR=$reporoot git log --pretty=oneline -n1`;
+ if (!$lastchange || $lastchange =~ /^(\w+)/) {
+ info ("can't git log for ".$$repository->uri);
+ return;
+ }
+ my $rev = $1;
+
+ if ($$repository->vcsrev eq $rev && $$repository->lastcheck) {
+ debug ("no need to refresh");
+ return;
+ }
+
+ GIT_DIR=/srv/git.debian.org/git/kernel/klibc-old.git/ git show HEAD:debian/changelog
+ foreach my $uri (@uri) {
+ # look for packages
+ $poolImportPkg->add($repository,$uri,$tarballonrepository) or die "Fucked\n";
+ }
+
+ $$repository->vcsrev($rev);
+ $$repository->lastcheck('now');
+ $$repository->update();
+
+ print "end import Repo\n";
+}
+
+###################################################
+###################################################
+###################################################
+###################################################
+###################################################
+############### MAIN ##############################
+###################################################
+###################################################
+
+info ("starting");
+########## THREAD POOLS #####
+$poolImportPkg = Thread::Pool::Simple->new(
+ min => 5,
+ max => 5,
+ load => 3,
+ do => [\&importPkg],
+ lifespan => 1
+ );
+
+########
+
+# Import packages
+my $repository_rs = $schema->resultset('Repository')->search({enabled => 'true'});
+while (my $repository = $repository_rs->next) {
+ next unless $repository->vcs_id && $repository->vcs_id->name eq "git";
+ importRepository(\$repository) or warn "importRepository failed for ".$repository->name."\n";
+}
+$poolImportPkg->join;
+
+# Purge the removed packages
+# This semems to be fucked?
+#my $interval = "< repository_id.lastcheck - interval '1 day'";
+#my $package_rs = $schema->resultset('Package')->search({'me.lastcheck' => \$interval} , {join => => 'repository_id'});
+#$package_rs->delete_all;
+
+info ("stopping");
+
Property changes on: svnbuildstat/trunk/script/svnbuildstat_update-db-git.pl
___________________________________________________________________
Name: svn:executable
+ *
Modified: svnbuildstat/trunk/script/svnbuildstat_update-db-svn.pl
===================================================================
--- svnbuildstat/trunk/script/svnbuildstat_update-db-svn.pl 2007-11-10 14:11:07 UTC (rev 504)
+++ svnbuildstat/trunk/script/svnbuildstat_update-db-svn.pl 2007-11-11 16:33:23 UTC (rev 505)
@@ -8,23 +8,17 @@
use File::Basename;
use Time::Local 'timelocal_nocheck';
-use threads;
-use threads::shared;
-use Thread::Pool::Simple;
-
-use Thread::Pool::Simple;
use lib '/home/sites/svnbuildstat.debian.net/svnbuildstat/lib';
use SvnBuildStat::Schema;
use SvnBuildStat::Config;
use Logger::Syslog;
+use SvnBuildStat::Common;
+use SvnBuildStat::Common::Svn;
my $config;
my $schema;
-my $poolImportPkg;
-
-
sub createTarballUrlFromTarballlayout {
my ($repository, $package) = @_;
@@ -39,45 +33,7 @@
return $tarballuri;
}
-sub getOnDebianData {
- my $package = shift;
- return unless $$package->name;
- return unless $$package->svndebrelease;
- my $svndebrelease = $$package->svndebrelease;
- $svndebrelease =~ s/^\d+://; # remove the EPOCH
- my $isindebian = 'f';
- my $tarballuri;
- my $isnative = 'f';
-
- my $ua = LWP::UserAgent->new;
- $ua->agent("SvnBuildStat/0.1 ");
-
- if ($$package->svndebrelease !~ /-[\d+\.]+$/) {
- $isnative = 't';
- } else {
- my $debmirror = 'http://ftp.debian.org/debian';
- my $debdiff .= $$package->name."_".$svndebrelease.".diff.gz";
- my $tarball = SvnBuildStat::Common::mkTarballFromPackage($package);
-
- if ($tarball) {
- foreach my $section (qw/main contrib non-free/) {
- my $tmp = "$1/".$$package->name if $$package->name =~ /^(lib.|.)/;
- my $debdiffuri = $debmirror.'/pool/'.$section.'/'.$tmp.'/'.$debdiff;
- my $tmp_tarballuri = $debmirror.'/pool/'.$section.'/'.$tmp.'/'.$tarball;
- if (testUrl($debdiffuri)) {
- $isindebian = 't';
- }
- if (testUrl($tmp_tarballuri)) {
- $tarballuri = $tmp_tarballuri;
- $tarballuri =~ s/^$debmirror/\@DEBMIRROR@/;
- }
- }
- }
- }
- return { tarballuri => $tarballuri, isindebian => $isindebian, isnative => $isnative };
-}
-
sub getUscanData {
my $package = shift;
my $cmd;
@@ -139,13 +95,13 @@
$last_rev = $build_rs->first->rev if ($build_rs->first);
}
- return unless $last_rev;
- print "LAST REV:". $last_rev."\n";
+ $last_rev = $$package->vcsrev unless $last_rev;
my $cmd = "LC_ALL=C svn log -r ".$last_rev.":".$$package->vcsrev." ".$$package->vcsuri;
my $begin;
my $entry;
foreach (`$cmd`) {
+ print $_;
if (/^------------------------------------------------------------------------/) {
$begin = 1;
$entry->update if $entry;
@@ -181,7 +137,7 @@
my $packagesrc;
my $svndebrelease;
my $tarballuri;
- my $rev = SvnBuildStat::Svn::getRev($uri);
+ my $rev = SvnBuildStat::Common::Svn::getRev($uri);
my $todo;
my $currentpendingbug;
my $currentchangelogentry;
@@ -192,44 +148,21 @@
my $amd64 = 'f';
- return unless $rev;
-
- my @control = `svn cat $uri/debian/control`;
- return unless @control;
- foreach (@control) {
- $packagesrc = $1 if /^Source:\ *(.*)/;
- if (/^(Maintainer|Uploaders):\ *(.*)/) {
- my $tmp = $2;
- foreach (split /,/, $tmp) {
- if (/(.+)<(.+)>/) {
- my $name = $1;
- my $email = $2;
- $name =~ s/^\ *//;
- $name =~ s/\ $//;
- my $maintainer = $schema->resultset('Maintainer')->find_or_create({email=>$email});
- $maintainer->name($name);
- $maintainer->update();
- push @maintainer, $maintainer;
- }
-
- }
- } elsif (/^Architecture: (.*)/) {
- my $arch = $1;
- $i386 = 't' if $arch =~ /(any|all|i386)/;
- $powerpc = 't' if $arch =~ /(any|all|powerpc)/;
- $sparc = 't' if $arch =~ /(any|all|sparc)/;
- $amd64 = 't' if $arch =~ /(any|all|amd64)/;
- }
+ if (!$rev) {
+ info ("Can't get current svn revision");
+ return;
}
- if (!$packagesrc) {
- print "Parse error: $uri/debian/control";
+ my $control = SvnBuildStat::Common::parseControl(`svn cat $uri/debian/control`);
+ if (!$control) {
+ info ("Failed to parse $uri/debian/control");
return;
-
}
- my $package = $schema->resultset('Package')->find_or_create({name => $packagesrc});
+
+ my $package = $schema->resultset('Package')->find_or_create({name => $control->{packagesrc}});
if (defined($rev) && defined ($package->vcsrev) && $rev eq $package->vcsrev) {
+ debug ("no need to refresh");
return;
}
print "GO ".$package->name."\n";
@@ -284,18 +217,22 @@
$package->svndebrelease ($svndebrelease);
$package->vcsuri($uri);
$package->repository_id($$repository);
- foreach my $maintainer (@maintainer) {
+ foreach (@{$control->{maintainers}}) {
+
+ my $maintainer = $schema->resultset('Maintainer')->find_or_create({email=>$_->email});
+ $maintainer->name($_->name);
+
my $package_maintainer =
$package->find_or_create_related('package_maintainers', {
'maintainer_id' => $maintainer->id});
}
-# updateChangelog(\$package);
+ updateChangelog(\$package);
# Search for the tarball
my $tarball = SvnBuildStat::Common::mkTarballFromPackage(\$package);
my $uscandata = getUscanData(\$package);
- my $ondebiandata = getOnDebianData(\$package);
+ my $ondebiandata = SvnBuildStat::Common::getDataFromDebianFtp(\$package);
my $tarballurlfromtarballlayout = createTarballUrlFromTarballlayout($repository,\$package);
$package->iswatchfilebroken($uscandata->{iswatchfilebroken});
# Is the tarball on a Debian mirror?
@@ -332,10 +269,10 @@
$package->upstreamrelease($uscandata->{upstreamrelease});
$package->isindebian($ondebiandata->{isindebian});
- $package->i386($i386);
- $package->powerpc($powerpc);
- $package->sparc($sparc);
- $package->amd64($amd64);
+ $package->i386($control->{i386});
+ $package->powerpc($control->{powerpc});
+ $package->sparc($control->{sparc});
+ $package->amd64($control->{amd64});
$package->todo($todo);
$package->currentchangelogentry($currentchangelogentry);
$package->currentpendingbug($currentpendingbug);
@@ -359,27 +296,21 @@
my $tarballonrepository;
print "Repository: ".$$repository->name."\n";
- my $t = 'svn ls -R '.$$repository->uri;
- my $rev = SvnBuildStat::Svn::getRev($$repository->uri);
+ debug ("importing ".$$repository->name);
+ my $rev = SvnBuildStat::Common::Svn::getRev($$repository->uri);
if(! $rev) {
print "Failed to get the current revision of ".$$repository->name."\n";
return;
}
- if ($$repository->vcsrev eq $rev && $$repository->lastcheck) {
- # If he repository is up to date, a still do a refresh every 48h
- # this because of the tarball check
- #
- my ($year, $mon, $day, $hour, $min, $sec) = ($$repository->lastcheck =~ /(\d{4})-(\d+)-(\d+)\ (\d+):(\d+):(\d+)/);
- my $lastcheck = timelocal_nocheck($sec, $min, $hour, $day, $mon, $year);
- if ($lastcheck > time - 3600*48) {
- print "No need to refresh ".$$repository->name."\n";
- return;
- }
+ if ($$repository->vcsrev eq $rev) {
+ debug ("no need to refresh ".$$repository->name);
+ return;
}
my @uri;
-
+
+my $t = 'svn ls -R '.$$repository->uri;
foreach (`$t`) {
chomp;
my $uri = $$repository->uri.'/'.$_;
@@ -394,14 +325,14 @@
foreach my $uri (@uri) {
# look for packages
- $poolImportPkg->add($repository,$uri,$tarballonrepository) or die "Fucked\n";
+ importPkg($repository,$uri,1);
}
$$repository->vcsrev($rev);
$$repository->lastcheck('now');
$$repository->update();
- print "end import Repo\n";
+ debug( "end import Repo");
}
###################################################
###################################################
@@ -413,23 +344,16 @@
###################################################
info ("starting");
-########## THREAD POOLS #####
-$poolImportPkg = Thread::Pool::Simple->new(
- min => 5,
- max => 5,
- load => 3,
- do => [\&importPkg],
- lifespan => 1
- );
########
# Import packages
my $repository_rs = $schema->resultset('Repository')->search({enabled => 'true'});
while (my $repository = $repository_rs->next) {
+ next unless $repository->vcs_id && $repository->vcs_id->name eq "svn";
+ print $repository->name."\n";
importRepository(\$repository) or warn "importRepository failed for ".$repository->name."\n";
}
-$poolImportPkg->join;
# Purge the removed packages
# This semems to be fucked?
Modified: svnbuildstat/trunk/svnbuildstat_www.yml
===================================================================
--- svnbuildstat/trunk/svnbuildstat_www.yml 2007-11-10 14:11:07 UTC (rev 504)
+++ svnbuildstat/trunk/svnbuildstat_www.yml 2007-11-11 16:33:23 UTC (rev 505)
@@ -1,2 +1,4 @@
---
name: SvnBuildStat::WWW
+Controller::Packages:
+ totor: totor
More information about the Collab-qa-commits
mailing list