[Collab-qa-commits] r939 - in udd/src: . udd
neronus-guest at alioth.debian.org
neronus-guest at alioth.debian.org
Sat Jul 26 19:21:16 UTC 2008
Author: neronus-guest
Date: 2008-07-26 19:21:15 +0000 (Sat, 26 Jul 2008)
New Revision: 939
Added:
udd/src/udd/upload_history_gatherer.py
Modified:
udd/src/setup-db.sql
udd/src/test.yaml
udd/src/udd/bugs_gatherer.pl
Log:
Added code to import upload history
Removed print from bugs gatherer
Modified: udd/src/setup-db.sql
===================================================================
--- udd/src/setup-db.sql 2008-07-26 17:44:03 UTC (rev 938)
+++ udd/src/setup-db.sql 2008-07-26 19:21:15 UTC (rev 939)
@@ -42,6 +42,9 @@
CREATE TABLE bug_fixed_in
(id int, version text);
+CREATE TABLE upload_history
+ (package text, version text, date timestamp with time zone, changed_by text, maintainer text, nmu boolean, signed_by text, key_id text);
+
CREATE VIEW popcon_src_average AS
SELECT sources.package, avg(vote) AS vote, avg(olde) AS old, avg(recent) AS recent, avg(nofiles) as nofiles
FROM sources, popcon,
Modified: udd/src/test.yaml
===================================================================
--- udd/src/test.yaml 2008-07-26 17:44:03 UTC (rev 938)
+++ udd/src/test.yaml 2008-07-26 19:21:15 UTC (rev 939)
@@ -1,5 +1,5 @@
general:
- dbname: udd
+ dbname: udd-test
types:
sources: module udd.sources_gatherer
packages: module udd.packages_gatherer
@@ -8,6 +8,7 @@
src-pkg: module udd.src_and_pkg_gatherer
popcon: module udd.popcon_gatherer
testing-migrations: module udd.testing_migrations_gatherer
+ upload-history: module udd.upload_history_gatherer
#src-pkg: python sources_gatherer.py
debug: 1
@@ -128,4 +129,9 @@
testing-migrations:
type: testing-migrations
path: /tmp/migrations
- update-command: echo "test"; wget -O /tmp/migrations 'http://qa.debian.org/~lucas/testing-status.raw'
+ update-command: wget -O /tmp/migrations 'http://qa.debian.org/~lucas/testing-status.raw'
+
+upload-history:
+ type: upload-history
+ path: /tmp/upload-history
+ update-command: if [ ! -e /tmp/upload-history/ ]; then mkdir /tmp/upload-history/; fi; rm -rf /tmp/upload-history/*; wget -r --no-parent -nd -A debian-devel-* -P /tmp/upload-history http://qa.debian.org/~filippo/ddc/
Modified: udd/src/udd/bugs_gatherer.pl
===================================================================
--- udd/src/udd/bugs_gatherer.pl 2008-07-26 17:44:03 UTC (rev 938)
+++ udd/src/udd/bugs_gatherer.pl 2008-07-26 19:21:15 UTC (rev 939)
@@ -1,5 +1,5 @@
#!/usr/bin/perl
-# Last-Modified: <Sat Jul 26 12:01:01 2008>
+# Last-Modified: <Sat Jul 26 13:26:22 2008>
use strict;
use warnings;
@@ -51,6 +51,8 @@
$dbh->prepare("DELETE FROM bug_fixed_in")->execute();
$dbh->prepare("DELETE FROM bug_merged_with")->execute();
+ my %binarytosource = ();
+
# Read all bugs
foreach my $bug_nr (get_bugs()) {
# Fetch bug using Debbugs
@@ -62,7 +64,13 @@
map { $bug{$_} = $dbh->quote($bug{$_}) } qw(subject originator owner pending);
my @found_versions = map { $dbh->quote($_) } @{$bug{found_versions}};
my @fixed_versions = map { $dbh->quote($_) } @{$bug{fixed_versions}};
- my $source = binarytosource($bug{package});
+
+
+ if(not exists $binarytosource{$bug{package}}) {
+ $binarytosource{$bug{package}} = binarytosource($bug{package});
+ }
+ my $source = $binarytosource{$bug{package}};
+
if(not defined $source) {
$source = 'NULL';
} else {
Added: udd/src/udd/upload_history_gatherer.py
===================================================================
--- udd/src/udd/upload_history_gatherer.py (rev 0)
+++ udd/src/udd/upload_history_gatherer.py 2008-07-26 19:21:15 UTC (rev 939)
@@ -0,0 +1,81 @@
+# Last-Modified: <Sat Jul 26 18:20:06 2008>
+# This file is part of the Ultimate Debian Database Project
+
+from gatherer import gatherer
+import aux
+from glob import glob
+import gzip
+import psycopg2
+import sys
+
+date_translation = {
+ 'Deb': 'Feb',
+ 'Augl': 'Aug',
+ 'Fev': 'Feb' }
+
+def get_gatherer(config, connection):
+ return upload_history_gatherer(config, connection)
+
+class upload_history_gatherer(gatherer):
+ def __init__(self, connection, config):
+ gatherer.__init__(self, connection, config)
+
+ def run(self, source):
+ if not 'path' in self.config[source]:
+ raise aux.ConfigException('path not specified for source ' + source)
+ path = self.config[source]['path']
+
+ cursor = self.cursor()
+
+ cursor.execute("DELETE FROM upload_history")
+
+ cursor.execute("PREPARE uh_insert AS INSERT INTO upload_history VALUES \
+ ($1, $2, $3, $4, $5, $6, $7, $8)")
+
+ for name in glob(path + '/debian-devel-*'):
+ print name
+ f = None
+ if name.endswith(".gz"):
+ f = gzip.open(name)
+ else:
+ f = open(name)
+
+ current = {}
+ last_field = None
+ line_count = 0
+ for line in f.readlines():
+ line_count += 1
+ line = line.strip()
+ # Stupid multi-line maintainer fields *grml*
+ if line == '':
+ query = "EXECUTE uh_insert(%(Source)s, %(Version)s, %(Date)s, %(Changed-By)s, \
+ %(Maintainer)s, %(NMU)s, %(Key)s, %(Signed-By)s)" % current
+ try:
+ cursor.execute(query)
+ except psycopg2.ProgrammingError, s:
+ print "Error at line %d of file %s" % (line_count, name)
+ raise
+ current = {}
+ last_field = None
+ continue
+
+ if line.find(':') == -1:
+ if not last_field:
+ raise Exception, "Format error on line " + line_count + "of file " + name
+ current[last_field] += line
+ continue
+
+
+ (field, data) = line.split(':', 1)
+ data = data.strip()
+
+ if field != 'NMU':
+ current[field] = aux.quote(data)
+ else:
+ current[field] = data
+
+ last_field = field
+
+ cursor.execute("DEALLOCATE uh_insert")
+
+ sys.exit(1)
More information about the Collab-qa-commits
mailing list