[Collab-qa-commits] r1808 - in udd: . scripts sql udd
Lucas Nussbaum
lucas at alioth.debian.org
Sat Sep 25 10:25:28 UTC 2010
Author: lucas
Date: 2010-09-25 10:25:27 +0000 (Sat, 25 Sep 2010)
New Revision: 1808
Added:
udd/scripts/update-hints
udd/udd/hints_gatherer.py
Modified:
udd/config-org.yaml
udd/sql/setup.sql
Log:
added hints gatherer
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2010-09-25 06:56:46 UTC (rev 1807)
+++ udd/config-org.yaml 2010-09-25 10:25:27 UTC (rev 1808)
@@ -27,6 +27,7 @@
pts: module udd.pts_gatherer
history-daily: module udd.history_daily_gatherer
i18n-apps: module udd.i18n_apps_gatherer
+ hints: module udd.hints_gatherer
timestamp-dir: /org/udd.debian.org/timestamps
lock-dir: /org/udd.debian.org/locks
archs:
@@ -379,6 +380,11 @@
schema: upload_history
table: upload_history
+hints:
+ type: hints
+ path: /org/udd.debian.org/tmp/hints
+ update-command: /org/udd.debian.org/udd/scripts/update-hints
+
bugs:
type: bugs
archived: false
Added: udd/scripts/update-hints
===================================================================
--- udd/scripts/update-hints (rev 0)
+++ udd/scripts/update-hints 2010-09-25 10:25:27 UTC (rev 1808)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -x
+set -e
+
+mkdir -p /org/udd.debian.org/tmp/hints
+cd /org/udd.debian.org/tmp/hints
+rm * || true
+wget -q -r -np -nH -nd -nc http://release.debian.org/britney/hints/
+rm index.html README robots.txt
Property changes on: udd/scripts/update-hints
___________________________________________________________________
Added: svn:executable
+ *
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2010-09-25 06:56:46 UTC (rev 1807)
+++ udd/sql/setup.sql 2010-09-25 10:25:27 UTC (rev 1808)
@@ -736,4 +736,8 @@
);
GRANT SELECT ON history.sources_count TO public;
-
+CREATE TABLE hints
+ (source text, version debversion, architecture text,
+ type text, file text, comment text);
+CREATE INDEX hints_idx on hints(source, version);
+GRANT SELECT ON hints TO public;
Added: udd/udd/hints_gatherer.py
===================================================================
--- udd/udd/hints_gatherer.py (rev 0)
+++ udd/udd/hints_gatherer.py 2010-09-25 10:25:27 UTC (rev 1808)
@@ -0,0 +1,124 @@
+# Last-Modified: <Sun Aug 17 12:13:02 2008>
+# This file is part of the Ultimate Debian Database Project
+
+from gatherer import gatherer
+import aux
+from glob import glob
+import gzip
+import psycopg2
+import sys
+import email.Utils
+import os.path
+import re
+
+def get_gatherer(config, connection, source):
+ return hints_gatherer(config, connection, source)
+
+class Hint(object):
+ """Base clase for representing a hint.
+
+ It also acts as a factory: Hint(hintame, *args) will return an object of
+ the appropriate class (eg. UnstableToTestingHint).
+ """
+
+ def __init__(self, type, *args):
+ starts_with_letter = re.compile(r'(?i)^[a-z]')
+
+ self.type = type
+ self.pkgs = []
+ self.versionmap = {} # original versions on the hint
+ self.archmap = {}
+
+ for pkg in args:
+ if '/' in pkg:
+ pkg, ver = pkg.split('/', 1)
+ if starts_with_letter.search(ver):
+ # Package names start with digits, so assume
+ # this is an architecture. This will break if
+ # an architecture name ever starts with a digit
+ # but that seems unlikely.
+ if '/' in ver:
+ arch, ver = ver.split('/', 1)
+ else:
+ arch = ver
+ ver = ''
+ else:
+ arch = ''
+ else:
+ ver = ''
+ arch = ''
+
+ src = pkg
+ if src.startswith('-'):
+ src = src[1:]
+ self.pkgs.append(src)
+
+ self.versionmap[src] = ver
+ if len(arch) > 0:
+ if src not in self.archmap:
+ self.archmap[src] = set()
+ self.archmap[src].add(arch)
+
+class hints_gatherer(gatherer):
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ if not 'path' in self.my_config:
+ raise aux.ConfigException('path not specified for source ' + source)
+
+ def tables(self):
+ return [ 'hints' ]
+
+ RE_COMMENT = re.compile(r'^\s*#')
+ RE_BLANKLINE = re.compile(r'^\s*$')
+ def parse_hints(self, path):
+ files = glob(path + '/*')
+ files.sort()
+ hints = []
+ for file in files:
+ f = open(file)
+ comments = []
+ lc = 0
+ for line in f:
+ lc += 1
+ line = line.rstrip('\r\n')
+ if self.RE_COMMENT.search(line):
+ comments.append(line)
+ continue
+ elif self.RE_BLANKLINE.search(line):
+ comments = []
+ continue
+ elif line.startswith('finished'):
+ break
+ hint = Hint(*line.split())
+ hints.append([hint, os.path.basename(file), '\n'.join(comments)])
+ return hints
+
+ def run(self):
+ path = self.my_config['path']
+
+ hints = self.parse_hints(path)
+
+ cursor = self.cursor()
+ cursor.execute("PREPARE h_insert AS INSERT INTO hints (source, version, architecture, type, file, comment) VALUES ($1, $2 , $3, $4, $5, $6)")
+ cursor.execute("DELETE FROM hints")
+ query = "EXECUTE h_insert(%(source)s, %(version)s, %(arch)s, %(type)s, %(file)s, %(comment)s)"
+ hs = []
+ for hint in hints:
+ h = {}
+ h['file'] = hint[1]
+ h['comment'] = hint[2]
+ h['type'] = hint[0].type
+ for src in hint[0].pkgs:
+ h['source'] = src
+ if hint[0].versionmap.has_key(src):
+ h['version'] = hint[0].versionmap[src]
+ else:
+ h['version'] = None
+ if hint[0].archmap.has_key(src):
+ h['arch'] = ' '.join(hint[0].archmap[src])
+ else:
+ h['arch'] = None
+ hs.append(h)
+ cursor.executemany(query, hs)
+ cursor.execute("DEALLOCATE h_insert")
+ cursor.execute("ANALYZE hints")
More information about the Collab-qa-commits
mailing list