[Collab-qa-commits] r1704 - in udd: . scripts sql udd

[Lucas Nussbaum]

Author: lucas
Date: 2010-02-28 16:46:25 +0000 (Sun, 28 Feb 2010)
New Revision: 1704

Added:
   udd/udd/pts_gatherer.py
Modified:
   udd/config-org.yaml
   udd/scripts/dump-db.sh
   udd/sql/setup.sql
   udd/sql/upgrade.sql
Log:
PTS subscribers gatherer

Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml	2010-02-28 16:45:36 UTC (rev 1703)
+++ udd/config-org.yaml	2010-02-28 16:46:25 UTC (rev 1704)
@@ -23,6 +23,7 @@
     ldap: module udd.ldap_gatherer
     wannabuild: module udd.wannabuild_gatherer
     removals: module udd.removals_gatherer
+    pts: module udd.pts_gatherer
     history-daily: module udd.history_daily_gatherer
   timestamp-dir: /org/udd.debian.org/timestamps
   lock-dir: /org/udd.debian.org/locks
@@ -570,5 +571,10 @@
   table: package_removal
   schema: package_removal
 
+pts:
+  type: pts
+  update-command: ssh -i /org/udd.debian.org/.ssh/id_ptsdata qa at master.debian.org /org/packages.qa.debian.org/bin/export-data-to-udd > /org/udd.debian.org/tmp/pts.txt
+  path: /org/udd.debian.org/tmp/pts.txt
+
 history-daily:
   type: history-daily

Modified: udd/scripts/dump-db.sh
===================================================================
--- udd/scripts/dump-db.sh	2010-02-28 16:45:36 UTC (rev 1703)
+++ udd/scripts/dump-db.sh	2010-02-28 16:46:25 UTC (rev 1704)
@@ -5,5 +5,5 @@
 umask 022
 pg_dump --no-owner -p 5441 -n history udd | gzip > udd-history.sql.gz.new
 mv udd-history.sql.gz.new udd-history.sql.gz
-pg_dump --no-owner -p 5441 -T ldap -T really_active_dds udd | gzip > udd.sql.gz.new
+pg_dump --no-owner -p 5441 -T ldap -T really_active_dds -T pts udd | gzip > udd.sql.gz.new
 mv udd.sql.gz.new udd.sql.gz

Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql	2010-02-28 16:45:36 UTC (rev 1703)
+++ udd/sql/setup.sql	2010-02-28 16:46:25 UTC (rev 1704)
@@ -716,6 +716,21 @@
     SELECT DISTINCT carnivore_login.id, carnivore_login.login FROM carnivore_login, carnivore_keys, ldap WHERE ((((carnivore_keys.id = carnivore_login.id) AND (carnivore_keys.key_type = 'keyring'::text)) AND (carnivore_login.login = ldap.login)) AND (ldap.activity_pgp > '2009-01-01 00:00:00+00'::timestamp with time zone));
 GRANT SELECT ON TABLE really_active_dds TO guestdd;
 
+-- PTS
+CREATE TABLE pts (
+  source text,
+  email text,
+  PRIMARY KEY(source, email)
+);
+GRANT SELECT ON pts TO guestdd;
+
+CREATE TABLE pts_public (
+  source text,
+  email text,
+  PRIMARY KEY(source, email)
+);
+GRANT SELECT ON pts_public TO public;
+
 -- HISTORICAL DATA
 CREATE SCHEMA history;
 GRANT USAGE ON SCHEMA history TO public;

Modified: udd/sql/upgrade.sql
===================================================================
--- udd/sql/upgrade.sql	2010-02-28 16:45:36 UTC (rev 1703)
+++ udd/sql/upgrade.sql	2010-02-28 16:46:25 UTC (rev 1704)
@@ -309,3 +309,21 @@
 
 GRANT SELECT ON archived_bugs_blocks TO PUBLIC;
 GRANT SELECT ON archived_bugs_blockedby TO PUBLIC;
+
+-- 2010-02-28
+-- PTS
+CREATE TABLE pts (
+  source text,
+  email text,
+  PRIMARY KEY(source, email)
+);
+GRANT SELECT ON pts TO guestdd;
+
+CREATE TABLE pts_public (
+  source text,
+  email text,
+  PRIMARY KEY(source, email)
+);
+GRANT SELECT ON pts_public TO public;
+
+

Added: udd/udd/pts_gatherer.py
===================================================================
--- udd/udd/pts_gatherer.py	                        (rev 0)
+++ udd/udd/pts_gatherer.py	2010-02-28 16:46:25 UTC (rev 1704)
@@ -0,0 +1,43 @@
+# Last-Modified: <Sun Aug 10 12:16:12 2008>
+
+# This file is a part of the Ultimate Debian Database Project
+
+from gatherer import gatherer
+from aux import ConfigException, quote
+from time import strptime
+
+ZERO_DATE = '0000-01-01'
+
+def get_gatherer(config, connection, source):
+  return pts_gatherer(config, connection, source)
+
+
+class pts_gatherer(gatherer):
+  """This class imports PTS subscribers"""
+
+  def __init__(self, connection, config, source):
+    gatherer.__init__(self, connection, config, source)
+    self.assert_my_config('path')
+
+  def run(self):
+    src_cfg = self.my_config
+
+    c = self.connection.cursor()
+
+    c.execute("DELETE FROM pts")
+
+    c.execute("PREPARE pts_insert AS INSERT INTO pts (source, email) VALUES ($1, $2)")
+      
+    f = open(src_cfg['path'])
+    for line in f:
+      (package, subs) = line.split("\t")
+      for sub in subs.split(", "):
+        sub = sub.strip()
+        c.execute("EXECUTE pts_insert(%s, %s)", (package, sub))
+
+    c.execute("DELETE FROM pts_public")
+    c.execute("INSERT INTO pts_public SELECT source, md5(lower(email)) FROM pts")
+    c.execute("DEALLOCATE pts_insert")
+    c.execute("ANALYZE pts")
+    c.execute("ANALYZE pts_public")
+