[Collab-qa-commits] r1538 - in udd: . sql udd

[Lucas Nussbaum]

Author: lucas
Date: 2009-07-24 12:05:21 +0000 (Fri, 24 Jul 2009)
New Revision: 1538

Added:
   udd/udd/dehs_gatherer.py
Modified:
   udd/config-org.yaml
   udd/sql/setup.sql
   udd/sql/upgrade.sql
Log:
DEHS gatherer

Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml	2009-07-24 12:04:21 UTC (rev 1537)
+++ udd/config-org.yaml	2009-07-24 12:05:21 UTC (rev 1538)
@@ -19,6 +19,7 @@
     ddtp: module udd.ddtp_gatherer
     ftpnew: module udd.ftpnew_gatherer
     screenshots: module udd.screenshot_gatherer
+    dehs: module udd.dehs_gatherer
   debug: 1
   timestamp-dir: /org/udd.debian.org/timestamps
   lock-dir: /org/udd.debian.org/locks
@@ -377,6 +378,11 @@
   table: lintian
   schema: lintian
 
+dehs:
+  type: dehs
+  update-command: rm -f /org/udd.debian.org/mirrors/dehs.txt && wget -q http://dehs.alioth.debian.org/udd_dump.txt  -O /org/udd.debian.org/mirrors/dehs.txt
+  path: /org/udd.debian.org/mirrors/dehs.txt
+
 debtags:
   type: debtags
   update-command: rm -f /org/udd.debian.org/mirrors/debtags.txt && wget -q http://svn.debian.org/viewsvn/*checkout*/debtags/tagdb/tags -O /org/udd.debian.org/mirrors/debtags.txt

Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql	2009-07-24 12:04:21 UTC (rev 1537)
+++ udd/sql/setup.sql	2009-07-24 12:05:21 UTC (rev 1538)
@@ -465,3 +465,21 @@
 AND key_type = 'keyring';
 
 GRANT SELECT ON active_dds TO PUBLIC;
+
+-- DEHS
+CREATE TYPE dehs_status AS ENUM('error', 'uptodate', 'outdated');
+CREATE TABLE dehs (
+  source TEXT NOT NULL,
+  unstable_version debversion,
+  unstable_upstream text,
+  unstable_parsed_version text,
+  unstable_status dehs_status,
+  experimental_version debversion,
+  experimental_upstream text,
+  experimental_parsed_version text,
+  experimental_status dehs_status,
+  PRIMARY KEY (source)
+);
+GRANT SELECT ON lintian TO PUBLIC;
+
+

Modified: udd/sql/upgrade.sql
===================================================================
--- udd/sql/upgrade.sql	2009-07-24 12:04:21 UTC (rev 1537)
+++ udd/sql/upgrade.sql	2009-07-24 12:05:21 UTC (rev 1538)
@@ -186,3 +186,21 @@
 
 -- 2009-07-24
 ALTER TABLE lintian ADD information text;
+
+-- DEHS
+CREATE TYPE dehs_status AS ENUM('error', 'uptodate', 'outdated');
+CREATE TABLE dehs (
+  source TEXT NOT NULL,
+  unstable_version debversion,
+  unstable_upstream text,
+  unstable_parsed_version text,
+  unstable_status dehs_status,
+  experimental_version debversion,
+  experimental_upstream text,
+  experimental_parsed_version text,
+  experimental_status dehs_status,
+  PRIMARY KEY (source)
+);
+GRANT SELECT ON lintian TO PUBLIC;
+
+

Added: udd/udd/dehs_gatherer.py
===================================================================
--- udd/udd/dehs_gatherer.py	                        (rev 0)
+++ udd/udd/dehs_gatherer.py	2009-07-24 12:05:21 UTC (rev 1538)
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+"""
+This script imports dehs status into the database
+See dehs.alioth.debian.org
+
+Query used (on DEHS' side) to generate the data:
+SELECT COALESCE(u.name, e.name) AS source,
+u.up_version AS unstable_upstream, e.up_version AS experimental_upstream,
+u.version AS unstable_version, e.version AS experimental_version,
+u.dversionmangled AS unstable_parsed_version, e.dversionmangled AS experimental_parsed_version,
+u.updated AS unstable_uptodate, e.updated AS experimental_uptodate,
+u.watch_warn <> '' AS  unstable_failed, e.watch_warn <> '' AS experimental_failed
+FROM (select * from pkgs where dist='unstable' AND watch IS NOT NULL) AS u
+FULL JOIN (select * from pkgs where dist='experimental' AND watch IS NOT NULL) AS e ON u.name=e.name
+WHERE (u.dist='unstable' OR u.dist IS NULL) AND (e.dist='experimental' OR e.dist IS NULL)
+"""
+
+from aux import quote
+from gatherer import gatherer
+import re
+
+def get_gatherer(connection, config, source):
+  return dehs_gatherer(connection, config, source)
+
+class dehs_gatherer(gatherer):
+
+  def __init__(self, connection, config, source):
+    gatherer.__init__(self, connection, config, source)
+    self.assert_my_config('path')
+
+  def run(self):
+    my_config = self.my_config
+
+    #start harassing the DB, preparing the final inserts and making place
+    #for the new data:
+    cur = self.cursor()
+
+    cur.execute("DELETE FROM dehs")
+
+    cur.execute("""PREPARE dehs_insert 
+      AS INSERT INTO dehs
+      (source, unstable_version, unstable_upstream, unstable_parsed_version, unstable_status,
+      experimental_version, experimental_upstream, experimental_parsed_version, experimental_status)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)""")
+
+    line_number = 0
+    entries = []
+    for line in file(my_config['path']):
+      line_number += 1
+
+      src, uu, eu, uv, ev, upv, epv, u_utd, e_utd, uf, ef = line.rstrip().split('|')
+      ustat = 'error' if uf == 't' else ('uptodate' if u_utd == 't' else ('outdated' if u_utd == 'f' else None))
+      estat = 'error' if ef == 't' else ('uptodate' if e_utd == 't' else ('outdated' if e_utd == 'f' else None))
+      entries.append((src, uv, uu, upv, ustat, ev, eu, epv, estat))
+
+    cur.executemany("EXECUTE dehs_insert (%s, %s, %s, %s, %s, %s, %s, %s, %s)", entries)
+    cur.execute("DEALLOCATE dehs_insert")
+    cur.execute("ANALYZE dehs")
+
+if __name__ == '__main__':
+  main()
+
+# vim:set et tabstop=2: