[Collab-qa-commits] r1514 - udd/udd

[Lucas Nussbaum]

Author: lucas
Date: 2009-07-13 21:28:53 +0000 (Mon, 13 Jul 2009)
New Revision: 1514

Modified:
   udd/udd/orphaned_packages_gatherer.py
   udd/udd/ubuntu_bugs_gatherer.py
Log:
better error handling

Modified: udd/udd/orphaned_packages_gatherer.py
===================================================================
--- udd/udd/orphaned_packages_gatherer.py	2009-07-13 21:28:03 UTC (rev 1513)
+++ udd/udd/orphaned_packages_gatherer.py	2009-07-13 21:28:53 UTC (rev 1514)
@@ -7,6 +7,7 @@
 import aux
 from gatherer import gatherer
 import re
+from psycopg2 import IntegrityError 
 
 def get_gatherer(connection, config, source):
   return orphaned_packages_gatherer(connection, config, source)
@@ -55,14 +56,19 @@
       if m == None:
         print "Invalid bug: #" + str(row[0]) + ": " + row[1]
       else:
+        #print "bug: #" + str(row[0]) + ": " + row[1]
         time_orphaned = self.get_time_orphaned(row[0])
-        if time_orphaned == None:
-          cur2.execute("EXECUTE opkgs_insert(%s,%s,%s,%s,%s)", (
-            m.group(2), m.group(1), row[0],
-            m.group(4), row[2]))
-        else:
-          cur2.execute("EXECUTE opkgs_insert(%s,%s,%s,%s,%s::abstime)", (
-            m.group(2), m.group(1), row[0],
-            m.group(4), time_orphaned))
+        try:
+          if time_orphaned == None:
+            cur2.execute("EXECUTE opkgs_insert(%s,%s,%s,%s,%s)", (
+              m.group(2), m.group(1), row[0],
+              m.group(4), row[2]))
+          else:
+            cur2.execute("EXECUTE opkgs_insert(%s,%s,%s,%s,%s::abstime)", (
+              m.group(2), m.group(1), row[0],
+              m.group(4), time_orphaned))
+        except IntegrityError, message:
+          print "Integrity Error inserting bug " + str(row[0]) + " " + m.group(2)
+          continue
 
 # vim:set et tabstop=2:

Modified: udd/udd/ubuntu_bugs_gatherer.py
===================================================================
--- udd/udd/ubuntu_bugs_gatherer.py	2009-07-13 21:28:03 UTC (rev 1513)
+++ udd/udd/ubuntu_bugs_gatherer.py	2009-07-13 21:28:53 UTC (rev 1514)
@@ -4,8 +4,9 @@
 This script imports the Ubuntu bugs from Launchpad
 """
 
+import sys
 from aux import quote
-import sys
+from os import _exit
 from gatherer import gatherer
 import re
 import urllib
@@ -26,7 +27,7 @@
 
   def run(self):
     my_config = self.my_config
-    num_fetchers = 3
+    num_fetchers = 4
     num_writers = 1
     bugs = self.fetch_all_bugs()
     httpq = Queue()
@@ -46,8 +47,11 @@
       t.start()
 
     c = self.cursor()
-    c.execute("truncate ubuntu_bugs_subscribers, ubuntu_bugs_duplicates, ubuntu_bugs_tags, ubuntu_bugs_tasks, ubuntu_bugs")
-
+    c.execute("delete from ubuntu_bugs_subscribers")
+    c.execute("delete from ubuntu_bugs_duplicates")
+    c.execute("delete from ubuntu_bugs_tags")
+    c.execute("delete from ubuntu_bugs_tasks")
+    c.execute("delete from ubuntu_bugs")
     ok = True
     while ok:
       try:
@@ -82,6 +86,7 @@
 #        print "Duplicate bug: " +  str(n)
       on = n
     fh.close()
+    #print nbugs
     return nbugs
 
   # "worker". Fetch a specific bug as text from launchpad.
@@ -97,7 +102,7 @@
           return
         except:
           print "Other exception raised in bugfetcher. exiting."
-          exit(1)
+          _exit(1)
 
         try:
           conn.request('GET', 'https://launchpad.net/bugs/' + str(b) + '/+text')
@@ -114,7 +119,7 @@
             print "[", currentThread().getName(), "] Bug ", b, ": Wrong status: ", r.status, " ", r.reason
             if r.status == 302:
               print "Exiting."
-              exit(1)
+              _exit(1)
             ok = False
             hq.put(b)
         except httplib.BadStatusLine, line:
@@ -123,6 +128,10 @@
           print r.read()
           ok = False
           hq.put(b)
+        except:
+          print "[", currentThread().getName(), "] Bug ", b, ": error, exiting"
+          print sys.exc_info()
+          _exit(1)
 
   parre = re.compile('^\s*(.*) \(([^(]*)\)$')
   def splitpar(self, text):