[Piuparts-commits] [SCM] piuparts git repository branch, master, updated. eda668423fa87898c59d1075118693714aa5a053

[Holger Levsen]

The following commit has been merged in the master branch:
commit 8a1e3b79a6bd7fa07b628b9b75089a5f8b24d22f
Author: Scott Schaefer <saschaefer at neurodiverse.org>
Date:   Wed Sep 28 20:45:16 2011 -0400

    Kill children to insure test doesn't run "forever"  (Closes: #640647)

diff --git a/TODO b/TODO
index f7b1cc6..730b9b2 100644
--- a/TODO
+++ b/TODO
@@ -48,6 +48,9 @@ for 0.43:
 - sometimes a chroot doesn't get removed on piatti - find out
   why and fix it.
 
+- Adjust report scripts to detect this condition;
+          text in log="*** Process KILLED - exceed maximum run time ***"
+
 
 for 0.44:
 
@@ -112,16 +115,8 @@ for 0.47 and later:
 - a redirect of http://piuparts.d.o/foo to
   http://p.d.o/source/f/foo.html would be nice
 
-- monitor:
-  - write slave-watcher to monitor hanging processes, eg. 
-    looping dpkg-preconfigure
-  - Check for and kill extraneous processes afterwards. Perhaps 
-    by checking whether their working directory is in the 
-    chroot.
-    Introduce a whitelist of processes to wait for and assume
-    it's an error if those haven't been killed after $time, ie 
-    10min. see #387428
-  - master.log grows to fast and there is no mechanism to stop it
+- monitor: master.log grows to fast and there is no mechanism to stop it
+  - use logrotate
 
 
 for 0.50 and later:
diff --git a/debian/changelog b/debian/changelog
index ff671e1..e7ecb9c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -36,6 +36,8 @@ piuparts (0.42) UNRELEASED; urgency=low
       a) Existing diversions removed/modified, and/or
       b) Installed diversions not removed by purge:
     (Closes: #588313)
+  * Kill children (hard-coded value, 45 minutes) to insure test doesn't
+    run "forever" (Closes: #640647)
 
   [ Andreas Beckmann ]
   * *.py: Add vim modeline.
diff --git a/piuparts-slave.py b/piuparts-slave.py
index 5ccf6fe..f0e430c 100644
--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -28,6 +28,7 @@ import sys
 import stat
 import time
 import logging
+from signal import alarm, signal, SIGALRM, SIGKILL
 import subprocess
 import ConfigParser
 
@@ -36,7 +37,7 @@ import piupartslib.packagesdb
 
 
 CONFIG_FILE = "/etc/piuparts/piuparts.conf"
-
+MAX_WAIT_TEST_RUN = 45*60
 
 def setup_logging(log_level, log_file_name):
     logger = logging.getLogger()
@@ -83,6 +84,12 @@ class Config(piupartslib.conf.Config):
             }, "")
 
 
+class Alarm(Exception):
+    pass
+
+def alarm_handler(signum, frame):
+    raise Alarm
+
 class MasterNotOK(Exception):
 
     def __init__(self):
@@ -346,6 +353,41 @@ def upgrade_testable(config, package, packages_files):
     else:
         return False
 
+def get_process_children(pid):
+    p = subprocess.Popen('ps --no-headers -o pid --ppid %d' % pid,
+           shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    return [int(p) for p in stdout.split()]
+
+def run_test_with_timeout(cmd, maxwait, kill_all):
+      logging.debug("Executing: %s" % cmd)
+
+      stdout = ""
+      sh_cmd = "{ %s; } 2>&1" % cmd
+      p = subprocess.Popen(sh_cmd, shell=True,
+                     stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+      if maxwait > 0:
+          signal(SIGALRM, alarm_handler)
+          alarm(maxwait)
+      try:
+          stdout, stderr = p.communicate()
+          if maxwait > 0:
+              alarm(0)
+      except Alarm:
+          pids = [p.pid]
+          if kill_all:
+              pids.extend(get_process_children(p.pid))
+          for pid in pids:
+              if pid > 0:
+                  try: 
+                      os.kill(pid, SIGKILL)
+                  except OSError:
+                      pass
+          return -1,stdout
+
+      return p.returncode,stdout 
+
+
 def test_package(config, package, packages_files):
     logging.info("Testing package %s/%s %s" % (config.section, package["Package"], package["Version"]))
 
@@ -360,54 +402,46 @@ def test_package(config, package, packages_files):
     output.write("\n")
 
     # omit distro test if chroot-tgz is not specified.
+    ret = 0
     if config["chroot-tgz"]: 
-      command = "%(piuparts-cmd)s -ad %(distro)s -b %(chroot-tgz)s" % \
-                  config
-      if config["keep-sources-list"] in ["yes", "true"]:
-          command += " --keep-sources-list "
+        command = "%(piuparts-cmd)s -ad %(distro)s -b %(chroot-tgz)s" % \
+                    config
+        if config["keep-sources-list"] in ["yes", "true"]:
+            command += " --keep-sources-list "
+        if config["mirror"]:
+            command += " --mirror %s " % config["mirror"]
+        command += " " + package["Package"]
 
-      if config["mirror"]:
-          command += " --mirror %s " % config["mirror"]
-      command += " " + package["Package"]
-
-      logging.debug("Executing: %s" % command)
-      output.write("Executing: %s\n" % command)
-      f = os.popen("{ %s; } 2>&1" % command, "r")
-      for line in f:
-          output.write(line)
-      status = f.close()
-      if status is None:
-          status = 0
-    else:
-          status = 0
+        output.write("Executing: %s\n" % command)
+        ret,f = run_test_with_timeout(command, MAX_WAIT_TEST_RUN, True)
+        if ret < 0:
+            output.write(f + "\n *** Process KILLED - exceed maximum run time ***\n")
+        else:
+            output.write(f)
 
-    if status == 0 and upgrade_testable(config, package, packages_files):
+    if ret == 0 and upgrade_testable(config, package, packages_files):
         distros = config["upgrade-test-distros"].split()
         distros = ["-d " + distro.strip() for distro in distros]
         distros = " ".join(distros)
         command = "%(piuparts-cmd)s -ab %(upgrade-test-chroot-tgz)s " % config
         command += distros
-
         if config["mirror"]:
           command += " --mirror %s " % config["mirror"]
-
         command += " " + package["Package"]
 
-        logging.debug("Executing: %s" % command)
-        output.write("\nExecuting: %s\n" % command)
-        f = os.popen("{ %s; } 2>&1" % command, "r")
-        for line in f:
-            output.write(line)
+        output.write("Executing: %s\n" % cmd)
+        ret,f = run_test_with_timeout(command, MAX_WAIT_TEST_RUN, True)
+        if ret < 0:
+            output.write(" *** Process KILLED - exceed maximum run time ***\n")
+        else:
+            output.write(f)
             output.flush()
-        status = f.close()
-        if status is None:
-            status = 0
 
     output.write("\n")
     output.write(time.strftime("End: %Y-%m-%d %H:%M:%S %Z\n", 
                                time.gmtime()))
     output.close()
-    if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
+    if ret != 0:
         subdir = "fail"
     else:
         subdir = "pass"

-- 
piuparts git repository