[Piuparts-commits] [SCM] piuparts git repository branch, master, updated. eda668423fa87898c59d1075118693714aa5a053

Holger Levsen holger at layer-acht.org
Fri Dec 23 10:28:00 UTC 2011


The following commit has been merged in the master branch:
commit 9bcbfe0fa48aba1da806fecfea79e3aa2d7bdb27
Merge: 241c05545defefcf0ad6e0a34684bc6f5dfc0897 c27ed5ec8dbed50bb1b50010fd5c6e9bac1e74f8
Author: Holger Levsen <holger at layer-acht.org>
Date:   Fri Dec 2 14:29:58 2011 +0100

    Merge branch 'feature/report' into develop

diff --combined piuparts-report.py
index 217c37e,bc89546..0cb3d6c
--- a/piuparts-report.py
+++ b/piuparts-report.py
@@@ -445,7 -445,6 +445,7 @@@ def emphasize_reason(reason)
          "dependency-cannot-be-tested",
          "dependency-does-not-exist",
          "circular-dependency",
 +        "does-not-exist",
          "unknown",
          "unknown-preferred-alternative",
          "no-dependency-from-alternatives-exists",
@@@ -662,7 -661,7 +662,7 @@@ class Section
          return link
  
      def link_to_state_page(self, section, package_name, link_target):
 -        state = self._binary_db.state_by_name(package_name)
 +        state = self._binary_db.get_package_state(package_name)
          if state != "unknown":
              link = "<a href=\"/%s/%s\">%s</a>" % (
                  section,
@@@ -726,30 -725,33 +726,30 @@@
          return total
  
  
 -    def merge_maintainer_templates(self, templates):
 -        for maint_tpl in templates:
 -            tpl = os.path.join(self._output_directory,"maintainer",maintainer_subdir(maint_tpl),maint_tpl)
 -            lines = read_file(tpl)
 +    def create_maintainer_summaries(self, maintainers, source_data):
 +        logging.debug("Writing maintainer summaries in %s" % self._output_directory)
 +        states = ["fail", "unknown", "pass"]
 +        for maintainer in maintainers.keys():
 +            sources = maintainers[maintainer]
              rows = ""
 -            for line in lines:
 -              state, count, packages = line.split(",")
 -              if packages == "none\n":
 -                links = " "
 -              else:
 -                links = ""
 -                for package in packages.split(" "):
 -                  links += "<a href=\"#%s\">%s</a> " % (package,package)
 -              rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
 -                       (state, count, links)
 -            os.unlink(tpl)
 -            template_path = tpl[:-len("_tpl")]
 -
 -            for state in ("fail","unkn","pass"):
 -                filename = template_path+"_"+state
 -                if os.path.isfile(filename):
 -                     f = file(filename, "r")
 -                     rows += file.read(f)
 -                     f.close()
 -                     os.unlink(filename)
 -
 -            maintainer = os.path.basename(maint_tpl[:-len("_tpl")])
 +            package_rows = ""
 +            packages = {}
 +            for state in states:
 +                packages[state] = []
 +            for source in sorted(sources):
 +                (state, sourcerows, binaryrows) = source_data[source]
 +                packages[state].append(source)
 +                package_rows += sourcerows + binaryrows
 +
 +            for state in states:
 +                if len(packages[state]) > 0:
 +                    links = ""
 +                    for package in packages[state]:
 +                        links += "<a href=\"#%s\">%s</a> " % (package, package)
 +                else:
 +                    links = " "
 +                rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
 +                        (state, len(packages[state]), links)
  
              distrolinks = "<tr class=\"normalrow\"><td class=\"labelcell\">other distributions: </td><td class=\"contentcell2\" colspan=\"5\">"
              for section in self._section_names:
@@@ -758,7 -760,7 +758,7 @@@
              distrolinks += "</td></tr>"
  
              htmlpage = string.Template(HTML_HEADER + MAINTAINER_BODY_TEMPLATE + HTML_FOOTER)
 -            filename = template_path+".html"
 +            filename = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer), maintainer + ".html")
              f = file(filename, "w")
              f.write(htmlpage.safe_substitute( {
                 "page_title": html_protect("Status of "+maintainer+" packages in "+self._config.section),
@@@ -766,11 -768,10 +766,11 @@@
                 "distrolinks": distrolinks,
                 "section_navigation": create_section_navigation(self._section_names,self._config.section),
                 "time": time.strftime("%Y-%m-%d %H:%M %Z"),
 -               "rows": rows,
 +               "rows": rows + package_rows,
               }))
              f.close()
  
 +
      def create_source_summary (self, source, logs_by_dir):
          source_version = self._source_db.get_control_header(source, "Version")
          binaries = self._source_db.get_control_header(source, "Binary")
@@@ -781,7 -782,7 +781,7 @@@
          failed = False
          binaryrows = ""
          for binary in sorted(binaries.split(", ")):
 -          state = self._binary_db.state_by_name(binary)
 +          state = self._binary_db.get_package_state(binary)
            if state == "unknown":
              # Don't track udebs and binary packages on other archs. 
              # The latter is a FIXME which needs parsing the Packages files from other archs too
@@@ -833,30 -834,61 +833,30 @@@
  
          return sourcerows, binaryrows, source_state, maintainer, uploaders
  
 -    def create_maintainer_templates_for_source(self,source, source_state, sourcerows, binaryrows, maintainer, uploaders):
 -        maintainer_pages = [] 
 -        maintainer_pages.append(get_email_address(maintainer))
 -        for uploader in uploaders.split(", "):
 -          if uploader:
 -            maintainer_pages.append(get_email_address(uploader))
 -        for maintainer_page in maintainer_pages:
 -          maintainer_summary_page_path = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer_page))
 -
 -          if not os.path.exists(maintainer_summary_page_path):
 -            os.makedirs(maintainer_summary_page_path)
 -          filename = os.path.join(maintainer_summary_page_path, (maintainer_page + "_tpl"))
 -          maintainer_package_count = {}
 -          maintainer_packages = {}
 -          if os.path.isfile(filename):
 -            lines = read_file(filename)
 -            for line in lines:
 -              state, count, packages = line.split(",")
 -              maintainer_package_count[state]=int(count)
 -              maintainer_packages[state]=packages[:-1]
 -            if maintainer_packages[source_state] == "none":
 -              maintainer_packages[source_state] = source
 -            else:
 -              maintainer_packages[source_state] = "%s %s" % (maintainer_packages[source_state],source)
 -          else:
 -            maintainer_package_count["fail"] = 0
 -            maintainer_package_count["unknown"] = 0
 -            maintainer_package_count["pass"] = 0
 -            for state in "fail", "unknown", "pass":
 -              maintainer_packages[state] = "none"
 -            maintainer_packages[source_state] = source
 -          if source_state == "fail":
 -            maintainer_package_count["fail"]+=1
 -          elif source_state == "unknown":
 -            maintainer_package_count["unknown"]+=1
 -          else:
 -            maintainer_package_count["pass"]+=1
 -          lines = ""
 -          for state in "fail", "unknown", "pass":
 -            lines +=  "%s,%s,%s\n" % (state,maintainer_package_count[state],maintainer_packages[state])
 -          write_file(filename,lines)
 -          append_file(filename[:-4]+"_"+source_state[:4],sourcerows+binaryrows)
  
      def create_package_summaries(self, logs_by_dir):
          logging.debug("Writing package templates in %s" % self._config.section)    
  
 +        maintainers = {}
 +        source_binary_rows = {}
          sources = ""
          for source in self._source_db.get_all_packages():
              (sourcerows, binaryrows, source_state, maintainer, uploaders) = self.create_source_summary(source, logs_by_dir)
              if source_state != "udeb":
 -              sources += "%s: %s\n" % (source, source_state)
 -              self.create_maintainer_templates_for_source(source, source_state, sourcerows, binaryrows, maintainer, uploaders)
 +                sources += "%s: %s\n" % (source, source_state)
 +                source_binary_rows[source] = (source_state, sourcerows, binaryrows)
 +                for maint in [maintainer] + uploaders.split(","):
 +                    if maint.strip():
 +                        email = get_email_address(maint.strip())
 +                        if not "INVALID" in email:
 +                            if not email in maintainers:
 +                                maintainers[email] = []
 +                            maintainers[email].append(source)
  
          write_file(os.path.join(self._output_directory, "sources.txt"), sources)
  
 +        self.create_maintainer_summaries(maintainers, source_binary_rows)
 +
  
      def make_stats_graph(self):
          countsfile = os.path.join(self._output_directory, "counts.txt")
@@@ -969,12 -1001,21 +969,21 @@@
                                           package["Package"],
                                           self.link_to_source_summary(package["Package"]),
                                           html_protect(package["Maintainer"]))
-                 if package.dependencies():
+                 all_deps = package.all_dependencies()
+                 if all_deps:
                      vlist += "\n<ul>\n"
-                     for dep in package.dependencies():
+                     for alternatives in all_deps:
+                         dep = alternatives[0]
                          vlist += "<li>dependency %s is %s</li>\n" % \
                                    (self.link_to_state_page(self._config.section,dep,dep), 
 -                                  emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
 +                                  emphasize_reason(html_protect(self._binary_db.get_package_state(dep))))
+                         if len(alternatives) > 1:
+                             vlist += "\n<ul>\n"
+                             for dep in alternatives[1:]:
+                                 vlist += "<li>alternative dependency %s is %s</li>\n" % \
+                                           (self.link_to_state_page(self._config.section,dep,dep),
+                                           emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
+                             vlist += "</ul>\n"
                      vlist += "</ul>\n"
                  vlist += "</li>\n"
              htmlpage = string.Template(HTML_HEADER + STATE_BODY_TEMPLATE + HTML_FOOTER)
@@@ -987,25 -1028,6 +996,25 @@@
                                          "list": vlist
                                         }))
  
 +
 +    def archive_logfile(self, vdir, log):
 +        archivedir = os.path.join("archive", vdir)
 +        if not os.path.exists(archivedir):
 +            os.makedirs(archivedir)
 +        os.rename(os.path.join(vdir, log), os.path.join("archive", vdir, log))
 +
 +
 +    def cleanup_removed_packages(self, logs_by_dir):
 +        for vdir in logs_by_dir.keys():
 +            for log in sorted(logs_by_dir[vdir]):
 +                if log.endswith(".log"):
 +                    package, version = log[:-len(".log")].split("_")
 +                    if not self._binary_db.has_package(package):
 +                        logging.debug("Package %s was removed, archiving %s/%s" % (package, vdir, log))
 +                        self.archive_logfile(vdir, log)
 +                        logs_by_dir[vdir].remove(log)
 +
 +
      def generate_html(self):
          logging.debug("Finding log files")
          dirs = ["pass", "fail", "bugged", "reserved", "untestable"]
@@@ -1013,9 -1035,6 +1022,9 @@@
          for vdir in dirs:
              logs_by_dir[vdir] = find_files_with_suffix(vdir, ".log")
  
 +        logging.debug("Archiving logs of obsolete packages")
 +        self.cleanup_removed_packages(logs_by_dir)
 +
          logging.debug("Copying log files")
          copy_logs(logs_by_dir, self._output_directory)
  
@@@ -1030,6 -1049,9 +1039,6 @@@
          if self._config["sources-url"]:
              self.create_package_summaries(logs_by_dir)
  
 -            logging.debug("Merging maintainer summaries in %s" % self._output_directory)    
 -            self.merge_maintainer_templates(find_files_with_suffix(self._output_directory+"/maintainer/", "_tpl"))
 -
          logging.debug("Writing section index page")    
          self.write_section_index_page(dirs, total_packages)
  
@@@ -1053,7 -1075,6 +1062,7 @@@
          self.generate_html()
          os.chdir(oldcwd)
  
 +
  def main():
      setup_logging(logging.DEBUG, None)
  
diff --combined piupartslib/packagesdb.py
index 175d547,dcd2b1a..3cfd351
--- a/piupartslib/packagesdb.py
+++ b/piupartslib/packagesdb.py
@@@ -54,6 -54,7 +54,7 @@@ class Package(UserDict.UserDict)
              name, value = header.split(":", 1)
              self[name.strip()] = value.strip()
          self._parsed_deps = {}
+         self._parsed_alt_deps = {}
  
      def _parse_dependencies(self, header_name):
          if header_name in self._parsed_deps:
@@@ -65,6 -66,17 +66,17 @@@
              self._parsed_deps[header_name] = depends
          return depends
  
+     def _parse_alternative_dependencies(self, header_name):
+         if header_name in self._parsed_alt_deps:
+             depends = self._parsed_alt_deps[header_name]
+         else:
+             parser = DependencyParser(self[header_name])
+             depends = parser.get_dependencies()
+             depends = [[alt.name for alt in alternatives] for alternatives in depends]
+             self._parsed_alt_deps[header_name] = depends
+         return depends
+ 
+     # first alternative only - [package_name...]
      def dependencies(self):
          vlist = []
          for header in ["Depends", "Pre-Depends"]:
@@@ -72,6 -84,14 +84,14 @@@
                  vlist += self._parse_dependencies(header)
          return vlist
  
+     # all alternatives - [[package_name...]...]
+     def all_dependencies(self):
+         vlist = []
+         for header in ["Depends", "Pre-Depends"]:
+             if header in self:
+                 vlist += self._parse_alternative_dependencies(header)
+         return vlist
+ 
      def depends_with_alts(self, header_name):
          vlist = []
          if header_name in self:
@@@ -194,7 -214,6 +214,7 @@@ class LogDB
  
  class PackagesDB:
  
 +    # keep in sync with piuparts-report.py: emphasize_reason()
      _states = [
          "successfully-tested",
          "failed-testing",
@@@ -206,7 -225,6 +226,7 @@@
          "dependency-cannot-be-tested",
          "dependency-does-not-exist",
          "circular-dependency",
 +        #"does-not-exist",  # can only happen as query result for a dependency
          "unknown",
          "unknown-preferred-alternative",
          "no-dependency-from-alternatives-exists",
@@@ -221,7 -239,6 +241,7 @@@
          "dependency-cannot-be-tested": "dependency-cannot-be-tested",
          "dependency-does-not-exist": "dependency-does-not-exist",
          "circular-dependency": "circular-dependency",
 +        "does-not-exist": "dependency-does-not-exist",
          "unknown-preferred-alternative": "unknown-preferred-alternative",
          "no-dependency-from-alternatives-exists": "dependency-cannot-be-tested",
      }
@@@ -271,16 -288,13 +291,16 @@@
      def _find_all_packages(self):
          if self._packages is None:
              self._packages = {}
 +            self._virtual_packages = {}
              for pf in self._packages_files:
                  for p in pf.values():
                      self._packages[p["Package"]] = p
              for p in self._packages.values():
                  for provided in p.provides():
 -                    if provided not in self._packages:
 -                        self._packages[provided] = p
 +                    if provided != p["Package"]:
 +                        if provided not in self._virtual_packages:
 +                            self._virtual_packages[provided] = []
 +                        self._virtual_packages[provided].append(p["Package"])
  
      def _get_recursive_dependencies(self, package, break_circles=True):
          assert self._packages is not None
@@@ -293,8 -307,6 +313,8 @@@
                  deps.append(dep)
                  if dep in self._packages:
                      more += self._packages[dep].dependencies()
 +                elif dep in self._virtual_packages:
 +                    more += self._packages[self._virtual_packages[dep][0]].dependencies()
  
          # Break circular dependencies
          if break_circles and package["Package"] in deps:
@@@ -339,9 -351,9 +359,9 @@@
                      prefer_alt = None
                      for alternative in alt_deps[d]:
                          dep = alternative.name
 -                        if dep in self._package_state:
 +                        altdep_state = self.get_package_state(dep)
 +                        if altdep_state != "does-not-exist":
                              alt_found += 1
 -                            altdep_state = self._package_state[dep]
                              if prefer_alt_score < 3 and altdep_state == "essential-required":
                                  prefer_alt = alternative
                                  prefer_alt_idx = d
@@@ -378,15 -390,18 +398,15 @@@
               return state
  
          for dep in package.dependencies():
 -            if dep not in self._package_state:
 -                return "dependency-does-not-exist"
 -            dep_state = self._package_state[dep]
 -            if dep_state is None:
 -                return "unknown"
 -            elif dep_state in self._dep_state_to_state:
 +            dep_state = self.get_package_state(dep)
 +            if dep_state in self._dep_state_to_state:
                  return self._dep_state_to_state[dep_state]
  
          state = "waiting-to-be-tested"
          for dep in package.dependencies():
 -            if self._package_state[dep] not in \
 -               ["successfully-tested", "essential-required"]:
 +            dep_state = self.get_package_state(dep)
 +            if dep_state not in \
 +                    ["successfully-tested", "essential-required"]:
                  state = "unknown"
                  break
          if state == "waiting-to-be-tested":
@@@ -398,21 -413,20 +418,21 @@@
              if pkg in deps:
                  deps.remove(pkg)
          if package["Package"] in deps:
 -            return "circular-dependency" # actually, it's a unknown circular-dependency
 +            return "circular-dependency"  # actually, it's an unknown circular-dependency
  
          # treat circular-dependencies as testable (for the part of the circle)
          state = "unknown" 
          if package["Package"] in self._known_circular_depends:
 -          for dep in package.dependencies():
 -            if dep not in self._known_circular_depends and self._package_state[dep] not in \
 -               ["successfully-tested", "essential-required"]:
 -                state = "unknown"
 -                break
 -            if dep in self._known_circular_depends and self._package_state[dep] not in \
 -               ["failed-testing","dependency-failed-testing"]:
 -                state = "waiting-to-be-tested"
 -                continue
 +            for dep in package.dependencies():
 +                dep_state = self.get_package_state(dep)
 +                if dep not in self._known_circular_depends and dep_state not in \
 +                        ["successfully-tested", "essential-required"]:
 +                    state = "unknown"
 +                    break
 +                if dep in self._known_circular_depends and dep_state not in \
 +                        ["failed-testing", "dependency-failed-testing"]:
 +                    state = "waiting-to-be-tested"
 +                    continue
          return state
  
      def _compute_package_states(self):
@@@ -460,8 -474,6 +480,8 @@@
  
          self._in_state["unknown"] = todo
          self._in_state["unknown-preferred-alternative"] = unpreferred_alt
 +        for package_name in unpreferred_alt:
 +            self._package_state[package_name] = "unknown-preferred-alternative"
  
          for state in self._states:
              self._in_state[state].sort()
@@@ -474,17 -486,11 +494,17 @@@
          return set(self._in_state[state])
  
      def has_package(self, name):
 +        self._find_all_packages()
          return name in self._packages
  
      def get_package(self, name):
          return self._packages[name]
  
 +    def get_providers(self, name):
 +        if name in self._virtual_packages:
 +            return self._virtual_packages[name]
 +        return []
 +
      def get_all_packages(self):
          self._find_all_packages()
          return self._packages
@@@ -515,16 -521,14 +535,16 @@@
          else:
            return self._packages[package_name][header]
  
 -    def get_package_state(self, package_name):
 -        return self._package_state[package_name]
 -
 -    def state_by_name(self, package_name):
 +    def get_package_state(self, package_name, resolve_virtual=True):
          if package_name in self._package_state:
              return self._package_state[package_name]
 -        else:
 -            return "unknown"
 +        if package_name in self._virtual_packages:
 +            if resolve_virtual:
 +                provider = self._virtual_packages[package_name][0]
 +                return self._package_state[provider]
 +            else:
 +                return "virtual"
 +        return "does-not-exist"
  
      def _find_packages_ready_for_testing(self):
          return self.get_pkg_names_in_state("waiting-to-be-tested")

-- 
piuparts git repository



More information about the Piuparts-commits mailing list