[Piuparts-commits] [piuparts] 01/02: add debiman-piuparts-distill
Holger Levsen
holger at layer-acht.org
Thu Aug 3 03:35:01 UTC 2017
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch develop
in repository piuparts.
commit d697f2991fddd4e3220b23f60a895a0f01b90fc5
Author: Michael Stapelberg <stapelberg at debian.org>
Date: Thu Jun 1 21:28:15 2017 +0200
add debiman-piuparts-distill
see https://bugs.debian.org/863089
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
Makefile | 2 +
debian/control | 5 +-
debian/piuparts-master.install | 1 +
debiman-piuparts-distill/atomically.go | 75 +++++++++++++++++
debiman-piuparts-distill/piuparts.go | 145 +++++++++++++++++++++++++++++++++
5 files changed, 226 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index 6bf7f49..0ecc983 100644
--- a/Makefile
+++ b/Makefile
@@ -51,6 +51,7 @@ build: build-stamp
build-stamp: $(SCRIPTS_GENERATED) $(DOCS_GENERATED) Makefile
$(MAKE) -C instances
$(MAKE) python-syntax-check
+ (cd debiman-piuparts-distill && go build)
touch $@
build-doc: $(DOCS_GENERATED)
@@ -110,6 +111,7 @@ install-conf-4-running-from-git: build-stamp
install: build-stamp
install -d $(DESTDIR)$(sbindir)
install -m 0755 piuparts $(DESTDIR)$(sbindir)/
+ install -m 0755 debiman-piuparts-distill/debiman-piuparts-distill $(DESTDIR)$(sbindir)/
install -d $(DESTDIR)$(sharedir)/piuparts
install -m 0755 piuparts-slave piuparts-master piuparts-master-backend piuparts-report piuparts-analyze $(DESTDIR)$(sharedir)/piuparts/
diff --git a/debian/control b/debian/control
index f65fd2f..d58a822 100644
--- a/debian/control
+++ b/debian/control
@@ -16,7 +16,8 @@ Build-Depends:
python-debianbts,
python-yaml,
python-mox3,
- python-lzma
+ python-lzma,
+ golang-any
Build-Depends-Indep:
asciidoc,
git,
@@ -55,7 +56,7 @@ Description: .deb package installation, upgrading, and removal testing tool
packages to test them before they upload them to the Debian package archive.
Package: piuparts-master
-Architecture: all
+Architecture: any
Depends:
piuparts-common (= ${binary:Version}),
adduser,
diff --git a/debian/piuparts-master.install b/debian/piuparts-master.install
index 5e41fa9..c5cb009 100644
--- a/debian/piuparts-master.install
+++ b/debian/piuparts-master.install
@@ -7,3 +7,4 @@ usr/share/piuparts/piuparts-analyze
usr/share/piuparts/piuparts-master
usr/share/piuparts/piuparts-master-backend
usr/share/piuparts/piuparts-report
+usr/sbin/debiman-piuparts-distill
diff --git a/debiman-piuparts-distill/atomically.go b/debiman-piuparts-distill/atomically.go
new file mode 100644
index 0000000..c1b2d2d
--- /dev/null
+++ b/debiman-piuparts-distill/atomically.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+ "bufio"
+ "compress/gzip"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+)
+
+func tempDir(dest string) string {
+ tempdir := os.Getenv("TMPDIR")
+ if tempdir == "" {
+ // Convenient for development: decreases the chance that we
+ // cannot move files due to /tmp being on a different file
+ // system.
+ tempdir = filepath.Dir(dest)
+ }
+ return tempdir
+}
+
+func writeAtomically(dest string, compress bool, write func(w io.Writer) error) (err error) {
+ f, err := ioutil.TempFile(tempDir(dest), "debiman-")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ // Remove the tempfile if an error occurred
+ if err != nil {
+ os.Remove(f.Name())
+ }
+ }()
+ defer f.Close()
+
+ bufw := bufio.NewWriter(f)
+
+ w := io.Writer(bufw)
+ var gzipw *gzip.Writer
+ if compress {
+ // NOTE(stapelberg): gzip’s decompression phase takes the same
+ // time, regardless of compression level. Hence, we invest the
+ // maximum CPU time once to achieve the best compression.
+ gzipw, err = gzip.NewWriterLevel(bufw, gzip.BestCompression)
+ if err != nil {
+ return err
+ }
+ defer gzipw.Close()
+ w = gzipw
+ }
+
+ if err := write(w); err != nil {
+ return err
+ }
+
+ if compress {
+ if err := gzipw.Close(); err != nil {
+ return err
+ }
+ }
+
+ if err := bufw.Flush(); err != nil {
+ return err
+ }
+
+ if err := f.Chmod(0644); err != nil {
+ return err
+ }
+
+ if err := f.Close(); err != nil {
+ return err
+ }
+
+ return os.Rename(f.Name(), dest)
+}
diff --git a/debiman-piuparts-distill/piuparts.go b/debiman-piuparts-distill/piuparts.go
new file mode 100644
index 0000000..73f773d
--- /dev/null
+++ b/debiman-piuparts-distill/piuparts.go
@@ -0,0 +1,145 @@
+// debiman-piuparts-distill extracts slave alternative links from
+// LOG-ALTERNATIVES lines found in piuparts logs.
+//
+// See https://github.com/Debian/debiman/issues/12 for more details.
+package main
+
+import (
+ "bufio"
+ "encoding/json"
+ "flag"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "sync"
+)
+
+var (
+ logsDir = flag.String("logs_dir",
+ "",
+ "Directory containing piuparts logfiles")
+
+ output = flag.String("output",
+ "",
+ "Path to write the (gzip-compressed, json-encoded) distilled links file to")
+
+ parallel = flag.Int("parallel",
+ 10,
+ "Number of logfiles to read in parallel")
+)
+
+var (
+ logAlternativesRe = regexp.MustCompile(`LOG-ALTERNATIVES: dpkg=([^:]+): piuparts=(?:[^:]+): (.*)`)
+ slaveParamsRe = regexp.MustCompile(`--slave ([^ ]+) (?:[^ ]+) ([^ ]+)`)
+)
+
+type link struct {
+ Pkg string `json:"binpackage"`
+ From string `json:"from"`
+ To string `json:"to"`
+}
+
+// process reads the piuparts logfile at path. links are extracted from each
+// LOG-ALTERNATIVES line and written to the links channel.
+func process(path string, links chan<- link) error {
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if !strings.HasPrefix(line, "LOG-ALTERNATIVES: ") {
+ continue
+ }
+ matches := logAlternativesRe.FindStringSubmatch(line)
+ if matches == nil {
+ continue
+ }
+ for _, param := range slaveParamsRe.FindAllStringSubmatch(line, -1) {
+ links <- link{
+ Pkg: matches[1],
+ From: param[1],
+ To: param[2],
+ }
+ }
+ }
+ return scanner.Err()
+}
+
+// byPkg is a helper type for sorting the results slice by binary package. Once
+// Go 1.8 becomes available on piuparts.debian.org, we can switch to sort.Slice.
+type byPkg []link
+
+func (p byPkg) Len() int { return len(p) }
+func (p byPkg) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+func (p byPkg) Less(i, j int) bool { return p[i].Pkg < p[j].Pkg }
+
+func main() {
+ flag.Parse()
+
+ if *output == "" {
+ log.Fatal("-output must be specified")
+ }
+
+ if *logsDir == "" {
+ log.Fatal("-logs_dir must be specified")
+ }
+
+ // Spawn -parallel worker goroutines, waiting for work
+ work := make(chan string)
+ linksChan := make(chan link)
+ var wg sync.WaitGroup
+ for i := 0; i < *parallel; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for path := range work {
+ if err := process(path, linksChan); err != nil {
+ log.Printf("error processing %q: %v", path, err)
+ }
+ }
+ }()
+ }
+ // Collect results from all workers into linksMap
+ linksMap := make(map[link]bool)
+ go func() {
+ for l := range linksChan {
+ linksMap[l] = true
+ }
+ }()
+ // Walk through *logsDir, enqueue all .log files onto the work channel
+ if err := filepath.Walk(*logsDir, func(path string, info os.FileInfo, err error) error {
+ if strings.HasSuffix(path, ".log") && info.Mode().IsRegular() {
+ work <- path
+ }
+ return nil
+ }); err != nil {
+ log.Fatal(err)
+ }
+ // Close the channel, signaling termination to the worker goroutines
+ close(work)
+ // Wait for the worker goroutines to terminate
+ wg.Wait()
+ close(linksChan)
+ // Convert the unsorted linksMap into a slice for sorting
+ links := make([]link, 0, len(linksMap))
+ for l := range linksMap {
+ log.Printf("l = %+v", l)
+ links = append(links, l)
+ }
+ // for easier debugging of the resulting file:
+ sort.Stable(byPkg(links))
+
+ if err := writeAtomically(*output, true, func(w io.Writer) error {
+ return json.NewEncoder(w).Encode(&links)
+ }); err != nil {
+ log.Fatal(err)
+ }
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git
More information about the Piuparts-commits
mailing list