[Piuparts-commits] [piuparts] 06/11: lib: new DecompressedStream for transparent stream decompression

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch develop
in repository piuparts.

commit b459fcbac8aceac5fd82753bab884d4374cd0bec
Author: Andreas Beckmann <anbe at debian.org>
Date:   Mon Nov 25 13:14:05 2013 +0100

    lib: new DecompressedStream for transparent stream decompression
    
    new class DecompressedStream to transparently download and
    decompress something while it is being read to avoid storing the
    complete compressed input or decompressed output in memory
    
    Signed-off-by: Andreas Beckmann <anbe at debian.org>
---
 debian/changelog        |  1 +
 piupartslib/__init__.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index eb30184..0971b31 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -14,6 +14,7 @@ piuparts (0.56) UNRELEASED; urgency=low
   * piupartslib/__init__.py:
     - open_packages_url(): Try Packages.gz if Packages.bz2 is not available.
       (Closes: #711157)
+    - Add new class DecompressedStream.
   * piupartslib/packagesdb.py:
     - PackagesFile: Allow restricting to a set of package names.
   * piuparts-slave.py:
diff --git a/piupartslib/__init__.py b/piupartslib/__init__.py
index fe9337c..b2278ba 100644
--- a/piupartslib/__init__.py
+++ b/piupartslib/__init__.py
@@ -29,6 +29,54 @@ import dependencyparser
 import packagesdb
 
 
+class DecompressedStream():
+    def __init__(self, fileobj, decompressor=None):
+        self._input = fileobj
+        self._decompressor = decompressor
+        self._buffer = ""
+        self._line_buffer = []
+        self._i = 0
+        self._end = 0
+
+    def _refill(self):
+        if self._input is None:
+            return False
+        while True:
+            # repeat until decompressor yields some output or input is exhausted
+            chunk = self._input.read(4096)
+            if not chunk:
+                self.close()
+                return False
+            if self._decompressor:
+                chunk = self._decompressor.decompress(chunk)
+            self._buffer = self._buffer + chunk
+            if chunk:
+                return True
+
+    def readline(self):
+        while not self._i < self._end:
+            self._i = self._end = 0
+            self._line_buffer = None
+            empty = not self._refill()
+            if not self._buffer:
+                break
+            self._line_buffer = self._buffer.splitlines(True)
+            self._end = len(self._line_buffer)
+            self._buffer = ""
+            if not self._line_buffer[-1].endswith("\n") and not empty:
+                self._buffer = self._line_buffer[-1]
+                self._end = self._end - 1
+        if self._i < self._end:
+            self._i = self._i + 1
+            return self._line_buffer[self._i - 1]
+        return ""
+
+    def close(self):
+        if self._input:
+            self._input.close()
+        self._input = self._decompressor = None
+
+
 def open_packages_url(url):
     """Open a Packages.bz2 file pointed to by a URL"""
     socket = None

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git