[game-data-packager] 03/03: Move progress reporting to gdp.command_line
Simon McVittie
smcv at debian.org
Sun Jan 24 15:01:46 UTC 2016
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 7e3fcb77ba25d90300f8480762e4165cb0bd7d87
Author: Simon McVittie <smcv at debian.org>
Date: Sun Jan 24 14:59:39 2016 +0000
Move progress reporting to gdp.command_line
---
game_data_packager/build.py | 55 ++++++++++++++++++---------
game_data_packager/command_line.py | 45 ++++++++++++++++++++++
game_data_packager/data.py | 76 ++++++++++++++++++--------------------
tests/hashed_file.py | 47 ++++++++++++++++++++++-
4 files changed, 164 insertions(+), 59 deletions(-)
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index d188f16..a2160c6 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -121,15 +121,6 @@ class DownloadNotAllowed(Exception):
class CDRipFailed(Exception):
pass
-def _ensure_hashes(hashes, path, size):
- if hashes is not None:
- return hashes
-
- if size > QUITE_LARGE:
- logger.info('identifying %s', path)
- return HashedFile.from_file(path, open(path, 'rb'), size=size,
- progress=(size > QUITE_LARGE))
-
def choose_mirror(wanted):
mirrors = []
mirror = os.environ.get('GDP_MIRROR')
@@ -252,6 +243,9 @@ class PackagingTask(object):
# list: arbitrary options (e.g. -z9 -Zgz -Sfixed)
self.compress_deb = game.compress_deb
+ # Factory for a progress report (or None).
+ self.progress_factory = lambda: None
+
self.game.load_file_data()
def __del__(self):
@@ -307,8 +301,12 @@ class PackagingTask(object):
if hashes is None:
if size > QUITE_LARGE:
logger.info('checking %s', path)
+ progress = self.progress_factory()
+ else:
+ progress = None
+
hashes = HashedFile.from_file(path, open(path, 'rb'), size=size,
- progress=(size > QUITE_LARGE))
+ progress=progress)
for wanted in remaining:
if not wanted.skip_hash_matching and not hashes.matches(wanted):
@@ -379,7 +377,7 @@ class PackagingTask(object):
# if a file (as opposed to a directory) is specified on the
# command-line, try harder to match it to something
if really_should_match_something:
- hashes = _ensure_hashes(None, path, size)
+ hashes = self.__ensure_hashes(None, path, size)
else:
hashes = None
@@ -387,7 +385,7 @@ class PackagingTask(object):
if match_path.endswith('/' + look_for):
candidates = [self.game.files[c] for c in candidates]
if candidates:
- hashes = _ensure_hashes(hashes, path, size)
+ hashes = self.__ensure_hashes(hashes, path, size)
if self.use_file('possible "%s"' % look_for, candidates,
path, hashes):
return
@@ -395,7 +393,7 @@ class PackagingTask(object):
if size in self.game.known_sizes:
candidates = self.game.known_sizes[size]
if candidates:
- hashes = _ensure_hashes(hashes, path, size)
+ hashes = self.__ensure_hashes(hashes, path, size)
candidates = [self.game.files[c] for c in candidates]
if self.use_file('file of size %d' % size,
candidates, path, hashes):
@@ -622,14 +620,15 @@ class PackagingTask(object):
wf = open(tmp, 'wb')
if entry.size is not None and entry.size > QUITE_LARGE:
- large = True
+ progress = self.progress_factory()
logger.info('extracting %s from %s', entry.name, name)
else:
- large = False
+ progress = None
logger.debug('extracting %s from %s', entry.name, name)
+
hf = HashedFile.from_file(
name + '//' + entry.name, entryfile, wf,
- size=entry.size, progress=large)
+ size=entry.size, progress=progress)
wf.close()
if entry.mtime is not None:
@@ -668,9 +667,15 @@ class PackagingTask(object):
yield open(self.found[provider.name], 'rb')
for p in other_parts:
yield open(self.found[p], 'rb')
+
+ if wanted.size >= QUITE_LARGE:
+ progress = self.progress_factory()
+ else:
+ progress = None
+
hasher = HashedFile.from_concatenated_files(wanted.name,
open_files(), writer, size=wanted.size,
- progress=(wanted.size > QUITE_LARGE))
+ progress=progress)
orig_time = os.stat(self.found[provider.name]).st_mtime
os.utime(path, (orig_time, orig_time))
self.use_file(wanted.name, (wanted,), path, hasher)
@@ -772,7 +777,8 @@ class PackagingTask(object):
wf = open(tmp, 'wb')
logger.info('downloading %s', url)
hf = HashedFile.from_file(url, rf, wf,
- size=wanted.size, progress=True)
+ size=wanted.size,
+ progress=self.progress_factory())
wf.close()
if self.use_file(wanted.name, (wanted,), tmp, hf):
@@ -2659,3 +2665,16 @@ class PackagingTask(object):
logger.warning('installing these packages might help:\n' +
'%s %s', ' '.join(self.packaging.INSTALL_CMD),
' '.join(sorted(packages)))
+
+ def __ensure_hashes(self, hashes, path, size):
+ if hashes is not None:
+ return hashes
+
+ if size > QUITE_LARGE:
+ logger.info('identifying %s', path)
+ progress = self.progress_factory()
+ else:
+ progress = None
+
+ return HashedFile.from_file(path, open(path, 'rb'), size=size,
+ progress=progress)
diff --git a/game_data_packager/command_line.py b/game_data_packager/command_line.py
index ae55bd1..7648783 100644
--- a/game_data_packager/command_line.py
+++ b/game_data_packager/command_line.py
@@ -20,13 +20,16 @@ import argparse
import logging
import os
import sys
+import time
import zipfile
from . import (load_games)
from .config import (read_config)
+from .data import (ProgressCallback)
from .gog import (run_gog_meta_mode)
from .paths import (DATADIR)
from .steam import (run_steam_meta_mode)
+from .util import (human_size)
logging.basicConfig()
logger = logging.getLogger(__name__)
@@ -38,6 +41,45 @@ if os.environ.get('DEBUG') or os.environ.get('GDP_DEBUG'):
else:
logging.getLogger().setLevel(logging.INFO)
+class TerminalProgress(ProgressCallback):
+ def __init__(self, interval=0.2):
+ """Constructor.
+
+ Progress will update at most once per @interval seconds, unless we
+ are at a "checkpoint".
+ """
+ self.pad = ' '
+ self.interval = interval
+ self.ts = time.time()
+
+ def __call__(self, done=None, total=None, checkpoint=False):
+ ts = time.time()
+
+ if done is None or (ts < self.ts + self.interval and not checkpoint):
+ return
+
+ if done is None or total == 0:
+ s = ''
+ elif total is None or total == 0:
+ s = human_size(done)
+ else:
+ s = '%.0f%% %s/%s' % (100 * done / total,
+ human_size(done),
+ human_size(total))
+
+ self.ts = ts
+
+ if len(self.pad) <= len(s):
+ self.pad = ' ' * len(s)
+
+ print(' %s \r %s\r' % (self.pad, s), end='', file=sys.stderr)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, et=None, ev=None, tb=None):
+ self()
+
def run_command_line():
logger.debug('Arguments: %r', sys.argv)
@@ -244,6 +286,9 @@ def run_command_line():
raise AssertionError('could not find %s' % parsed.shortname)
with game.construct_task() as task:
+ if sys.stderr.isatty():
+ task.progress_factory = TerminalProgress()
+
task.run_command_line(parsed)
if __name__ == '__main__':
diff --git a/game_data_packager/data.py b/game_data_packager/data.py
index 433d65f..da2603c 100644
--- a/game_data_packager/data.py
+++ b/game_data_packager/data.py
@@ -18,10 +18,24 @@
import hashlib
import io
-import sys
-import time
-from .util import (human_size)
+class ProgressCallback:
+ """API for a progress report."""
+
+ def __call__(self, done, total=None, checkpoint=False):
+ """Update progress: we have done @done bytes out of @total
+ (None if unknown).
+
+ If @checkpoint is True, it is a hint that this particular
+ update is important (for instance the end of a file).
+ """
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, et=None, ev=None, tb=None):
+ pass
class HashedFile:
def __init__(self, name):
@@ -32,53 +46,35 @@ class HashedFile:
self.skip_hash_matching = False
@classmethod
- def from_file(cls, name, f, write_to=None, size=None, progress=False):
+ def from_file(cls, name, f, write_to=None, size=None, progress=None):
return cls.from_concatenated_files(name, [f], write_to, size, progress)
@classmethod
def from_concatenated_files(cls, name, fs, write_to=None, size=None,
- progress=False):
+ progress=None):
md5 = hashlib.new('md5')
sha1 = hashlib.new('sha1')
sha256 = hashlib.new('sha256')
done = 0
- if progress and sys.stderr.isatty():
- pad = [' ']
- def update_progress(s):
- ts = time.time()
- if ts < update_progress.ts + 0.2 and not s.startswith('100%'):
- return
- update_progress.ts = ts
- if len(pad[0]) <= len(s):
- pad[0] = ' ' * len(s)
- print(' %s \r %s\r' % (pad[0], s), end='', file=sys.stderr)
- update_progress.ts = time.time()
- else:
- update_progress = lambda s: None
+ with (progress or ProgressCallback()) as update_progress:
+ for f in fs:
+ while True:
+ update_progress(done, size)
- for f in fs:
- while True:
- if size is None:
- update_progress(human_size(done))
- else:
- update_progress('%.0f%% %s/%s' % (
- 100 * done / size if size != 0 else 100,
- human_size(done),
- human_size(size)))
-
- blob = f.read(io.DEFAULT_BUFFER_SIZE)
- if not blob:
- update_progress('')
- break
-
- done += len(blob)
-
- md5.update(blob)
- sha1.update(blob)
- sha256.update(blob)
- if write_to is not None:
- write_to.write(blob)
+ blob = f.read(io.DEFAULT_BUFFER_SIZE)
+
+ if not blob:
+ update_progress(done, size, checkpoint=True)
+ break
+
+ done += len(blob)
+
+ md5.update(blob)
+ sha1.update(blob)
+ sha256.update(blob)
+ if write_to is not None:
+ write_to.write(blob)
self = cls(name)
self.md5 = md5.hexdigest()
diff --git a/tests/hashed_file.py b/tests/hashed_file.py
index 2650ea1..5b1f2ad 100755
--- a/tests/hashed_file.py
+++ b/tests/hashed_file.py
@@ -17,9 +17,23 @@
import hashlib
import io
+import sys
import unittest
-from game_data_packager.build import HashedFile
+from game_data_packager.command_line import (TerminalProgress)
+from game_data_packager.data import (HashedFile)
+
+class ZeroReader:
+ def __init__(self, total):
+ self.done = 0
+ self.total = total
+
+ def read(self, max_bytes):
+ ret = min(max_bytes, self.total - self.done)
+ self.total -= ret
+ return b'\x00' * ret
+
+SIZE = 30 * 1024 * 1024
class HashedFileTestCase(unittest.TestCase):
def setUp(self):
@@ -82,6 +96,37 @@ class HashedFileTestCase(unittest.TestCase):
self.assertIs(first.matches(second), False)
self.assertIs(second.matches(first), False)
+ def test_progress(self):
+ print('', file=sys.stderr)
+ HashedFile.from_file('progress.bin', ZeroReader(SIZE),
+ size=SIZE,
+ progress=TerminalProgress(interval=0.1))
+ print('', file=sys.stderr)
+ HashedFile.from_file('progress.bin', ZeroReader(SIZE),
+ progress=TerminalProgress(interval=0.1))
+ print('', file=sys.stderr)
+
+ HashedFile.from_file('progress.bin', ZeroReader(SIZE),
+ size=SIZE)
+ HashedFile.from_file('progress.bin', ZeroReader(SIZE))
+
+ print('', file=sys.stderr)
+ HashedFile.from_concatenated_files('concatenated.bin',
+ [ZeroReader(SIZE), ZeroReader(SIZE)],
+ size=2 * SIZE,
+ progress=TerminalProgress(interval=0.1))
+ print('', file=sys.stderr)
+ HashedFile.from_concatenated_files('concatenated.bin',
+ [ZeroReader(SIZE), ZeroReader(SIZE)],
+ progress=TerminalProgress(interval=0.1))
+ print('', file=sys.stderr)
+
+ HashedFile.from_concatenated_files('concatenated.bin',
+ [ZeroReader(SIZE), ZeroReader(SIZE)],
+ size=2 * SIZE)
+ HashedFile.from_concatenated_files('concatenated.bin',
+ [ZeroReader(SIZE), ZeroReader(SIZE)])
+
def tearDown(self):
pass
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list