[game-data-packager] 05/24: Add support for downloading patches etc.
Simon McVittie
smcv at debian.org
Tue Dec 30 01:32:03 UTC 2014
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit e00a4495356a4e463e88c0a734f66e483f892138
Author: Simon McVittie <smcv at debian.org>
Date: Mon Dec 29 14:01:06 2014 +0000
Add support for downloading patches etc.
---
lib/game_data_packager/__init__.py | 121 +++++++++++++++++++++++++++++++------
lib/game_data_packager/__main__.py | 10 +--
lib/game_data_packager/util.py | 5 ++
lib/via-python | 8 +++
4 files changed, 121 insertions(+), 23 deletions(-)
diff --git a/lib/game_data_packager/__init__.py b/lib/game_data_packager/__init__.py
index 400a3d3..0e6a73b 100644
--- a/lib/game_data_packager/__init__.py
+++ b/lib/game_data_packager/__init__.py
@@ -22,15 +22,17 @@ import hashlib
import io
import logging
import os
+import random
import re
import shutil
import subprocess
import tarfile
import tempfile
+import urllib.request
import yaml
-from .util import TemporaryUmask
+from .util import TemporaryUmask, mkdir_p
logging.basicConfig()
logger = logging.getLogger('game-data-packager')
@@ -115,7 +117,7 @@ class WantedFile(HashedFile):
super(WantedFile, self).__init__(name)
self.distinctive_name = True
self.distinctive_size = False
- self.downloadable = False
+ self.download = None
self.install = False
self._install_as = None
self._look_for = []
@@ -161,7 +163,7 @@ class WantedFile(HashedFile):
return {
'distinctive_name': self.distinctive_name,
'distinctive_size': self.distinctive_size,
- 'downloadable': self.downloadable,
+ 'download': self.download,
'install': self.install,
'install_as': self.install_as,
'look_for': list(self.look_for),
@@ -173,11 +175,17 @@ class WantedFile(HashedFile):
}
class GameDataPackage(object):
- def __init__(self, name, datadir='/usr/share/games/game-data-packager',
+ def __init__(self,
+ name,
+ datadir='/usr/share/games/game-data-packager',
+ etcdir='/etc/game-data-packager',
workdir=None):
# The name of the binary package
self.name = name
+ # game-data-packager's configuration directory
+ self.etcdir = etcdir
+
# game-data-packager's data directory.
self.datadir = datadir
@@ -207,6 +215,9 @@ class GameDataPackage(object):
# { 'baseq3/pak1.pk3' => '/usr/share/games/quake3/baseq3/pak1.pk3' }
self.found = {}
+ # Failed downloads
+ self.download_failed = set()
+
# Where we install files.
# For instance, if this is 'usr/share/games/quake3' and we have
# a WantedFile with install_as='baseq3/pak1.pk3' then we would
@@ -272,6 +283,7 @@ class GameDataPackage(object):
if self.workdir is None:
self.workdir = tempfile.mkdtemp(prefix='gdptmp.')
self._cleanup_dirs.add(self.workdir)
+ return self.workdir
def to_yaml(self):
files = {}
@@ -303,7 +315,7 @@ class GameDataPackage(object):
for k in (
'distinctive_name',
'distinctive_size',
- 'downloadable',
+ 'download',
'install',
'install_as',
'look_for',
@@ -372,21 +384,25 @@ class GameDataPackage(object):
def consider_file(self, path, really_should_match_something):
match_path = '/' + path.lower()
+ if really_should_match_something:
+ logger.debug('considering %s', path)
+ hashes = HashedFile.from_file(path, open(path, 'rb'))
+ else:
+ hashes = None
+
for wanted in self.files.values():
for lf in wanted.look_for:
if match_path.endswith('/' + lf):
- self.use_file(wanted, path, None)
+ self.use_file(wanted, path, hashes)
if wanted.distinctive_name:
return
if wanted.distinctive_size:
size = os.path.getsize(path)
if wanted.size == size:
- self.use_file(wanted, path, None)
-
- if really_should_match_something:
- hashes = HashedFile.from_file(path, open(path, 'rb'))
+ self.use_file(wanted, path, hashes)
+ if hashes is not None:
if hashes.matches(wanted):
self.use_file(wanted, path, hashes)
return
@@ -402,11 +418,14 @@ class GameDataPackage(object):
for fn in filenames:
path = os.path.join(dirpath, fn)
self.consider_file(path, False)
+ else:
+ logger.warning('file "%s" does not exist or is not a file or ' +
+ 'directory', path)
- def fill_gaps(self):
+ def fill_gaps(self, download=False):
for (filename, wanted) in self.files.items():
if wanted.install and filename not in self.found:
- self.fill_gap(wanted)
+ self.fill_gap(wanted, download=download)
def consider_tar_stream(self, name, tar):
for entry in tar:
@@ -429,7 +448,7 @@ class GameDataPackage(object):
tmp = os.path.join(self.get_workdir(),
'tmp', wanted.name)
tmpdir = os.path.dirname(tmp)
- os.makedirs(tmpdir)
+ mkdir_p(tmpdir)
wf = open(tmp, 'wb')
hf = HashedFile.from_file(
@@ -439,11 +458,75 @@ class GameDataPackage(object):
if not self.use_file(wanted, tmp, hf):
os.remove(tmp)
- def fill_gap(self, wanted):
+ def choose_mirror(self, wanted):
+ mirrors = []
+ for mirror_list, details in wanted.download.items():
+ try:
+ f = open(os.path.join(self.etcdir, mirror_list))
+ for line in f:
+ url = line.strip()
+ if url.startswith('#'):
+ continue
+ if details.get('path', '.') != '.':
+ if not url.endswith('/'):
+ url = url + '/'
+ url = url + details['path']
+ if not url.endswith('/'):
+ url = url + '/'
+ url = url + details.get('name', wanted.name)
+ mirrors.append(url)
+ except:
+ logger.warning('Could not open mirror list "%s"', mirror_list,
+ exc_info=True)
+ if not mirrors:
+ logger.error('Could not select a mirror for "%s"', wanted.name)
+ return []
+ random.shuffle(mirrors)
+ return mirrors
+
+ def fill_gap(self, wanted, download=False):
logger.debug('could not find %s, trying to derive it...', wanted.name)
for provider_name in self.providers.get(wanted.name, ()):
+ provider = self.files[provider_name]
+
+ if (download and provider_name not in self.found and
+ provider.download):
+ logger.debug('trying to download %s to provide %s...',
+ provider.name, wanted.name)
+ urls = self.choose_mirror(provider)
+ for url in urls:
+ if url in self.download_failed:
+ logger.debug('... no, it already failed')
+ continue
+
+ logger.debug('... %s', url)
+ tmp = None
+
+ try:
+ rf = urllib.request.urlopen(url)
+ if rf is None:
+ continue
+
+ tmp = os.path.join(self.get_workdir(),
+ 'tmp', provider.name)
+ tmpdir = os.path.dirname(tmp)
+ mkdir_p(tmpdir)
+ wf = open(tmp, 'wb')
+ hf = HashedFile.from_file(url, rf, wf)
+ wf.close()
+
+ if self.use_file(provider, tmp, hf):
+ break
+ else:
+ os.remove(tmp)
+ except Exception as e:
+ logger.warning('Failed to download "%s": %s', url,
+ e)
+ self.download_failed.add(url)
+ if tmp is not None:
+ os.remove(tmp)
+
if provider_name in self.found:
- provider = self.files[provider_name]
found_name = self.found[provider_name]
logger.debug('trying provider %s found at %s',
provider_name, found_name)
@@ -453,7 +536,7 @@ class GameDataPackage(object):
tmp = os.path.join(self.get_workdir(),
'tmp', wanted.name)
tmpdir = os.path.dirname(tmp)
- os.makedirs(tmpdir)
+ mkdir_p(tmpdir)
rf = open(found_name, 'rb')
contents = rf.read()
@@ -502,7 +585,7 @@ class GameDataPackage(object):
return False
docdir = os.path.join(destdir, 'usr/share/doc', self.name)
- os.makedirs(docdir)
+ mkdir_p(docdir)
shutil.copyfile(os.path.join(self.datadir, self.name + '.copyright'),
os.path.join(docdir, 'copyright'))
shutil.copyfile(os.path.join(self.datadir, 'changelog.gz'),
@@ -513,7 +596,7 @@ class GameDataPackage(object):
# The shell script code puts slipstream.unpacked in workdir.
assert destdir == self.workdir + '/slipstream.unpacked'
debdir = os.path.join(self.workdir, 'DEBIAN')
- os.makedirs(debdir)
+ mkdir_p(debdir)
shutil.copyfile(os.path.join(self.datadir, self.name + '.control'),
os.path.join(debdir, 'control'))
@@ -532,7 +615,7 @@ class GameDataPackage(object):
copy_to_dir = os.path.dirname(copy_to)
logger.debug('Copying to %s', copy_to)
if not os.path.isdir(copy_to_dir):
- os.makedirs(copy_to_dir)
+ mkdir_p(copy_to_dir)
# Use cp(1) so we can make a reflink if source and
# destination happen to be the same btrfs volume
subprocess.check_call(['cp', '--reflink=auto', copy_from,
diff --git a/lib/game_data_packager/__main__.py b/lib/game_data_packager/__main__.py
index 7adfa5e..ea7d3af 100644
--- a/lib/game_data_packager/__main__.py
+++ b/lib/game_data_packager/__main__.py
@@ -29,8 +29,11 @@ def go(argv):
metavar='DIRECTORY|FILE')
args = parser.parse_args(argv[1:])
- with GameDataPackage(argv[0], datadir=os.environ['DATADIR'],
- workdir=os.environ['WORKDIR']) as package:
+ with GameDataPackage(argv[0],
+ datadir=os.environ['DATADIR'],
+ workdir=os.environ['WORKDIR'],
+ etcdir=os.environ['ETCDIR'],
+ ) as package:
if args.repack:
args.paths.insert(0, '/' + package.install_to)
@@ -39,8 +42,7 @@ def go(argv):
package.consider_file_or_dir(arg)
package.fill_gaps()
-
- # FIXME: try to download any missing files and do another extract pass
+ package.fill_gaps(download=True)
if not package.fill_dest_dir(os.environ['DESTDIR']):
sys.exit(1)
diff --git a/lib/game_data_packager/util.py b/lib/game_data_packager/util.py
index 48b7c1a..d946551 100644
--- a/lib/game_data_packager/util.py
+++ b/lib/game_data_packager/util.py
@@ -35,3 +35,8 @@ class TemporaryUmask(object):
def __exit__(self, et, ev, tb):
os.umask(self.saved_mask)
+
+def mkdir_p(path):
+ if not os.path.isdir(path):
+ with TemporaryUmask(0o022):
+ os.makedirs(path)
diff --git a/lib/via-python b/lib/via-python
index bced649..1351ba9 100644
--- a/lib/via-python
+++ b/lib/via-python
@@ -21,7 +21,14 @@ gdp_data_driven () {
shift
fi
+ ETCDIR=/etc/game-data-packager
+
+ if [ "$0" = "./game-data-packager" ]; then
+ ETCDIR="./etc"
+ fi
+
(
+ export ETCDIR
export SHORTNAME
export DATADIR
export DESTDIR
@@ -64,5 +71,6 @@ gdp_data_driven () {
( cd "$WORKDIR" && slipstream_repack "$OUTFILE" )
rm -f "$WORKDIR/DO-NOT-COMPRESS"
+ rm -fr "$WORKDIR/tmp"
rm -fr "$DESTDIR"
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list