[game-data-packager] 02/02: urllib: use same Agent everywhere

[Alexandre Detiste]

This is an automated email from the git hooks/post-receive script.

detiste-guest pushed a commit to branch master
in repository game-data-packager.

commit 401df8c877cf4ccfe7016df5b6b6435e5d4f8728
Author: Alexandre Detiste <alexandre.detiste at gmail.com>
Date:   Wed Sep 30 16:25:23 2015 +0200

    urllib: use same Agent everywhere
---
 game_data_packager/__init__.py | 5 +----
 game_data_packager/steam.py    | 3 ++-
 game_data_packager/util.py     | 5 +++++
 tools/spider.py                | 3 ++-
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 10205a3..833851a 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -46,6 +46,7 @@ from .config import read_config
 from .gog import GOG
 from .paths import DATADIR, ETCDIR
 from .util import (MEBIBYTE,
+        AGENT,
         TemporaryUmask,
         copy_with_substitutions,
         mkdir_p,
@@ -58,10 +59,6 @@ from .util import (MEBIBYTE,
 from .steam import get_steam_id, owned_steam_games
 from .version import GAME_PACKAGE_VERSION
 
-AGENT = ('Debian Game-Data-Packager/%s (%s %s;'
-         ' +http://wiki.debian.org/Games/GameDataPackager)' %
-        (GAME_PACKAGE_VERSION, os.uname()[0], os.uname()[4]) )
-
 logging.basicConfig()
 logger = logging.getLogger('game-data-packager')
 
diff --git a/game_data_packager/steam.py b/game_data_packager/steam.py
index d39be1b..5e38b4d 100644
--- a/game_data_packager/steam.py
+++ b/game_data_packager/steam.py
@@ -19,6 +19,7 @@ import glob
 import os
 import xml.etree.ElementTree
 import urllib.request
+from .util import AGENT
 
 def parse_acf(path):
     for manifest in glob.glob(path + '/*.acf'):
@@ -45,7 +46,7 @@ def parse_acf(path):
 
 def owned_steam_games(steam_id):
     url = "http://steamcommunity.com/profiles/" + steam_id + "/games?xml=1"
-    html = urllib.request.urlopen(url)
+    html = urllib.request.urlopen(urllib.request.Request(url, headers={'User-Agent': AGENT}))
     tree = xml.etree.ElementTree.ElementTree()
     tree.parse(html)
     games_xml = tree.getiterator('game')
diff --git a/game_data_packager/util.py b/game_data_packager/util.py
index 766ec65..bab69c5 100644
--- a/game_data_packager/util.py
+++ b/game_data_packager/util.py
@@ -21,10 +21,15 @@ import shutil
 import stat
 import subprocess
 import sys
+from .version import GAME_PACKAGE_VERSION
 
 KIBIBYTE = 1024
 MEBIBYTE = KIBIBYTE * KIBIBYTE
 
+AGENT = ('Debian Game-Data-Packager/%s (%s %s;'
+         ' +http://wiki.debian.org/Games/GameDataPackager)' %
+        (GAME_PACKAGE_VERSION, os.uname()[0], os.uname()[4]) )
+
 class TemporaryUmask(object):
     """Context manager to set the umask. Not thread-safe.
 
diff --git a/tools/spider.py b/tools/spider.py
index a966135..5eed6b6 100755
--- a/tools/spider.py
+++ b/tools/spider.py
@@ -24,6 +24,7 @@ import time
 import urllib.request
 from bs4 import BeautifulSoup
 from game_data_packager import load_games
+from game_data_packager.util import AGENT
 
 CSV = 'data/wikipedia.csv'
 
@@ -52,7 +53,7 @@ for shortname, game in load_games(None, game=todo).items():
 
     print('processing %s ...' % shortname)
     url = game.wikibase + game.wiki
-    html = urllib.request.urlopen(url)
+    html = urllib.request.urlopen(urllib.request.Request(url,headers={'User-Agent': AGENT}))
     soup = BeautifulSoup(html, 'lxml')
     for tag in soup.find_all(href=is_wikipedia):
         print('  ' + tag['href'])

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git