[pyosmium] 01/06: New upstream version 2.13.0
Bas Couwenberg
sebastic at debian.org
Thu Aug 31 22:24:43 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository pyosmium.
commit 3a9ac3ade3cea65af599e4ffad13e5183be8bdd5
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Thu Aug 31 23:40:27 2017 +0200
New upstream version 2.13.0
---
.travis.yml | 19 +++-
CHANGELOG.md | 39 ++++---
README.md | 2 +-
appveyor.yml | 4 +-
lib/generic_handler.hpp | 14 +--
lib/merged_input.hpp | 42 ++++++-
lib/osm.cc | 22 +++-
lib/osmium.cc | 10 +-
src/osmium/osm/__init__.py | 56 ++++++++++
src/osmium/replication/server.py | 19 +++-
src/osmium/version.py | 6 +-
test/helpers.py | 38 ++++++-
test/test_geom.py | 23 +++-
test/test_nodelist.py | 1 +
test/test_osm.py | 23 ++--
test/test_replication.py | 229 +++++++++++++++++++++++++++++++++++++++
test/test_taglist.py | 3 +-
tools/pyosmium-get-changes | 3 +
18 files changed, 489 insertions(+), 64 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 8a275d5..88006e3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,7 @@
language: cpp
sudo: false
+dist: trusty
matrix:
include:
@@ -25,7 +26,7 @@ matrix:
- os: osx
osx_image: xcode6.4
compiler: clang
- env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX= BOOST_PYTHON_OPTION=
+ env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX=2 BOOST_PYTHON_OPTION=
- os: osx
osx_image: xcode6.4
compiler: clang
@@ -44,14 +45,15 @@ addons:
apt:
sources:
- boost-latest
- - ubuntu-toolchain-r-test
packages:
- g++-4.8
- gcc-4.8
- libboost-python1.55-dev
- libboost1.55-dev
- libsparsehash-dev
+ - python-dev
- python-nose
+ - python-mock
- python3
- python3-dev
- python3-nose
@@ -62,7 +64,7 @@ install:
- if [ "$TRAVIS_OS_NAME" = 'osx' ]; then
brew install python$PYTHON_SUFFIX google-sparsehash;
brew install boost-python $BOOST_PYTHON_OPTION;
- pip$PYTHON_SUFFIX install -q nose;
+ pip$PYTHON_SUFFIX install -q nose mock;
fi
script:
@@ -70,8 +72,13 @@ script:
CXX=g++-4.8;
CC=gcc-4.8;
fi
- - python${USE_PYTHON_VERSION} --version
- - python${USE_PYTHON_VERSION} setup.py build
+ - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then
+ PYTHON=python${USE_PYTHON_VERSION};
+ else
+ PYTHON=/usr/bin/python${USE_PYTHON_VERSION};
+ fi
+ - $PYTHON --version
+ - $PYTHON setup.py build
- cd test
- - python${USE_PYTHON_VERSION} run_tests.py
+ - $PYTHON run_tests.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4467604..1da72ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,26 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
+## [2.13.0] - 2017-08-31
+
+### Added
+
+- tests for WKB factories and replication server
+- str() and repr() implementations for all classes in osmium.osm
+- when applying diffs to a handler, a location cache may be used
+
+### Changed
+
+- use new MultipolygonManager for building areas
+- allow to access nodes in a NodeRefList with negative index
+- use current libosmium
+
+### Fixed
+
+- pyosmium-get-changes exits with an error when no start sequence can
+ be found
+
+
## [2.12.4] - 2017-08-19
### Added
@@ -21,7 +41,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
- make apply_reader_simple a template again
-- minor fised to documentation
+- minor fixes to documentation
+
## [2.12.3] - 2017-05-25
@@ -201,20 +222,4 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- Exception not caught in test.
-[unreleased]: https://github.com/osmcode/pyosmium/compare/v2.12.3...HEAD
-[2.12.3]: https://github.com/osmcode/pyosmium/compare/v2.12.2...v2.12.3
-[2.12.2]: https://github.com/osmcode/pyosmium/compare/v2.12.1...v2.12.2
-[2.12.1]: https://github.com/osmcode/pyosmium/compare/v2.12.0...v2.12.1
-[2.12.0]: https://github.com/osmcode/pyosmium/compare/v2.11.0...v2.12.0
-[2.11.0]: https://github.com/osmcode/pyosmium/compare/v2.10.2...v2.11.0
-[2.10.2]: https://github.com/osmcode/pyosmium/compare/v2.9.0...v2.10.2
-[2.9.0]: https://github.com/osmcode/pyosmium/compare/v2.8.0...v2.9.0
-[2.8.0]: https://github.com/osmcode/pyosmium/compare/v2.7.1...v2.8.0
-[2.7.1]: https://github.com/osmcode/pyosmium/compare/v2.6.0...v2.7.1
-[2.6.0]: https://github.com/osmcode/pyosmium/compare/v2.5.4...v2.6.0
-[2.5.4]: https://github.com/osmcode/pyosmium/compare/v2.5.3...v2.5.4
-[2.5.3]: https://github.com/osmcode/pyosmium/compare/v2.4.1...v2.5.3
-[2.4.1]: https://github.com/osmcode/pyosmium/compare/v2.3.0...v2.4.1
-[2.3.0]: https://github.com/osmcode/pyosmium/compare/v2.2.0...v2.3.0
-[2.2.0]: https://github.com/osmcode/pyosmium/compare/v2.1.0...v2.2.0
diff --git a/README.md b/README.md
index 2a56bb6..fcfb39f 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ There is a small test suite in the test directory. This provides regression
test for the python bindings, it is not meant to be a test suite for Libosmium.
You'll need the Python `nose` module. On Debian/Ubuntu install the package
-`python-nose`.
+`python-nose`. For Python2 `mock` is required as well (package `python-mock`).
The suite can be run with:
diff --git a/appveyor.yml b/appveyor.yml
index 8ce1bfa..b0a35c4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -3,8 +3,10 @@ environment:
matrix:
- PYTHON: "C:\\Python27-x64"
DEVPACK: pyosmium_libs27-b63.7z
+ PIPINSTALLS: nose wheel mock
- PYTHON: "C:\\Python36-x64"
DEVPACK: pyosmium_libs36-b63.7z
+ PIPINSTALLS: nose wheel
os: Visual Studio 2015
@@ -24,7 +26,7 @@ install:
- 7z x c:\dev\%DEVPACK%
- dir c:\libs\include
- dir c:\libs\lib
- - pip install nose wheel
+ - pip install %PIPINSTALLS%
# scripts that are called at very beginning, before repo cloning
init:
diff --git a/lib/generic_handler.hpp b/lib/generic_handler.hpp
index beaa7df..61a5ec8 100644
--- a/lib/generic_handler.hpp
+++ b/lib/generic_handler.hpp
@@ -2,7 +2,7 @@
#define PYOSMIUM_GENERIC_HANDLER_HPP
#include <osmium/area/assembler.hpp>
-#include <osmium/area/multipolygon_collector.hpp>
+#include <osmium/area/multipolygon_manager.hpp>
#include <osmium/handler.hpp>
#include <osmium/handler/node_locations_for_ways.hpp>
#include <osmium/index/map/all.hpp>
@@ -44,7 +44,7 @@ void apply_with_location(osmium::io::Reader &r, const std::string &idx) {
}
void apply_with_area(osmium::io::Reader &r,
- osmium::area::MultipolygonCollector<osmium::area::Assembler> &collector,
+ osmium::area::MultipolygonManager<osmium::area::Assembler> &mp_manager,
const std::string &idx) {
const auto& map_factory = osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>::instance();
std::unique_ptr<index_type> index = map_factory.create_map(idx);
@@ -52,7 +52,7 @@ void apply_with_area(osmium::io::Reader &r,
location_handler.ignore_errors();
osmium::apply(r, location_handler, *this,
- collector.handler([this](const osmium::memory::Buffer& area_buffer) {
+ mp_manager.handler([this](const osmium::memory::Buffer& area_buffer) {
osmium::apply(area_buffer, *this);
})
);
@@ -82,14 +82,12 @@ void apply(const osmium::io::File &file, osmium::osm_entity_bits::type types,
case area_handler:
{
osmium::area::Assembler::config_type assembler_config;
- osmium::area::MultipolygonCollector<osmium::area::Assembler> collector(assembler_config);
+ osmium::area::MultipolygonManager<osmium::area::Assembler> mp_manager{assembler_config};
- osmium::io::Reader reader1(file);
- collector.read_relations(reader1);
- reader1.close();
+ osmium::relations::read_relations(file, mp_manager);
osmium::io::Reader reader2(file);
- apply_with_area(reader2, collector, idx);
+ apply_with_area(reader2, mp_manager, idx);
reader2.close();
break;
}
diff --git a/lib/merged_input.hpp b/lib/merged_input.hpp
index 4d88575..c844fd5 100644
--- a/lib/merged_input.hpp
+++ b/lib/merged_input.hpp
@@ -10,6 +10,8 @@
#include <osmium/io/any_output.hpp>
#include <osmium/io/output_iterator.hpp>
#include <osmium/handler.hpp>
+#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/index/map/all.hpp>
#include <osmium/object_pointer_collection.hpp>
#include <osmium/visitor.hpp>
@@ -50,8 +52,7 @@ namespace {
namespace pyosmium {
class MergeInputReader {
-public:
- void apply(BaseHandler& handler, bool simplify = true) {
+ void apply_without_location(BaseHandler& handler, bool simplify = true) {
handler.apply_start();
if (simplify) {
objects.sort(osmium::object_order_type_id_reverse_version());
@@ -73,6 +74,43 @@ public:
changes.clear();
}
+ void apply_with_location(BaseHandler& handler, const std::string &idx,
+ bool simplify = true) {
+ const auto& map_factory = osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>::instance();
+ std::unique_ptr<index_type> index = map_factory.create_map(idx);
+ osmium::handler::NodeLocationsForWays<index_type> location_handler(*index);
+ location_handler.ignore_errors();
+
+ handler.apply_start();
+ if (simplify) {
+ objects.sort(osmium::object_order_type_id_reverse_version());
+ osmium::item_type prev_type = osmium::item_type::undefined;
+ osmium::object_id_type prev_id = 0;
+ for (auto &item: objects) {
+ if (item.type() != prev_type || item.id() != prev_id) {
+ prev_type = item.type();
+ prev_id = item.id();
+ osmium::apply_item(item, location_handler, handler);
+ }
+ }
+ } else {
+ objects.sort(osmium::object_order_type_id_version());
+ osmium::apply(objects.begin(), objects.end(), location_handler, handler);
+ }
+
+ objects = osmium::ObjectPointerCollection();
+ changes.clear();
+ }
+
+public:
+ void apply(BaseHandler& handler, const std::string &idx = "",
+ bool simplify = true) {
+ if (idx.empty())
+ apply_without_location(handler, simplify);
+ else
+ apply_with_location(handler, idx, simplify);
+ }
+
void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer,
bool with_history = true) {
auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(reader);
diff --git a/lib/osm.cc b/lib/osm.cc
index 3491237..9263381 100644
--- a/lib/osm.cc
+++ b/lib/osm.cc
@@ -29,6 +29,18 @@ inline const char *get_tag_by_key_with_none(osmium::TagList const& obj,
return key ? obj.get_value_by_key(key) : nullptr;
}
+inline const osmium::NodeRef &get_item_nodereflist(osmium::NodeRefList const& list, ssize_t idx)
+{
+ auto sz = list.size();
+ osmium::NodeRefList::size_type iout = (idx >= 0 ? idx : (ssize_t) sz + idx);
+
+ if (iout >= sz) {
+ PyErr_SetString(PyExc_IndexError, "Bad index.");
+ boost::python::throw_error_already_set();
+ }
+
+ return list[iout];
+}
inline bool taglist_contains_tag(osmium::TagList const& obj, const char *key)
{
@@ -88,6 +100,12 @@ BOOST_PYTHON_MODULE(_osm)
.def("valid", &osmium::Location::valid, args("self"),
"Check that the location is a valid WGS84 coordinate, i.e. "
"that it is within the usual bounds.")
+ .def("lat_without_check", &osmium::Location::lat_without_check, args("self"),
+ "Return latitude (y coordinate) without checking if the location "
+ "is valid.")
+ .def("lon_without_check", &osmium::Location::lon_without_check, args("self"),
+ "Return longitude (x coordinate) without checking if the location "
+ "is valid.")
;
class_<osmium::Box>("Box",
"A bounding box around a geographic area. It is defined by an "
@@ -184,8 +202,8 @@ BOOST_PYTHON_MODULE(_osm)
"is normally not used directly, use one of its subclasses instead.",
no_init)
.def("__len__", &osmium::NodeRefList::size)
- .def("__getitem__",
- make_function(static_cast<const osmium::NodeRef& (osmium::NodeRefList::*)(osmium::NodeRefList::size_type) const>(&osmium::NodeRefList::operator[]), return_value_policy<reference_existing_object>()))
+ .def("__getitem__", make_function(&get_item_nodereflist,
+ return_value_policy<reference_existing_object>()))
.def("__iter__", iterator<osmium::NodeRefList,return_internal_reference<>>())
.def("is_closed", &osmium::NodeRefList::is_closed, args("self"),
"True if the start and end node are the same (synonym for "
diff --git a/lib/osmium.cc b/lib/osmium.cc
index 62d787f..0398b13 100644
--- a/lib/osmium.cc
+++ b/lib/osmium.cc
@@ -174,10 +174,16 @@ BOOST_PYTHON_MODULE(_osmium)
"Collects data from multiple input files and sorts and optionally "
"deduplicates the data before applying it to a handler.")
.def("apply", &pyosmium::MergeInputReader::apply,
- (arg("self"), arg("handler"), arg("simplify")=true),
+ (arg("self"), arg("handler"), arg("idx")="", arg("simplify")=true),
"Apply collected data to a handler. The data will be sorted first. "
"If `simplify` is true (default) then duplicates will be eliminated "
- "and only the newest version of each object kept. After the data "
+ "and only the newest version of each object kept. If `idx` is given "
+ "a node location cache with the given type will be created and "
+ "applied when creating the ways. Note that a diff file normally does "
+ "not contain all node locations to reconstruct changed ways. If the "
+ "full way geometries are needed, create a persistent node location "
+ "cache during initial import of the area and reuse it when processing "
+ "diffs. After the data "
"has been applied the buffer of the MergeInputReader is empty and "
"new data can be added for the next round of application.")
.def("apply_to_reader", &pyosmium::MergeInputReader::apply_to_reader,
diff --git a/src/osmium/osm/__init__.py b/src/osmium/osm/__init__.py
index 22712b4..c988227 100644
--- a/src/osmium/osm/__init__.py
+++ b/src/osmium/osm/__init__.py
@@ -25,3 +25,59 @@ def create_mutable_relation(rel, **args):
Node.replace = create_mutable_node
Way.replace = create_mutable_way
Relation.replace = create_mutable_relation
+
+Location.__repr__ = lambda l : 'osmium.osm.Location(x=%r, y=%r)' \
+ % (l.x, l.y) \
+ if l.valid() else 'osmium.osm.Location()'
+Location.__str__ = lambda l : '%f/%f' % (l.lon_without_check(), l.lat_without_check()) \
+ if l.valid() else 'invalid'
+
+Box.__repr__ = lambda b : 'osmium.osm.Box(bottom_left=%r, top_right=%r)' \
+ % (b.bottom_left, b.top_right)
+Box.__str__ = lambda b : '(%s %s)' % (b.bottom_left, b.top_right)
+
+Tag.__repr__ = lambda t : 'osmium.osm.Tag(k=%s, v=%s)' % (t.k, t.v)
+Tag.__str__ = lambda t : '%s=%s' % (t.k, t.v)
+
+TagList.__repr__ = lambda t : "osmium.osm.TagList({%s})" \
+ % ",".join(["%r=%r" % (i.k, i.v) for i in t])
+TagList.__str__ = lambda t : "{%s}" % ",".join([str(i) for i in t])
+
+NodeRef.__repr__ = lambda n : 'osmium.osm.NodeRef(ref=%r, location=%r)' % (n.ref, n.location)
+NodeRef.__str__ = lambda n : '%s@%s' % (n.ref, n.location) if n.location.valid() \
+ else str(n.ref)
+
+NodeRefList.__repr__ = lambda t : "%s([%s])" % (t.__class__.__name__,
+ ",".join([repr(i) for i in t]))
+NodeRefList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t])
+
+RelationMember.__repr__ = lambda r : 'osmium.osm.RelationMember(ref=%r, type=%r, role=%r)' \
+ % (r.ref, r.type, r.role)
+RelationMember.__str__ = lambda r : '%s%d@%s' % (r.type, r.ref, r.role) \
+ if r.role else '%s%d' % (r.type, r.ref)
+
+RelationMemberList.__repr__ = lambda t : "osmium.osm.RelationMemberList([%s])" \
+ % ",".join([repr(i) for i in t])
+RelationMemberList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t])
+
+OSMObject.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags)
+
+def _str_ellipse(s, length=50):
+ s = str(s)
+ return s if len(s) <= length else (s[:length - 4] + '...' + s[-1])
+
+Node.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, location=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.location)
+Node.__str__ = lambda n : 'n%d: location=%s tags=%s' \
+ % (n.id, n.location, _str_ellipse(n.tags))
+
+Way.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, nodes=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.nodes)
+Way.__str__ = lambda o : 'w%d: nodes=%s tags=%s' \
+ % (o.id, _str_ellipse(o.nodes), _str_ellipse(o.tags))
+
+Relation.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, members=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.members)
+Relation.__str__ = lambda o : 'r%d: members=%s, tags=%s' \
+ % (o.id, _str_ellipse(o.members), _str_ellipse(o.tags))
+
+Changeset.__repr__ = lambda o : '%s(id=%r, uid=%r, created_at=%r, closed_at=%r, open=%r, num_changes=%r, bounds=%r, user=%r, tags=%s)' %(o.__class__.__name__, o.id, o.uid, o.created_at, o.closed_at, o.open, o.num_changes, o.bounds, o.user, o.tags)
+Changeset.__str__ = lambda o : 'c%d: closed_at=%s, bounds=%s, tags=%s' \
+ % (o.id, o.closed_at, o.bounds, _str_ellipse(o.tags))
diff --git a/src/osmium/replication/server.py b/src/osmium/replication/server.py
index 345b333..f8f6373 100644
--- a/src/osmium/replication/server.py
+++ b/src/osmium/replication/server.py
@@ -78,13 +78,28 @@ class ReplicationServer(object):
return DownloadResult(current_id - 1, rd, newest.sequence)
- def apply_diffs(self, handler, start_id, max_size=1024, simplify=True):
+ def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True):
""" Download diffs starting with sequence id `start_id`, merge them
together and then apply them to handler `handler`. `max_size`
restricts the number of diffs that are downloaded. The download
stops as soon as either a diff cannot be downloaded or the
unpacked data in memory exceeds `max_size` kB.
+ If `idx` is set, a location cache will be created and applied to
+ the way nodes. You should be aware that diff files usually do not
+ contain the complete set of nodes when a way is modified. That means
+ that you cannot just create a new location cache, apply it to a diff
+ and expect to get complete way geometries back. Instead you need to
+ do an initial data import using a persistent location cache to
+ obtain a full set of node locations and then reuse this location
+ cache here when applying diffs.
+
+ Diffs may contain multiple versions of the same object when it was
+ changed multiple times during the period covered by the diff. If
+ `simplify` is set to False then all versions are returned. If it
+ is True (the default) then only the most recent version will be
+ sent to the handler.
+
The function returns the sequence id of the last diff that was
downloaded or None if the download failed completely.
"""
@@ -93,7 +108,7 @@ class ReplicationServer(object):
if diffs is None:
return None
- diffs.reader.apply(handler, simplify)
+ diffs.reader.apply(handler, idx=idx, simplify=simplify)
return diffs.id
diff --git a/src/osmium/version.py b/src/osmium/version.py
index 9b0bd33..0d61057 100644
--- a/src/osmium/version.py
+++ b/src/osmium/version.py
@@ -3,9 +3,9 @@ Version information.
"""
# the major version
-pyosmium_major = '2.12'
+pyosmium_major = '2.13'
# current release (Pip version)
-pyosmium_release = '2.12.4'
+pyosmium_release = '2.13.0'
# libosmium version shipped with the Pip release
-libosmium_version = '2.12.2'
+libosmium_version = '2.13.1'
diff --git a/test/helpers.py b/test/helpers.py
index df0862d..13ea5b2 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -3,8 +3,20 @@
import random
import tempfile
import os
+import sys
from textwrap import dedent
import osmium
+from datetime import datetime
+
+if sys.version_info[0] == 3:
+ from datetime import timezone
+
+ def mkdate(*args):
+ return datetime(*args, tzinfo=timezone.utc)
+else:
+ def mkdate(*args):
+ return datetime(*args)
+
def _complete_object(o):
"""Takes a hash with an incomplete OSM object description and returns a
@@ -102,6 +114,8 @@ def osmobj(kind, **args):
ret['type'] = kind
return ret
+def check_repr(o):
+ return not str(o).startswith('<') and not repr(o).startswith('<')
class HandlerTestBase:
@@ -115,7 +129,29 @@ class HandlerTestBase:
fn = create_opl_file(self.data)
try:
- self.Handler().apply_file(fn, self.apply_locations, self.apply_idx)
+ self.handler = self.Handler()
+ self.handler.apply_file(fn, self.apply_locations, self.apply_idx)
finally:
os.remove(fn)
+ if hasattr(self, "check_result"):
+ self.check_result()
+
+
+class CountingHandler(osmium.SimpleHandler):
+
+ def __init__(self):
+ super(CountingHandler, self).__init__()
+ self.counts = [0, 0, 0, 0]
+
+ def node(self, _):
+ self.counts[0] += 1
+
+ def way(self, _):
+ self.counts[1] += 1
+
+ def relation(self, _):
+ self.counts[2] += 1
+
+ def area(self, _):
+ self.counts[3] += 1
diff --git a/test/test_geom.py b/test/test_geom.py
index 74bdd84..1c4e9bc 100644
--- a/test/test_geom.py
+++ b/test/test_geom.py
@@ -10,8 +10,12 @@ class TestWkbCreateNode(HandlerTestBase, unittest.TestCase):
data = [osmobj('N', id=1)]
class Handler(o.SimpleHandler):
+ wkbs = []
def node(self, n):
- wkb = wkbfab.create_point(n)
+ self.wkbs.append(wkbfab.create_point(n))
+
+ def check_result(self):
+ assert_equals(1, len(self.handler.wkbs))
class TestWkbCreateWay(HandlerTestBase, unittest.TestCase):
data = [osmobj('N', id=1, lat=0, lon=0),
@@ -21,10 +25,14 @@ class TestWkbCreateWay(HandlerTestBase, unittest.TestCase):
apply_locations = True
class Handler(o.SimpleHandler):
+ wkbs = []
def way(self, w):
- wkb = wkbfab.create_linestring(w)
- wkb = wkbfab.create_linestring(w, direction=o.geom.direction.BACKWARD)
- wkb = wkbfab.create_linestring(w, use_nodes=o.geom.use_nodes.ALL)
+ self.wkbs.append(wkbfab.create_linestring(w))
+ self.wkbs.append(wkbfab.create_linestring(w, direction=o.geom.direction.BACKWARD))
+ self.wkbs.append(wkbfab.create_linestring(w, use_nodes=o.geom.use_nodes.ALL))
+
+ def check_result(self):
+ assert_equals(3, len(self.handler.wkbs))
class TestWkbCreatePoly(HandlerTestBase, unittest.TestCase):
data = [osmobj('N', id=1, lat=0, lon=0),
@@ -36,5 +44,10 @@ class TestWkbCreatePoly(HandlerTestBase, unittest.TestCase):
apply_locations = True
class Handler(o.SimpleHandler):
+ wkbs = []
+
def area(self, a):
- wkb = wkbfab.create_multipolygon(a)
+ self.wkbs.append(wkbfab.create_multipolygon(a))
+
+ def check_result(self):
+ assert_equals(1, len(self.handler.wkbs))
diff --git a/test/test_nodelist.py b/test/test_nodelist.py
index 46937a2..362abed 100644
--- a/test/test_nodelist.py
+++ b/test/test_nodelist.py
@@ -32,6 +32,7 @@ class TestNodeIds(HandlerTestBase, unittest.TestCase):
eq_(34359737784, w.nodes[2].ref)
eq_(-34, w.nodes[3].ref)
eq_(0, w.nodes[4].ref)
+ eq_(0, w.nodes[-1].ref)
class TestMissingRef(HandlerTestBase, unittest.TestCase):
data = """\
diff --git a/test/test_osm.py b/test/test_osm.py
index ea5c2c6..288196c 100644
--- a/test/test_osm.py
+++ b/test/test_osm.py
@@ -2,19 +2,7 @@ from nose.tools import *
import unittest
import os
import sys
-from datetime import datetime
-
-if sys.version_info[0] == 3:
- from datetime import timezone
-
- def mkdate(*args):
- return datetime(*args, tzinfo=timezone.utc)
-else:
- def mkdate(*args):
- return datetime(*args)
-
-
-from helpers import create_osm_file, osmobj, HandlerTestBase
+from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase, mkdate
import osmium as o
@@ -23,6 +11,7 @@ class TestLocation(unittest.TestCase):
def test_invalid_location(self):
loc = o.osm.Location()
assert_false(loc.valid())
+ assert_true(check_repr(loc))
def test_valid_location(self):
loc = o.osm.Location(1,10)
@@ -30,6 +19,8 @@ class TestLocation(unittest.TestCase):
assert_equals(loc.lat, 10, 0.00001)
assert_equals(loc.x, 10000000)
assert_equals(loc.y, 100000000)
+ assert_true(check_repr(loc))
+
class TestNodeAttributes(HandlerTestBase, unittest.TestCase):
data = [osmobj('N', id=1, version=5, changeset=58674, uid=42,
@@ -47,6 +38,7 @@ class TestNodeAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 1)
+ assert_true(check_repr(n))
class TestNodePositiveId(HandlerTestBase, unittest.TestCase):
@@ -91,6 +83,8 @@ class TestWayAttributes(HandlerTestBase, unittest.TestCase):
assert_false(n.is_closed())
assert_false(n.ends_have_same_id())
assert_false(n.ends_have_same_location())
+ assert_true(check_repr(n))
+ assert_true(check_repr(n.nodes))
class TestRelationAttributes(HandlerTestBase, unittest.TestCase):
data = [osmobj('R', id=1, version=5, changeset=58674, uid=42,
@@ -109,6 +103,8 @@ class TestRelationAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 1)
+ assert_true(check_repr(n))
+ assert_true(check_repr(n.members))
class TestAreaFromWayAttributes(HandlerTestBase, unittest.TestCase):
data = [osmobj('N', id=1, lat=0, lon=0),
@@ -167,3 +163,4 @@ class TestChangesetAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(515288620, c.bounds.top_right.y)
assert_equals(-1465242, c.bounds.bottom_left.x)
assert_equals(515288506, c.bounds.bottom_left.y)
+ assert_true(check_repr(c))
diff --git a/test/test_replication.py b/test/test_replication.py
new file mode 100644
index 0000000..9ca1fa0
--- /dev/null
+++ b/test/test_replication.py
@@ -0,0 +1,229 @@
+from nose.tools import *
+import unittest
+from io import BytesIO
+from textwrap import dedent
+from helpers import mkdate, CountingHandler
+
+try:
+ from urllib.error import URLError
+except ImportError:
+ from urllib2 import URLError
+
+try:
+ from unittest.mock import MagicMock, patch
+except ImportError:
+ from mock import MagicMock, patch
+
+import osmium as o
+import osmium.replication.server as rserv
+
+class UrllibMock(MagicMock):
+
+ def set_result(self, s):
+ self.return_value = BytesIO(dedent(s).encode())
+
+ def set_script(self, files):
+ self.side_effect = [BytesIO(dedent(s).encode()) for s in files]
+
+def test_get_state_url():
+ svr = rserv.ReplicationServer("http://text.org")
+
+ data = [
+ (None, 'http://text.org/state.txt'),
+ (1, 'http://text.org/000/000/001.state.txt'),
+ (999, 'http://text.org/000/000/999.state.txt'),
+ (1000, 'http://text.org/000/001/000.state.txt'),
+ (573923, 'http://text.org/000/573/923.state.txt'),
+ (3290012, 'http://text.org/003/290/012.state.txt'),
+ ]
+
+ for i, o in data:
+ assert_equals(o, svr.get_state_url(i))
+
+def test_get_diff_url():
+ svr = rserv.ReplicationServer("https://who.is/me/")
+
+ data = [
+ (1, 'https://who.is/me//000/000/001.osc.gz'),
+ (500, 'https://who.is/me//000/000/500.osc.gz'),
+ (83750, 'https://who.is/me//000/083/750.osc.gz'),
+ (999999999, 'https://who.is/me//999/999/999.osc.gz'),
+ ]
+
+ for i, o in data:
+ assert_equals(o, svr.get_diff_url(i))
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_get_state_valid(mock):
+ mock.set_result("""\
+ #Sat Aug 26 11:04:04 UTC 2017
+ txnMaxQueried=1219304113
+ sequenceNumber=2594669
+ timestamp=2017-08-26T11\:04\:02Z
+ txnReadyList=
+ txnMax=1219304113
+ txnActiveList=1219303583,1219304054,1219304104""")
+
+ res = rserv.ReplicationServer("http://test.io").get_state_info()
+
+ assert_is_not_none(res)
+ assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))
+ assert_equals(res.sequence, 2594669)
+
+ assert_equal(mock.call_count, 1)
+
+ at patch('osmium.replication.server.urlrequest.urlopen')
+def test_get_state_server_timeout(mock):
+ mock.side_effect = URLError(reason='Mock')
+
+ svr = rserv.ReplicationServer("http://test.io")
+ assert_is_none(svr.get_state_info())
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_diffs_count(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1
+ w1
+ r1
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ h = CountingHandler()
+ assert_equals(100, svr.apply_diffs(h, 100, 10000))
+
+ assert_equals(h.counts, [1, 1, 1, 0])
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_diffs_without_simplify(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 v23
+ n1 v24
+ w1
+ r1
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ h = CountingHandler()
+ assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False))
+ assert_equals([2, 1, 1, 0], h.counts)
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_diffs_with_simplify(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 v23
+ n1 v24
+ w1
+ r1
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ h = CountingHandler()
+ assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=True))
+ assert_equals([1, 1, 1, 0], h.counts)
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_with_location(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 x10.0 y23.0
+ w1 Nn1,n2
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ class Handler(CountingHandler):
+ def way(self, w):
+ self.counts[1] += 1
+ assert_equals(2, len(w.nodes))
+ assert_equals(1, w.nodes[0].ref)
+ assert_equals(10, w.nodes[0].location.lon)
+ assert_equals(23, w.nodes[0].location.lat)
+ assert_equals(2, w.nodes[1].ref)
+ assert_false(w.nodes[1].location.valid())
+
+ h = Handler()
+ assert_equals(100, svr.apply_diffs(h, 100, 10000, idx="flex_mem"))
+
+ assert_equals(h.counts, [1, 1, 0, 0])
+
+
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_reader_without_simplify(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 v23
+ n1 v24
+ w1
+ r1
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ h = CountingHandler()
+
+ diffs = svr.collect_diffs(100, 100000)
+ assert_is_not_none(diffs)
+
+ diffs.reader.apply(h, simplify=False)
+ assert_equals([2, 1, 1, 0], h.counts)
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_reader_with_simplify(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 v23
+ n1 v24
+ w1
+ r1
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ h = CountingHandler()
+ diffs = svr.collect_diffs(100, 100000)
+ assert_is_not_none(diffs)
+
+ diffs.reader.apply(h, simplify=True)
+ assert_equals([1, 1, 1, 0], h.counts)
+
+ at patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)
+def test_apply_reader_with_location(mock):
+ mock.set_script(("""\
+ sequenceNumber=100
+ timestamp=2017-08-26T11\:04\:02Z
+ """, """
+ n1 x10.0 y23.0
+ w1 Nn1,n2
+ """))
+ svr = rserv.ReplicationServer("http://test.io", "opl")
+
+ class Handler(CountingHandler):
+ def way(self, w):
+ self.counts[1] += 1
+ assert_equals(2, len(w.nodes))
+ assert_equals(1, w.nodes[0].ref)
+ assert_equals(10, w.nodes[0].location.lon)
+ assert_equals(23, w.nodes[0].location.lat)
+ assert_equals(2, w.nodes[1].ref)
+ assert_false(w.nodes[1].location.valid())
+
+ h = Handler()
+ diffs = svr.collect_diffs(100, 100000)
+ assert_is_not_none(diffs)
+
+ diffs.reader.apply(h, idx="flex_mem")
+
+ assert_equals(h.counts, [1, 1, 0, 0])
diff --git a/test/test_taglist.py b/test/test_taglist.py
index 0437644..c59f4cf 100644
--- a/test/test_taglist.py
+++ b/test/test_taglist.py
@@ -4,7 +4,7 @@ import os
import sys
from datetime import datetime
-from helpers import create_osm_file, osmobj, HandlerTestBase
+from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase
import osmium as o
@@ -72,6 +72,7 @@ class TestTagContains(HandlerTestBase, unittest.TestCase):
assert_not_in("x", n.tags)
assert_not_in(None, n.tags)
assert_not_in("", n.tags)
+ assert_true(check_repr(n.tags))
class TestTagIndexOp(HandlerTestBase, unittest.TestCase):
data = "n234 Tabba=x,2=vvv,xx=abba"
diff --git a/tools/pyosmium-get-changes b/tools/pyosmium-get-changes
index dac4403..f86ffe3 100755
--- a/tools/pyosmium-get-changes
+++ b/tools/pyosmium-get-changes
@@ -188,6 +188,9 @@ if __name__ == '__main__':
svr = rserv.ReplicationServer(url)
startseq = options.start.get_sequence(svr)
+ if startseq is None:
+ log.error("Cannot read state file from server. Is the URL correct?")
+ exit(1)
if options.outfile is None:
write_end_sequence(options.seq_file, startseq)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/pyosmium.git
More information about the Pkg-grass-devel
mailing list