[Git][debian-gis-team/osm2pgsql][upstream] New upstream version 1.9.2+ds
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Mon Sep 4 14:13:40 BST 2023
Bas Couwenberg pushed to branch upstream at Debian GIS Project / osm2pgsql
Commits:
8af8eff1 by Bas Couwenberg at 2023-09-04T15:06:45+02:00
New upstream version 1.9.2+ds
- - - - -
28 changed files:
- .github/workflows/ci.yml
- CMakeLists.txt
- docs/osm2pgsql-replication.1
- docs/osm2pgsql.1
- flex-config/bbox.lua
- src/command-line-parser.cpp
- src/gen/gen-rivers.cpp
- src/gen/gen-tile-builtup.cpp
- src/gen/gen-tile-raster.cpp
- src/gen/osm2pgsql-gen.cpp
- src/gen/tracer.cpp
- src/middle-pgsql.cpp
- src/middle-ram.cpp
- src/options.hpp
- src/osm2pgsql.cpp
- src/output-flex.cpp
- src/output-flex.hpp
- + src/output-requirements.hpp
- src/output.cpp
- src/output.hpp
- src/tagtransform-lua.cpp
- src/wkb.cpp
- src/wkb.hpp
- tests/bdd/flex/lua-expire-output-definitions.feature
- tests/bdd/flex/lua-expire.feature
- tests/test-expire-from-geometry.cpp
- tests/test-middle.cpp
- tests/test-wkb.cpp
Changes:
=====================================
.github/workflows/ci.yml
=====================================
@@ -196,7 +196,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu20-pg13-clang10-cpp17:
+ ubuntu20-pg15-clang10:
runs-on: ubuntu-20.04
env:
@@ -204,7 +204,7 @@ jobs:
CXX: clang++-10
LUA_VERSION: 5.3
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
USE_PROJ_LIB: 6
BUILD_TYPE: Debug
@@ -249,7 +249,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-clang14-jit:
+ ubuntu22-pg15-clang14-jit:
runs-on: ubuntu-22.04
env:
@@ -257,7 +257,7 @@ jobs:
CXX: clang++-14
LUA_VERSION: 5.4
LUAJIT_OPTION: ON
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
BUILD_TYPE: Debug
PSYCOPG: 2
@@ -267,7 +267,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-clang14-proj6:
+ ubuntu22-pg15-clang14-proj6:
runs-on: ubuntu-22.04
env:
@@ -275,7 +275,7 @@ jobs:
CXX: clang++-14
LUA_VERSION: 5.4
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
USE_PROJ_LIB: 6
BUILD_TYPE: Debug
@@ -286,7 +286,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-clang14-noproj:
+ ubuntu22-pg15-clang14-noproj:
runs-on: ubuntu-22.04
env:
@@ -294,7 +294,7 @@ jobs:
CXX: clang++-14
LUA_VERSION: 5.3
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
USE_PROJ_LIB: off
BUILD_TYPE: Debug
@@ -305,7 +305,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-clang14-cpp17:
+ ubuntu22-pg15-clang14:
runs-on: ubuntu-22.04
env:
@@ -313,7 +313,7 @@ jobs:
CXX: clang++-14
LUA_VERSION: 5.4
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
USE_PROJ_LIB: 6
BUILD_TYPE: Debug
@@ -324,7 +324,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-gcc12-release:
+ ubuntu22-pg15-gcc12-release:
runs-on: ubuntu-22.04
env:
@@ -333,7 +333,7 @@ jobs:
EXTRA_FLAGS: -Wno-stringop-overread
LUA_VERSION: 5.4
LUAJIT_OPTION: ON
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
BUILD_TYPE: Release
PSYCOPG: 2
@@ -343,14 +343,14 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-gcc12-release-nolua:
+ ubuntu22-pg15-gcc12-release-nolua:
runs-on: ubuntu-22.04
env:
CC: gcc-12
CXX: g++-12
EXTRA_FLAGS: -Wno-stringop-overread
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
BUILD_TYPE: Release
PSYCOPG: 2
@@ -360,7 +360,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-clang14-cpp20:
+ ubuntu22-pg15-clang14-cpp20:
runs-on: ubuntu-22.04
env:
@@ -368,7 +368,7 @@ jobs:
CXX: clang++-14
LUA_VERSION: 5.3
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
CPP_VERSION: 20
BUILD_TYPE: Debug
@@ -379,7 +379,7 @@ jobs:
- uses: ./.github/actions/ubuntu-prerequisites
- uses: ./.github/actions/build-and-test
- ubuntu22-pg14-gcc12-cpp20:
+ ubuntu22-pg15-gcc12-cpp20:
runs-on: ubuntu-22.04
env:
@@ -387,7 +387,7 @@ jobs:
CXX: g++-12
LUA_VERSION: 5.3
LUAJIT_OPTION: OFF
- POSTGRESQL_VERSION: 14
+ POSTGRESQL_VERSION: 15
POSTGIS_VERSION: 3
CPP_VERSION: 20
BUILD_TYPE: Debug
=====================================
CMakeLists.txt
=====================================
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.8.0)
-project(osm2pgsql VERSION 1.9.1 LANGUAGES CXX C)
+project(osm2pgsql VERSION 1.9.2 LANGUAGES CXX C)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
=====================================
docs/osm2pgsql-replication.1
=====================================
@@ -1,4 +1,4 @@
-.TH "OSM2PGSQL-REPLICATION" "1" "1.9.1" "" ""
+.TH "OSM2PGSQL-REPLICATION" "1" "1.9.2" "" ""
.SH NAME
osm2pgsql-replication \- osm2pgsql database updater
.SH SYNOPSIS
=====================================
docs/osm2pgsql.1
=====================================
@@ -1,4 +1,4 @@
-.TH "OSM2PGSQL" "1" "1.9.1" "" ""
+.TH "OSM2PGSQL" "1" "1.9.2" "" ""
.SH NAME
.PP
osm2pgsql - OpenStreetMap data to PostgreSQL converter
=====================================
flex-config/bbox.lua
=====================================
@@ -82,7 +82,7 @@ end
-- Format the bounding box we get from calling get_bbox() on the parameter
-- in the way needed for the PostgreSQL/PostGIS box2d type.
local function format_bbox(object)
- local xmin, ymin, xmax, ymax = object.get_bbox()
+ local xmin, ymin, xmax, ymax = object:get_bbox()
if xmin == nil then
return nil
end
=====================================
src/command-line-parser.cpp
=====================================
@@ -486,7 +486,7 @@ static void check_options(options_t *options)
throw std::runtime_error{
"RAM node cache can only be disabled in slim mode."};
}
- if (options->flat_node_file.empty()) {
+ if (options->flat_node_file.empty() && !options->append) {
log_warn("RAM cache is disabled. This will likely slow down "
"processing a lot.");
}
=====================================
src/gen/gen-rivers.cpp
=====================================
@@ -258,7 +258,7 @@ SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}"
if (!name.empty()) {
names.emplace(id, name);
}
- auto const geom = ewkb_to_geom(decode_hex(result.get_value(i, 3)));
+ auto const geom = ewkb_to_geom(decode_hex(result.get(i, 3)));
if (geom.is_linestring()) {
auto const &ls = geom.get<geom::linestring_t>();
=====================================
src/gen/gen-tile-builtup.cpp
=====================================
@@ -173,7 +173,7 @@ static void draw_from_db(double margin, canvas_list_t *canvas_list,
box.max_x(), box.max_y());
for (int n = 0; n < result.num_tuples(); ++n) {
- auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0)));
+ auto const geom = ewkb_to_geom(decode_hex(result.get(n, 0)));
cc.canvas.draw(geom, tile);
}
}
=====================================
src/gen/gen-tile-raster.cpp
=====================================
@@ -163,7 +163,7 @@ static void draw_from_db(double margin, unsigned int image_extent,
for (int n = 0; n < result.num_tuples(); ++n) {
std::string param = result.get_value(n, 1);
- auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0)));
+ auto const geom = ewkb_to_geom(decode_hex(result.get(n, 0)));
auto const [it, success] = canvas_list->try_emplace(
std::move(param), image_extent, image_buffer);
=====================================
src/gen/osm2pgsql-gen.cpp
=====================================
@@ -296,7 +296,7 @@ public:
int app_run_gen()
{
- log_debug("Running configured generalizer (run {})...", ++m_gen_run);
+ log_debug("Configuring generalizer...");
if (lua_type(lua_state(), 1) != LUA_TSTRING) {
throw std::runtime_error{"Argument #1 to 'run_gen' must be a "
@@ -326,8 +326,8 @@ public:
auto generalizer =
create_generalizer(strategy, &db_connection, ¶ms);
- log_debug("Generalizer '{}' ({}) initialized.", generalizer->name(),
- generalizer->strategy());
+ log_info("Running generalizer '{}' ({})...", generalizer->name(),
+ generalizer->strategy());
if (m_append) {
params.set("delete_existing", true);
@@ -335,6 +335,7 @@ public:
write_to_debug_log(params, "Params (after initialization):");
+ util::timer_t timer_gen;
if (generalizer->on_tiles()) {
process_tiles(db_connection, params, generalizer.get());
} else {
@@ -353,8 +354,8 @@ public:
std::chrono::duration_cast<std::chrono::milliseconds>(
timer.elapsed())));
}
- log_debug("Finished generalizer '{}' (run {}).", generalizer->name(),
- m_gen_run);
+ log_info("Finished generalizer '{}' in {}.", generalizer->name(),
+ util::human_readable_duration(timer_gen.stop()));
return 0;
}
@@ -368,16 +369,64 @@ public:
std::string const description =
luaX_get_table_string(lua_state(), "description", 1, "Argument #1");
- std::string const sql =
- luaX_get_table_string(lua_state(), "sql", 1, "Argument #1");
- log_debug("Running SQL command: {}.", description);
+ bool const transaction = luaX_get_table_bool(lua_state(), "transaction",
+ 1, "Argument #1", false);
+
+ std::string const if_has_rows = luaX_get_table_string(
+ lua_state(), "if_has_rows", 1, "Argument #1", "");
+
+ std::vector<std::string> queries;
+ if (transaction) {
+ queries.emplace_back("BEGIN");
+ }
+
+ lua_getfield(lua_state(), 1, "sql");
+ int const ltype = lua_type(lua_state(), -1);
+ if (ltype == LUA_TSTRING) {
+ queries.emplace_back(lua_tostring(lua_state(), -1));
+ } else if (ltype == LUA_TTABLE) {
+ if (!luaX_is_array(lua_state())) {
+ throw std::runtime_error{
+ "Table in 'sql' field must be an array."};
+ }
+ luaX_for_each(lua_state(), [&]() {
+ if (lua_type(lua_state(), -1) != LUA_TSTRING) {
+ throw std::runtime_error{
+ "Table in 'sql' field must only contain strings."};
+ }
+ queries.emplace_back(lua_tostring(lua_state(), -1));
+ });
+ } else {
+ throw std::runtime_error{
+ "Argument #1 must contain a 'sql' string or table field."};
+ }
+
+ if (transaction) {
+ queries.emplace_back("COMMIT");
+ }
- util::timer_t timer_sql;
pg_conn_t const db_connection{m_conninfo};
- db_connection.exec(sql);
- log_debug("SQL command took {}.",
- util::human_readable_duration(timer_sql.stop()));
+
+ if (m_append && !if_has_rows.empty()) {
+ auto const result = db_connection.exec(if_has_rows);
+ if (result.num_tuples() == 0) {
+ log_info("Not running SQL command: {} (no rows in "
+ "condition result).",
+ description);
+ return 0;
+ }
+ }
+
+ log_info("Running SQL commands: {}.", description);
+
+ util::timer_t timer_sql;
+ for (auto const &query : queries) {
+ log_debug("Running sql: {}", query);
+ db_connection.exec(query);
+ }
+ log_info("Finished SQL commands in {}.",
+ util::human_readable_duration(timer_sql.stop()));
return 0;
}
@@ -504,7 +553,6 @@ private:
std::string m_conninfo;
std::string m_dbschema;
- std::size_t m_gen_run = 0;
uint32_t m_jobs;
bool m_append;
bool m_updatable;
@@ -595,6 +643,7 @@ void genproc_t::run()
}
}
+// NOLINTNEXTLINE(bugprone-exception-escape)
int main(int argc, char *argv[])
{
try {
=====================================
src/gen/tracer.cpp
=====================================
@@ -41,6 +41,15 @@ void tracer_t::reset()
m_num_points = 0;
}
+static potrace_word bit_squeeze(potrace_word w, unsigned char const *d) noexcept
+{
+ return (0x80U & d[0]) | (0x40U & d[1]) | (0x20U & d[2]) | (0x10U & d[3]) |
+ (0x08U & d[4]) | (0x04U & d[5]) | (0x02U & d[6]) | (0x01U & d[7]) |
+ w;
+}
+
+static_assert(sizeof(potrace_word) == 8);
+
void tracer_t::prepare(canvas_t const &canvas) noexcept
{
std::size_t const size = canvas.size();
@@ -48,14 +57,17 @@ void tracer_t::prepare(canvas_t const &canvas) noexcept
m_bits.reserve((size * size) / bits_per_word);
- unsigned char const *d = canvas.begin();
- while (d != canvas.end()) {
- potrace_word w = 0x1U & *d++;
- for (std::size_t n = 1; n < bits_per_word; ++n) {
- w <<= 1U;
- assert(d != canvas.end());
- w |= 0x1U & *d++;
- }
+ for (unsigned char const *d = canvas.begin(); d != canvas.end(); d += 8) {
+ auto w = bit_squeeze(0, d);
+
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+ w = bit_squeeze(w << 8U, d += 8);
+
m_bits.push_back(w);
}
=====================================
src/middle-pgsql.cpp
=====================================
@@ -73,8 +73,8 @@ static void load_id_list(pg_conn_t const &db_connection,
std::string const &table,
osmium::index::IdSetSmall<osmid_t> *ids)
{
- auto const res =
- db_connection.exec(fmt::format("SELECT id FROM {} ORDER BY id", table));
+ auto const res = db_connection.exec(
+ fmt::format("SELECT DISTINCT id FROM {} ORDER BY id", table));
for (int n = 0; n < res.num_tuples(); ++n) {
ids->set(osmium::string_to_object_id(res.get_value(n, 0)));
}
@@ -822,48 +822,69 @@ void middle_pgsql_t::get_node_parents(
send_id_list(m_db_connection, "osm2pgsql_changed_nodes", changed_nodes);
- m_db_connection.exec("ANALYZE osm2pgsql_changed_nodes");
+ std::vector<std::string> queries;
+
+ queries.emplace_back("ANALYZE osm2pgsql_changed_nodes");
bool const has_bucket_index =
check_bucket_index(&m_db_connection, m_options->prefix);
if (has_bucket_index) {
- m_db_connection.exec(build_sql(*m_options, R"(
-WITH changed_buckets AS (
- SELECT array_agg(id) AS node_ids, id >> {way_node_index_id_shift} AS bucket
- FROM osm2pgsql_changed_nodes GROUP BY id >> {way_node_index_id_shift}
-)
-INSERT INTO osm2pgsql_changed_ways
- SELECT DISTINCT w.id
- FROM {schema}"{prefix}_ways" w, changed_buckets b
- WHERE w.nodes && b.node_ids
- AND {schema}"{prefix}_index_bucket"(w.nodes)
- && ARRAY[b.bucket];
- )"));
+ // The query to get the parent ways of changed nodes is "hidden"
+ // inside a PL/pgSQL function so that the query planner only sees
+ // a single node id that is being queried for. If we ask for all
+ // nodes at the same time the query planner sometimes thinks it is
+ // better to do a full table scan which totally destroys performance.
+ // This is due to the PostgreSQL statistics on ARRAYs being way off.
+ queries.emplace_back(R"(
+CREATE OR REPLACE FUNCTION osm2pgsql_find_changed_ways() RETURNS void AS $$
+DECLARE
+ changed_buckets RECORD;
+BEGIN
+ FOR changed_buckets IN
+ SELECT array_agg(id) AS node_ids, id >> {way_node_index_id_shift} AS bucket
+ FROM osm2pgsql_changed_nodes GROUP BY id >> {way_node_index_id_shift}
+ LOOP
+ INSERT INTO osm2pgsql_changed_ways
+ SELECT DISTINCT w.id
+ FROM {schema}"{prefix}_ways" w
+ WHERE w.nodes && changed_buckets.node_ids
+ AND {schema}"{prefix}_index_bucket"(w.nodes)
+ && ARRAY[changed_buckets.bucket];
+ END LOOP;
+END;
+$$ LANGUAGE plpgsql
+)");
+ queries.emplace_back("SELECT osm2pgsql_find_changed_ways()");
+ queries.emplace_back("DROP FUNCTION osm2pgsql_find_changed_ways()");
} else {
- m_db_connection.exec(build_sql(*m_options, R"(
+ queries.emplace_back(R"(
INSERT INTO osm2pgsql_changed_ways
- SELECT DISTINCT w.id
+ SELECT w.id
FROM {schema}"{prefix}_ways" w, osm2pgsql_changed_nodes n
WHERE w.nodes && ARRAY[n.id]
- )"));
+ )");
}
if (m_options->middle_database_format == 1) {
- m_db_connection.exec(build_sql(*m_options, R"(
+ queries.emplace_back(R"(
INSERT INTO osm2pgsql_changed_relations
- SELECT DISTINCT r.id
+ SELECT r.id
FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes n
WHERE r.parts && ARRAY[n.id]
AND r.parts[1:way_off] && ARRAY[n.id]
- )"));
+ )");
} else {
- m_db_connection.exec(build_sql(*m_options, R"(
+ queries.emplace_back(R"(
INSERT INTO osm2pgsql_changed_relations
- SELECT DISTINCT r.id
+ SELECT r.id
FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes c
WHERE {schema}"{prefix}_member_ids"(r.members, 'N'::char) && ARRAY[c.id];
- )"));
+ )");
+ }
+
+ for (auto const &query : queries) {
+ m_db_connection.exec(build_sql(*m_options, query));
}
load_id_list(m_db_connection, "osm2pgsql_changed_ways", parent_ways);
=====================================
src/middle-ram.cpp
=====================================
@@ -10,6 +10,7 @@
#include "logging.hpp"
#include "middle-ram.hpp"
#include "options.hpp"
+#include "output-requirements.hpp"
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/util/delta.hpp>
=====================================
src/options.hpp
=====================================
@@ -49,31 +49,6 @@ struct database_options_t
std::string build_conninfo(database_options_t const &opt);
-/**
- * Outputs can signal their requirements to the middle by setting these fields.
- */
-struct output_requirements
-{
- /**
- * Need full node objects with tags, attributes (only if --extra-attributes
- * is set) and locations. If false, only node locations are needed.
- */
- bool full_nodes = false;
-
- /**
- * Need full way objects with tags, attributes (only if --extra-attributes
- * is set) and way nodes. If false, only way nodes are needed.
- */
- bool full_ways = false;
-
- /**
- * Need full relation objects with tags, attributes (only if
- * --extra-attributes is set) and members. If false, no data from relations
- * is needed.
- */
- bool full_relations = false;
-};
-
/**
* Structure for storing command-line and other options
*/
=====================================
src/osm2pgsql.cpp
=====================================
@@ -348,6 +348,7 @@ static void check_and_set_style(options_t *options)
}
}
+// NOLINTNEXTLINE(bugprone-exception-escape)
int main(int argc, char *argv[])
{
try {
@@ -404,6 +405,9 @@ int main(int argc, char *argv[])
} catch (std::exception const &e) {
log_error("{}", e.what());
return 1;
+ } catch (...) {
+ log_error("Unknown exception.");
+ return 1;
}
return 0;
=====================================
src/output-flex.cpp
=====================================
@@ -48,6 +48,9 @@
#include <stdexcept>
#include <string>
+// Mutex used to coordinate access to Lua code
+static std::mutex lua_mutex;
+
// Lua can't call functions on C++ objects directly. This macro defines simple
// C "trampoline" functions which are called from Lua which get the current
// context (the output_flex_t object) and call the respective function on the
@@ -895,9 +898,6 @@ int output_flex_t::expire_output_table()
void output_flex_t::call_lua_function(prepared_lua_function_t func,
osmium::OSMObject const &object)
{
- static std::mutex lua_mutex;
- std::lock_guard<std::mutex> const guard{lua_mutex};
-
m_calling_context = func.context();
lua_pushvalue(lua_state(), func.index()); // the function to call
@@ -914,6 +914,13 @@ void output_flex_t::call_lua_function(prepared_lua_function_t func,
m_calling_context = calling_context::main;
}
+void output_flex_t::get_mutex_and_call_lua_function(
+ prepared_lua_function_t func, osmium::OSMObject const &object)
+{
+ std::lock_guard<std::mutex> const guard{lua_mutex};
+ call_lua_function(func, object);
+}
+
void output_flex_t::pending_way(osmid_t id)
{
if (!m_process_way) {
@@ -926,7 +933,7 @@ void output_flex_t::pending_way(osmid_t id)
way_delete(id);
- call_lua_function(m_process_way, m_way_cache.get());
+ get_mutex_and_call_lua_function(m_process_way, m_way_cache.get());
}
void output_flex_t::select_relation_members()
@@ -935,6 +942,9 @@ void output_flex_t::select_relation_members()
return;
}
+ // We can not use get_mutex_and_call_lua_function() here, because we need
+ // the mutex to stick around as long as we are looking at the Lua stack.
+ std::lock_guard<std::mutex> const guard{lua_mutex};
call_lua_function(m_select_relation_members, m_relation_cache.get());
// If the function returned nil there is nothing to be marked.
@@ -1014,7 +1024,8 @@ void output_flex_t::pending_relation(osmid_t id)
delete_from_tables(osmium::item_type::relation, id);
if (m_process_relation) {
- call_lua_function(m_process_relation, m_relation_cache.get());
+ get_mutex_and_call_lua_function(m_process_relation,
+ m_relation_cache.get());
}
}
@@ -1029,7 +1040,7 @@ void output_flex_t::pending_relation_stage1c(osmid_t id)
}
m_disable_add_row = true;
- call_lua_function(m_process_relation, m_relation_cache.get());
+ get_mutex_and_call_lua_function(m_process_relation, m_relation_cache.get());
m_disable_add_row = false;
}
@@ -1104,7 +1115,7 @@ void output_flex_t::node_add(osmium::Node const &node)
}
m_context_node = &node;
- call_lua_function(m_process_node, node);
+ get_mutex_and_call_lua_function(m_process_node, node);
m_context_node = nullptr;
}
@@ -1117,7 +1128,7 @@ void output_flex_t::way_add(osmium::Way *way)
}
m_way_cache.init(way);
- call_lua_function(m_process_way, m_way_cache.get());
+ get_mutex_and_call_lua_function(m_process_way, m_way_cache.get());
}
void output_flex_t::relation_add(osmium::Relation const &relation)
@@ -1128,7 +1139,7 @@ void output_flex_t::relation_add(osmium::Relation const &relation)
m_relation_cache.init(relation);
select_relation_members();
- call_lua_function(m_process_relation, relation);
+ get_mutex_and_call_lua_function(m_process_relation, relation);
}
void output_flex_t::delete_from_table(table_connection_t *table_connection,
@@ -1518,7 +1529,7 @@ void output_flex_t::reprocess_marked()
}
way_delete(id);
if (m_process_way) {
- call_lua_function(m_process_way, m_way_cache.get());
+ get_mutex_and_call_lua_function(m_process_way, m_way_cache.get());
}
}
=====================================
src/output-flex.hpp
=====================================
@@ -185,12 +185,15 @@ private:
/**
* Call a Lua function that was "prepared" earlier with the OSMObject
- * as its only parameter. Uses a mutex internally to make access to the
- * Lua environment thread safe.
+ * as its only parameter.
*/
void call_lua_function(prepared_lua_function_t func,
osmium::OSMObject const &object);
+ /// Aquire the lua_mutex and the call `call_lua_function()`.
+ void get_mutex_and_call_lua_function(prepared_lua_function_t func,
+ osmium::OSMObject const &object);
+
void init_lua(std::string const &filename);
// Get the flex table that is as first parameter on the Lua stack.
=====================================
src/output-requirements.hpp
=====================================
@@ -0,0 +1,38 @@
+#ifndef OSM2PGSQL_OUTPUT_REQUIREMENTS_HPP
+#define OSM2PGSQL_OUTPUT_REQUIREMENTS_HPP
+
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2023 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+/**
+ * Outputs can signal their requirements to the middle by setting these fields.
+ */
+struct output_requirements
+{
+ /**
+ * Need full node objects with tags, attributes (only if --extra-attributes
+ * is set) and locations. If false, only node locations are needed.
+ */
+ bool full_nodes = false;
+
+ /**
+ * Need full way objects with tags, attributes (only if --extra-attributes
+ * is set) and way nodes. If false, only way nodes are needed.
+ */
+ bool full_ways = false;
+
+ /**
+ * Need full relation objects with tags, attributes (only if
+ * --extra-attributes is set) and members. If false, no data from relations
+ * is needed.
+ */
+ bool full_relations = false;
+};
+
+#endif // OSM2PGSQL_OUTPUT_REQUIREMENTS_HPP
=====================================
src/output.cpp
=====================================
@@ -7,12 +7,14 @@
* For a full list of authors see the git log.
*/
+#include "output.hpp"
+
#include "db-copy.hpp"
#include "format.hpp"
+#include "options.hpp"
#include "output-gazetteer.hpp"
#include "output-null.hpp"
#include "output-pgsql.hpp"
-#include "output.hpp"
#ifdef HAVE_LUA
# include "output-flex.hpp"
=====================================
src/output.hpp
=====================================
@@ -18,13 +18,14 @@
#include <osmium/index/id_set.hpp>
-#include "options.hpp"
#include "osmtypes.hpp"
+#include "output-requirements.hpp"
class db_copy_thread_t;
class thread_pool_t;
struct middle_query_t;
+struct options_t;
class output_t
{
=====================================
src/tagtransform-lua.cpp
=====================================
@@ -110,8 +110,8 @@ bool lua_tagtransform_t::filter_tags(osmium::OSMObject const &o, bool *polygon,
lua_pushstring(lua_state(), t.key.c_str());
lua_pushstring(lua_state(), t.value.c_str());
lua_rawset(lua_state(), -3);
+ ++sz;
}
- sz += tags.size();
}
lua_pushinteger(lua_state(), sz);
=====================================
src/wkb.cpp
=====================================
@@ -576,32 +576,35 @@ geom::geometry_t ewkb_to_geom(std::string_view wkb)
return geom;
}
-unsigned char decode_hex_char(char c)
-{
- if (c >= '0' && c <= '9') {
- return c - '0';
- }
- if (c >= 'A' && c <= 'F') {
- return c - 'A' + 10;
- }
- if (c >= 'a' && c <= 'f') {
- return c - 'a' + 10;
- }
+static constexpr std::array<char, 256> const hex_table = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0,
+
+ 0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
- throw std::runtime_error{"Invalid wkb: Not a hex character"};
+unsigned char decode_hex_char(char c) noexcept
+{
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
+ return hex_table[static_cast<std::size_t>(static_cast<unsigned char>(c))];
}
-std::string decode_hex(char const *hex)
+std::string decode_hex(std::string_view hex_string)
{
+ if (hex_string.size() % 2 != 0) {
+ throw std::runtime_error{"Invalid wkb: Not a valid hex string"};
+ }
+
std::string wkb;
+ wkb.reserve(hex_string.size() / 2);
- while (*hex != '\0') {
+ // NOLINTNEXTLINE(llvm-qualified-auto, readability-qualified-auto)
+ for (auto hex = hex_string.begin(); hex != hex_string.end();) {
unsigned int const c = decode_hex_char(*hex++);
-
- if (*hex == '\0') {
- throw std::runtime_error{"Invalid wkb: Not a valid hex string"};
- }
-
wkb += static_cast<char>((c << 4U) | decode_hex_char(*hex++));
}
=====================================
src/wkb.hpp
=====================================
@@ -41,15 +41,15 @@
[[nodiscard]] geom::geometry_t ewkb_to_geom(std::string_view wkb);
/**
- * Decode one hex character (0-9A-F or 0-9a-f) and return its value. Throw
- * an exception if not a valid hex character.
+ * Decode one hex character (0-9A-F or 0-9a-f) and return its value.
+ * Returns 0 for characters that are not hex characters.
*/
-[[nodiscard]] unsigned char decode_hex_char(char c);
+[[nodiscard]] unsigned char decode_hex_char(char c) noexcept;
/**
* Decode a string of hex characters. Throws an exception if the input is not
* a valid hex encoding.
*/
-[[nodiscard]] std::string decode_hex(char const *hex);
+[[nodiscard]] std::string decode_hex(std::string_view hex);
#endif // OSM2PGSQL_WKB_HPP
=====================================
tests/bdd/flex/lua-expire-output-definitions.feature
=====================================
@@ -17,7 +17,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
filename = false
})
"""
@@ -32,7 +31,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
table = 'bar',
schema = false
})
@@ -48,7 +46,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
table = false
})
"""
@@ -63,7 +60,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
maxzoom = 'bar',
filename = 'somewhere'
})
@@ -79,7 +75,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
maxzoom = 12,
minzoom = 'bar',
filename = 'somewhere'
@@ -96,7 +91,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
maxzoom = 123,
filename = 'somewhere'
})
@@ -112,7 +106,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
maxzoom = 12,
minzoom = -3,
filename = 'somewhere'
@@ -129,7 +122,6 @@ Feature: Expire output definitions in Lua file
And the lua style
"""
osm2pgsql.define_expire_output({
- name = 'foo',
maxzoom = 12,
minzoom = 14,
filename = 'somewhere'
=====================================
tests/bdd/flex/lua-expire.feature
=====================================
@@ -72,7 +72,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ some = object:as_point() })
end
"""
When running osm2pgsql flex
@@ -93,7 +93,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ some = object:as_point() })
end
"""
When running osm2pgsql flex
@@ -136,7 +136,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ some = object:as_point() })
end
"""
Then running osm2pgsql flex fails
@@ -184,7 +184,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ some = object:as_point() })
end
"""
When running osm2pgsql flex
@@ -208,7 +208,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ some = object:as_point() })
end
"""
When running osm2pgsql flex
@@ -231,7 +231,7 @@ Feature: Expire configuration in Lua file
})
function osm2pgsql.process_node(object)
- t:insert({})
+ t:insert({ geom = object:as_point() })
end
"""
When running osm2pgsql flex
=====================================
tests/test-expire-from-geometry.cpp
=====================================
@@ -390,13 +390,6 @@ TEST_CASE("expire multilinestring geometry", "[NoDB]")
et.from_geometry(geom, expire_config);
}
- SECTION("geom with check")
- {
- geom::geometry_t geom{std::move(ml)};
- geom.set_srid(3857);
- et.from_geometry_if_3857(geom, expire_config);
- }
-
auto const tiles = et.get_tiles();
REQUIRE(tiles.size() == 3);
CHECK(tile_t::from_quadkey(tiles[0], zoom) == tile_t{zoom, 2049, 2046});
@@ -440,13 +433,6 @@ TEST_CASE("expire multipolygon geometry", "[NoDB]")
et.from_geometry(geom, expire_config);
}
- SECTION("geom with check")
- {
- geom::geometry_t geom{std::move(mp)};
- geom.set_srid(3857);
- et.from_geometry_if_3857(geom, expire_config);
- }
-
auto const tiles = et.get_tiles();
REQUIRE(tiles.size() == 17);
@@ -476,18 +462,8 @@ TEST_CASE("expire geometry collection", "[NoDB]")
collection.add_geometry(geom::geometry_t{
geom::linestring_t{{15000.0, 15000.0}, {25000.0, 15000.0}}});
- SECTION("geom")
- {
- geom::geometry_t const geom{std::move(collection)};
- et.from_geometry(geom, expire_config);
- }
-
- SECTION("geom with check")
- {
- geom::geometry_t geom{std::move(collection)};
- geom.set_srid(3857);
- et.from_geometry_if_3857(geom, expire_config);
- }
+ geom::geometry_t const geom{std::move(collection)};
+ et.from_geometry(geom, expire_config);
auto const tiles = et.get_tiles();
REQUIRE(tiles.size() == 6);
@@ -499,6 +475,19 @@ TEST_CASE("expire geometry collection", "[NoDB]")
CHECK(tile_t::from_quadkey(tiles[5], zoom) == tile_t{zoom, 2048, 2048});
}
+TEST_CASE("expire works if in 3857", "[NoDB]")
+{
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
+
+ geom::geometry_t geom{geom::point_t{0.0, 0.0}};
+ geom.set_srid(3857);
+ et.from_geometry_if_3857(geom, expire_config);
+
+ auto const tiles = et.get_tiles();
+ REQUIRE(tiles.size() == 4);
+}
+
TEST_CASE("expire doesn't do anything if not in 3857", "[NoDB]")
{
expire_config_t const expire_config;
=====================================
tests/test-middle.cpp
=====================================
@@ -18,6 +18,7 @@
#include "dependency-manager.hpp"
#include "middle-pgsql.hpp"
#include "middle-ram.hpp"
+#include "output-requirements.hpp"
#include "common-buffer.hpp"
#include "common-cleanup.hpp"
=====================================
tests/test-wkb.cpp
=====================================
@@ -189,7 +189,7 @@ TEST_CASE("wkb: geometrycollection", "[NoDB]")
TEST_CASE("wkb: invalid", "[NoDB]") { REQUIRE_THROWS(ewkb_to_geom("INVALID")); }
-TEST_CASE("wkb hex decode of valid hex characters")
+TEST_CASE("wkb hex decode of valid and invalid hex characters")
{
REQUIRE(decode_hex_char('0') == 0);
REQUIRE(decode_hex_char('9') == 9);
@@ -197,7 +197,11 @@ TEST_CASE("wkb hex decode of valid hex characters")
REQUIRE(decode_hex_char('f') == 0x0f);
REQUIRE(decode_hex_char('A') == 0x0a);
REQUIRE(decode_hex_char('F') == 0x0f);
- REQUIRE_THROWS(decode_hex_char('x'));
+ REQUIRE(decode_hex_char('#') == 0);
+ REQUIRE(decode_hex_char('@') == 0);
+ REQUIRE(decode_hex_char('g') == 0);
+ REQUIRE(decode_hex_char('G') == 0);
+ REQUIRE(decode_hex_char(0x7f) == 0);
}
TEST_CASE("wkb hex decode of valid hex string")
@@ -216,11 +220,6 @@ TEST_CASE("wkb hex decode of valid hex string")
REQUIRE(result == data);
}
-TEST_CASE("wkb hex decode of invalid hex string")
-{
- REQUIRE_THROWS(decode_hex("no"));
-}
-
TEST_CASE("wkb hex decode of empty string is okay")
{
std::string const hex{};
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/8af8eff1c55971869800d5cd5fbad17d352b0a26
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/8af8eff1c55971869800d5cd5fbad17d352b0a26
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20230904/a2a46fb8/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list