diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..19b6e19 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[run] +source = src +omit = + src/example.py + src/osm_osw_reformatter/version.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 53de4a9..9e4cb27 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -60,6 +60,14 @@ jobs: python -m coverage run --source=src/osm_osw_reformatter -m unittest discover -v tests/unit_tests >> $log_file 2>&1 echo -e "\nCoverage Report\n" >> $log_file coverage report >> $log_file + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: false - name: Check coverage run: | @@ -73,4 +81,4 @@ jobs: connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} container_name: 'osm-osw-reformatter-package' clean_destination_folder: false - delete_if_exists: false \ No newline at end of file + delete_if_exists: false diff --git a/CHANGELOG.md b/CHANGELOG.md index adacb26..f781e04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change log +### 0.3.5 +- [BUG-3665](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3665) - Fix OSM→OSW export so zone boundary nodes are retained in `nodes.geojson` and zone `_w_id` references resolve to remapped sequential `_id`s, restoring OSW validation compliance for pedestrian-area geometries. +- Add regression coverage that converts a `highway=pedestrian` plaza fixture and asserts `python-osw-validation` reports zero issues. +- [ISSUE-3191](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3191/) - Fixed Documentation + ### 0.3.4 - Fix OSM→OSW conversion when OSM Way contains consecutive duplicate nodes (bug 3286). Instead of generating an invalid 0 length geometry, the segment is ignored. diff --git a/README.md b/README.md index 52f8a64..bd62adf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # TDEI python lib formatter library + +[![codecov](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/branch/main/graph/badge.svg)](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter) This python package designed to convert geospatial data from one format to another. In this case, it converts data from OpenStreetMap (OSM) format to OpenSideWalks (OSW) format and OpenSideWalks (OSW) format to OpenStreetMap (OSM) format. Let's break down the key components and processes involved in this converter: @@ -30,7 +32,7 @@ The conversion of OSW data to OSM is beneficial for incorporating detailed pedes Running the code base requires a proper Python environment set up. The following lines of code helps one establish such env named `tdei-osw`. replace `tdei-osw` with the name of your choice. ``` -conda create -n tdei-osw python==3.10.3 gdal +conda create -n tdei-osw python=3.10 gdal conda activate tdei-osw pip install -r requirements.txt ``` @@ -92,24 +94,25 @@ To install the GDAL library (Geospatial Data Abstraction Library) on your system ```python import asyncio -from osm-osw-reformatter import Formatter +from osm_osw_reformatter import Formatter async def osm_convert(): f = Formatter(workdir=, file_path=) - await f.osm2osw() + return await f.osm2osw() # Uncomment below line to clean up the generated files # f.cleanup() def osw_convert(): f = Formatter(workdir=, file_path=) - f.osw2osm() + return f.osw2osm() # Uncomment below line to clean up the generated files # f.cleanup() if __name__ == '__main__': - asyncio.run(osm_convert()) + results = asyncio.run(osm_convert()) + print(results.generated_files) osw_convert() ``` @@ -141,103 +144,12 @@ folder. - The terminal will show the output of coverage like this ```shell -> coverage run --source=src -m unittest discover -v tests/unit_tests -test_construct_geometries (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_nodes_counter (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_points_counter (helpers.test_osm.TestOSMHelper) ... ok -test_count_entities_with_ways_counter (helpers.test_osm.TestOSMHelper) ... ok -test_get_osm_graph (helpers.test_osm.TestOSMHelper) ... ok -test_osw_node_filter (helpers.test_osm.TestOSMHelper) ... ok -test_osw_point_filter (helpers.test_osm.TestOSMHelper) ... ok -test_osw_way_filter (helpers.test_osm.TestOSMHelper) ... ok -test_simplify_og (helpers.test_osm.TestOSMHelper) ... ok -test_cleanup_of_temp_files (helpers.test_osw.TestOSWHelper) ... ok -test_construct_geometries (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_nodes_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_points_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_entities_with_ways_counter (helpers.test_osw.TestOSWHelper) ... ok -test_count_lines (helpers.test_osw.TestOSWHelper) ... ok -test_count_nodes (helpers.test_osw.TestOSWHelper) ... ok -test_count_points (helpers.test_osw.TestOSWHelper) ... ok -test_count_polygons (helpers.test_osw.TestOSWHelper) ... ok -test_count_ways (helpers.test_osw.TestOSWHelper) ... ok -test_count_zones (helpers.test_osw.TestOSWHelper) ... ok -test_get_osm_graph (helpers.test_osw.TestOSWHelper) ... ok -test_merge (helpers.test_osw.TestOSWHelper) ... ok -test_missing_files (helpers.test_osw.TestOSWHelper) ... ok -test_osw_node_filter (helpers.test_osw.TestOSWHelper) ... ok -test_osw_point_filter (helpers.test_osw.TestOSWHelper) ... ok -test_osw_way_filter (helpers.test_osw.TestOSWHelper) ... ok -test_simplify_og (helpers.test_osw.TestOSWHelper) ... ok -test_unzip (helpers.test_osw.TestOSWHelper) ... ok -test_unzip_should_return_3_files (helpers.test_osw.TestOSWHelper) ... ok -test_custom_values (helpers.test_response.TestResponseClass) ... ok -test_default_values (helpers.test_response.TestResponseClass) ... ok -test_error_none (helpers.test_response.TestResponseClass) ... ok -test_error_string (helpers.test_response.TestResponseClass) ... ok -test_generated_files_list (helpers.test_response.TestResponseClass) ... ok -test_generated_files_string (helpers.test_response.TestResponseClass) ... ok -test_cleanup_existing_files (test_formatter.TestFormatter) ... ok -test_cleanup_non_existent_files (test_formatter.TestFormatter) ... ok -test_osm2osw_error (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Open failed for 'test.pbf': No such file or directory -ok -test_osm2osw_successful (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_workdir_already_exists (test_formatter.TestFormatter) ... ok -test_workdir_creation (test_formatter.TestFormatter) ... ok -test_convert_error (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Open failed for 'test.pbf': No such file or directory -ok -test_convert_successful (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_3_files (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_files_are_string (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_generated_files_include_nodes_points_edges (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets... -Creating networks from region extracts... -Created OSW files! -ok -test_convert_error (test_osw2osm.test_osw2osm.TestOSW2OSM) ... [Errno 2] No such file or directory: 'test.zip' -ok -test_convert_generated_files_are_string (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_convert_successful (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_generated_file (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_generated_file_should_be_xml (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok -test_crossing_markings (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_incline (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_surface (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_tactile_paving (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok -test_is_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_is_kerb_invalid (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_normalize_invalid_node (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_normalize_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok -test_is_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_is_powerpole_invalid (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_normalize_invalid_point (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_normalize_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok -test_is_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_footway (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_living_street (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_pedestrian (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_stairs (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_is_traffic_island (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_invalid_way (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok -test_normalize_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok +> python -m coverage run --source=src -m unittest discover tests/unit_tests +................................. +.................................. ---------------------------------------------------------------------- -Ran 73 tests in 79.494s +Ran 225 tests in 44.601s OK ``` diff --git a/requirements.txt b/requirements.txt index f075ae7..133877e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ shapely~=2.0.2 pyproj~=3.6.1 coverage~=7.5.1 ogr2osm==1.2.0 -python-osw-validation==0.3.5 \ No newline at end of file +python-osw-validation==0.4.0 \ No newline at end of file diff --git a/setup.py b/setup.py index e040e16..ce66339 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'asyncio~=3.4.3', 'networkx~=3.2', 'shapely~=2.0.2', + 'pyproj~=3.6.1', 'ogr2osm==1.2.0' ], packages=find_packages(where='src'), diff --git a/src/osm_osw_reformatter/serializer/osm/osm_graph.py b/src/osm_osw_reformatter/serializer/osm/osm_graph.py index ea5b965..88045eb 100644 --- a/src/osm_osw_reformatter/serializer/osm/osm_graph.py +++ b/src/osm_osw_reformatter/serializer/osm/osm_graph.py @@ -588,6 +588,21 @@ def construct_geometries(self, progressbar: Optional[callable] = None) -> None: if progressbar: progressbar.update(1) + # Protect zone boundary nodes: even if they ended up in internal_nodes + # due to circular-way simplification edge cases, they must not be removed + # because zones' _w_id still references them and to_geojson() needs to + # remap those IDs to sequential _id values. + zone_boundary_ids = set() + for _n, _d in self.G.nodes(data=True): + w_ids = _d.get("_w_id") + if isinstance(w_ids, list): + for ref in w_ids: + try: + zone_boundary_ids.add(int(ref)) + except (TypeError, ValueError): + pass + if zone_boundary_ids: + internal_nodes = [n for n in internal_nodes if n not in zone_boundary_ids] self.G.remove_nodes_from(internal_nodes) def to_undirected(self): @@ -675,6 +690,20 @@ def _remap_node_ref(ref, node_id_map): zone_features = [] polygon_features = [] node_id_map = {} + zone_node_refs = set() + for _, d in self.G.nodes(data=True): + if OSWZoneNormalizer.osw_zone_filter(d): + w_ids = d.get("_w_id", d.get("ndref", [])) + if not isinstance(w_ids, list): + w_ids = [w_ids] + for ref in w_ids: + zone_node_refs.add(ref) + zone_node_refs.add(str(ref)) + try: + zone_node_refs.add(int(ref)) + except (TypeError, ValueError): + pass + for n, d in self.G.nodes(data=True): d_copy = {**d} source_id = _source_id(n) @@ -682,8 +711,9 @@ def _remap_node_ref(ref, node_id_map): geometry = mapping(geometry_obj) geometry_type = geometry_obj.geom_type is_topology_node = geometry_type == "Point" and self.G.degree(n) > 0 + is_zone_node = geometry_type == "Point" and n in zone_node_refs - if is_topology_node: + if is_topology_node or is_zone_node: _assign_ids(d_copy, node_id_counter, source_id) node_id_map[n] = d_copy["_id"] node_id_counter += 1 diff --git a/src/osm_osw_reformatter/version.py b/src/osm_osw_reformatter/version.py index bfeb9e7..40ed83d 100644 --- a/src/osm_osw_reformatter/version.py +++ b/src/osm_osw_reformatter/version.py @@ -1 +1 @@ -__version__ = '0.3.4' +__version__ = '0.3.5' diff --git a/tests/unit_tests/test_files/zone_boundary.xml b/tests/unit_tests/test_files/zone_boundary.xml new file mode 100644 index 0000000..cf9f2a3 --- /dev/null +++ b/tests/unit_tests/test_files/zone_boundary.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/unit_tests/test_osm_compliance/test_osm_compliance.py b/tests/unit_tests/test_osm_compliance/test_osm_compliance.py index c21f1b7..841fbca 100644 --- a/tests/unit_tests/test_osm_compliance/test_osm_compliance.py +++ b/tests/unit_tests/test_osm_compliance/test_osm_compliance.py @@ -10,6 +10,7 @@ ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(ROOT_DIR)), 'output') TEST_DATA_WITH_INCLINE_ZIP_FILE = os.path.join(ROOT_DIR, 'test_files/dataset_with_incline.zip') +TEST_ZONE_BOUNDARY_FILE = os.path.join(ROOT_DIR, 'test_files/zone_boundary.xml') class TestOSMCompliance(unittest.IsolatedAsyncioTestCase): @@ -37,6 +38,25 @@ async def test_output_is_osm_compliant(self): os.remove(zip_path) formatter.cleanup() + async def test_osm2osw_zone_boundary_is_osw_compliant(self): + formatter = Formatter(workdir=OUTPUT_DIR, file_path=TEST_ZONE_BOUNDARY_FILE, prefix='zone_boundary') + res = await formatter.osm2osw() + osw_files = res.generated_files + + zip_path = os.path.join(OUTPUT_DIR, 'zone_boundary_osw.zip') + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for f in osw_files: + zipf.write(f, os.path.basename(f)) + + validator = OSWValidation(zipfile_path=zip_path) + result = validator.validate() + self.assertEqual(len(result.issues), 0, f'OSW Validation issues: {json.dumps(result.issues)}') + + for f in osw_files: + os.remove(f) + os.remove(zip_path) + formatter.cleanup() + async def test_incline_tag_preserved(self): osw2osm = OSW2OSM( zip_file_path=TEST_DATA_WITH_INCLINE_ZIP_FILE, diff --git a/tests/unit_tests/test_serializer/test_osm_graph.py b/tests/unit_tests/test_serializer/test_osm_graph.py index 7560f5f..ea6aef5 100644 --- a/tests/unit_tests/test_serializer/test_osm_graph.py +++ b/tests/unit_tests/test_serializer/test_osm_graph.py @@ -1058,6 +1058,58 @@ def test_to_geojson_zones_reference_remapped_node_ids_in_w_id(self): zone_w_ids = zone_data["features"][0]["properties"]["_w_id"] self.assertTrue(all(wid in node_ids for wid in zone_w_ids)) + def test_to_geojson_zone_boundary_custom_points_stay_in_nodes_file(self): + graph = nx.MultiDiGraph() + graph.add_node( + 10, + geometry=Point(0, 0), + lon=0.0, + lat=0.0, + **{"ext:entrance": "yes"}, + ) + graph.add_node(20, geometry=Point(1, 0), lon=1.0, lat=0.0) + graph.add_node( + "z100", + geometry=Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]), + highway="pedestrian", + _w_id=["10", "20"], + ) + + osm_graph = OSMGraph(G=graph) + + with TemporaryDirectory() as tmpdir: + nodes_path = os.path.join(tmpdir, 'nodes.geojson') + edges_path = os.path.join(tmpdir, 'edges.geojson') + points_path = os.path.join(tmpdir, 'points.geojson') + lines_path = os.path.join(tmpdir, 'lines.geojson') + zones_path = os.path.join(tmpdir, 'zones.geojson') + polygons_path = os.path.join(tmpdir, 'polygons.geojson') + + osm_graph.to_geojson( + nodes_path, + edges_path, + points_path, + lines_path, + zones_path, + polygons_path, + ) + + with open(nodes_path) as f: + node_data = json.load(f) + with open(zones_path) as f: + zone_data = json.load(f) + + node_ids = {feat["properties"]["_id"] for feat in node_data["features"]} + node_ext_ids = { + feat["properties"].get("ext:osm_id") + for feat in node_data["features"] + } + zone_w_ids = zone_data["features"][0]["properties"]["_w_id"] + + self.assertIn("10", node_ext_ids) + self.assertFalse(os.path.exists(points_path)) + self.assertTrue(all(wid in node_ids for wid in zone_w_ids)) + def test_to_undirected_on_simple_graph(self): g = nx.Graph() g.add_edge(1, 2)