[Main] Fix OSM→OSW zone _w_id references missing from nodes

.coveragerc

@@ -0,0 +1,5 @@
+[run]
+source = src
+omit =
+    src/example.py
+    src/osm_osw_reformatter/version.py

.github/workflows/unit_tests.yml

@@ -60,6 +60,14 @@ jobs:
           python -m coverage run --source=src/osm_osw_reformatter -m unittest discover -v tests/unit_tests >> $log_file 2>&1
           echo -e "\nCoverage Report\n" >> $log_file
           coverage report >> $log_file
+          coverage xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          fail_ci_if_error: false
 
       - name: Check coverage
         run: |
@@ -73,4 +81,4 @@ jobs:
           connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
           container_name: 'osm-osw-reformatter-package'
           clean_destination_folder: false
-          delete_if_exists: false
+          delete_if_exists: false

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change log
 
+### 0.3.5
+- [BUG-3665](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3665) - Fix OSM→OSW export so zone boundary nodes are retained in `nodes.geojson` and zone `_w_id` references resolve to remapped sequential `_id`s, restoring OSW validation compliance for pedestrian-area geometries.
+- Add regression coverage that converts a `highway=pedestrian` plaza fixture and asserts `python-osw-validation` reports zero issues.
+- [ISSUE-3191](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3191/) - Fixed Documentation  
+
 ### 0.3.4
 - Fix OSM→OSW conversion when OSM Way contains consecutive duplicate nodes (bug 3286). Instead of generating an invalid 0 length geometry, the segment is ignored.
 

README.md

@@ -1,6 +1,8 @@
 
 
 # TDEI python lib formatter library  
+
+[![codecov](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter/branch/main/graph/badge.svg)](https://codecov.io/gh/TaskarCenterAtUW/TDEI-python-lib-osw-formatter)
 
 This python package designed to convert geospatial data from one format to another. In this case, it converts data from OpenStreetMap (OSM) format to OpenSideWalks (OSW) format and OpenSideWalks (OSW) format to OpenStreetMap (OSM) format. Let's break down the key components and processes involved in this converter:    
 
@@ -30,7 +32,7 @@ The conversion of OSW data to OSM is beneficial for incorporating detailed pedes
 Running the code base requires a proper Python environment set up. The following lines of code helps one establish such env named `tdei-osw`. replace `tdei-osw` with the name of your choice.
 
 ```
-conda create -n tdei-osw python==3.10.3 gdal
+conda create -n tdei-osw python=3.10 gdal
 conda activate tdei-osw
 pip install -r requirements.txt
 ```
@@ -92,24 +94,25 @@ To install the GDAL library (Geospatial Data Abstraction Library) on your system
 
 ```python  
 import asyncio
-from osm-osw-reformatter import Formatter  
+from osm_osw_reformatter import Formatter  
 
 async def osm_convert():
     f = Formatter(workdir=<OUTPUT_DIR>, file_path=<OSM_INPUT_FILE>)
-    await f.osm2osw()
+    return await f.osm2osw()
     # Uncomment below line to clean up the generated files
     # f.cleanup()
 
 
 def osw_convert():
     f = Formatter(workdir=<OUTPUT_DIR>, file_path=<OSW_INPUT_FILE>)
-    f.osw2osm()
+    return f.osw2osm()
     # Uncomment below line to clean up the generated files
     # f.cleanup()
 
 
 if __name__ == '__main__':
-    asyncio.run(osm_convert())
+    results = asyncio.run(osm_convert())
+    print(results.generated_files)
     osw_convert()  
 ```  
 
@@ -141,103 +144,12 @@ folder.
 - The terminal will show the output of coverage like this  
 
 ```shell  
->  coverage run --source=src -m unittest discover -v tests/unit_tests  
-test_construct_geometries (helpers.test_osm.TestOSMHelper) ... ok
-test_count_entities_with_nodes_counter (helpers.test_osm.TestOSMHelper) ... ok
-test_count_entities_with_points_counter (helpers.test_osm.TestOSMHelper) ... ok
-test_count_entities_with_ways_counter (helpers.test_osm.TestOSMHelper) ... ok
-test_get_osm_graph (helpers.test_osm.TestOSMHelper) ... ok
-test_osw_node_filter (helpers.test_osm.TestOSMHelper) ... ok
-test_osw_point_filter (helpers.test_osm.TestOSMHelper) ... ok
-test_osw_way_filter (helpers.test_osm.TestOSMHelper) ... ok
-test_simplify_og (helpers.test_osm.TestOSMHelper) ... ok
-test_cleanup_of_temp_files (helpers.test_osw.TestOSWHelper) ... ok
-test_construct_geometries (helpers.test_osw.TestOSWHelper) ... ok
-test_count_entities_with_nodes_counter (helpers.test_osw.TestOSWHelper) ... ok
-test_count_entities_with_points_counter (helpers.test_osw.TestOSWHelper) ... ok
-test_count_entities_with_ways_counter (helpers.test_osw.TestOSWHelper) ... ok
-test_count_lines (helpers.test_osw.TestOSWHelper) ... ok
-test_count_nodes (helpers.test_osw.TestOSWHelper) ... ok
-test_count_points (helpers.test_osw.TestOSWHelper) ... ok
-test_count_polygons (helpers.test_osw.TestOSWHelper) ... ok
-test_count_ways (helpers.test_osw.TestOSWHelper) ... ok
-test_count_zones (helpers.test_osw.TestOSWHelper) ... ok
-test_get_osm_graph (helpers.test_osw.TestOSWHelper) ... ok
-test_merge (helpers.test_osw.TestOSWHelper) ... ok
-test_missing_files (helpers.test_osw.TestOSWHelper) ... ok
-test_osw_node_filter (helpers.test_osw.TestOSWHelper) ... ok
-test_osw_point_filter (helpers.test_osw.TestOSWHelper) ... ok
-test_osw_way_filter (helpers.test_osw.TestOSWHelper) ... ok
-test_simplify_og (helpers.test_osw.TestOSWHelper) ... ok
-test_unzip (helpers.test_osw.TestOSWHelper) ... ok
-test_unzip_should_return_3_files (helpers.test_osw.TestOSWHelper) ... ok
-test_custom_values (helpers.test_response.TestResponseClass) ... ok
-test_default_values (helpers.test_response.TestResponseClass) ... ok
-test_error_none (helpers.test_response.TestResponseClass) ... ok
-test_error_string (helpers.test_response.TestResponseClass) ... ok
-test_generated_files_list (helpers.test_response.TestResponseClass) ... ok
-test_generated_files_string (helpers.test_response.TestResponseClass) ... ok
-test_cleanup_existing_files (test_formatter.TestFormatter) ... ok
-test_cleanup_non_existent_files (test_formatter.TestFormatter) ... ok
-test_osm2osw_error (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Open failed for 'test.pbf': No such file or directory
-ok
-test_osm2osw_successful (test_formatter.TestFormatter) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Creating networks from region extracts...
-Created OSW files!
-ok
-test_workdir_already_exists (test_formatter.TestFormatter) ... ok
-test_workdir_creation (test_formatter.TestFormatter) ... ok
-test_convert_error (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Open failed for 'test.pbf': No such file or directory
-ok
-test_convert_successful (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Creating networks from region extracts...
-Created OSW files!
-ok
-test_generated_3_files (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Creating networks from region extracts...
-Created OSW files!
-ok
-test_generated_files_are_string (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Creating networks from region extracts...
-Created OSW files!
-ok
-test_generated_files_include_nodes_points_edges (test_osm2osw.test_osm2osw.TestOSM2OSW) ... Estimating number of ways, nodes, points, lines, zones and polygons in datasets...
-Creating networks from region extracts...
-Created OSW files!
-ok
-test_convert_error (test_osw2osm.test_osw2osm.TestOSW2OSM) ... [Errno 2] No such file or directory: 'test.zip'
-ok
-test_convert_generated_files_are_string (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok
-test_convert_successful (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok
-test_generated_file (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok
-test_generated_file_should_be_xml (test_osw2osm.test_osw2osm.TestOSW2OSM) ... ok
-test_crossing_markings (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok
-test_incline (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok
-test_surface (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok
-test_tactile_paving (test_serializer.test_osw_normalizer.TestCommonFunctions) ... ok
-test_is_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok
-test_is_kerb_invalid (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok
-test_normalize_invalid_node (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok
-test_normalize_kerb (test_serializer.test_osw_normalizer.TestOSWNodeNormalizer) ... ok
-test_is_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok
-test_is_powerpole_invalid (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok
-test_normalize_invalid_point (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok
-test_normalize_powerpole (test_serializer.test_osw_normalizer.TestOSWPointNormalizer) ... ok
-test_is_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_footway (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_living_street (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_pedestrian (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_stairs (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_is_traffic_island (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_normalize_crossing (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_normalize_invalid_way (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
-test_normalize_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalizer) ... ok
+>  python -m coverage run --source=src -m unittest discover tests/unit_tests  
+.................................
+..................................
 
 ----------------------------------------------------------------------
-Ran 73 tests in 79.494s
+Ran 225 tests in 44.601s
 
 OK
 ```

requirements.txt

@@ -5,4 +5,4 @@ shapely~=2.0.2
 pyproj~=3.6.1
 coverage~=7.5.1
 ogr2osm==1.2.0
-python-osw-validation==0.3.5
+python-osw-validation==0.4.0

setup.py

@@ -31,6 +31,7 @@
         'asyncio~=3.4.3',
         'networkx~=3.2',
         'shapely~=2.0.2',
+        'pyproj~=3.6.1',
         'ogr2osm==1.2.0'
     ],
     packages=find_packages(where='src'),

src/osm_osw_reformatter/serializer/osm/osm_graph.py

@@ -588,6 +588,21 @@ def construct_geometries(self, progressbar: Optional[callable] = None) -> None:
                 if progressbar:
                     progressbar.update(1)
 
+        # Protect zone boundary nodes: even if they ended up in internal_nodes
+        # due to circular-way simplification edge cases, they must not be removed
+        # because zones' _w_id still references them and to_geojson() needs to
+        # remap those IDs to sequential _id values.
+        zone_boundary_ids = set()
+        for _n, _d in self.G.nodes(data=True):
+            w_ids = _d.get("_w_id")
+            if isinstance(w_ids, list):
+                for ref in w_ids:
+                    try:
+                        zone_boundary_ids.add(int(ref))
+                    except (TypeError, ValueError):
+                        pass
+        if zone_boundary_ids:
+            internal_nodes = [n for n in internal_nodes if n not in zone_boundary_ids]
         self.G.remove_nodes_from(internal_nodes)
 
     def to_undirected(self):
@@ -675,15 +690,30 @@ def _remap_node_ref(ref, node_id_map):
         zone_features = []
         polygon_features = []
         node_id_map = {}
+        zone_node_refs = set()
+        for _, d in self.G.nodes(data=True):
+            if OSWZoneNormalizer.osw_zone_filter(d):
+                w_ids = d.get("_w_id", d.get("ndref", []))
+                if not isinstance(w_ids, list):
+                    w_ids = [w_ids]
+                for ref in w_ids:
+                    zone_node_refs.add(ref)
+                    zone_node_refs.add(str(ref))
+                    try:
+                        zone_node_refs.add(int(ref))
+                    except (TypeError, ValueError):
+                        pass
+
         for n, d in self.G.nodes(data=True):
             d_copy = {**d}
             source_id = _source_id(n)
             geometry_obj = d_copy.pop("geometry")
             geometry = mapping(geometry_obj)
             geometry_type = geometry_obj.geom_type
             is_topology_node = geometry_type == "Point" and self.G.degree(n) > 0
+            is_zone_node = geometry_type == "Point" and n in zone_node_refs
 
-            if is_topology_node:
+            if is_topology_node or is_zone_node:
                 _assign_ids(d_copy, node_id_counter, source_id)
                 node_id_map[n] = d_copy["_id"]
                 node_id_counter += 1

src/osm_osw_reformatter/version.py

@@ -1 +1 @@
-__version__ = '0.3.4'
+__version__ = '0.3.5'

tests/unit_tests/test_files/zone_boundary.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<osm version="0.6" generator="osw-zone-boundary-repro">
+  <bounds minlat="47.6190" minlon="-122.3490" maxlat="47.6200" maxlon="-122.3480"/>
+
+  <!--
+    Repro shape:
+    Node 1001 is both a pedestrian-zone boundary vertex and an ext:* custom
+    point. Before the zone-boundary export fix, it was written to points.geojson
+    instead of nodes.geojson, while zones.geojson still referenced it in _w_id.
+  -->
+  <node id="1001" visible="true" version="1" lat="47.6190" lon="-122.3490">
+    <tag k="ext:entrance" v="yes"/>
+  </node>
+  <node id="1002" visible="true" version="1" lat="47.6190" lon="-122.3480"/>
+  <node id="1003" visible="true" version="1" lat="47.6200" lon="-122.3485"/>
+
+  <way id="2001" visible="true" version="1">
+    <nd ref="1001"/>
+    <nd ref="1002"/>
+    <nd ref="1003"/>
+    <nd ref="1001"/>
+    <tag k="highway" v="pedestrian"/>
+    <tag k="area" v="yes"/>
+    <tag k="name" v="Zone Boundary Custom Point Repro"/>
+  </way>
+</osm>

tests/unit_tests/test_osm_compliance/test_osm_compliance.py

@@ -10,6 +10,7 @@
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 OUTPUT_DIR = os.path.join(os.path.dirname(os.path.dirname(ROOT_DIR)), 'output')
 TEST_DATA_WITH_INCLINE_ZIP_FILE = os.path.join(ROOT_DIR, 'test_files/dataset_with_incline.zip')
+TEST_ZONE_BOUNDARY_FILE = os.path.join(ROOT_DIR, 'test_files/zone_boundary.xml')
 
 
 class TestOSMCompliance(unittest.IsolatedAsyncioTestCase):
@@ -37,6 +38,25 @@ async def test_output_is_osm_compliant(self):
         os.remove(zip_path)
         formatter.cleanup()
 
+    async def test_osm2osw_zone_boundary_is_osw_compliant(self):
+        formatter = Formatter(workdir=OUTPUT_DIR, file_path=TEST_ZONE_BOUNDARY_FILE, prefix='zone_boundary')
+        res = await formatter.osm2osw()
+        osw_files = res.generated_files
+
+        zip_path = os.path.join(OUTPUT_DIR, 'zone_boundary_osw.zip')
+        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+            for f in osw_files:
+                zipf.write(f, os.path.basename(f))
+
+        validator = OSWValidation(zipfile_path=zip_path)
+        result = validator.validate()
+        self.assertEqual(len(result.issues), 0, f'OSW Validation issues: {json.dumps(result.issues)}')
+
+        for f in osw_files:
+            os.remove(f)
+        os.remove(zip_path)
+        formatter.cleanup()
+
     async def test_incline_tag_preserved(self):
         osw2osm = OSW2OSM(
             zip_file_path=TEST_DATA_WITH_INCLINE_ZIP_FILE,

tests/unit_tests/test_serializer/test_osm_graph.py

@@ -1058,6 +1058,58 @@ def test_to_geojson_zones_reference_remapped_node_ids_in_w_id(self):
             zone_w_ids = zone_data["features"][0]["properties"]["_w_id"]
             self.assertTrue(all(wid in node_ids for wid in zone_w_ids))
 
+    def test_to_geojson_zone_boundary_custom_points_stay_in_nodes_file(self):
+        graph = nx.MultiDiGraph()
+        graph.add_node(
+            10,
+            geometry=Point(0, 0),
+            lon=0.0,
+            lat=0.0,
+            **{"ext:entrance": "yes"},
+        )
+        graph.add_node(20, geometry=Point(1, 0), lon=1.0, lat=0.0)
+        graph.add_node(
+            "z100",
+            geometry=Polygon([(0, 0), (1, 0), (0, 1), (0, 0)]),
+            highway="pedestrian",
+            _w_id=["10", "20"],
+        )
+
+        osm_graph = OSMGraph(G=graph)
+
+        with TemporaryDirectory() as tmpdir:
+            nodes_path = os.path.join(tmpdir, 'nodes.geojson')
+            edges_path = os.path.join(tmpdir, 'edges.geojson')
+            points_path = os.path.join(tmpdir, 'points.geojson')
+            lines_path = os.path.join(tmpdir, 'lines.geojson')
+            zones_path = os.path.join(tmpdir, 'zones.geojson')
+            polygons_path = os.path.join(tmpdir, 'polygons.geojson')
+
+            osm_graph.to_geojson(
+                nodes_path,
+                edges_path,
+                points_path,
+                lines_path,
+                zones_path,
+                polygons_path,
+            )
+
+            with open(nodes_path) as f:
+                node_data = json.load(f)
+            with open(zones_path) as f:
+                zone_data = json.load(f)
+
+            node_ids = {feat["properties"]["_id"] for feat in node_data["features"]}
+            node_ext_ids = {
+                feat["properties"].get("ext:osm_id")
+                for feat in node_data["features"]
+            }
+            zone_w_ids = zone_data["features"][0]["properties"]["_w_id"]
+
+            self.assertIn("10", node_ext_ids)
+            self.assertFalse(os.path.exists(points_path))
+            self.assertTrue(all(wid in node_ids for wid in zone_w_ids))
+
     def test_to_undirected_on_simple_graph(self):
         g = nx.Graph()
         g.add_edge(1, 2)