Skip to content

Commit eeb30e8

Browse files
authored
Merge pull request #251 from aodn/feature/7080-download-estimator-test
Feature/7080 download estimator test
2 parents fcd6c2a + abc0438 commit eeb30e8

4 files changed

Lines changed: 238 additions & 27 deletions

File tree

server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import au.org.aodn.ogcapi.server.core.util.GeometryUtils;
88
import org.ehcache.config.builders.*;
99
import org.ehcache.config.units.MemoryUnit;
10+
import org.ehcache.impl.config.persistence.DefaultPersistenceConfiguration;
1011
import org.ehcache.jsr107.EhcacheCachingProvider;
1112
import org.locationtech.jts.geom.Geometry;
1213
import org.locationtech.jts.geom.prep.PreparedGeometry;
@@ -19,6 +20,7 @@
1920
import javax.cache.Caching;
2021
import java.io.File;
2122
import java.io.IOException;
23+
import java.math.BigInteger;
2224
import java.nio.file.Files;
2325
import java.nio.file.Path;
2426
import java.time.Duration;
@@ -31,7 +33,10 @@ public class CacheConfig {
3133
public static final String CACHE_WMS_MAP_TILE = "cache-wms-map_tile";
3234
public static final String GET_CAPABILITIES_WMS_LAYERS = "get-capabilities-wms-layers";
3335
public static final String GET_CAPABILITIES_WFS_FEATURE_TYPES = "get-capabilities-wfs-feature-types";
36+
3437
public static final String DOWNLOADABLE_FIELDS = "downloadable-fields";
38+
public static final String DOWNLOADABLE_SIZE = "downloadable-size";
39+
3540
public static final String ALL_NO_LAND_GEOMETRY = "all-noland-geometry";
3641
public static final String ALL_PARAM_VOCABS = "parameter-vocabs";
3742
public static final String ELASTIC_SEARCH_UUID_ONLY = "elastic-search-uuid-only";
@@ -53,7 +58,7 @@ public JCacheCacheManager cacheManager() throws IOException {
5358

5459
org.ehcache.config.Configuration config = ConfigurationBuilder
5560
.newConfigurationBuilder()
56-
.withService(new org.ehcache.impl.config.persistence.DefaultPersistenceConfiguration(storagePath))
61+
.withService(new DefaultPersistenceConfiguration(storagePath))
5762
.withCache(CACHE_WMS_MAP_TILE,
5863
CacheConfigurationBuilder.newCacheConfigurationBuilder(
5964
Object.class, byte[].class,
@@ -81,6 +86,12 @@ public JCacheCacheManager cacheManager() throws IOException {
8186
ResourcePoolsBuilder.heap(200)
8287
).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24)))
8388
)
89+
.withCache(DOWNLOADABLE_SIZE,
90+
CacheConfigurationBuilder.newCacheConfigurationBuilder(
91+
Object.class, BigInteger.class,
92+
ResourcePoolsBuilder.heap(100)
93+
).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24)))
94+
)
8495
.withCache(ELASTIC_SEARCH_UUID_ONLY,
8596
CacheConfigurationBuilder.newCacheConfigurationBuilder(
8697
Object.class, Object.class,

server/src/main/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataService.java

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,22 @@
11
package au.org.aodn.ogcapi.server.core.service.geoserver.wfs;
22

3+
import au.org.aodn.ogcapi.server.core.configuration.CacheConfig;
34
import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest;
45
import au.org.aodn.ogcapi.server.core.util.DatetimeUtils;
56
import lombok.extern.slf4j.Slf4j;
7+
import net.opengis.ows10.ExceptionReportType;
68
import net.opengis.wfs.FeatureCollectionType;
79
import org.geotools.wfs.v1_1.WFSConfiguration;
810
import org.geotools.xsd.Parser;
11+
import org.springframework.cache.annotation.Cacheable;
912
import org.springframework.http.*;
1013
import org.springframework.web.client.RestTemplate;
1114
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
15+
import org.xml.sax.SAXException;
1216

17+
import javax.xml.parsers.ParserConfigurationException;
1318
import java.io.ByteArrayOutputStream;
19+
import java.io.IOException;
1420
import java.io.InputStream;
1521
import java.io.StringReader;
1622
import java.math.BigInteger;
@@ -19,12 +25,12 @@
1925

2026
@Slf4j
2127
public class DownloadWfsDataService {
22-
private final WfsServer wfsServer;
23-
private final RestTemplate restTemplate;
24-
private final HttpEntity<?> pretendUserEntity;
25-
private final int chunkSize;
26-
private static final WFSConfiguration CONFIG = new WFSConfiguration();
27-
private static final int SAMPLES_SIZE = 500;
28+
protected final WfsServer wfsServer;
29+
protected final RestTemplate restTemplate;
30+
protected final HttpEntity<?> pretendUserEntity;
31+
protected final int chunkSize;
32+
protected static final WFSConfiguration WFS_CONFIG = new WFSConfiguration();
33+
protected static final int SAMPLES_SIZE = 500; // A not too small sample for download size estimation
2834

2935
public DownloadWfsDataService(
3036
WfsServer wfsServer,
@@ -88,14 +94,15 @@ public String prepareWfsRequestUrl(
8894
* b. Issue a query with data download but then limit the records size, and do a liner interpolation
8995
* @return The estimated file size
9096
*/
97+
@Cacheable(CacheConfig.DOWNLOADABLE_SIZE)
9198
public BigInteger estimateDownloadSize(
9299
String uuid,
93100
String layerName,
94101
String startDate,
95102
String endDate,
96103
Object multiPolygon,
97104
List<String> fields,
98-
String outputFormat) {
105+
String outputFormat) throws IllegalArgumentException {
99106

100107
// Just get number of record, the reply will always in XML
101108
String wfsRequestUrl = prepareWfsRequestUrl(
@@ -105,7 +112,7 @@ public BigInteger estimateDownloadSize(
105112
ResponseEntity<String> response = restTemplate.exchange(wfsRequestUrl, HttpMethod.GET, pretendUserEntity, String.class);
106113

107114
if(response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
108-
Parser parser = new Parser(CONFIG);
115+
Parser parser = new Parser(WFS_CONFIG);
109116
parser.setValidating(false);
110117
parser.setFailOnValidationError(false);
111118

@@ -126,8 +133,11 @@ public BigInteger estimateDownloadSize(
126133
.divide(BigInteger.valueOf(SAMPLES_SIZE));
127134
}
128135
}
136+
else if(o instanceof ExceptionReportType report) {
137+
throw new IllegalArgumentException(String.join(",", report.getException().stream().map(ex -> ex.getExceptionText().toString()).toList()));
138+
}
129139
}
130-
catch(Exception e) {
140+
catch(IOException | SAXException | ParserConfigurationException e) {
131141
log.error("Fail to convert wfs hits result", e);
132142
}
133143
}

server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -288,22 +288,34 @@ public SseEmitter downloadWfsDataWithSse(String uuid,
288288
);
289289
}
290290
else {
291-
BigInteger est = downloadWfsDataService.estimateDownloadSize(
292-
uuid,
293-
layerName,
294-
startDate,
295-
endDate,
296-
multiPolygon,
297-
fields,
298-
outputFormat
299-
);
300-
emitter.send(SseEmitter.event()
301-
.name(est != null ? "estimate-complete" : "estimate-failed")
302-
.data(Map.of(
303-
"size", est != null ? est : "",
304-
"timestamp", System.currentTimeMillis()
305-
)));
306-
emitter.complete();
291+
try {
292+
BigInteger est = downloadWfsDataService.estimateDownloadSize(
293+
uuid,
294+
layerName,
295+
startDate,
296+
endDate,
297+
multiPolygon,
298+
fields,
299+
outputFormat
300+
);
301+
emitter.send(SseEmitter.event()
302+
.name(est != null ? "estimate-complete" : "estimate-failed")
303+
.data(Map.of(
304+
"size", est != null ? est : "",
305+
"timestamp", System.currentTimeMillis()
306+
)));
307+
}
308+
catch(IllegalArgumentException iae) {
309+
emitter.send(SseEmitter.event()
310+
.name("estimate-failed")
311+
.data(Map.of(
312+
"message", iae.getMessage(),
313+
"timestamp", System.currentTimeMillis()
314+
)));
315+
}
316+
finally {
317+
emitter.complete();
318+
}
307319
}
308320
} catch (Exception e) {
309321
WfsErrorHandler.handleError(e, uuid, emitter, cleanupWfsResources);

server/src/test/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataServiceTest.java

Lines changed: 179 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,13 @@
1818
import org.springframework.boot.context.properties.EnableConfigurationProperties;
1919
import org.springframework.boot.test.context.SpringBootTest;
2020
import org.springframework.http.HttpEntity;
21+
import org.springframework.http.HttpMethod;
22+
import org.springframework.http.HttpStatus;
23+
import org.springframework.http.ResponseEntity;
2124
import org.springframework.test.context.TestPropertySource;
2225
import org.springframework.web.client.RestTemplate;
2326

27+
import java.math.BigInteger;
2428
import java.util.ArrayList;
2529
import java.util.HashMap;
2630
import java.util.List;
@@ -264,7 +268,6 @@ public void verifyRequestUrlGenerateCorrect() {
264268
);
265269
assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=shape-zip&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))", result, "Correct url 1");
266270
}
267-
268271
/**
269272
* Make sure the url generated contains the correct polygon
270273
*
@@ -299,4 +302,179 @@ public void verifyRequestUrlGenerateCorrectWithPolygon() throws JsonProcessingEx
299302
result,
300303
"Correct url 1");
301304
}
305+
/**
306+
* Verify estimate size on success request
307+
*/
308+
@Test
309+
void shouldReturnEstimatedSizeWhenBothRequestsSucceed() {
310+
String uuid = "lyr-123";
311+
String layer = "water_bodies";
312+
String start = "2024-01-01";
313+
String end = "2024-12-31";
314+
Object multiPolygon = new Object(); // or real geometry
315+
List<String> fields = List.of("name", "area");
316+
String format = "application/json";
317+
318+
// 1. Hits response (XML)
319+
String hitsXml = """
320+
<?xml version="1.0" encoding="UTF-8"?>
321+
<wfs:FeatureCollection
322+
xmlns:xs="http://www.w3.org/2001/XMLSchema"
323+
xmlns:wfs="http://www.opengis.net/wfs"
324+
xmlns:gml="http://www.opengis.net/gml"
325+
xmlns:ogc="http://www.opengis.net/ogc"
326+
xmlns:ows="http://www.opengis.net/ows"
327+
xmlns:xlink="http://www.w3.org/1999/xlink"
328+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
329+
numberOfFeatures="227193"
330+
timeStamp="2026-03-01T22:28:56.206Z"
331+
xsi:schemaLocation="http://www.opengis.net/wfs http://schemas.opengis.net/wfs/1.1.0/wfs.xsd"
332+
/>
333+
""";
334+
ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
335+
336+
// 2. Sample response (small payload)
337+
byte[] sampleBytes = "fake data".getBytes();
338+
ResponseEntity<byte[]> sampleResponse = new ResponseEntity<>(sampleBytes, HttpStatus.OK);
339+
340+
doReturn(hitsResponse)
341+
.when(restTemplate).exchange(
342+
argThat((String url) -> url != null && url.contains("resultType=hits")),
343+
eq(HttpMethod.GET),
344+
any(HttpEntity.class),
345+
eq(String.class));
346+
347+
doReturn(sampleResponse)
348+
.when(restTemplate).exchange(
349+
argThat((String url) -> url != null && url.contains("maxFeatures=" + DownloadWfsDataService.SAMPLES_SIZE)),
350+
eq(HttpMethod.GET), any(), eq(byte[].class));
351+
352+
doReturn(Optional.of("http://dummy.com/wfs"))
353+
.when(wfsServer).getFeatureServerUrl(eq(uuid), anyString());
354+
355+
WfsFields fs = WfsFields.builder()
356+
.fields(List.of(
357+
WfsField.builder().type("dateTime").name("time").build()
358+
))
359+
.build();
360+
361+
doReturn(fs)
362+
.when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class));
363+
364+
BigInteger size = downloadWfsDataService.estimateDownloadSize(
365+
uuid, layer, start, end, multiPolygon, fields, format);
366+
367+
// Should have call with resultType=hits to get number of record
368+
verify(restTemplate).exchange(
369+
eq("https://dummy.com/wfs?VERSION=1.1.0&typeName=water_bodies&SERVICE=WFS&REQUEST=GetFeature&resultType=hits&propertyName=name,area&cql_filter=((time DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))"), // or contains(...)
370+
eq(HttpMethod.GET),
371+
any(),
372+
eq(String.class) // or byte[].class etc.
373+
);
374+
375+
// Should also call with maxFeatures
376+
verify(restTemplate).exchange(
377+
eq("https://dummy.com/wfs?REQUEST=GetFeature&propertyName=name,area&VERSION=1.0.0&typeName=water_bodies&SERVICE=WFS&outputFormat=application/json&maxFeatures=500&cql_filter=((time DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))"), // or contains(...)
378+
eq(HttpMethod.GET),
379+
any(),
380+
eq(byte[].class) // or byte[].class etc.
381+
);
382+
383+
// numberOfFeatures="227193"
384+
long expected = 227193L * sampleBytes.length / DownloadWfsDataService.SAMPLES_SIZE;
385+
assertEquals(BigInteger.valueOf(expected), size, "Size match");
386+
}
387+
/**
388+
* Expect illegal exception when param is wrong
389+
*/
390+
@Test
391+
void throwExceptionWhenHitsRequestFails() {
392+
393+
String uuid = "lyr-123";
394+
String layer = "imos:aatams_sattag_dm_profile_map1";
395+
String start = "2024-01-01";
396+
String end = "2024-12-31";
397+
Object multiPolygon = new Object(); // or real geometry
398+
List<String> fields = List.of("name", "area");
399+
String format = "application/json";
400+
401+
// 1. Hits response (XML), indicate error
402+
String hitsXml = """
403+
<?xml version="1.0" encoding="UTF-8"?>
404+
<ows:ExceptionReport xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ows="http://www.opengis.net/ows" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0" xsi:schemaLocation="http://www.opengis.net/ows https://geoserver-123.aodn.org.au/geoserver/schemas/ows/1.0.0/owsExceptionReport.xsd">
405+
<ows:Exception exceptionCode="InvalidParameterValue" locator="typeName">
406+
<ows:ExceptionText>Feature type imos:aatams_sattag_dm_profile_map1 unknown</ows:ExceptionText>
407+
</ows:Exception>
408+
</ows:ExceptionReport>
409+
""";
410+
ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
411+
doReturn(hitsResponse)
412+
.when(restTemplate).exchange(
413+
argThat((String url) -> url != null && url.contains("resultType=hits")),
414+
eq(HttpMethod.GET),
415+
any(HttpEntity.class),
416+
eq(String.class));
417+
418+
doReturn(Optional.of("http://dummy.com/wfs"))
419+
.when(wfsServer).getFeatureServerUrl(eq(uuid), anyString());
420+
421+
WfsFields fs = WfsFields.builder()
422+
.fields(List.of(
423+
WfsField.builder().type("dateTime").name("time").build()
424+
))
425+
.build();
426+
427+
doReturn(fs)
428+
.when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class));
429+
430+
assertThrows(IllegalArgumentException.class,
431+
() -> downloadWfsDataService.estimateDownloadSize(
432+
uuid, layer, start, end, multiPolygon, fields, format)
433+
);
434+
}
435+
436+
@Test
437+
void returnsNullWhenParserThrowsException() {
438+
String uuid = "lyr-123";
439+
String layer = "imos:aatams_sattag_dm_profile_map1";
440+
String start = "2024-01-01";
441+
String end = "2024-12-31";
442+
Object multiPolygon = new Object(); // or real geometry
443+
List<String> fields = List.of("name", "area");
444+
String format = "application/json";
445+
446+
// 1. A syntax error XML, should not see this but just incase
447+
String hitsXml = """
448+
<?xml version="1.0" encoding="UTF-8"?>
449+
<ows:ExceptionReport23433 xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ows="http://www.opengis.net/ows" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0" xsi:schemaLocation="http://www.opengis.net/ows https://geoserver-123.aodn.org.au/geoserver/schemas/ows/1.0.0/owsExceptionReport.xsd">
450+
<ows:Exception exceptionCode="InvalidParameterValue" locator="typeName">
451+
<ows:ExceptionText>Feature type imos:aatams_sattag_dm_profile_map1 unknown</ows:ExceptionText>
452+
</ows:Exception>
453+
</ows:ExceptionReport>
454+
""";
455+
ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
456+
doReturn(hitsResponse)
457+
.when(restTemplate).exchange(
458+
argThat((String url) -> url != null && url.contains("resultType=hits")),
459+
eq(HttpMethod.GET),
460+
any(HttpEntity.class),
461+
eq(String.class));
462+
463+
doReturn(Optional.of("http://dummy.com/wfs"))
464+
.when(wfsServer).getFeatureServerUrl(eq(uuid), anyString());
465+
466+
WfsFields fs = WfsFields.builder()
467+
.fields(List.of(
468+
WfsField.builder().type("dateTime").name("time").build()
469+
))
470+
.build();
471+
472+
doReturn(fs)
473+
.when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class));
474+
475+
BigInteger size = downloadWfsDataService.estimateDownloadSize(
476+
uuid, layer, start, end, multiPolygon, fields, format);
477+
478+
assertNull(size, "Size should be null");
479+
}
302480
}

0 commit comments

Comments
 (0)