diff --git a/app/gui/build.gradle b/app/gui/build.gradle index 90ae72d09a..8e7aef5e93 100644 --- a/app/gui/build.gradle +++ b/app/gui/build.gradle @@ -25,6 +25,7 @@ mainClassName = 'org.mobilitydata.gtfsvalidator.app.gui.Main' dependencies { implementation project(':core') implementation project(':main') + implementation libs.guava implementation libs.flogger implementation libs.flogger.system.backend testImplementation libs.junit diff --git a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java index 9447cc60c9..b43089301f 100644 --- a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java +++ b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java @@ -15,6 +15,7 @@ */ package org.mobilitydata.gtfsvalidator.app.gui; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import java.awt.Color; import java.awt.Component; @@ -45,7 +46,9 @@ import javax.swing.JFrame; import javax.swing.JLabel; import javax.swing.JPanel; +import javax.swing.JScrollPane; import javax.swing.JSpinner; +import javax.swing.JTextArea; import javax.swing.JTextField; import javax.swing.event.DocumentEvent; import javax.swing.event.DocumentListener; @@ -78,6 +81,8 @@ public class GtfsValidatorApp extends JFrame { private final JSpinner numThreadsSpinner = new JSpinner(); private final JTextField countryCodeField = new JTextField("", 3); + private final JTextArea httpHeadersField = new JTextArea(4, TEXT_FIELD_COLUMN_WIDTH); + private final JLabel httpHeadersErrorLabel = new JLabel(); private final MonitoredValidationRunner validationRunner; private final ValidationDisplay validationDisplay; @@ -129,6 +134,17 @@ public String getCountryCode() { return countryCodeField.getText(); } + public void setHttpHeaders(String httpHeaders) { + httpHeadersField.setText(httpHeaders); + } + + /** + * Returns the raw text from the HTTP headers field (newline-separated {@code Name: Value} lines). + */ + public String getHttpHeaders() { + return httpHeadersField.getText(); + } + void addPreValidationCallback(Runnable callback) { preValidationCallbacks.add(callback); } @@ -291,6 +307,34 @@ private void constructAdvancedOptionsPanel(JPanel parent) { fieldConstraints.gridy = 1; panel.add(countryCodeField, fieldConstraints); + // HTTP Headers — spans both columns so the text area gets full width + GridBagConstraints fullRowConstraints = new GridBagConstraints(); + fullRowConstraints.gridx = 0; + fullRowConstraints.gridwidth = 2; + fullRowConstraints.anchor = GridBagConstraints.NORTHWEST; + fullRowConstraints.fill = GridBagConstraints.HORIZONTAL; + fullRowConstraints.weightx = 1.0; + + fullRowConstraints.gridy = 2; + panel.add(new JLabel(bundle.getString("http_headers")), fullRowConstraints); + + httpHeadersField.setLineWrap(false); + JScrollPane headersScrollPane = new JScrollPane(httpHeadersField); + fullRowConstraints.gridy = 3; + panel.add(headersScrollPane, fullRowConstraints); + + httpHeadersErrorLabel.setForeground(Color.RED); + httpHeadersErrorLabel.setVisible(false); + fullRowConstraints.gridy = 4; + panel.add(httpHeadersErrorLabel, fullRowConstraints); + + fullRowConstraints.gridy = 5; + panel.add(new JLabel(bundle.getString("http_headers_description")), fullRowConstraints); + + httpHeadersField + .getDocument() + .addDocumentListener(documentChangeListener(this::updateValidationButtonStatus)); + advancedOptionsPanel.setVisible(false); } @@ -342,6 +386,13 @@ private void constructValidateButton(JPanel panel) { } private void updateValidationButtonStatus() { + String headerError = validateHttpHeadersText(httpHeadersField.getText()); + if (headerError != null) { + httpHeadersErrorLabel.setText(headerError); + httpHeadersErrorLabel.setVisible(true); + } else { + httpHeadersErrorLabel.setVisible(false); + } validateButton.setEnabled(isValidationReadyToRun()); } @@ -352,6 +403,9 @@ private boolean isValidationReadyToRun() { if (outputDirectoryField.getText().isBlank()) { return false; } + if (validateHttpHeadersText(httpHeadersField.getText()) != null) { + return false; + } return true; } @@ -373,7 +427,7 @@ private ValidationRunnerConfig buildConfig() throws URISyntaxException { config.setPrettyJson(true); config.setStdoutOutput(false); - String gtfsInput = gtfsInputField.getText(); + String gtfsInput = gtfsInputField.getText().strip(); if (gtfsInput.isBlank()) { throw new IllegalStateException("gtfsInputField is blank"); } @@ -383,7 +437,7 @@ private ValidationRunnerConfig buildConfig() throws URISyntaxException { config.setGtfsSource(Path.of(gtfsInput).toUri()); } - String outputDirectory = outputDirectoryField.getText(); + String outputDirectory = outputDirectoryField.getText().strip(); if (outputDirectory.isBlank()) { throw new IllegalStateException("outputDirectoryField is blank"); } @@ -399,9 +453,50 @@ private ValidationRunnerConfig buildConfig() throws URISyntaxException { config.setCountryCode(CountryCode.forStringOrUnknown(countryCode)); } + config.setHttpHeaders(parseHttpHeaders(httpHeadersField.getText())); + return config.build(); } + /** + * Validates newline-separated {@code Name: Value} header lines. + * + * @return {@code null} if all lines are valid, or a human-readable error message for the first + * invalid line. + */ + static String validateHttpHeadersText(String text) { + for (String line : text.split("\n")) { + if (line.isBlank()) { + continue; + } + int colon = line.indexOf(':'); + if (colon <= 0) { + return "Invalid header (expected \u201cName: Value\u201d): " + line.trim(); + } + } + return null; + } + + /** + * Parses newline-separated {@code Name: Value} header lines into an {@link ImmutableMap}. Blank + * lines are skipped. Colons inside the value are preserved. + * + *

Callers must ensure the text is valid via {@link #validateHttpHeadersText} first. + */ + static ImmutableMap parseHttpHeaders(String text) { + // Use LinkedHashMap so duplicate header names keep the last value (last-wins semantics) + // rather than throwing, which is a more forgiving user experience. + java.util.LinkedHashMap map = new java.util.LinkedHashMap<>(); + for (String line : text.split("\n")) { + if (line.isBlank()) { + continue; + } + int colon = line.indexOf(':'); + map.put(line.substring(0, colon).trim(), line.substring(colon + 1).trim()); + } + return ImmutableMap.copyOf(map); + } + private static Font createBoldFont() { JLabel label = new JLabel(); Font baseFont = label.getFont(); diff --git a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferences.java b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferences.java index 964a40bdc6..48713df7e7 100644 --- a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferences.java +++ b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferences.java @@ -15,6 +15,7 @@ public class GtfsValidatorPreferences { private static final String KEY_OUTPUT_DIRECTORY = "output_directory"; private static final String KEY_NUM_THREADS = "num_threads"; private static final String KEY_COUNTRY_CODE = "country_code"; + private static final String KEY_HTTP_HEADERS = "http_headers"; private final Preferences prefs; @@ -27,6 +28,9 @@ public void loadPreferences(GtfsValidatorApp app) { loadPathSetting(KEY_OUTPUT_DIRECTORY, app::setOutputDirectory); loadIntSetting(KEY_NUM_THREADS, app::setNumThreads); loadStringSetting(KEY_COUNTRY_CODE, app::setCountryCode); + // HTTP headers are intentionally NOT loaded and any previously stored value is deleted to + // avoid leaking credentials (tokens, passwords) across sessions. + prefs.remove(KEY_HTTP_HEADERS); } public void savePreferences(GtfsValidatorApp app) { @@ -34,6 +38,7 @@ public void savePreferences(GtfsValidatorApp app) { saveStringSetting(app::getOutputDirectory, KEY_OUTPUT_DIRECTORY); saveIntSetting(app::getNumThreads, KEY_NUM_THREADS); saveStringSetting(app::getCountryCode, KEY_COUNTRY_CODE); + // HTTP headers are intentionally NOT saved — they may contain credentials. } private void loadStringSetting(String key, Consumer setter) { diff --git a/app/gui/src/main/resources/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.properties b/app/gui/src/main/resources/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.properties index 0334634911..658e7f2eee 100644 --- a/app/gui/src/main/resources/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.properties +++ b/app/gui/src/main/resources/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.properties @@ -15,5 +15,7 @@ advanced=Advanced advanced_options=Advanced Options number_of_threads=Number of threads used to run the validator: country_code=Country Code (for phone validation): +http_headers=Custom HTTP Headers (one per line, Name: Value format): +http_headers_description=Only applied when downloading from a URL. Example: Authorization: Bearer token validate=Validate \ No newline at end of file diff --git a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java index 6a2e710ab9..7601bc6d05 100644 --- a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java +++ b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java @@ -4,6 +4,7 @@ import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.verify; +import com.google.common.collect.ImmutableMap; import java.awt.GraphicsEnvironment; import java.net.URI; import java.net.URISyntaxException; @@ -101,4 +102,101 @@ public void testPreValidationCallback() throws URISyntaxException { verify(callback).run(); } + + @Test + public void testHttpHeadersPassedToConfig() throws URISyntaxException { + app.setGtfsSource("http://transit/gtfs.zip"); + app.setOutputDirectory(Path.of("/path/to/output")); + app.setHttpHeaders("Authorization: Bearer token123\nUser-Agent: my-app/2.0"); + + app.getValidateButtonForTesting().doClick(); + + verify(runner).run(configCaptor.capture(), Mockito.same(app)); + + ValidationRunnerConfig config = configCaptor.getValue(); + assertThat(config.httpHeaders()) + .isEqualTo( + ImmutableMap.of( + "Authorization", "Bearer token123", + "User-Agent", "my-app/2.0")); + } + + @Test + public void testParseHttpHeaders_empty() { + assertThat(GtfsValidatorApp.parseHttpHeaders("")).isEmpty(); + assertThat(GtfsValidatorApp.parseHttpHeaders(" \n \n")).isEmpty(); + } + + @Test + public void testParseHttpHeaders_duplicateKeyKeepsLast() { + ImmutableMap result = + GtfsValidatorApp.parseHttpHeaders( + "Authorization: Bearer first\nAuthorization: Bearer second"); + assertThat(result).isEqualTo(ImmutableMap.of("Authorization", "Bearer second")); + } + + @Test + public void testParseHttpHeaders_valueContainsColon() { + ImmutableMap result = + GtfsValidatorApp.parseHttpHeaders("X-Endpoint: http://trace.example.com/id"); + assertThat(result).isEqualTo(ImmutableMap.of("X-Endpoint", "http://trace.example.com/id")); + } + + @Test + public void testNoHttpHeadersGivesEmptyMap() throws URISyntaxException { + app.setGtfsSource("http://transit/gtfs.zip"); + app.setOutputDirectory(Path.of("/path/to/output")); + + app.getValidateButtonForTesting().doClick(); + + verify(runner).run(configCaptor.capture(), Mockito.same(app)); + assertThat(configCaptor.getValue().httpHeaders()).isEmpty(); + } + + // --- Validation tests --- + + @Test + public void testValidateHttpHeadersText_validHeaders() { + assertThat(GtfsValidatorApp.validateHttpHeadersText("")).isNull(); + assertThat(GtfsValidatorApp.validateHttpHeadersText(" \n ")).isNull(); + assertThat(GtfsValidatorApp.validateHttpHeadersText("Authorization: Bearer token")).isNull(); + assertThat( + GtfsValidatorApp.validateHttpHeadersText( + "Authorization: Bearer token\nUser-Agent: app/1.0")) + .isNull(); + } + + @Test + public void testValidateHttpHeadersText_missingColon() { + // Value-only line (the bug that was reported) + String error = + GtfsValidatorApp.validateHttpHeadersText( + "Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ=="); + assertThat(error).isNotNull(); + assertThat(error).contains("Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ=="); + } + + @Test + public void testValidateHttpHeadersText_colonAtStart() { + // Colon at position 0 → name is empty → invalid + String error = GtfsValidatorApp.validateHttpHeadersText(": value"); + assertThat(error).isNotNull(); + } + + @Test + public void testInvalidHeaderDisablesValidateButton() { + app.setGtfsSource("http://transit/gtfs.zip"); + app.setOutputDirectory(Path.of("/path/to/output")); + // Valid so far — button should be enabled + assertThat(app.getValidateButtonForTesting().isEnabled()).isTrue(); + + // Set an invalid header — button should become disabled + app.setHttpHeaders("Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ=="); + assertThat(app.getValidateButtonForTesting().isEnabled()).isFalse(); + + // Fix the header — button re-enabled + app.setHttpHeaders( + "Authorization: Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ=="); + assertThat(app.getValidateButtonForTesting().isEnabled()).isTrue(); + } } diff --git a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferencesTest.java b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferencesTest.java index f37ecb7b0d..760a9faad0 100644 --- a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferencesTest.java +++ b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorPreferencesTest.java @@ -56,4 +56,26 @@ public void testEndToEnd() { assertThat(dest.getCountryCode()).isEqualTo("CA"); } } + + @Test + public void testHttpHeadersAreNotPersisted() { + // Headers must not survive a save/load cycle (security: they may contain credentials). + { + GtfsValidatorApp source = new GtfsValidatorApp(runner, display); + source.setGtfsSource("http://gtfs.org/gtfs.zip"); + source.setOutputDirectory(Path.of("/tmp/gtfs")); + source.setHttpHeaders("Authorization: Bearer secret-token"); + + GtfsValidatorPreferences prefs = new GtfsValidatorPreferences(); + prefs.savePreferences(source); + } + + { + GtfsValidatorPreferences prefs = new GtfsValidatorPreferences(); + GtfsValidatorApp dest = new GtfsValidatorApp(runner, display); + prefs.loadPreferences(dest); + + assertThat(dest.getHttpHeaders()).isEmpty(); + } + } } diff --git a/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java b/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java index 7b80fd8e86..6a07f428a5 100644 --- a/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java +++ b/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java @@ -17,12 +17,15 @@ package org.mobilitydata.gtfsvalidator.cli; import com.beust.jcommander.Parameter; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; import org.mobilitydata.gtfsvalidator.input.CountryCode; import org.mobilitydata.gtfsvalidator.runner.ValidationRunnerConfig; @@ -114,6 +117,14 @@ public class Arguments { description = "Output JSON report to stdout instead of writing to files (conflicts with -o)") private boolean stdoutOutput = false; + @Parameter( + names = {"--http_header"}, + description = + "Custom HTTP header to send when downloading a GTFS feed from a URL, in the format" + + " 'Name: Value'. May be repeated to set multiple headers. A 'User-Agent' header" + + " overrides the default validator User-Agent.") + private List httpHeaders = new ArrayList<>(); + ValidationRunnerConfig toConfig() throws URISyntaxException { ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); if (input != null) { @@ -149,9 +160,22 @@ ValidationRunnerConfig toConfig() throws URISyntaxException { builder.setPrettyJson(pretty); builder.setSkipValidatorUpdate(skipValidatorUpdate); builder.setStdoutOutput(stdoutOutput); + builder.setHttpHeaders(parseHttpHeaders(httpHeaders)); return builder.build(); } + private static ImmutableMap parseHttpHeaders(List rawHeaders) { + // Use LinkedHashMap so duplicate header names keep the last value (last-wins semantics) + // rather than throwing, which is a more forgiving user experience. + java.util.LinkedHashMap map = new java.util.LinkedHashMap<>(); + for (String raw : rawHeaders) { + int colon = raw.indexOf(':'); + // validate() already guarantees colon > 0 before toConfig() is called. + map.put(raw.substring(0, colon).trim(), raw.substring(colon + 1).trim()); + } + return ImmutableMap.copyOf(map); + } + public String getOutputBase() { return outputBase; } @@ -212,6 +236,14 @@ public boolean validate() { return false; } + for (String raw : httpHeaders) { + int colon = raw.indexOf(':'); + if (colon <= 0) { + logger.atSevere().log("Invalid --http_header value (expected 'Name: Value'): %s", raw); + return false; + } + } + return true; } diff --git a/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java b/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java index 29f2960e53..b5708aafba 100644 --- a/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java +++ b/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import com.beust.jcommander.JCommander; +import com.google.common.collect.ImmutableMap; import java.io.File; import java.net.URI; import java.net.URISyntaxException; @@ -296,13 +297,112 @@ public void testStdoutOutputWithoutInput() { } @Test - public void exportNoticesSchema_schemaOnlyWithoutOutputBase_isNotValid() { - String[] cliArguments = {"--export_notices_schema"}; - Arguments args = new Arguments(); - new JCommander(args).parse(cliArguments); + public void httpHeader_singleHeader() throws URISyntaxException { + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", "X-Custom-Header: my-value" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + ValidationRunnerConfig config = underTest.toConfig(); + assertThat(config.httpHeaders()).isEqualTo(ImmutableMap.of("X-Custom-Header", "my-value")); + } - assertThat(args.validate()).isFalse(); - assertThat(args.getExportNoticeSchema()).isTrue(); - assertThat(args.abortAfterNoticeSchemaExport()).isTrue(); + @Test + public void httpHeader_multipleHeaders() throws URISyntaxException { + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", "Authorization: Bearer token123", + "--http_header", "User-Agent: my-custom-agent/1.0" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + ValidationRunnerConfig config = underTest.toConfig(); + assertThat(config.httpHeaders()) + .isEqualTo( + ImmutableMap.of( + "Authorization", "Bearer token123", + "User-Agent", "my-custom-agent/1.0")); + } + + @Test + public void httpHeader_headerValueContainsColon() throws URISyntaxException { + // Values like URLs that contain colons should be preserved fully. + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", "X-Endpoint: http://trace.example.com/id" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + ValidationRunnerConfig config = underTest.toConfig(); + assertThat(config.httpHeaders()) + .isEqualTo(ImmutableMap.of("X-Endpoint", "http://trace.example.com/id")); + } + + @Test + public void httpHeader_noHeadersGivesEmptyMap() throws URISyntaxException { + String[] args = {"--input", "/tmp/gtfs.zip", "--output_base", "/tmp/out"}; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + ValidationRunnerConfig config = underTest.toConfig(); + assertThat(config.httpHeaders()).isEmpty(); } + + @Test + public void httpHeader_duplicateKeyKeepsLast() throws URISyntaxException { + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", "Authorization: Bearer first", + "--http_header", "Authorization: Bearer second" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + assertTrue(underTest.validate()); + ValidationRunnerConfig config = underTest.toConfig(); + assertThat(config.httpHeaders()).isEqualTo(ImmutableMap.of("Authorization", "Bearer second")); + } + + @Test + public void httpHeader_valueOnlyNoColen_isNotValid() { + // Reproduces the reported bug: user pastes just the credential value without "Name: " + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", "Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ==" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + assertFalse(underTest.validate()); + } + + @Test + public void httpHeader_colonAtStartNullName_isNotValid() { + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", ": value" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + assertFalse(underTest.validate()); + } + + @Test + public void httpHeader_validHeaderPassesValidation() { + String[] args = { + "--url", "http://example.com/gtfs.zip", + "--output_base", "/tmp/out", + "--http_header", + "Authorization: Basic YXBpQG1vYmlsaXR5ZGF0YS5vcmc6cWN2M3RoaypOWk00cGFmX3V5YQ==" + }; + Arguments underTest = new Arguments(); + new JCommander(underTest).parse(args); + assertTrue(underTest.validate()); + } + + // --- end of class --- } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java index 4291735f15..62464a533b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java @@ -23,6 +23,7 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.commons.compress.archivers.zip.ZipFile; @@ -112,12 +113,17 @@ private static boolean containsGtfsFileInSubfolder(ZipInputStream zipInputStream * @param targetPath the path to store the downloaded GTFS archive * @param noticeContainer * @param validatorVersion + * @param httpHeaders additional HTTP headers; a {@code "User-Agent"} entry overrides the default * @return the {@code GtfsInput} created after download of the GTFS archive * @throws IOException if GTFS archive cannot be stored at the specified location * @throws URISyntaxException if URL is malformed */ public static GtfsInput createFromUrl( - URL sourceUrl, Path targetPath, NoticeContainer noticeContainer, String validatorVersion) + URL sourceUrl, + Path targetPath, + NoticeContainer noticeContainer, + String validatorVersion, + Map httpHeaders) throws IOException, URISyntaxException { // getParent() may return null if there is no parent, so call toAbsolutePath() first. Path targetDirectory = targetPath.toAbsolutePath().getParent(); @@ -125,26 +131,45 @@ public static GtfsInput createFromUrl( Files.createDirectories(targetDirectory); } try (OutputStream outputStream = Files.newOutputStream(targetPath)) { - HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion); + HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion, httpHeaders); } return createFromPath(targetPath, noticeContainer); } + /** + * Creates a specific GtfsInput to read a GTFS ZIP archive from the given URL, using the default + * validator User-Agent. + * + * @deprecated Use {@link #createFromUrl(URL, Path, NoticeContainer, String, Map)} to support + * custom HTTP headers. + */ + @Deprecated + public static GtfsInput createFromUrl( + URL sourceUrl, Path targetPath, NoticeContainer noticeContainer, String validatorVersion) + throws IOException, URISyntaxException { + return createFromUrl(sourceUrl, targetPath, noticeContainer, validatorVersion, Map.of()); + } + /** * Creates a specific GtfsInput to read data from the given URL. The loaded ZIP file is kept in * memory. * * @param sourceUrl the fully qualified URL to download of the resource to download * @param noticeContainer + * @param validatorVersion + * @param httpHeaders additional HTTP headers; a {@code "User-Agent"} entry overrides the default * @return the {@code GtfsInput} created after download of the GTFS archive * @throws IOException if no file could not be found at the specified location * @throws URISyntaxException if URL is malformed */ public static GtfsInput createFromUrlInMemory( - URL sourceUrl, NoticeContainer noticeContainer, String validatorVersion) + URL sourceUrl, + NoticeContainer noticeContainer, + String validatorVersion, + Map httpHeaders) throws IOException, URISyntaxException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion); + HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion, httpHeaders); File zipFile = new File(sourceUrl.toString()); String fileName = zipFile.getName().replace(".zip", ""); if (containsGtfsFileInSubfolder( @@ -156,6 +181,20 @@ public static GtfsInput createFromUrlInMemory( } } + /** + * Creates a specific GtfsInput to read data from the given URL, using the default validator + * User-Agent. The loaded ZIP file is kept in memory. + * + * @deprecated Use {@link #createFromUrlInMemory(URL, NoticeContainer, String, Map)} to support + * custom HTTP headers. + */ + @Deprecated + public static GtfsInput createFromUrlInMemory( + URL sourceUrl, NoticeContainer noticeContainer, String validatorVersion) + throws IOException, URISyntaxException { + return createFromUrlInMemory(sourceUrl, noticeContainer, validatorVersion, Map.of()); + } + /** * Lists all files inside the GTFS dataset, even if they are not CSV and do not have .txt * extension. diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java index 4e77d30ede..69e7ab6cdf 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java @@ -20,6 +20,7 @@ import java.io.OutputStream; import java.net.URISyntaxException; import java.net.URL; +import java.util.Map; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -49,15 +50,40 @@ public static String getUserAgent(String validatorVersion) { * @param sourceUrl the fully qualified URL * @param outputStream the output stream * @param validatorVersion the version of the validator + * @param extraHeaders additional HTTP headers to send; a {@code "User-Agent"} entry overrides the + * default validator User-Agent */ - public static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) + public static void loadFromUrl( + URL sourceUrl, + OutputStream outputStream, + String validatorVersion, + Map extraHeaders) throws IOException, URISyntaxException { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(sourceUrl.toString()); + // Apply default User-Agent first, then let extraHeaders override any header including it. request.addHeader("User-Agent", getUserAgent(validatorVersion)); + for (Map.Entry entry : extraHeaders.entrySet()) { + request.setHeader(entry.getKey(), entry.getValue()); + } try (CloseableHttpResponse response = httpClient.execute(request)) { response.getEntity().writeTo(outputStream); } } } + + /** + * Downloads data from network using the default validator User-Agent. + * + * @param sourceUrl the fully qualified URL + * @param outputStream the output stream + * @param validatorVersion the version of the validator + * @deprecated Use {@link #loadFromUrl(URL, OutputStream, String, Map)} to support custom HTTP + * headers. + */ + @Deprecated + public static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) + throws IOException, URISyntaxException { + loadFromUrl(sourceUrl, outputStream, validatorVersion, Map.of()); + } } diff --git a/docs/USAGE.md b/docs/USAGE.md index b549ceb8d3..e02aa74ada 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -28,7 +28,8 @@ | `-p` | `--pretty` | Optional | Pretty JSON validation report. If specified, the JSON validation report will be printed using JSON Pretty print. This does not impact data parsing. | | `--stdout` | `--stdout` | Optional | Output JSON report to stdout instead of writing to files. Use with `-i` or `-u` but not with `-o`. Enables piping to tools like `jq`. | | `-d` | `--date` | Optional | The date used to validate the feed for time-based rules, e.g feed_expiration_30_days, in ISO_LOCAL_DATE format like '2001-01-30'. By default, the current date is used. | -| `-svu` | `--skip_validator_update` | Optional | Skip GTFS version validation update check. If specified, the GTFS version validation will be skipped. By default, the GTFS version validation will be performed. | +| `-svu` | `--skip_validator_update` | Optional | Skip GTFS version validation update check. If specified, the GTFS version validation will be skipped. By default, the GTFS version validation will be performed. | +| *(none)* | `--http_header` | Optional | Custom HTTP header to send when downloading a GTFS feed from a URL, in the format `Name: Value`. May be repeated to set multiple headers. A `User-Agent` header overrides the default validator User-Agent (e.g. `--http_header "Authorization: Bearer token"`). Only used with `-u` / `--url`. | ⚠️ Note that exactly one of the following options must be provided: `--url` or `--input`. @@ -62,6 +63,29 @@ java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip -o relative/outp 1. Validate the GTFS data and output the results to the directory located at `relative/output/path`. Validation results are exported to JSON by default. Please note that since downloading will take time, we recommend validating repeatedly on a local file. +### with custom HTTP headers + +Use `--http_header` to send custom headers when downloading a feed from a URL. The flag can be repeated for multiple headers. + +Override the default `User-Agent`: +``` +java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip -o relative/output/path --http_header "User-Agent: my-app/2.0 (contact@example.com)" +``` + +Add an authorization token: +``` +java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip -o relative/output/path --http_header "Authorization: Bearer " +``` + +Combine multiple headers: +``` +java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip -o relative/output/path \ + --http_header "User-Agent: my-app/2.0" \ + --http_header "X-Trace-Id: abc123" +``` + +⚠️ Note that `--http_header` is only used when downloading from a URL (`-u`/`--url`). It has no effect with `-i`/`--input`. + ## via stdout output (for scripting and piping) The `--stdout` option outputs JSON directly to stdout instead of writing files, making it ideal for scripting and piping to other tools. diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index a99687951b..245cfdd161 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -401,13 +401,15 @@ private static GtfsInput createGtfsInput( } if (config.storageDirectory().isEmpty()) { - return GtfsInput.createFromUrlInMemory(source.toURL(), noticeContainer, validatorVersion); + return GtfsInput.createFromUrlInMemory( + source.toURL(), noticeContainer, validatorVersion, config.httpHeaders()); } else { return GtfsInput.createFromUrl( source.toURL(), config.storageDirectory().get().resolve(GTFS_ZIP_FILENAME), noticeContainer, - validatorVersion); + validatorVersion, + config.httpHeaders()); } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java index 096d8e5ab2..3c86fb370d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java @@ -16,6 +16,7 @@ package org.mobilitydata.gtfsvalidator.runner; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; import java.net.URI; import java.nio.file.Path; import java.time.LocalDate; @@ -69,6 +70,10 @@ public Path systemErrorsReportPath() { // If true, output JSON report to stdout instead of writing to files public abstract boolean stdoutOutput(); + // Custom HTTP headers to include when downloading a GTFS feed from a URL. + // A "User-Agent" entry overrides the default validator User-Agent. + public abstract ImmutableMap httpHeaders(); + public static Builder builder() { // Set reasonable defaults where appropriate. return new AutoValue_ValidationRunnerConfig.Builder() @@ -80,7 +85,8 @@ public static Builder builder() { .setCountryCode(CountryCode.forStringOrUnknown(CountryCode.ZZ)) .setDateForValidation(LocalDate.now()) .setSkipValidatorUpdate(false) - .setStdoutOutput(false); + .setStdoutOutput(false) + .setHttpHeaders(ImmutableMap.of()); } @AutoValue.Builder @@ -109,6 +115,8 @@ public abstract static class Builder { public abstract Builder setStdoutOutput(boolean stdoutOutput); + public abstract Builder setHttpHeaders(ImmutableMap httpHeaders); + public abstract ValidationRunnerConfig build(); } } diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index d4b2146a47..3404f42bc3 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -121,7 +121,7 @@ public void saveJobFileFromUrl(String jobId, String url, String validatorVersion blobInfo, 1, TimeUnit.HOURS, Storage.SignUrlOption.httpMethod(HttpMethod.POST)); try (WriteChannel writer = storage.writer(signedURL)) { OutputStream outputStream = Channels.newOutputStream(writer); - HttpGetUtil.loadFromUrl(new URL(url), outputStream, validatorVersion); + HttpGetUtil.loadFromUrl(new URL(url), outputStream, validatorVersion, Map.of()); } }