From 157cd653df8d72d0560e04b1f69d1e39d47f050b Mon Sep 17 00:00:00 2001 From: Lindsay Bec Date: Mon, 22 Jun 2026 15:36:47 +1000 Subject: [PATCH 1/3] allow multiple files or directories to be supplied --- .../Configuration/CommandLineOptions.cs | 12 ++++---- src/OtelImporter/Program.cs | 30 +++++++++++-------- .../CommandLineParserTests.cs | 13 ++++---- .../OtelImporter.Tests/InputResolverTests.cs | 20 +++++++++++++ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/OtelImporter/Configuration/CommandLineOptions.cs b/src/OtelImporter/Configuration/CommandLineOptions.cs index db48a29..9059fd3 100644 --- a/src/OtelImporter/Configuration/CommandLineOptions.cs +++ b/src/OtelImporter/Configuration/CommandLineOptions.cs @@ -2,7 +2,7 @@ namespace OtelImporter.Configuration; internal sealed record CommandLineOptions { - public string? InputFile { get; init; } + public IReadOnlyList InputFiles { get; init; } = []; public string? Endpoint { get; init; } public OtlpProtocol? Protocol { get; init; } public double? MaxBatchesPerSecond { get; init; } @@ -26,7 +26,7 @@ internal sealed record CommandLineParseResult(CommandLineOptions? Options, strin // Hand-rolled argument parsing keeps the dependency surface (and AOT footprint) small. // Supported: -// positional, a *.jsonl/*.jsonl.zst trace file or a directory of them +// [...] positional, one or more trace files or directories // --endpoint, -e upstream OTLP endpoint (overrides environment variables) // --protocol, -p grpc | http (overrides port sniffing) // --max-rate, -r throttle: maximum batches per second @@ -44,7 +44,7 @@ internal static class CommandLineParser { public static CommandLineParseResult Parse(string[] args) { - string? inputFile = null; + var inputFiles = new List(); string? endpoint = null; OtlpProtocol? protocol = null; double? maxBatchesPerSecond = null; @@ -164,9 +164,7 @@ public static CommandLineParseResult Parse(string[] args) default: if (arg.StartsWith('-')) return CommandLineParseResult.Failure($"Unknown option '{arg}'."); - if (inputFile is not null) - return CommandLineParseResult.Failure($"Unexpected extra argument '{arg}'. Only one input file is supported."); - inputFile = arg; + inputFiles.Add(arg); break; } } @@ -179,7 +177,7 @@ public static CommandLineParseResult Parse(string[] args) return CommandLineParseResult.Success(new CommandLineOptions { - InputFile = inputFile, + InputFiles = inputFiles, Endpoint = endpoint, Protocol = protocol, MaxBatchesPerSecond = maxBatchesPerSecond, diff --git a/src/OtelImporter/Program.cs b/src/OtelImporter/Program.cs index acabc17..912ce5d 100644 --- a/src/OtelImporter/Program.cs +++ b/src/OtelImporter/Program.cs @@ -31,23 +31,27 @@ public static async Task RunAsync(string[] args) return ExitCode.Success; } - if (string.IsNullOrWhiteSpace(options.InputFile)) + if (options.InputFiles.Count == 0) { - Console.Error.WriteLine("error: no input file specified."); + Console.Error.WriteLine("error: no input specified."); Console.Error.WriteLine(); PrintUsage(Console.Error); return ExitCode.UsageError; } - // The positional argument may be a single file or a directory of trace files. - var resolution = InputResolver.Resolve(options.InputFile!); - if (resolution.Error is not null) + var allFiles = new List(); + foreach (var input in options.InputFiles) { - Console.Error.WriteLine($"error: {resolution.Error}"); - return ExitCode.UsageError; + var resolution = InputResolver.Resolve(input); + if (resolution.Error is not null) + { + Console.Error.WriteLine($"error: {resolution.Error}"); + return ExitCode.UsageError; + } + allFiles.AddRange(resolution.Files); } - var inputFiles = resolution.Files; + var inputFiles = allFiles.Distinct(StringComparer.Ordinal).ToList(); var filter = SpanTimeFilter.Create(options.From, options.To); // --inspect is a read-only pass: no export, so endpoint/protocol/rate/retry are all ignored. @@ -382,14 +386,14 @@ static void PrintUsage(TextWriter writer) writer.WriteLine("OtelImporter - stream OpenTelemetry trace files to an OTLP endpoint."); writer.WriteLine(); writer.WriteLine("Usage:"); - writer.WriteLine(" OtelImporter [--endpoint ] [--protocol ]"); + writer.WriteLine(" OtelImporter [...] [--endpoint ] [--protocol ]"); writer.WriteLine(" [--max-rate ] [--max-retries ] [--max-batch-size ]"); - writer.WriteLine(" OtelImporter --inspect"); + writer.WriteLine(" OtelImporter [...] --inspect"); writer.WriteLine(); writer.WriteLine("Arguments:"); - writer.WriteLine(" Path to a .jsonl or .jsonl.zst OTLP trace file, or a"); - writer.WriteLine(" directory; every .jsonl/.jsonl.zst file directly inside"); - writer.WriteLine(" it is processed in name order."); + writer.WriteLine(" [...] One or more paths to .jsonl/.jsonl.zst OTLP trace files,"); + writer.WriteLine(" or directories; every .jsonl/.jsonl.zst file directly"); + writer.WriteLine(" inside each directory is processed in name order."); writer.WriteLine(); writer.WriteLine("Options:"); writer.WriteLine(" -e, --endpoint Upstream OTLP endpoint. Overrides environment variables."); diff --git a/tests/OtelImporter.Tests/CommandLineParserTests.cs b/tests/OtelImporter.Tests/CommandLineParserTests.cs index 5a5063b..53f7904 100644 --- a/tests/OtelImporter.Tests/CommandLineParserTests.cs +++ b/tests/OtelImporter.Tests/CommandLineParserTests.cs @@ -10,7 +10,7 @@ public void ParsesPositionalInputFile() var result = CommandLineParser.Parse(["traces.jsonl"]); Assert.Null(result.Error); - Assert.Equal("traces.jsonl", result.Options!.InputFile); + Assert.Equal("traces.jsonl", result.Options!.InputFiles[0]); Assert.Null(result.Options.Endpoint); Assert.Null(result.Options.Protocol); } @@ -47,7 +47,7 @@ public void ParsesInspectFlag(string flag) Assert.Null(result.Error); Assert.True(result.Options!.Inspect); - Assert.Equal("traces.jsonl", result.Options.InputFile); + Assert.Equal("traces.jsonl", result.Options.InputFiles[0]); } [Fact] @@ -233,7 +233,7 @@ public void FlagsCanPrecedePositionalArgument() var result = CommandLineParser.Parse(["-e", "http://host:4317", "-p", "grpc", "traces.jsonl"]); Assert.Null(result.Error); - Assert.Equal("traces.jsonl", result.Options!.InputFile); + Assert.Equal("traces.jsonl", result.Options!.InputFiles[0]); Assert.Equal("http://host:4317", result.Options.Endpoint); Assert.Equal(OtlpProtocol.Grpc, result.Options.Protocol); } @@ -316,10 +316,11 @@ public void RejectsMissingFlagValue() } [Fact] - public void RejectsTwoPositionalArguments() + public void ParsesMultiplePositionalArguments() { - var result = CommandLineParser.Parse(["one.jsonl", "two.jsonl"]); + var result = CommandLineParser.Parse(["one.jsonl", "two.jsonl", "three/"]); - Assert.NotNull(result.Error); + Assert.Null(result.Error); + Assert.Equal(["one.jsonl", "two.jsonl", "three/"], result.Options!.InputFiles); } } diff --git a/tests/OtelImporter.Tests/InputResolverTests.cs b/tests/OtelImporter.Tests/InputResolverTests.cs index 724c70f..33f2168 100644 --- a/tests/OtelImporter.Tests/InputResolverTests.cs +++ b/tests/OtelImporter.Tests/InputResolverTests.cs @@ -100,6 +100,26 @@ public void FailsWhenDirectoryHasNoTraceFiles() Assert.Empty(result.Files); } + [Fact] + public void SameFilenameInDifferentDirectoriesAreNotDeduplicated() + { + // Dedup in Program.cs uses the full path, so dir1/file.jsonl and dir2/file.jsonl + // are distinct even though the filename is the same. + var dir1 = Directory.CreateDirectory(Path.Combine(_dir, "dir1")).FullName; + var dir2 = Directory.CreateDirectory(Path.Combine(_dir, "dir2")).FullName; + var file1 = Path.Combine(dir1, "traces.jsonl"); + var file2 = Path.Combine(dir2, "traces.jsonl"); + File.WriteAllText(file1, ""); + File.WriteAllText(file2, ""); + + var allFiles = new List(); + allFiles.AddRange(InputResolver.Resolve(dir1).Files); + allFiles.AddRange(InputResolver.Resolve(dir2).Files); + var deduped = allFiles.Distinct(StringComparer.Ordinal).ToList(); + + Assert.Equal([file1, file2], deduped); + } + [Fact] public void FailsWhenPathDoesNotExist() { From ee6230e72a5e2b0ab535ea7073c08711c4ba7bbe Mon Sep 17 00:00:00 2001 From: Lindsay Bec Date: Mon, 22 Jun 2026 15:53:14 +1000 Subject: [PATCH 2/3] create cold replay job using otelimporter --- k8s/replay-job.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 k8s/replay-job.yaml diff --git a/k8s/replay-job.yaml b/k8s/replay-job.yaml new file mode 100644 index 0000000..687bab1 --- /dev/null +++ b/k8s/replay-job.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: otel-replay +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: otelimporter + image: ghcr.io/octopusdeploy/otelimporter:latest + args: + - /traces/2024-01 + - /traces/2024-02 + - /traces/2024-03 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://otel-collector:4318" + # Optional: auth header if your collector requires it + # - name: OTEL_EXPORTER_OTLP_HEADERS + # valueFrom: + # secretKeyRef: + # name: otel-replay-secret + # key: headers # format: key1=value1,key2=value2 + volumeMounts: + - name: traces + mountPath: /traces + readOnly: true + volumes: + - name: traces + persistentVolumeClaim: + claimName: traces-pvc From b6ea8b6bdb7913ae585cf6276260fad1d97eccce Mon Sep 17 00:00:00 2001 From: Lindsay Bec Date: Tue, 23 Jun 2026 09:58:59 +1000 Subject: [PATCH 3/3] remove k8s testing file --- k8s/replay-job.yaml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 k8s/replay-job.yaml diff --git a/k8s/replay-job.yaml b/k8s/replay-job.yaml deleted file mode 100644 index 687bab1..0000000 --- a/k8s/replay-job.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: otel-replay -spec: - backoffLimit: 0 - template: - spec: - restartPolicy: Never - containers: - - name: otelimporter - image: ghcr.io/octopusdeploy/otelimporter:latest - args: - - /traces/2024-01 - - /traces/2024-02 - - /traces/2024-03 - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://otel-collector:4318" - # Optional: auth header if your collector requires it - # - name: OTEL_EXPORTER_OTLP_HEADERS - # valueFrom: - # secretKeyRef: - # name: otel-replay-secret - # key: headers # format: key1=value1,key2=value2 - volumeMounts: - - name: traces - mountPath: /traces - readOnly: true - volumes: - - name: traces - persistentVolumeClaim: - claimName: traces-pvc