Skip to content

feat(tracing): validate spans mapper against live Transaction Search #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gibbsie merged 1 commit into from
Aug 14, 2026
Merged

feat(tracing): validate spans mapper against live Transaction Search #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 96 additions & 24 deletions backend/src/lambda/__tests__/trace-query-handler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,11 +526,16 @@ describe("TRACE_BACKEND=spans dispatch (design §3 dual-backend, §1 query mecha
status: "Complete",
results: [
[
{ field: "traceId", value: "1-5f84c7c1-000000000000000000000001" },
{ field: "spanId", value: "root-1" },
{ field: "name", value: "root-op" },
{ field: "startTimeUnixNano", value: "1000000000000" },
{ field: "endTimeUnixNano", value: "1001000000000" },
{
field: "@message",
value: JSON.stringify({
traceId: "5f84c7c1000000000000000000000001",
spanId: "root-1",
name: "root-op",
startTimeUnixNano: 1000000000000,
endTimeUnixNano: 1001000000000,
}),
},
],
],
});
Expand All @@ -551,7 +556,7 @@ describe("TRACE_BACKEND=spans dispatch (design §3 dual-backend, §1 query mecha
expect(body).toHaveProperty("truncated");
expect(body).toHaveProperty("meta");
expect(body.traces).toHaveLength(1);
expect(body.traces[0].traceId).toBe("1-5f84c7c1-000000000000000000000001");
expect(body.traces[0].traceId).toBe("5f84c7c1000000000000000000000001");
});

test("TRACE_BACKEND=spans, query Complete with zero rows + entry fresh -> status:indexing", async () => {
Expand Down Expand Up @@ -688,22 +693,27 @@ describe("TRACE_BACKEND=spans dispatch (design §3 dual-backend, §1 query mecha
expect(startCall).toBeDefined();
const input = (startCall!.args[0] as StartQueryCommand).input;
expect(input.queryString).toContain(
'filter `annotation.run_id` = "run-22222222-2222-2222-2222-222222222222"',
'filter `attributes.run_id` = "run-22222222-2222-2222-2222-222222222222"',
);
});

test("TRACE_BACKEND=spans, admin raw traceId route queries aws/spans by traceId, response shape unchanged", async () => {
test("TRACE_BACKEND=spans, admin raw traceId route with an X-Ray-format id normalizes to 32-hex before querying aws/spans (finding a3d8a2ea #9)", async () => {
process.env.TRACE_BACKEND = "spans";
logsMock.on(StartQueryCommand).resolves({ queryId: "q-h8" });
logsMock.on(GetQueryResultsCommand).resolves({
status: "Complete",
results: [
[
{ field: "traceId", value: "1-5f84c7c1-000000000000000000000002" },
{ field: "spanId", value: "root-2" },
{ field: "name", value: "root-op-2" },
{ field: "startTimeUnixNano", value: "1000000000000" },
{ field: "endTimeUnixNano", value: "1001000000000" },
{
field: "@message",
value: JSON.stringify({
traceId: "5f84c7c1000000000000000000000002",
spanId: "root-2",
name: "root-op-2",
startTimeUnixNano: 1000000000000,
endTimeUnixNano: 1001000000000,
}),
},
],
],
});
Expand All @@ -720,6 +730,58 @@ describe("TRACE_BACKEND=spans dispatch (design §3 dual-backend, §1 query mecha
expect(body.status).toBe("ready");
expect(body.traces).toHaveLength(1);
expect(xrayMock.calls()).toHaveLength(0);

const startCall = logsMock
.calls()
.find((c) => c.args[0] instanceof StartQueryCommand);
expect(startCall).toBeDefined();
const input = (startCall!.args[0] as StartQueryCommand).input;
// Normalized to plain 32-hex — no `1-` prefix, no dash separators.
expect(input.queryString).toContain(
'filter traceId = "5f84c7c1000000000000000000000002"',
);
expect(input.queryString).not.toContain("1-5f84c7c1");
});

test("TRACE_BACKEND=spans, admin raw traceId route with an already-32-hex id passes it through unchanged (finding a3d8a2ea #9, both formats)", async () => {
process.env.TRACE_BACKEND = "spans";
logsMock.on(StartQueryCommand).resolves({ queryId: "q-h8b" });
logsMock.on(GetQueryResultsCommand).resolves({
status: "Complete",
results: [
[
{
field: "@message",
value: JSON.stringify({
traceId: "6a7e5de027c150316d0ff197004e14b1",
spanId: "021348f2ab124f06",
name: "root-op-3",
startTimeUnixNano: 1000000000000,
endTimeUnixNano: 1001000000000,
}),
},
],
],
});

const event = makeEvent(
"GET /traces/{traceId}",
{ traceId: "6a7e5de027c150316d0ff197004e14b1" },
{ "custom:organization": "org-1", "custom:role": "admin" },
);

const res = await handler(event);
expect(res.statusCode).toBe(200);
const body = JSON.parse(res.body!);
expect(body.traces).toHaveLength(1);

const startCall = logsMock
.calls()
.find((c) => c.args[0] instanceof StartQueryCommand);
const input = (startCall!.args[0] as StartQueryCommand).input;
expect(input.queryString).toContain(
'filter traceId = "6a7e5de027c150316d0ff197004e14b1"',
);
});

test("TRACE_BACKEND=spans, non-admin raw traceId route -> still 403, zero Logs Insights calls (invariant 2 unchanged)", async () => {
Expand Down Expand Up @@ -1197,11 +1259,16 @@ describe("TRACE_BACKEND=spans — defensive filter rejects, failed-status mappin
// SPANS_QUERY_ROW_LIMIT is 1000 — return exactly 1000 rows so
// runSpanQuery reports truncated (rows.length >= limit).
const results = Array.from({ length: 1000 }, (_, i) => [
{ field: "traceId", value: "1-5f84c7c1-00000000000000000000000e" },
{ field: "spanId", value: `span-${i}` },
{ field: "name", value: `op-${i}` },
{ field: "startTimeUnixNano", value: "1000000000000" },
{ field: "endTimeUnixNano", value: "1001000000000" },
{
field: "@message",
value: JSON.stringify({
traceId: "5f84c7c100000000000000000000000e",
spanId: `span-${i}`,
name: `op-${i}`,
startTimeUnixNano: 1000000000000,
endTimeUnixNano: 1001000000000,
}),
},
]);
logsMock.on(StartQueryCommand).resolves({ queryId: "q-trunc" });
logsMock
Expand All @@ -1223,12 +1290,17 @@ describe("TRACE_BACKEND=spans — defensive filter rejects, failed-status mappin
});

const metadataRow = [
{ field: "traceId", value: "1-5f84c7c1-00000000000000000000000f" },
{ field: "spanId", value: "meta-span-1" },
{ field: "name", value: "meta-op" },
{ field: "startTimeUnixNano", value: "1000000000000" },
{ field: "endTimeUnixNano", value: "1001000000000" },
{ field: "attributes.custom.stage", value: "prod" },
{
field: "@message",
value: JSON.stringify({
traceId: "5f84c7c100000000000000000000000f",
spanId: "meta-span-1",
name: "meta-op",
startTimeUnixNano: 1000000000000,
endTimeUnixNano: 1001000000000,
attributes: { custom: { stage: "prod" } },
}),
},
];

test("spans: includeMetadata=1 as ADMIN -> metadata bag included on spans (admin + explicit opt-in honored)", async () => {
Expand Down
28 changes: 26 additions & 2 deletions backend/src/lambda/trace-query-handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ const xrayClient = new XRayClient({});
* start; a warm invocation reflects whatever the env held at that
* cold-start snapshot, matching how every other env-driven Lambda config
* in this codebase (e.g. AGENT_MODEL in services-stack.ts) is read. */
/** Matches an X-Ray-format trace id: `1-{8hex}-{24hex}` (e.g.
* `1-5f84c7c1-000000000000000000000001`). */
const XRAY_TRACE_ID_RE = /^1-([0-9a-f]{8})-([0-9a-f]{24})$/i;

/**
* Normalizes an X-Ray-format traceId (`1-{8hex}-{24hex}`) to the plain
* 32-hex form aws/spans stores its `traceId` field as (verified: all
* sampled aws/spans traceIds are 32-hex with no `1-` prefix — evidence
* report finding a3d8a2ea, verdict #1). Any other shape (already 32-hex,
* or unrecognized) is passed through unchanged, so existing links minted
* before this normalization, and ids that are already in the spans-native
* form, keep working identically.
*/
export function normalizeToSpansTraceId(traceId: string): string {
const match = XRAY_TRACE_ID_RE.exec(traceId);
if (!match) return traceId;
return `${match[1]}${match[2]}`;
}

function traceBackend(): "xray" | "spans" {
return process.env.TRACE_BACKEND === "spans" ? "spans" : "xray";
}
Expand Down Expand Up @@ -440,7 +459,12 @@ async function handleRawTraceId(
// form, checked at the route level by the admin gate above, never a
// user-supplied filter target for annotation purposes here) and is
// the natural Logs Insights equivalent of BatchGetTraces([traceId]).
const filter = buildSpanCorrelationFilter(traceId);
// Old links (minted while TRACE_BACKEND=xray) carry the X-Ray-format
// `1-{8hex}-{24hex}` id; aws/spans stores plain 32-hex (verified,
// evidence report finding a3d8a2ea) — normalize before filtering so
// those links keep resolving under the spans backend too.
const spansTraceId = normalizeToSpansTraceId(traceId);
const filter = buildSpanCorrelationFilter(spansTraceId);
if (!filter.ok) {
return json(200, {
query: { kind: "traceId", id: traceId, correlationId: null },
Expand All @@ -454,7 +478,7 @@ async function handleRawTraceId(
// Filter on traceId itself, not the correlation-id annotation — build
// the clause directly rather than reusing the annotation-targeted
// builder's field name.
const traceIdClause = `filter traceId = "${traceId}"`;
const traceIdClause = `filter traceId = "${spansTraceId}"`;
const { traces, queryStatus } = await fetchTracesBySpanFilter(
traceIdClause,
new Date(Date.now() - DEFAULT_WINDOW_MS).toISOString(),
Expand Down
Loading