Skip to content

Make status telemetry available in simulation and fix schema coverage #13847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tclinkenbeard-oai wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Make status telemetry available in simulation and fix schema coverage #13847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
166c213
Skip unreachable status schema coverage requirements
tclinkenbeard-oai Aug 11, 2026
6f36956
Correct canonical status schema output values
tclinkenbeard-oai Aug 11, 2026
8b68328
Exercise canonical storage engines in status simulation
tclinkenbeard-oai Aug 11, 2026
60dd94b
Preserve consistency checks for targeted status simulations
tclinkenbeard-oai Aug 11, 2026
0dc998c
Populate simulated process and machine status telemetry
tclinkenbeard-oai Aug 11, 2026
5265cad
Fix status telemetry locality and storage coverage regressions
tclinkenbeard-oai Aug 11, 2026
9822bad
Retry required status telemetry after simulated faults
tclinkenbeard-oai Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 8 additions & 22 deletions documentation/sphinx/source/mr-status-json-schemas.rst.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@
"version":"3.0.0", // a process version will not be reported if it is not protocol-compatible; it will be absent from status
"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece",
"locality":{ // This will contain any locality fields that are provided on the command line
"$map_key=localityName":"value"
"$map_key=localityName":"value" // A locality value can also be null when unset.
},
"class_source":{
"$enum":[
Expand All @@ -110,7 +110,8 @@
"commit_proxy",
"grv_proxy",
"master",
"test"
"test",
"blob_worker"
]
},
"degraded":true,
Expand Down Expand Up @@ -151,6 +152,8 @@
"cluster_controller",
"data_distributor",
"ratekeeper",
"consistency_scan",
"backupworker",
"router",
"coordinator"
]
Expand All @@ -161,15 +164,12 @@
"created_time_timestamp": 0,
"storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree",
"unknown"
]}
Expand Down Expand Up @@ -823,43 +823,32 @@
"log_version":2,
"log_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
"ssd-sharded-rocksdb"
]},
"log_spill":1,
"storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
]},
"tss_count":1,
"tss_storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
]},
"coordinators_count":1,
Expand All @@ -883,16 +872,13 @@
"perpetual_storage_wiggle_locality":"0",
"perpetual_storage_wiggle_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree-beta",
"memory-radixtree",
"none"
]},
"storage_migration_type":{
Expand Down Expand Up @@ -987,7 +973,7 @@
"address":"1.2.3.4",
"machine_id":"6344abf1813eb05b",
"locality":{ // This will contain any locality fields that are provided on the command line
"$map_key=localityName":"value"
"$map_key=localityName":"value" // A locality value can also be null when unset.
},
"cpu":{
"logical_core_utilization":0.4 // computed as cpu_seconds / elapsed_seconds; value may be capped at 0.5 due to hyper-threading
Expand Down
24 changes: 3 additions & 21 deletions fdbclient/Schemas.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,8 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
"cluster_controller",
"data_distributor",
"ratekeeper",
"blob_manager",
"blob_worker",
"consistency_scan",
"backupworker",
"router",
"coordinator"
]
Expand All @@ -146,15 +145,12 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
"created_time_timestamp": 0,
"storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree",
"unknown"
]}
Expand Down Expand Up @@ -814,43 +810,32 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
"log_version":2,
"log_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
"ssd-sharded-rocksdb"
]},
"log_spill":1,
"storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
]},
"tss_count":1,
"tss_storage_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree"
]},
"coordinators_count":1,
Expand All @@ -874,16 +859,13 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
"perpetual_storage_wiggle_locality":"0",
"perpetual_storage_wiggle_engine":{
"$enum":[
"ssd",
"ssd-1",
"ssd-2",
"ssd-redwood-1",
"ssd-rocksdb-v1",
"ssd-sharded-rocksdb",
"memory",
"memory-1",
"memory-2",
"memory-radixtree-beta",
"memory-radixtree",
"none"
]},
"storage_migration_type": {
Expand Down
4 changes: 2 additions & 2 deletions fdbclient/SpecialKeySpace.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -989,8 +989,8 @@ Future<bool> checkExclusion(Database db,
// Iterate over all excluded localites and check if the process is excluded based on the locality.
for (const auto& [localityKey, localityVec] : parsedLocalities) {
std::string localityValue;
if (!localityObj.get(localityKey, localityValue)) {
// If the locality doesn't exist in the locality object skip over it.
if (!localityObj.tryGet(localityKey, localityValue)) {
// Missing or unset locality values cannot match an exclusion.
continue;
}

Expand Down
3 changes: 3 additions & 0 deletions fdbclient/include/fdbclient/JsonBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once

#include <cstddef>
#include <string>
#include <vector>
#include <cmath>
Expand Down Expand Up @@ -88,6 +89,8 @@ class JsonBuilder {
};
}

void writeValue(std::nullptr_t) { write("null"); }

void writeValue(const bool& val) { write(val ? "true" : "false"); }

template <typename T>
Expand Down
90 changes: 75 additions & 15 deletions fdbserver/SimulatedCluster.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,63 @@ struct BasicSimulationConfig {

DatabaseConfiguration db;
};

constexpr int64_t simulatedMemoryLimit = 500'000'000;

void emitSimulatedStatusMetrics(LocalityData const& localities) {
const std::string machineId = printable(localities.machineId());
const std::string zoneId = printable(localities.zoneId());

TraceEvent("ProcessMetrics")
.detail("Elapsed", 1.0)
.detail("CPUSeconds", 0.0)
.detail("UptimeSeconds", 1.0)
.detail("Memory", simulatedMemoryLimit / 2)
.detail("ResidentMemory", simulatedMemoryLimit / 4)
.detail("UnusedAllocatedMemory", 0)
.detail("MbpsSent", 0.0)
.detail("MbpsReceived", 0.0)
.detail("DiskTotalBytes", simulatedMemoryLimit * 4)
.detail("DiskFreeBytes", simulatedMemoryLimit * 2)
.detail("DiskIdleSeconds", 1.0)
.detail("DiskReads", 0)
.detail("DiskWrites", 0)
.detail("DiskReadsCount", 0)
.detail("DiskWritesCount", 0)
.detail("DiskReadSectors", 0)
.detail("DiskWriteSectors", 0)
.detail("CacheHits", 0)
.detail("CacheMisses", 0)
.detail("ZoneID", zoneId)
.detail("MachineID", machineId)
.detail("CurrentConnections", 0)
.detail("ConnectionsEstablished", 0.0)
.detail("ConnectionsClosed", 0.0)
.detail("ConnectionErrors", 0.0)
.detail("TLSPolicyFailures", 0.0)
.trackLatest("ProcessMetrics");

TraceEvent("MachineMetrics")
.detail("Elapsed", 1.0)
.detail("MbpsSent", 0.0)
.detail("MbpsReceived", 0.0)
.detail("RetransSegs", 0)
.detail("CPUSeconds", 0.0)
.detail("TotalMemory", simulatedMemoryLimit * 2)
.detail("CommittedMemory", simulatedMemoryLimit)
.detail("AvailableMemory", simulatedMemoryLimit)
.detail("ZoneID", zoneId)
.detail("MachineID", machineId)
.trackLatest("MachineMetrics");

TraceEvent("NetworkMetrics")
.detail("Elapsed", 1.0)
.detail("PriorityStarvedBelow1", 0.0)
.detail("ZoneID", zoneId)
.detail("MachineID", machineId)
.trackLatest("NetworkMetrics");
}

constexpr bool hasRocksDB =
#ifdef WITH_ROCKSDB
true
Expand Down Expand Up @@ -786,20 +843,6 @@ Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConnectionR
.detail("Excluded", process->excluded)
.detail("UsingSSL", sslEnabled)
.detail("ProcessMode", processMode);
TraceEvent("ProgramStart")
.detail("Cycles", cycles)
.detail("RandomId", randomId)
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detail("DataFolder", *dataFolder)
.detail("TLogSpillFolder", tLogSpillFolder)
.detail("ConnectionString", connRecord ? connRecord->getConnectionString().toString() : "")
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
.detail("CommandLine", "fdbserver -r simulation")
.detail("BuggifyEnabled", isGeneralBuggifyEnabled())
.detail("Simulated", true)
.trackLatest("ProgramStart");

try {
// SOMEDAY: test lower memory limits, without making them too small and causing the database to stop
Expand All @@ -817,14 +860,31 @@ Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<IClusterConnectionR
NetworkAddress n(ip, listenPort, true, sslEnabled && listenPort == port);
futures.push_back(FlowTransport::transport().bind(n, n));
}
TraceEvent("ProgramStart")
.detail("Cycles", cycles)
.detail("RandomId", randomId)
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detail("DataFolder", *dataFolder)
.detail("TLogSpillFolder", tLogSpillFolder)
.detail("ConnectionString", connRecord ? connRecord->getConnectionString().toString() : "")
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
.detail("CommandLine", "fdbserver -r simulation")
.detail("MemoryLimit", simulatedMemoryLimit)
.detail("BuggifyEnabled", isGeneralBuggifyEnabled())
.detail("Simulated", true)
.trackLatest("ProgramStart");

if (processRunFDBD(processMode)) {
emitSimulatedStatusMetrics(localities);
futures.push_back(fdbd(connRecord,
localities,
processClass,
*dataFolder,
tLogSpillFolder,
*coordFolder,
500e6,
simulatedMemoryLimit,
"",
"",
-1,
Expand Down
15 changes: 15 additions & 0 deletions fdbserver/clustercontroller/Status.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3587,6 +3587,21 @@ TEST_CASE("/status/json/builder") {
JsonBuilder json;
ASSERT(checkJson(json, "null"));

JsonBuilderObject nullObject;
nullObject["missing"] = nullptr;
ASSERT(checkJson(nullObject, R"({"missing":null})"));

JsonBuilderArray nullArray;
nullArray.push_back(nullptr);
ASSERT(checkJson(nullArray, "[null]"));

LocalityData locality(Optional<Standalone<StringRef>>(),
Standalone<StringRef>("zone"_sr),
Standalone<StringRef>("machine"_sr),
Optional<Standalone<StringRef>>());
ASSERT(checkJson(locality.toJSON<JsonBuilderObject>(),
R"({"dcid":null,"machineid":"machine","processid":null,"zoneid":"zone"})"));

JsonBuilderArray array;
ASSERT(checkJson(array, "[]"));

Expand Down
Loading
Loading