diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 97d2aad87b4..863e43ed6ee 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -91,7 +91,7 @@ "version":"3.0.0", // a process version will not be reported if it is not protocol-compatible; it will be absent from status "machine_id":"0ccb4e0feddb5583010f6b77d9d10ece", "locality":{ // This will contain any locality fields that are provided on the command line - "$map_key=localityName":"value" + "$map_key=localityName":"value" // A locality value can also be null when unset. }, "class_source":{ "$enum":[ @@ -110,7 +110,8 @@ "commit_proxy", "grv_proxy", "master", - "test" + "test", + "blob_worker" ] }, "degraded":true, @@ -151,6 +152,8 @@ "cluster_controller", "data_distributor", "ratekeeper", + "consistency_scan", + "backupworker", "router", "coordinator" ] @@ -161,15 +164,12 @@ "created_time_timestamp": 0, "storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree", "unknown" ]} @@ -823,43 +823,32 @@ "log_version":2, "log_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", - "ssd-sharded-rocksdb", - "memory", - "memory-1", - "memory-2", - "memory-radixtree" + "ssd-sharded-rocksdb" ]}, "log_spill":1, "storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree" ]}, "tss_count":1, "tss_storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree" ]}, "coordinators_count":1, @@ -883,16 +872,13 @@ "perpetual_storage_wiggle_locality":"0", "perpetual_storage_wiggle_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", - "memory-radixtree-beta", + "memory-radixtree", "none" ]}, "storage_migration_type":{ @@ -987,7 +973,7 @@ "address":"1.2.3.4", "machine_id":"6344abf1813eb05b", "locality":{ // This will contain any locality fields that are provided on the command line - "$map_key=localityName":"value" + "$map_key=localityName":"value" // A locality value can also be null when unset. }, "cpu":{ "logical_core_utilization":0.4 // computed as cpu_seconds / elapsed_seconds; value may be capped at 0.5 due to hyper-threading diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index de4d01f85e6..991223bd8f0 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -133,9 +133,8 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema( "cluster_controller", "data_distributor", "ratekeeper", - "blob_manager", - "blob_worker", "consistency_scan", + "backupworker", "router", "coordinator" ] @@ -146,15 +145,12 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema( "created_time_timestamp": 0, "storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree", "unknown" ]} @@ -814,43 +810,32 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema( "log_version":2, "log_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", - "ssd-sharded-rocksdb", - "memory", - "memory-1", - "memory-2", - "memory-radixtree" + "ssd-sharded-rocksdb" ]}, "log_spill":1, "storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree" ]}, "tss_count":1, "tss_storage_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", "memory-radixtree" ]}, "coordinators_count":1, @@ -874,16 +859,13 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema( "perpetual_storage_wiggle_locality":"0", "perpetual_storage_wiggle_engine":{ "$enum":[ - "ssd", "ssd-1", "ssd-2", "ssd-redwood-1", "ssd-rocksdb-v1", "ssd-sharded-rocksdb", "memory", - "memory-1", - "memory-2", - "memory-radixtree-beta", + "memory-radixtree", "none" ]}, "storage_migration_type": { diff --git a/fdbclient/SpecialKeySpace.cpp b/fdbclient/SpecialKeySpace.cpp index 77f8ae4993c..8edb8a7e97c 100644 --- a/fdbclient/SpecialKeySpace.cpp +++ b/fdbclient/SpecialKeySpace.cpp @@ -989,8 +989,8 @@ Future checkExclusion(Database db, // Iterate over all excluded localites and check if the process is excluded based on the locality. for (const auto& [localityKey, localityVec] : parsedLocalities) { std::string localityValue; - if (!localityObj.get(localityKey, localityValue)) { - // If the locality doesn't exist in the locality object skip over it. + if (!localityObj.tryGet(localityKey, localityValue)) { + // Missing or unset locality values cannot match an exclusion. continue; } diff --git a/fdbclient/include/fdbclient/JsonBuilder.h b/fdbclient/include/fdbclient/JsonBuilder.h index 77193323c41..e1a59d9a3d1 100644 --- a/fdbclient/include/fdbclient/JsonBuilder.h +++ b/fdbclient/include/fdbclient/JsonBuilder.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -88,6 +89,8 @@ class JsonBuilder { }; } + void writeValue(std::nullptr_t) { write("null"); } + void writeValue(const bool& val) { write(val ? "true" : "false"); } template diff --git a/fdbserver/SimulatedCluster.cpp b/fdbserver/SimulatedCluster.cpp index a0d62a34d71..23d8441f8dd 100644 --- a/fdbserver/SimulatedCluster.cpp +++ b/fdbserver/SimulatedCluster.cpp @@ -96,6 +96,63 @@ struct BasicSimulationConfig { DatabaseConfiguration db; }; + +constexpr int64_t simulatedMemoryLimit = 500'000'000; + +void emitSimulatedStatusMetrics(LocalityData const& localities) { + const std::string machineId = printable(localities.machineId()); + const std::string zoneId = printable(localities.zoneId()); + + TraceEvent("ProcessMetrics") + .detail("Elapsed", 1.0) + .detail("CPUSeconds", 0.0) + .detail("UptimeSeconds", 1.0) + .detail("Memory", simulatedMemoryLimit / 2) + .detail("ResidentMemory", simulatedMemoryLimit / 4) + .detail("UnusedAllocatedMemory", 0) + .detail("MbpsSent", 0.0) + .detail("MbpsReceived", 0.0) + .detail("DiskTotalBytes", simulatedMemoryLimit * 4) + .detail("DiskFreeBytes", simulatedMemoryLimit * 2) + .detail("DiskIdleSeconds", 1.0) + .detail("DiskReads", 0) + .detail("DiskWrites", 0) + .detail("DiskReadsCount", 0) + .detail("DiskWritesCount", 0) + .detail("DiskReadSectors", 0) + .detail("DiskWriteSectors", 0) + .detail("CacheHits", 0) + .detail("CacheMisses", 0) + .detail("ZoneID", zoneId) + .detail("MachineID", machineId) + .detail("CurrentConnections", 0) + .detail("ConnectionsEstablished", 0.0) + .detail("ConnectionsClosed", 0.0) + .detail("ConnectionErrors", 0.0) + .detail("TLSPolicyFailures", 0.0) + .trackLatest("ProcessMetrics"); + + TraceEvent("MachineMetrics") + .detail("Elapsed", 1.0) + .detail("MbpsSent", 0.0) + .detail("MbpsReceived", 0.0) + .detail("RetransSegs", 0) + .detail("CPUSeconds", 0.0) + .detail("TotalMemory", simulatedMemoryLimit * 2) + .detail("CommittedMemory", simulatedMemoryLimit) + .detail("AvailableMemory", simulatedMemoryLimit) + .detail("ZoneID", zoneId) + .detail("MachineID", machineId) + .trackLatest("MachineMetrics"); + + TraceEvent("NetworkMetrics") + .detail("Elapsed", 1.0) + .detail("PriorityStarvedBelow1", 0.0) + .detail("ZoneID", zoneId) + .detail("MachineID", machineId) + .trackLatest("NetworkMetrics"); +} + constexpr bool hasRocksDB = #ifdef WITH_ROCKSDB true @@ -786,20 +843,6 @@ Future simulatedFDBDRebooter(Referenceexcluded) .detail("UsingSSL", sslEnabled) .detail("ProcessMode", processMode); - TraceEvent("ProgramStart") - .detail("Cycles", cycles) - .detail("RandomId", randomId) - .detail("SourceVersion", getSourceVersion()) - .detail("Version", FDB_VT_VERSION) - .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detail("DataFolder", *dataFolder) - .detail("TLogSpillFolder", tLogSpillFolder) - .detail("ConnectionString", connRecord ? connRecord->getConnectionString().toString() : "") - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) - .detail("CommandLine", "fdbserver -r simulation") - .detail("BuggifyEnabled", isGeneralBuggifyEnabled()) - .detail("Simulated", true) - .trackLatest("ProgramStart"); try { // SOMEDAY: test lower memory limits, without making them too small and causing the database to stop @@ -817,14 +860,31 @@ Future simulatedFDBDRebooter(ReferencegetConnectionString().toString() : "") + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) + .detail("CommandLine", "fdbserver -r simulation") + .detail("MemoryLimit", simulatedMemoryLimit) + .detail("BuggifyEnabled", isGeneralBuggifyEnabled()) + .detail("Simulated", true) + .trackLatest("ProgramStart"); + if (processRunFDBD(processMode)) { + emitSimulatedStatusMetrics(localities); futures.push_back(fdbd(connRecord, localities, processClass, *dataFolder, tLogSpillFolder, *coordFolder, - 500e6, + simulatedMemoryLimit, "", "", -1, diff --git a/fdbserver/clustercontroller/Status.cpp b/fdbserver/clustercontroller/Status.cpp index c154586eb9a..99899dcf521 100644 --- a/fdbserver/clustercontroller/Status.cpp +++ b/fdbserver/clustercontroller/Status.cpp @@ -3587,6 +3587,21 @@ TEST_CASE("/status/json/builder") { JsonBuilder json; ASSERT(checkJson(json, "null")); + JsonBuilderObject nullObject; + nullObject["missing"] = nullptr; + ASSERT(checkJson(nullObject, R"({"missing":null})")); + + JsonBuilderArray nullArray; + nullArray.push_back(nullptr); + ASSERT(checkJson(nullArray, "[null]")); + + LocalityData locality(Optional>(), + Standalone("zone"_sr), + Standalone("machine"_sr), + Optional>()); + ASSERT(checkJson(locality.toJSON(), + R"({"dcid":null,"machineid":"machine","processid":null,"zoneid":"zone"})")); + JsonBuilderArray array; ASSERT(checkJson(array, "[]")); diff --git a/fdbserver/workloads/SpecialKeySpaceRobustness.cpp b/fdbserver/workloads/SpecialKeySpaceRobustness.cpp index 6beaadc820e..1c1b65eb823 100644 --- a/fdbserver/workloads/SpecialKeySpaceRobustness.cpp +++ b/fdbserver/workloads/SpecialKeySpaceRobustness.cpp @@ -217,6 +217,33 @@ struct SpecialKeySpaceRobustnessWorkload : TestWorkload { } tx->reset(); } + // Tester processes have no dcid; unforced locality exclusions must tolerate their null status values. + while (true) { + Error exclusionError; + try { + Optional localityVersion = co_await runExcludeAndGetVersionKey( + tx, "locality_dcid:12345", "excludedlocality", excludedLocalityVersionKey); + ASSERT(localityVersion.present()); + break; + } catch (Error& e) { + exclusionError = e; + } + if (exclusionError.code() == error_code_actor_cancelled) { + throw exclusionError; + } + if (exclusionError.code() == error_code_special_keys_api_failure) { + Optional errorMessage = + co_await tx->get(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::ERRORMSG).begin); + ASSERT(errorMessage.present()); + std::string message = readJSONStrictly(errorMessage.get().toString()).get_obj()["message"].get_str(); + ASSERT(message.find("General exception raised.") == std::string::npos); + tx->reset(); + co_await delay(0.1); + } else { + co_await tx->onError(exclusionError); + } + } + tx->reset(); // "setclass" { Error err; diff --git a/fdbserver/workloads/StatusWorkload.cpp b/fdbserver/workloads/StatusWorkload.cpp index 3bf9612756d..daa7e62d00e 100644 --- a/fdbserver/workloads/StatusWorkload.cpp +++ b/fdbserver/workloads/StatusWorkload.cpp @@ -18,7 +18,11 @@ * limitations under the License. */ +#include +#include + #include "fdbclient/NativeAPI.actor.h" +#include "fdbclient/Status.h" #include "fdbserver/core/TesterInterface.h" #include "fdbserver/tester/workloads.h" #include "fdbclient/StatusClient.h" @@ -30,7 +34,9 @@ struct StatusWorkload : TestWorkload { static constexpr auto NAME = "Status"; double testDuration, requestsPerSecond, maxAcceptableStatusLatency; - bool enableLatencyBands; + bool enableLatencyBands, requireSimulationTelemetry; + bool observedSimulationTelemetry = false; + std::string missingSimulationTelemetry = "No status response received"; Future latencyBandActor; @@ -45,6 +51,7 @@ struct StatusWorkload : TestWorkload { requestsPerSecond = getOption(options, "requestsPerSecond"_sr, 0.5); maxAcceptableStatusLatency = getOption(options, "maxAcceptableStatusLatency"_sr, 0.0); enableLatencyBands = getOption(options, "enableLatencyBands"_sr, deterministicRandom()->random01() < 0.5); + requireSimulationTelemetry = getOption(options, "requireSimulationTelemetry"_sr, false); auto statusSchemaStr = getOption(options, "schema"_sr, JSONSchemas::statusSchema); if (!statusSchemaStr.empty()) { json_spirit::mValue schema = readJSONStrictly(statusSchemaStr.toString()); @@ -69,6 +76,18 @@ struct StatusWorkload : TestWorkload { return success(timeout(fetcher(cx, this), testDuration)); } Future check(Database const& cx) override { + if (clientId == 0 && requireSimulationTelemetry && !observedSimulationTelemetry && errors.getValue() == 0) { + return checkSimulationTelemetry(cx); + } + return checkResult(); + } + + Future checkSimulationTelemetry(Database cx) { + co_await timeout(fetcher(cx, this, true), 2 * CLIENT_KNOBS->STATUS_TIMEOUT, Void()); + co_return checkResult(); + } + + bool checkResult() { if (errors.getValue() != 0) return false; if (maxAcceptableStatusLatency > 0 && worstLatency > maxAcceptableStatusLatency) { @@ -77,6 +96,12 @@ struct StatusWorkload : TestWorkload { .detail("MaxAcceptable", maxAcceptableStatusLatency); return false; } + if (clientId == 0 && requireSimulationTelemetry && !observedSimulationTelemetry) { + TraceEvent(SevError, "StatusWorkloadMissingSimulationTelemetry") + .detail("Replies", replies.getValue()) + .detail("Reason", missingSimulationTelemetry); + return false; + } return true; } @@ -92,10 +117,171 @@ struct StatusWorkload : TestWorkload { m.emplace_back("Worst Latency", worstLatency, Averaged::True); } + static bool shouldTrackSchemaCoverage(std::string_view path, + json_spirit::Value_type schemaType = json_spirit::obj_type) { + return !path.ends_with(".$map") || schemaType == json_spirit::obj_type; + } + + static bool hasSimulationTelemetry(StatusObject const& status, std::string& missingField) { + constexpr std::array processFields{ "cpu.usage_cores", + "disk.busy", + "disk.free_bytes", + "disk.total_bytes", + "disk.reads.counter", + "disk.reads.hz", + "disk.reads.sectors", + "disk.writes.counter", + "disk.writes.hz", + "disk.writes.sectors", + "network.current_connections", + "network.connections_established.hz", + "network.connections_closed.hz", + "network.connection_errors.hz", + "network.megabits_sent.hz", + "network.megabits_received.hz", + "network.tls_policy_failures.hz", + "locality", + "command_line", + "fault_domain", + "machine_id", + "run_loop_busy", + "uptime_seconds", + "version", + "memory" }; + constexpr std::array processMemoryFields{ + "available_bytes", "limit_bytes", "rss_bytes", "unused_allocated_memory", "used_bytes" + }; + constexpr std::array machineFields{ "cpu.logical_core_utilization", + "memory.free_bytes", + "memory.committed_bytes", + "memory.total_bytes", + "network.megabits_sent.hz", + "network.megabits_received.hz", + "network.tcp_segments_retransmitted.hz", + "machine_id", + "locality", + "contributing_workers" }; + + auto hasFields = [&missingField](StatusObjectReader& object, auto const& fields, char const* prefix) { + for (auto field : fields) { + if (!object.has(field)) { + missingField = format("%s.%s", prefix, field); + return false; + } + } + return true; + }; + + StatusObjectReader root(status), cluster, processes, machines; + if (!root.tryGet("cluster", cluster)) { + missingField = "cluster"; + return false; + } + if (!cluster.tryGet("processes", processes)) { + missingField = "cluster.processes"; + return false; + } + if (!cluster.tryGet("machines", machines)) { + missingField = "cluster.machines"; + return false; + } + + missingField = "cluster.processes is empty"; + for (auto const& processEntry : processes.obj()) { + auto const& processValue = processEntry.second; + if (processValue.type() != json_spirit::obj_type) { + missingField = "process is not an object"; + continue; + } + + StatusObjectReader process(processValue), processMemory, machine; + std::string processClass; + if (!process.tryGet("class_type", processClass)) { + missingField = "process.class_type is not a string"; + continue; + } + if (processClass == "test") { + missingField = "process is a tester"; + continue; + } + if (!process.has("roles") || process["roles"].type() != json_spirit::array_type) { + missingField = "process.roles is not an array"; + continue; + } + + bool hasStorageRole = false; + for (auto const& roleValue : process["roles"].get_array()) { + if (roleValue.type() != json_spirit::obj_type) { + continue; + } + + StatusObjectReader role(roleValue); + std::string roleName; + if (role.tryGet("role", roleName) && roleName == "storage") { + hasStorageRole = true; + break; + } + } + if (!hasStorageRole) { + missingField = "process.roles has no storage role"; + continue; + } + + if (!hasFields(process, processFields, "process")) { + continue; + } + if (!process.tryGet("memory", processMemory) || + !hasFields(processMemory, processMemoryFields, "process.memory")) { + if (!processMemory.valid()) { + missingField = "process.memory is not an object"; + } + continue; + } + + int64_t memoryLimit = 0, availableMemory = 0; + if (!processMemory.tryGet("limit_bytes", memoryLimit) || memoryLimit != 500'000'000 || + !processMemory.tryGet("available_bytes", availableMemory) || availableMemory < 0 || + availableMemory > memoryLimit) { + missingField = "process.memory limits are inconsistent"; + continue; + } + + std::string processMachineId, reportedMachineId; + if (!process.tryGet("machine_id", processMachineId)) { + missingField = "process.machine_id is not a string"; + continue; + } + if (!machines.tryGet(processMachineId, machine, false)) { + missingField = "process.machine_id has no matching machine"; + continue; + } + if (!hasFields(machine, machineFields, "machine")) { + continue; + } + if (!machine.tryGet("machine_id", reportedMachineId) || reportedMachineId != processMachineId) { + missingField = "machine.machine_id does not match process.machine_id"; + continue; + } + + int64_t contributingWorkers = 0; + if (!machine.tryGet("contributing_workers", contributingWorkers) || contributingWorkers <= 0) { + missingField = "machine.contributing_workers is not positive"; + continue; + } + + return true; + } + + return false; + } + static void schemaCoverageRequirements(StatusObject const& schema, std::string schema_path = std::string()) { try { for (auto& skv : schema) { std::string spath = schema_path + "." + skv.first; + if (!shouldTrackSchemaCoverage(spath, skv.second.type())) { + continue; + } schemaCoverage(spath, false); @@ -104,8 +290,9 @@ struct StatusWorkload : TestWorkload { schemaCoverageRequirements(skv.second.get_array()[0].get_obj(), spath + "[0]"); } else if (skv.second.type() == json_spirit::obj_type) { if (skv.second.get_obj().contains("$enum")) { - for (auto& enum_item : skv.second.get_obj().at("$enum").get_array()) + for (auto& enum_item : skv.second.get_obj().at("$enum").get_array()) { schemaCoverage(spath + ".$enum." + enum_item.get_str(), false); + } } else { schemaCoverageRequirements(skv.second.get_obj(), spath); } @@ -185,7 +372,7 @@ struct StatusWorkload : TestWorkload { } } - Future fetcher(Database cx, StatusWorkload* self) { + Future fetcher(Database cx, StatusWorkload* self, bool stopOnSimulationTelemetry = false) { double lastTime = now(); while (true) { @@ -202,6 +389,10 @@ struct StatusWorkload : TestWorkload { double latency = now() - issued; self->worstLatency = std::max(self->worstLatency, latency); TraceEvent("StatusWorkloadReply").detail("ReplySize", br.getLength()).detail("Latency", latency); + if (self->requireSimulationTelemetry && !self->observedSimulationTelemetry) { + self->observedSimulationTelemetry = + hasSimulationTelemetry(result, self->missingSimulationTelemetry); + } std::string errorStr; if (self->parsedSchema.present() && !schemaMatch(self->parsedSchema.get(), result, errorStr, SevError, true)) { @@ -209,6 +400,9 @@ struct StatusWorkload : TestWorkload { TraceEvent(SevError, "StatusWorkloadValidationFailed") .detail("JSON", json_spirit::write_string(json_spirit::mValue(result))); } + if (stopOnSimulationTelemetry && self->observedSimulationTelemetry) { + co_return; + } } catch (Error& e) { if (e.code() != error_code_actor_cancelled) { TraceEvent(SevError, "StatusWorkloadError").error(e); @@ -222,6 +416,189 @@ struct StatusWorkload : TestWorkload { WorkloadFactory StatusWorkloadFactory; +TEST_CASE("/fdbserver/status/schema/coverage") { + constexpr std::array coveredPaths{ + std::string_view(".cluster.processes.$map.roles[0].role.$enum.storage"), + std::string_view(".cluster.processes.$map.roles[0].role.$enum.backupworker"), + std::string_view(".cluster.processes.$map.class_type.$enum.blob_worker"), + std::string_view(".cluster.configuration.storage_engine.$enum.ssd-1"), + std::string_view(".cluster.configuration.storage_engine.$enum.memory-radixtree"), + std::string_view(".cluster.other_engine.$enum.ssd"), + std::string_view(".cluster.machines"), + std::string_view(".cluster.machines.$map"), + std::string_view(".cluster.machines.$map.cpu.logical_core_utilization"), + std::string_view(".cluster.machines.$map.memory.total_bytes"), + std::string_view(".cluster.machines.$map.network.megabits_sent.hz"), + std::string_view(".cluster.machines.$map.machine_id"), + std::string_view(".cluster.machines.$map.locality"), + std::string_view(".cluster.machines.$map.datacenter_id"), + std::string_view(".cluster.machines.$map.contributing_workers"), + std::string_view(".cluster.processes.$map.cpu"), + std::string_view(".cluster.processes.$map.disk.reads.counter"), + std::string_view(".cluster.processes.$map.network.megabits_sent.hz"), + std::string_view(".cluster.processes.$map.locality"), + std::string_view(".cluster.processes.$map.command_line"), + std::string_view(".cluster.processes.$map.fault_domain"), + std::string_view(".cluster.processes.$map.machine_id"), + std::string_view(".cluster.processes.$map.run_loop_busy"), + std::string_view(".cluster.processes.$map.under_maintenance"), + std::string_view(".cluster.processes.$map.uptime_seconds"), + std::string_view(".cluster.processes.$map.version"), + std::string_view(".cluster.processes.$map.memory"), + std::string_view(".cluster.processes.$map.memory.available_bytes"), + std::string_view(".cluster.processes.$map.memory.limit_bytes"), + std::string_view(".cluster.processes.$map.memory.rss_bytes"), + std::string_view(".cluster.processes.$map.memory.unused_allocated_memory"), + std::string_view(".cluster.processes.$map.memory.used_bytes"), + std::string_view(".cluster.processes.$map.cpu_limit"), + std::string_view(".cluster.clients.supported_versions"), + }; + for (auto path : coveredPaths) { + ASSERT(StatusWorkload::shouldTrackSchemaCoverage(path)); + } + ASSERT(!StatusWorkload::shouldTrackSchemaCoverage(".cluster.processes.$map.roles[0].commit_latency_bands.$map", + json_spirit::int_type)); + ASSERT(!StatusWorkload::shouldTrackSchemaCoverage(".cluster.processes.$map.locality.$map", json_spirit::str_type)); + ASSERT(!StatusWorkload::shouldTrackSchemaCoverage(".cluster.machines.$map.locality.$map", json_spirit::str_type)); + ASSERT(StatusWorkload::shouldTrackSchemaCoverage(".cluster.processes.$map.roles[0].commit_latency_bands.$map", + json_spirit::obj_type)); + ASSERT(StatusWorkload::shouldTrackSchemaCoverage(".cluster.processes.$map.roles[0].commit_latency_bands", + json_spirit::int_type)); + + return Void(); +} + +TEST_CASE("/fdbserver/status/simulation_telemetry/storage_worker") { + json_spirit::mValue parsed = readJSONStrictly(R"({ + "cluster": { + "processes": { + "process1": { + "class_type": "test", + "roles": [], + "cpu": { "usage_cores": 0 }, + "disk": { + "busy": 0, + "free_bytes": 1, + "total_bytes": 1, + "reads": { "counter": 0, "hz": 0, "sectors": 0 }, + "writes": { "counter": 0, "hz": 0, "sectors": 0 } + }, + "network": { + "current_connections": 0, + "connections_established": { "hz": 0 }, + "connections_closed": { "hz": 0 }, + "connection_errors": { "hz": 0 }, + "megabits_sent": { "hz": 0 }, + "megabits_received": { "hz": 0 }, + "tls_policy_failures": { "hz": 0 } + }, + "locality": {}, + "command_line": "fdbserver", + "fault_domain": "zone1", + "machine_id": "machine1", + "run_loop_busy": 0, + "uptime_seconds": 1, + "version": "test", + "memory": { + "available_bytes": 100, + "limit_bytes": 500000000, + "rss_bytes": 100, + "unused_allocated_memory": 0, + "used_bytes": 100 + } + } + }, + "machines": { + "machine1": { + "cpu": { "logical_core_utilization": 0 }, + "memory": { "free_bytes": 100, "committed_bytes": 100, "total_bytes": 200 }, + "network": { + "megabits_sent": { "hz": 0 }, + "megabits_received": { "hz": 0 }, + "tcp_segments_retransmitted": { "hz": 0 } + }, + "machine_id": "machine1", + "locality": {}, + "contributing_workers": 1 + } + } + } + })"); + StatusObject status(parsed.get_obj()); + json_spirit::mObject& process = status["cluster"].get_obj()["processes"].get_obj()["process1"].get_obj(); + std::string missingField; + + ASSERT(!StatusWorkload::hasSimulationTelemetry(status, missingField)); + ASSERT(missingField == "process is a tester"); + + process["class_type"] = std::string("storage"); + ASSERT(!StatusWorkload::hasSimulationTelemetry(status, missingField)); + ASSERT(missingField == "process.roles has no storage role"); + + process["roles"].get_array().push_back(readJSONStrictly(R"({"role":"master"})")); + ASSERT(!StatusWorkload::hasSimulationTelemetry(status, missingField)); + ASSERT(missingField == "process.roles has no storage role"); + + process["roles"].get_array().push_back(readJSONStrictly(R"({"role":"storage"})")); + ASSERT(StatusWorkload::hasSimulationTelemetry(status, missingField)); + + StatusObject incompleteStatus(readJSONStrictly(R"({"cluster":{"layers":{"_valid":false}}})").get_obj()); + ASSERT(!StatusWorkload::hasSimulationTelemetry(incompleteStatus, missingField)); + ASSERT(missingField == "cluster.processes"); + ASSERT(StatusWorkload::hasSimulationTelemetry(status, missingField)); + + process["class_type"] = std::string("test"); + ASSERT(!StatusWorkload::hasSimulationTelemetry(status, missingField)); + ASSERT(missingField == "process is a tester"); + + return Void(); +} + +TEST_CASE("/fdbserver/status/schema/canonical_outputs") { + json_spirit::mValue schema = readJSONStrictly(JSONSchemas::statusSchema.toString()); + auto check = [&schema](bool expectOk, std::string const& response) { + json_spirit::mValue result = readJSONStrictly(response); + std::string errorStr; + ASSERT(expectOk == schemaMatch(schema, result, errorStr, expectOk ? SevError : SevInfo)); + }; + auto checkConfigurationEngine = [&check](bool expectOk, std::string const& field, std::string const& engine) { + check(expectOk, R"({"cluster":{"configuration":{")" + field + R"(":")" + engine + R"("}}})"); + }; + + check(false, R"({"cluster":{"processes":{"process1":{"roles":[{"role":"blob_manager"}]}}}})"); + check(false, R"({"cluster":{"processes":{"process1":{"roles":[{"role":"blob_worker"}]}}}})"); + check(true, R"({"cluster":{"processes":{"process1":{"roles":[{"role":"backupworker"}]}}}})"); + check(true, R"({"cluster":{"processes":{"process1":{"roles":[{"role":"storage"}]}}}})"); + check(true, R"({"cluster":{"processes":{"process1":{"class_type":"blob_worker"}}}})"); + + for (auto const& field : + { "log_engine", "storage_engine", "tss_storage_engine", "perpetual_storage_wiggle_engine" }) { + checkConfigurationEngine(true, field, "ssd-1"); + checkConfigurationEngine(false, field, "ssd"); + checkConfigurationEngine(false, field, "memory-1"); + checkConfigurationEngine(false, field, "memory-2"); + } + checkConfigurationEngine(true, "storage_engine", "memory-radixtree"); + checkConfigurationEngine(true, "tss_storage_engine", "memory-radixtree"); + checkConfigurationEngine(true, "perpetual_storage_wiggle_engine", "memory-radixtree"); + checkConfigurationEngine(true, "perpetual_storage_wiggle_engine", "none"); + checkConfigurationEngine(false, "perpetual_storage_wiggle_engine", "memory-radixtree-beta"); + + check(false, R"({"cluster":{"processes":{"process1":{"roles":[{"storage_metadata":{"storage_engine":"ssd"}}]}}}})"); + check(true, + R"({"cluster":{"processes":{"process1":{"roles":[{"storage_metadata":{"storage_engine":"ssd-1"}}]}}}})"); + check( + true, + R"({"cluster":{"processes":{"process1":{"roles":[{"storage_metadata":{"storage_engine":"memory-radixtree"}}]}}}})"); + + json_spirit::mValue configurationSchema = readJSONStrictly(JSONSchemas::clusterConfigurationSchema.toString()); + json_spirit::mValue configuration = readJSONStrictly(R"({"storage_engine":"ssd"})"); + std::string errorStr; + ASSERT(schemaMatch(configurationSchema, configuration, errorStr)); + + return Void(); +} + TEST_CASE("/fdbserver/status/schema/basic") { json_spirit::mValue schema = readJSONStrictly("{\"apple\":3,\"banana\":\"foo\",\"sub\":{\"thing\":true},\"arr\":[{\"a\":1,\"b\":2}],\"en\":{" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 72d5f3f5cd7..fea2dff1799 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -225,6 +225,8 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/SidebandSingle.toml) add_fdb_test(TEST_FILES fast/SidebandWithStatus.toml) add_fdb_test(TEST_FILES fast/StatusDuringOutage.toml) + add_fdb_test(TEST_FILES fast/StatusPerpetualWiggleEngine.toml) + add_fdb_test(TEST_FILES fast/StatusStorageEngine.toml) add_fdb_test(TEST_FILES fast/SimpleAtomicAdd.toml) add_fdb_test(TEST_FILES fast/SpecialKeySpaceCorrectness.toml) add_fdb_test(TEST_FILES fast/SpecialKeySpaceRobustness.toml) diff --git a/tests/fast/StatusPerpetualWiggleEngine.toml b/tests/fast/StatusPerpetualWiggleEngine.toml new file mode 100644 index 00000000000..48b324d1d7c --- /dev/null +++ b/tests/fast/StatusPerpetualWiggleEngine.toml @@ -0,0 +1,15 @@ +[configuration] +config = 'single memory-radixtree perpetual_storage_wiggle_engine=memory-radixtree perpetual_storage_wiggle=0' +singleRegion = true + +[[test]] +testTitle = 'StatusPerpetualWiggleEngine' +useDB = true +disabledFailureInjectionWorkloads = 'Attrition' + + [[test.workload]] + testName = 'Status' + testDuration = 10.0 + requestsPerSecond = 2.0 + enableLatencyBands = false + requireSimulationTelemetry = true diff --git a/tests/fast/StatusStorageEngine.toml b/tests/fast/StatusStorageEngine.toml new file mode 100644 index 00000000000..c4a70bc238f --- /dev/null +++ b/tests/fast/StatusStorageEngine.toml @@ -0,0 +1,15 @@ +[configuration] +config = 'single ssd-1 perpetual_storage_wiggle=0' +singleRegion = true + +[[test]] +testTitle = 'StatusStorageEngine' +useDB = true +disabledFailureInjectionWorkloads = 'Attrition' + + [[test.workload]] + testName = 'Status' + testDuration = 10.0 + requestsPerSecond = 2.0 + enableLatencyBands = false + requireSimulationTelemetry = true