Argos collector - DRAFT / DO NOT REVIEW YET

sparta/example/CoreModel/CMakeLists.txt

@@ -63,8 +63,8 @@ sparta_regress(sparta_core_example)
 sparta_named_test(sparta_core_example_20000cycs sparta_core_example -r 200000)
 sparta_named_test(sparta_core_example_fetch_max_ipc sparta_core_example  -i 1M -p top.cpu.core0.fetch.params.fetch_max_ipc true)
 sparta_named_test(sparta_core_example_pipeout sparta_core_example -i 1000 -z pipeout -K layouts/cpu_layout.alf)
-sparta_named_test(sparta_core_example_pipeout_icount sparta_core_example -i 1000 --debug-on-icount 100 -z pipeout -K layouts/cpu_layout.alf)
-sparta_named_test(sparta_core_example_pipeout_cycle sparta_core_example -i 1000 --debug-on 100 -z pipeout -K layouts/cpu_layout.alf)
+sparta_named_test(sparta_core_example_pipeout_icount sparta_core_example -i 1000 --debug-on-icount 100 -z pipeout_icount -K layouts/cpu_layout.alf)
+sparta_named_test(sparta_core_example_pipeout_cycle sparta_core_example -i 1000 --debug-on 100 -z pipeout_cycle -K layouts/cpu_layout.alf)
 # Using a standard  YAML causes this test to fail.  Need to investigate Issue #4
 # sparta_named_test(sparta_core_example_preload sparta_core_example -i 1000 -p top.cpu.core0.preloader.params.preload_file sample_preload.yaml)
 sparta_named_test(sparta_core_example_multicore_yaml_opts_smoke_test sparta_core_example -i 10k --report top multicore_report_opts_basic.yaml out.txt --num-cores 2)
@@ -154,10 +154,10 @@ sparta_expect_failure_test(sparta_core_example_override_extensions_neg_test spar
 sparta_named_test(sparta_core_example_3d_nested_extension_params sparta_core_example --no-run --extension-file nested_vecs.yaml --write-final-config final.yaml)
 
 if(USING_SIMDB)
-sparta_named_test(sparta_core_example_simdb_apps1 sparta_core_example --simdb-apps simdb-reports cherry-pick-fast-checkpointer --simdb-database test.db --no-run)
-sparta_named_test(sparta_core_example_simdb_apps2 sparta_core_example --simdb-apps simdb-reports cherry-pick-fast-checkpointer --simdb-app-database simdb-reports reports.db --no-run)
+sparta_named_test(sparta_core_example_simdb_apps1 sparta_core_example --simdb-apps simdb-reports cherry-pick-fast-checkpointer --simdb-database simdb_apps1.db --no-run)
+sparta_named_test(sparta_core_example_simdb_apps2 sparta_core_example --simdb-apps simdb-reports cherry-pick-fast-checkpointer --simdb-app-database simdb-reports reports.db cherry-pick-fast-checkpointer chkptr.db --no-run)
 sparta_named_test(sparta_core_example_simdb_apps3 sparta_core_example --simdb-apps cherry-pick-fast-checkpointer --simdb-app-count cherry-pick-fast-checkpointer 2 --simdb-database autogen --no-run)
-sparta_named_test(sparta_core_example_simdb_apps4 sparta_core_example --simdb-apps cherry-pick-fast-checkpointer --simdb-app-count cherry-pick-fast-checkpointer top.cpu.core* --num-cores 2 --no-run)
+sparta_named_test(sparta_core_example_simdb_apps4 sparta_core_example --simdb-apps cherry-pick-fast-checkpointer --simdb-app-count cherry-pick-fast-checkpointer top.cpu.core* --num-cores 2 --simdb-database simdb_apps4.db --no-run)
 
 # Run all backwards compatibility tests for map_v2.1+ reports and their v2.0.* equivalents
 add_test(NAME sparta_simdb_report_verif_suite COMMAND python3 ../../../scripts/simdb/run_report_verif_suite.py --sim-exe-path sparta_core_example --serial --force --fail-fast)

sparta/sparta/app/CommandLineSimulator.hpp

@@ -55,7 +55,7 @@ class InformationWriter;
 
 namespace app {
 
-const constexpr char DefaultHeartbeat[]        = "0";
+const constexpr char DefaultHeartbeat[]        = "10";
 
 
 /*!
@@ -457,7 +457,6 @@ class CommandLineSimulator
      */
     std::unique_ptr<sparta::trigger::Triggerable> pipeline_collection_triggerable_;
     std::unique_ptr<sparta::trigger::Trigger>     pipeline_trigger_;
-    std::unique_ptr<sparta::InformationWriter>    info_out_;
 
     /*!
      * \brief Heartbeat period of pipeline collection file (before lexical cast

sparta/sparta/app/Simulation.hpp

@@ -1009,6 +1009,12 @@ class Simulation
      * \brief SimDB DatabaseManager/AppManager instances.
      */
      std::shared_ptr<simdb::AppManagers> app_managers_;
+
+     /*!
+      * \brief Flag telling us whether postSimLoopTeardown() API call
+      * is still required.
+      */
+     bool app_managers_require_teardown_ = true;
 };
 
 } // namespace app

sparta/sparta/app/SimulationConfiguration.hpp

@@ -471,6 +471,12 @@ class SimulationConfiguration
      */
     std::string pipeline_collection_file_prefix = NoPipelineCollectionStr;
 
+    /*!
+     * Set by CommandLineSimulator. Controls time/space tradeoffs for
+     * pipeline collection.
+     */
+    utils::ValidValue<size_t> pipeline_collection_heartbeat;
+
     /*!
      * Additional report descriptions
      */
@@ -661,6 +667,22 @@ class SimulationConfiguration
             return legacy_reports_enabled_;
         }
 
+        /*!
+         * \brief Call the setVerbose() method for all SimDB apps once created.
+         */
+        void enableVerboseMode(bool enable = true)
+        {
+            verbose_ = enable;
+        }
+
+        /*!
+         * \brief Get verbose flag.
+         */
+        bool verboseMode() const
+        {
+            return verbose_;
+        }
+
         /*!
          * \brief Add a SQLite PRAGMA to execute on database creation.
          * \note Applies to all app databases if different.
@@ -694,6 +716,7 @@ class SimulationConfiguration
         std::set<std::string> reused_db_files_;
         std::map<std::string, std::string> dbmgr_pragmas_;
         bool legacy_reports_enabled_ = true;
+        bool verbose_ = false;
     } simdb_config;
 
     /*!

sparta/sparta/collection/BitBucket.hpp

@@ -0,0 +1,229 @@
+// <BitBucket.hpp> -*- C++ -*-
+
+#pragma once
+
+#include "sparta/utils/ValidValue.hpp"
+#include "simdb/apps/argos/EntryPoint.hpp"
+#include <cstring>
+
+namespace sparta::collection {
+
+//! \class BitBucket
+//! \brief Base class for all collectable bit buckets (scalars, sparse/contig iterables).
+//! We use this class so that we can gather data from e.g. Pair objects without requiring
+//! that they hold onto their own collected bytes/values. When data values become available,
+//! they go directly into the BitBucket and are organized into data structures that SimDB
+//! expects (depends on the collected type).
+class BitBucket
+{
+public:
+    explicit BitBucket(simdb::TinyStrings<>* tiny_strings, simdb::argos::EnumInspector* enum_inspector) :
+        tiny_strings_(tiny_strings),
+        enum_inspector_(enum_inspector)
+    {}
+
+    virtual ~BitBucket() = default;
+    virtual void clear() = 0;
+    virtual void writeField(const void* data, uint32_t bytes, uint32_t field_id) = 0;
+    virtual void writeTo(simdb::argos::EntryPoint* entry_point) = 0;
+
+    template <typename T>
+    bool writeField(const T& val, uint32_t field_id) {
+        // Write bools as uint8_t
+        if constexpr (std::is_same_v<T, bool>) {
+            return writeField(val ? uint8_t(1) : uint8_t(0), field_id);
+        }
+
+        // Write strings as uint32_t via TinyStrings
+        else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<std::decay_t<T>, const char*>) {
+            return writeField(tiny_strings_->getStringID(val), field_id);
+        }
+
+        // Write enums as their underlying integer type.
+        else if constexpr (std::is_enum_v<T>) {
+            enum_inspector_->inspect(val);
+            using underlying_t = std::underlying_type_t<T>;
+            const underlying_t enum_int = static_cast<underlying_t>(val);
+            return writeField(enum_int, field_id);
+        }
+
+        // Write struct/class fields that provide exactly one cast-to-POD operator
+        else if constexpr (simdb::type_traits::is_pod_convertible_v<T> && !std::is_enum_v<T> &&
+                          (!std::is_trivial_v<T> || !std::is_standard_layout_v<T>)) {
+            using converted_t = simdb::type_traits::pod_convertible_t<T>();
+            static_assert(std::is_trivial_v<converted_t> && std::is_standard_layout_v<converted_t>);
+            auto converted_val = static_cast<converted_t>(val);
+            return writeField(converted_val, field_id);
+        }
+
+        // Write PODs (or TinyStrings uint32_t ID, or enums by their underlying type,
+        // or bools as uint8_t)
+        else if (std::is_trivial_v<T> && std::is_standard_layout_v<T>) {
+            writeField(&val, sizeof(T), field_id);
+            return true;
+        }
+
+        // Cannot collect this field. Let the caller decide what to do.
+        else {
+            return false;
+        }
+    }
+
+    simdb::TinyStrings<>* getTinyStrings() const {
+        return tiny_strings_;
+    }
+
+    simdb::argos::EnumInspector* getEnumInspector() const {
+        return enum_inspector_;
+    }
+
+private:
+    simdb::TinyStrings<>* tiny_strings_ = nullptr;
+    simdb::argos::EnumInspector* enum_inspector_ = nullptr;
+};
+
+//! BitBucket implementation for Collectable objects (whether "standalone"
+//! or inside an IterableCollector).
+class CollectableBitBucket : public BitBucket
+{
+public:
+    using BitBucket::BitBucket;
+
+    void clear() override final {
+        buffer_.clear();
+        buffer_.reserve(bytes_per_pass_);
+        bytes_per_pass_ = 0;
+    }
+
+    void writeField(const void* data, uint32_t bytes, uint32_t) override final {
+        auto src = static_cast<const char*>(data);
+        buffer_.resize(bytes_per_pass_ + bytes);
+        auto dst = &buffer_[bytes_per_pass_];
+        memcpy(dst, src, bytes);
+        bytes_per_pass_ += bytes;
+    }
+
+    //! Called when using a standalone Collectable
+    void writeTo(simdb::argos::EntryPoint* entry_point) override final {
+        entry_point->setScalarValueBytes(std::move(buffer_));
+        clear();
+    }
+
+    //! Called when inside an IterableCollector
+    void writeTo(std::vector<char> & dest) {
+        std::swap(dest, buffer_);
+        clear();
+    }
+
+private:
+    std::vector<char> buffer_;
+    size_t bytes_per_pass_ = 0;
+};
+
+template <bool Sparse>
+class IterableCollectorBitBucket;
+
+template <>
+class IterableCollectorBitBucket<true> : public BitBucket
+{
+public:
+    IterableCollectorBitBucket(simdb::TinyStrings<>* tiny_strings, simdb::argos::EnumInspector* enum_inspector, size_t capacity)
+        : BitBucket(tiny_strings, enum_inspector)
+        , capacity_(capacity)
+    {
+        while (capacity--)
+        {
+            bin_buckets_.emplace_back(std::make_unique<CollectableBitBucket>(tiny_strings, enum_inspector));
+        }
+    }
+
+    void clear() override final {
+        active_bin_idx_.clearValid();
+        all_bin_idxs_.clear();
+        all_bin_idxs_.reserve(capacity_);
+    }
+
+    void setActiveBinIdx(uint32_t bin_idx) {
+        assert(bin_idx <= UINT16_MAX);
+        assert(bin_idx < capacity_);
+        active_bin_idx_ = static_cast<uint16_t>(bin_idx);
+        all_bin_idxs_.emplace_back(bin_idx);
+    }
+
+    void writeField(const void* data, uint32_t bytes, uint32_t field_id) override final {
+        auto& bin_bucket = bin_buckets_[active_bin_idx_.getValue()];
+        assert(bin_bucket);
+        bin_bucket->writeField(data, bytes, field_id);
+    }
+
+    void writeTo(simdb::argos::EntryPoint* entry_point) override final {
+        for (auto bin_idx : all_bin_idxs_) {
+            bin_buckets_[bin_idx]->writeTo(all_bin_bytes_[bin_idx]);
+        }
+
+        entry_point->setSparseContainerBinBytes(std::move(all_bin_bytes_));
+        clear();
+    }
+
+private:
+    std::vector<std::unique_ptr<CollectableBitBucket>> bin_buckets_;
+    std::map<uint16_t, std::vector<char>> all_bin_bytes_;
+    utils::ValidValue<uint16_t> active_bin_idx_;
+    std::vector<uint16_t> all_bin_idxs_;
+    size_t capacity_ = 0;
+};
+
+template <>
+class IterableCollectorBitBucket<false> : public BitBucket
+{
+public:
+    IterableCollectorBitBucket(simdb::TinyStrings<>* tiny_strings, simdb::argos::EnumInspector* enum_inspector, size_t capacity)
+        : BitBucket(tiny_strings, enum_inspector)
+        , capacity_(capacity)
+    {
+        all_bin_bytes_.reserve(capacity_);
+        while (capacity--)
+        {
+            bin_buckets_.emplace_back(std::make_unique<CollectableBitBucket>(tiny_strings, enum_inspector));
+        }
+    }
+
+    void clear() override final {
+        container_size_ = 0;
+        all_bin_bytes_.reserve(capacity_);
+    }
+
+    void setActiveBinIdx(uint32_t bin_idx) {
+        assert(bin_idx <= UINT16_MAX);
+        assert(bin_idx == container_size_);
+        ++container_size_;
+    }
+
+    void writeField(const void* data, uint32_t bytes, uint32_t field_id) override final {
+        if (SPARTA_EXPECT_FALSE(container_size_ == 0)) {
+            setActiveBinIdx(0);
+        }
+        auto& bin_bucket = bin_buckets_[container_size_ - 1];
+        bin_bucket->writeField(data, bytes, field_id);
+    }
+
+    void writeTo(simdb::argos::EntryPoint* entry_point) override final {
+        all_bin_bytes_.resize(container_size_);
+        for (size_t i = 0; i < container_size_; ++i) {
+            auto & bin_bucket = bin_buckets_[i];
+            auto & bin_buffer = all_bin_bytes_[i];
+            bin_bucket->writeTo(bin_buffer);
+        }
+
+        entry_point->setContigContainerBinBytes(std::move(all_bin_bytes_));
+        clear();
+    }
+
+private:
+    std::vector<std::unique_ptr<CollectableBitBucket>> bin_buckets_;
+    std::vector<std::vector<char>> all_bin_bytes_;
+    uint16_t container_size_ = 0;
+    size_t capacity_ = 0;
+};
+
+} // sparta::collection