diff --git a/.github/actions/regress.sh b/.github/actions/regress.sh index 37da5177..22f191ac 100755 --- a/.github/actions/regress.sh +++ b/.github/actions/regress.sh @@ -16,3 +16,13 @@ if [ ${REGRESS_SIMDB} -ne 0 ]; then echo "ERROR: regress of SimDB FAILED!!!" exit 1 fi + +# simdb_regress includes ArgosSmoke, though it runs relatively quickly. +# For github checks, this will run for several minutes to fully bash +# collection and deserialization. +make ArgosSmoke_10k +REGRESS_ARGOS=$? +if [ ${REGRESS_ARGOS} -ne 0 ]; then + echo "ERROR: ArgosSmoke FAILED!!!" + exit 1 +fi diff --git a/docs/book/Makefile b/docs/book/Makefile new file mode 100644 index 00000000..fc0376ef --- /dev/null +++ b/docs/book/Makefile @@ -0,0 +1,61 @@ +# SimDB book — build with Asciidoctor (see README.md for install notes). +# +# make # same as make html +# make html # book.html +# make pdf # book.pdf (requires asciidoctor-pdf) + +BOOK := book.adoc +HTML := book.html +PDF := book.pdf +PARTS := $(wildcard parts/*.adoc) +IMAGES := $(wildcard images/*) +SRCS := $(BOOK) $(PARTS) $(IMAGES) + +ASCIIDOCTOR := asciidoctor +ASCIIDOCTOR_PDF := asciidoctor-pdf +ASCIIDOCTOR_OPTS := --failure-level=WARN + +.DEFAULT_GOAL := html + +.PHONY: html pdf check-asciidoctor check-asciidoctor-pdf + +html: check-asciidoctor $(HTML) + +pdf: check-asciidoctor-pdf $(PDF) + +$(HTML): $(SRCS) + $(ASCIIDOCTOR) $(ASCIIDOCTOR_OPTS) $(BOOK) + +$(PDF): $(SRCS) + $(ASCIIDOCTOR_PDF) $(ASCIIDOCTOR_OPTS) $(BOOK) + +check-asciidoctor: + @if ! command -v $(ASCIIDOCTOR) >/dev/null 2>&1; then \ + printf '\n*** Error: %s is not installed or not on PATH. ***\n\n' '$(ASCIIDOCTOR)'; \ + printf 'HTML builds need the Asciidoctor CLI.\n\n'; \ + printf 'Install (pick one):\n\n'; \ + printf ' Debian/Ubuntu package:\n'; \ + printf ' sudo apt-get install -y asciidoctor\n\n'; \ + printf ' Ruby gem (no sudo; installs under your home directory):\n'; \ + printf ' gem install --user-install asciidoctor\n'; \ + printf ' export PATH="$$(ruby -e '"'"'print Gem.user_dir'"'"')/bin:$$PATH"\n\n'; \ + printf ' Ruby gem (system-wide; needs sudo on most Linux setups):\n'; \ + printf ' sudo gem install asciidoctor\n\n'; \ + exit 1; \ + fi + +check-asciidoctor-pdf: + @if ! command -v $(ASCIIDOCTOR_PDF) >/dev/null 2>&1; then \ + printf '\n*** Error: %s is not installed or not on PATH. ***\n\n' '$(ASCIIDOCTOR_PDF)'; \ + printf 'PDF builds need the asciidoctor-pdf Ruby gem.\n'; \ + printf 'There is no asciidoctor-pdf package in Debian/Ubuntu apt — use gem.\n\n'; \ + printf 'Install (recommended; no sudo):\n\n'; \ + printf ' gem install --user-install asciidoctor-pdf\n'; \ + printf ' export PATH="$$(ruby -e '"'"'print Gem.user_dir'"'"')/bin:$$PATH"\n\n'; \ + printf 'Or system-wide:\n\n'; \ + printf ' sudo gem install asciidoctor-pdf\n\n'; \ + printf 'Plain "gem install ..." without --user-install often fails with:\n'; \ + printf ' Gem::FilePermissionError (cannot write to /var/lib/gems/...)\n\n'; \ + printf 'Then run: make pdf\n\n'; \ + exit 1; \ + fi diff --git a/docs/book/add-xrefs.py b/docs/book/add-xrefs.py new file mode 100644 index 00000000..f7f2f2ea --- /dev/null +++ b/docs/book/add-xrefs.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +"""Add chapter/part anchors and cross-reference links to SimDB book sources.""" + +from __future__ import annotations + +import re +from pathlib import Path + +BOOK_DIR = Path(__file__).resolve().parent +PART_FILES = [ + "parts/part1-why-simdb.adoc", + "parts/part2-getting-started.adoc", + "parts/part3-sqlite-interface.adoc", + "parts/part4-concurrent-pipelines.adoc", + "parts/part5-app-framework.adoc", + "parts/part6-advanced-pipelines.adoc", + "parts/part7-argos-case-study.adoc", + "parts/part8-patterns-cookbook.adoc", + "parts/part9-reference.adoc", +] + +PART_ANCHORS = { + 1: "part-i", + 2: "part-ii", + 3: "part-iii", + 4: "part-iv", + 5: "part-v", + 6: "part-vi", + 7: "part-vii", + 8: "part-viii", + 9: "part-ix", +} + +CHAPTER_ANCHORS = { + 1: "ch-01-introduction", + 2: "ch-02-simulation-data-problem", + 3: "ch-03-landscape", + 4: "ch-04-core-concepts", + 5: "ch-05-installing-simdb", + 6: "ch-06-building-cmake", + 7: "ch-07-first-program", + 8: "ch-08-regression-tests", + 9: "ch-09-defining-schema", + 10: "ch-10-database-manager", + 11: "ch-11-inserting-data", + 12: "ch-12-querying-data", + 13: "ch-13-transactions", + 14: "ch-14-blobs-compression", + 15: "ch-15-inspecting-dumping", + 16: "ch-16-pipeline-fundamentals", + 17: "ch-17-ports-concurrent-queue", + 18: "ch-18-building-pipeline", + 19: "ch-19-database-stages", + 20: "ch-20-move-semantics", + 21: "ch-21-pipeline-examples", + 22: "ch-22-simdb-app", + 23: "ch-23-app-manager", + 24: "ch-24-multiple-apps", + 25: "ch-25-app-factories", + 26: "ch-26-flushers", + 27: "ch-27-snoopers", + 28: "ch-28-disabling-pipelines", + 29: "ch-29-async-db-access", + 30: "ch-30-threads-scheduling", + 31: "ch-31-argos-is", + 32: "ch-32-argos-collector", + 33: "ch-33-on-disk-model", + 34: "ch-34-argos-viewer", + 35: "ch-35-end-to-end", + 36: "ch-36-schema-pipeline-together", + 37: "ch-37-stage-boundaries", + 38: "ch-38-performance-tuning", + 39: "ch-39-determinism-debugging", + 40: "ch-40-application-shapes", + 41: "ch-41-testing-simdb-apps", +} + +APPENDIX_ANCHORS = { + "Utilities Reference": "appendix-utilities-reference", + "API Quick Reference & Glossary": "appendix-api-glossary", + "FAQ & Troubleshooting": "appendix-faq", + "Migration Guide": "appendix-migration-guide", +} + +FAQ_SECTION_ANCHORS = { + '"Database is locked" / SQLite contention': "faq-database-locked", + '"Why is my pipeline sleeping?"': "faq-pipeline-sleeping", + '"Why don\'t I see my rows yet?"': "faq-rows-not-visible", + '"My database is partial or corrupt after a crash"': "faq-partial-database", + '"Why did CI fail in Release but not Debug?"': "faq-release-werror", + '"Snooper never finds my key"': "faq-snooper-key", + '"AsyncDatabaseAccessor eval hangs or times out"': "faq-async-eval-hangs", +} + +ROMAN = {1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII", 8: "VIII", 9: "IX"} + + +def part_number(rel: str) -> int: + m = re.search(r"part(\d+)", rel) + assert m, rel + return int(m.group(1)) + + +def ensure_anchors() -> None: + ch = 1 + for idx, rel in enumerate(PART_FILES, start=1): + path = BOOK_DIR / rel + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + out: list[str] = [] + part_num = part_number(rel) + part_anchor = PART_ANCHORS[idx] + if lines and not lines[0].strip().startswith(f"[#{part_anchor}]"): + out.append(f"[#{part_anchor}]\n") + for line in lines: + if line.strip().startswith(f"[#{part_anchor}]"): + continue + if line.startswith("== ") and not line.startswith("=== "): + title = line[3:].strip() + if part_num == 9 and title in APPENDIX_ANCHORS: + anchor = APPENDIX_ANCHORS[title] + else: + anchor = CHAPTER_ANCHORS[ch] + ch += 1 + if not out or not out[-1].strip().startswith(f"[#{anchor}]"): + out.append(f"[#{anchor}]\n") + if line.startswith("=== ") and part_num == 9: + title = line[4:].strip() + if title in FAQ_SECTION_ANCHORS: + anchor = FAQ_SECTION_ANCHORS[title] + if not out or not out[-1].strip().startswith(f"[#{anchor}]"): + out.append(f"[#{anchor}]\n") + out.append(line) + path.write_text("".join(out), encoding="utf-8") + + +def fix_part_titles(text: str) -> str: + for n in range(1, 10): + roman = ROMAN[n] + text = re.sub( + rf"^= <>:", + f"= Part {roman}:", + text, + flags=re.MULTILINE | re.IGNORECASE, + ) + # exact anchor ids + anchor = PART_ANCHORS[n] + text = re.sub( + rf"^= <<{anchor},Part {roman}>>:", + f"= Part {roman}:", + text, + flags=re.MULTILINE, + ) + return text + + +def dedupe_nested_xrefs(text: str) -> str: + prev = None + while prev != text: + prev = text + text = re.sub(r"<<([^,>]+),<<\1,([^>]+)>>>>", r"<<\1,\2>>", text) + return text + + +def fix_truncated_xrefs(text: str) -> str: + """Undo part/chapter xrefs split by an over-eager repair pass.""" + for num, anchor in CHAPTER_ANCHORS.items(): + digits = str(num) + for split in range(1, len(digits)): + text = text.replace( + f"<<{anchor},Chapter {digits[:split]}>>{digits[split:]}>>", + xref(anchor, f"Chapter {num}"), + ) + for n, anchor in PART_ANCHORS.items(): + roman = ROMAN[n] + for split in range(1, len(roman)): + prefix = roman[:split] + suffix = roman[split:] + text = text.replace( + f"<<{anchor},Part {prefix}>>{suffix}>>", + xref(anchor, f"Part {roman}"), + ) + return text + + +def repair_broken_xrefs(text: str) -> str: + """Close xrefs that lost their trailing `>>` during earlier passes.""" + part_label = r"Part (?:VIII|VII|III|IV|VI|II|IX|I|V)" + text = re.sub( + rf"<<([a-z0-9-]+),({part_label})(?![>A-Za-z])", + r"<<\1,\2>>", + text, + ) + text = re.sub( + r"<<([a-z0-9-]+),(Chapter \d+)(?![>0-9])", + r"<<\1,\2>>", + text, + ) + return text + + +def normalize_xref_syntax(text: str) -> str: + text = fix_truncated_xrefs(text) + text = repair_broken_xrefs(text) + text = dedupe_nested_xrefs(text) + while ">>>>" in text: + text = text.replace(">>>>", ">>") + return text + + +def strip_all_xrefs(text: str) -> str: + """Expand xrefs back to visible labels for a clean re-link pass.""" + text = normalize_xref_syntax(text) + text = re.sub(r"(?:part-[a-z0-9-]+,)+", "", text) + text = re.sub(r"<<([^,>]+),([^>]+)>>", r"\2", text) + text = re.sub(r"<<([^>]+)>>", r"\1", text) + text = re.sub( + r"\bChapters Chapter (\d+) and Chapter (\d+)\b", + r"Chapters \1 and \2", + text, + ) + return text + + +def xref(anchor: str, label: str) -> str: + return f"<<{anchor},{label}>>" + + +def linkify_compounds(text: str) -> str: + """Multi-part and part+chapter phrases (always emit complete xrefs).""" + replacements = [ + (r"Parts I--VIII", f"Parts {xref('part-i','Part I')}--{xref('part-viii','Part VIII')}"), + (r"Parts III--VI", f"Parts {xref('part-iii','Part III')}--{xref('part-vi','Part VI')}"), + (r"Parts IV through VI", f"Parts {xref('part-iv','Part IV')} through {xref('part-vi','Part VI')}"), + (r"Parts IV--VI", f"Parts {xref('part-iv','Part IV')}--{xref('part-vi','Part VI')}"), + (r"Parts II and III", f"Parts {xref('part-ii','Part II')} and {xref('part-iii','Part III')}"), + (r"Parts VIII and IX", f"Parts {xref('part-viii','Part VIII')} and {xref('part-ix','Part IX')}"), + (r"Parts III through V", f"Parts {xref('part-iii','Part III')} through {xref('part-v','Part V')}"), + (r"Part IV, Part VIII", f"{xref('part-iv','Part IV')}, {xref('part-viii','Part VIII')}"), + (r"Part III, Part VIII Ch 36", f"{xref('part-iii','Part III')}, {xref('part-viii','Part VIII')} {xref('ch-36-schema-pipeline-together','Chapter 36')}"), + ( + r"Part III, Part VIII Ch (\d+)", + lambda m: ( + f"{xref('part-iii','Part III')}, {xref('part-viii','Part VIII')} " + f"{xref(CHAPTER_ANCHORS[int(m.group(1))], f'Chapter {m.group(1)}')}" + ), + ), + (r"Part IV, Part VIII Ch 37", f"{xref('part-iv','Part IV')}, {xref('part-viii','Part VIII')} {xref('ch-37-stage-boundaries','Chapter 37')}"), + (r"Part V, Part VIII Ch 38", f"{xref('part-v','Part V')}, {xref('part-viii','Part VIII')} {xref('ch-38-performance-tuning','Chapter 38')}"), + ] + for pattern, repl in replacements: + text = re.sub(pattern, repl, text) + + text = re.sub( + r"Part VIII Ch (\d+)", + lambda m: f"{xref('part-viii','Part VIII')} {xref(CHAPTER_ANCHORS[int(m.group(1))], f'Chapter {m.group(1)}')}", + text, + ) + text = re.sub( + r"Part VIII, Chapter (\d+)", + lambda m: f"{xref('part-viii','Part VIII')}, {xref(CHAPTER_ANCHORS[int(m.group(1))], f'Chapter {m.group(1)}')}", + text, + ) + return text + + +def linkify_parts_and_chapters(text: str) -> str: + """Link lone Part/Chapter mentions in plain prose only.""" + text = re.sub( + r"Chapters (\d+) and (\d+)", + lambda m: ( + f"Chapters {xref(CHAPTER_ANCHORS[int(m.group(1))], f'Chapter {m.group(1)}')} " + f"and {xref(CHAPTER_ANCHORS[int(m.group(2))], f'Chapter {m.group(2)}')}" + ), + text, + ) + segments = re.split(r"(<<[^>]+>>)", text) + out: list[str] = [] + for seg in segments: + if seg.startswith("<<"): + out.append(seg) + continue + for n in range(9, 0, -1): + roman = ROMAN[n] + anchor = PART_ANCHORS[n] + seg = re.sub( + rf"\bPart {roman}'s\b", + f"{xref(anchor, f'Part {roman}')}'s", + seg, + ) + for num in sorted(CHAPTER_ANCHORS.keys(), reverse=True): + seg = re.sub( + rf"\bChapter {num}\b", + xref(CHAPTER_ANCHORS[num], f"Chapter {num}"), + seg, + ) + seg = re.sub( + r"\bCh\. (\d+)\b", + lambda m: xref(CHAPTER_ANCHORS[int(m.group(1))], f"Chapter {m.group(1)}"), + seg, + ) + for n in range(9, 0, -1): + roman = ROMAN[n] + anchor = PART_ANCHORS[n] + seg = re.sub(rf"\bPart {roman}\b", xref(anchor, f"Part {roman}"), seg) + faq_rows = xref("faq-rows-not-visible", "Why don't I see my rows yet?") + seg = seg.replace('FAQ\'s "Why don\'t I see my rows yet?"', f"FAQ's {faq_rows}") + out.append(seg) + return "".join(out) + + +def linkify_prose(text: str) -> str: + text = linkify_compounds(text) + segments = re.split(r"(<<[^>]+>>)", text) + return "".join( + seg if seg.startswith("<<") else linkify_parts_and_chapters(seg) for seg in segments + ) + + +def linkify_segment(text: str) -> str: + return linkify_prose(text) + + +def linkify_plain(text: str) -> str: + return linkify_segment(text) + + +def linkify_outside_xrefs(text: str) -> str: + return linkify_segment(text) + + +def process_book_intro() -> None: + path = BOOK_DIR / "book.adoc" + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + out: list[str] = [] + for line in lines: + if line.startswith("= SimDB") or line.startswith("include::"): + out.append(line) + continue + if line.strip().startswith(":") or line.strip().startswith("[") or line.strip().startswith("//"): + out.append(line) + continue + out.append(normalize_xref_syntax(linkify_segment(line))) + path.write_text("".join(out), encoding="utf-8") + + +def process_file(rel: str) -> None: + path = BOOK_DIR / rel + text = fix_part_titles(path.read_text(encoding="utf-8")) + lines = text.splitlines(keepends=True) + out: list[str] = [] + in_literal = False + source_pending = False + for line in lines: + if line.startswith("[source") or line.startswith("[,cpp]") or line.startswith("[,bash]"): + source_pending = True + out.append(line) + continue + if line.strip() == "----": + if source_pending: + source_pending = False + in_literal = True + elif in_literal: + in_literal = False + else: + in_literal = True + out.append(line) + continue + if line.startswith("...."): + in_literal = not in_literal + out.append(line) + continue + if in_literal: + out.append(line) + continue + if line.startswith("= ") and not line.startswith("=="): + out.append(line) + continue + if line.strip().startswith("[#"): + out.append(line) + continue + out.append(normalize_xref_syntax(linkify_outside_xrefs(line))) + path.write_text("".join(out), encoding="utf-8") + + +def main() -> None: + ensure_anchors() + for rel in PART_FILES: + process_file(rel) + process_book_intro() + print("Cross-references added.") + + +if __name__ == "__main__": + main() diff --git a/docs/book/book.adoc b/docs/book/book.adoc new file mode 100644 index 00000000..841ffcdb --- /dev/null +++ b/docs/book/book.adoc @@ -0,0 +1,64 @@ += SimDB: The Complete Guide +:doctype: book +:toc: left +:toclevels: 2 +:sectnums: +:sectnumlevels: 1 +:icons: font +:source-highlighter: highlight.js +:experimental: +:imagesdir: images + +[colophon] += About This Book + +SimDB is a high-performance, header-only C++17 module that unifies *concurrent +data pipelines* with *SQLite* to power scalable simulation data engines, +analysis tooling, and UI backends. + +This book is written for a specific reader: the *domain expert* -- a hardware +architect, performance modeler, or researcher -- who needs to move large volumes +of simulation data safely and quickly, but who does not write software for a +living. SimDB's guiding principle is that it should be *easy to use and hard to +misuse*. This book follows that same principle. + +We build understanding in layers: + +* *<>* explains _why_ SimDB exists and how it differs from the alternatives. +* *Parts II--III* get you productive with the standalone SQLite interface. +* *Parts <>--<>* introduce the concurrent pipeline and app frameworks. +* *<>* is an end-to-end case study of Argos, SimDB's flagship + collection-and-visualization application. +* *Parts VIII--IX* collect patterns, recipes, and reference material. + +[NOTE] +==== +Parts I--IX are drafted in prose with cross-reference links throughout. +==== + +// Part I +include::parts/part1-why-simdb.adoc[] + +// Part II +include::parts/part2-getting-started.adoc[] + +// Part III +include::parts/part3-sqlite-interface.adoc[] + +// Part IV +include::parts/part4-concurrent-pipelines.adoc[] + +// Part V +include::parts/part5-app-framework.adoc[] + +// Part VI +include::parts/part6-advanced-pipelines.adoc[] + +// Part VII +include::parts/part7-argos-case-study.adoc[] + +// Part VIII +include::parts/part8-patterns-cookbook.adoc[] + +// Part IX +include::parts/part9-reference.adoc[] diff --git a/docs/book/book.html b/docs/book/book.html new file mode 100644 index 00000000..5f688077 --- /dev/null +++ b/docs/book/book.html @@ -0,0 +1,8652 @@ + + + + + + + +SimDB: The Complete Guide + + + + + + + +
+
+

About This Book

+
+
+

SimDB is a high-performance, header-only C++17 module that unifies concurrent +data pipelines with SQLite to power scalable simulation data engines, +analysis tooling, and UI backends.

+
+
+

This book is written for a specific reader: the domain expert — a hardware +architect, performance modeler, or researcher — who needs to move large volumes +of simulation data safely and quickly, but who does not write software for a +living. SimDB’s guiding principle is that it should be easy to use and hard to +misuse. This book follows that same principle.

+
+
+

We build understanding in layers:

+
+
+
    +
  • +

    Part I explains why SimDB exists and how it differs from the alternatives.

    +
    • +
  • +

    Parts II—​III get you productive with the standalone SQLite interface.

    +
    • +
  • +

    Parts Part IV--Part VI introduce the concurrent pipeline and app frameworks.

    +
    • +
  • +

    Part VII is an end-to-end case study of Argos, SimDB’s flagship +collection-and-visualization application.

    +
    • +
  • +

    Parts VIII—​IX collect patterns, recipes, and reference material.

    +
    • +
+
+
+ + + + + +
+ + +
+

Parts I—​IX are drafted in prose with cross-reference links throughout.

+
+
+
+
+
+

Part I: Why SimDB

+
+
+This part answers the question a brand-new user asks first: "What is SimDB, and +why would I choose it over the tools I already know?" No code yet — just the +problem, the landscape, and the mental model you will carry through the rest of +the book. +
+
+
+

1. Introduction

+
+
+

Every simulation produces data, and sooner or later that data becomes the +project. Statistics pile up. Traces balloon. Someone wants a live view of what +the model is doing, someone else wants to replay a run from an hour ago, and a +third person wants to compare a thousand runs at once. The simulator was the +interesting part; moving its data around safely and quickly turns into a +second, unplanned engineering project.

+
+
+

SimDB exists to make that second project disappear.

+
+
+

The one-sentence version

+
+

SimDB is a header-only C++17 module that unifies concurrent data pipelines +with SQLite to power scalable simulation data engines, analysis tooling, and +user-interface backends.

+
+
+

"Header-only" means there is no library to build and link against separately: +you include SimDB’s headers and compile. "Concurrent data pipelines" means you +can process data on multiple threads without hand-writing the locking and +coordination that usually comes with that. And "SQLite" means everything your +simulation emits can land in a single, ordinary database file that any tool, +language, or teammate can open later.

+
+
+

The thesis of the whole project fits on one line:

+
+
+

SQLite + Concurrency + Flexibility — safe and fast.

+
+
+

Plenty of libraries give you concurrency. A different set of libraries give you +convenient access to SQLite. SimDB’s reason to exist is that it gives you both +at once, wired together so the fast path is also the safe path. We will make +that claim concrete when we survey the alternatives later in this part.

+
+
+
+

Who this book is for

+
+

This book is written for a specific reader: the domain expert. You might be a +hardware architect, a performance modeler, or a researcher. You know your +problem domain deeply, you have a real imagination for what you want to build, +and you write code to get your work done — but you do not write software for a +living, and you would rather not become a concurrency specialist just to save +your results.

+
+
+

That reader shaped SimDB’s single most important design imperative: it should be +easy to use and hard to misuse. SimDB was deliberately not built "for software +engineers, by software engineers". The sharp edges that usually come with +threading and databases — race conditions, lock contention, half-written files, +transactions that are too small to be fast or too large to be safe — are +handled for you by default. You are meant to spend your attention on what to +capture and how to look at it, not on the machinery that moves it.

+
+
+ + + + + +
+ + +
+

You do not need to be a database or threading expert to use this book. Where a +concept from either world matters, we introduce it in plain terms the first time +it comes up.

+
+
+
+
+

If you are a career software engineer, you are very welcome here too — SimDB +will not get in your way, and the later parts expose enough control for advanced +use. But when a tradeoff had to be made between raw flexibility and a design +that is hard to get wrong, SimDB chose the latter, and this book takes the same +stance.

+
+
+
+

The three pillars

+
+

SimDB’s value rests on three ideas that recur throughout the book.

+
+
+

Unified pipelines and SQLite. SimDB pairs a concurrent processing pipeline +with native SQLite integration, so database work is batched and routed +automatically onto a single database thread. You get the throughput of a +purpose-built pipeline and the convenience of a real, queryable database, +without the two fighting each other for access.

+
+
+

High performance. SimDB is built for the kind of throughput that fast +simulations demand — and it gets there by doing the heavy data work on its own +threads, so your simulator itself does not have to be multi-threaded to benefit. +It minimizes transaction overhead, eliminates database access contention, and +moves data through the pipeline by move (rather than copy) whenever you let it.

+
+
+

A clean break for complex codebases. Large simulation projects tend to +accumulate tangled data paths and "just write it to a file" habits that are hard +to unwind. SimDB replaces that sprawl with one unified, optimized path from +"data produced" to "data stored", which lets teams reach a performant solution +with fewer moving parts.

+
+
+
+

What you can build

+
+

SimDB is a foundation, not a single-purpose tool. The same building blocks +support a wide range of workflows:

+
+
+
    +
  • +

    Fast, scalable simulation statistics engines.

    +
    • +
  • +

    Backends for live dashboards or post-run analysis user interfaces.

    +
    • +
  • +

    Simulation state replayers that reconstruct what the model looked like at any +point in time.

    +
    • +
  • +

    Aggregate analysis across hundreds or thousands of simulation runs.

    +
    • +
+
+
+

If your work involves producing simulation data and then needing to store, +inspect, replay, or compare it, it very likely fits one of these shapes — and +Part VII walks through a complete, real example (Argos) that does several of +them at once.

+
+
+
+

How to read this book

+
+

The book is built in layers, and the parts are ordered so each one leans only on +what came before.

+
+
+
    +
  • +

    Part I (this part) makes the case for SimDB: the problem it solves, how it +compares to the alternatives, and the mental model you will carry everywhere +else.

    +
    • +
  • +

    Parts Part II and Part III get you productive with the standalone SQLite interface — installing SimDB, building a first program, and working with schemas, inserts, +queries, and transactions. This is where code snippets begin.

    +
    • +
  • +

    Parts Part IV through Part VI introduce the concurrent pipeline and the application +framework that composes pipelines and schemas into cohesive units.

    +
    • +
  • +

    Part VII is an end-to-end case study of Argos, SimDB’s flagship +collection-and-visualization system.

    +
    • +
  • +

    Parts Part VIII and Part IX collect patterns, recipes, and reference material for when +you are building in earnest.

    +
    • +
+
+
+ + + + + +
+ + +
+

This part is intentionally concept-first: there is no code in it. If you learn +best by running something, you can jump to Part II now and refer back here when +you want the "why" behind what you are doing.

+
+
+
+
+

Wherever you start, the next chapter is the place to understand the problem +SimDB was built to solve — because the shape of the solution follows directly +from it.

+
+
+
+
+
+

2. The Simulation Data Problem

+
+
+

Before we can appreciate what SimDB does, it helps to look squarely at the +problem it was built to solve. If you have written a simulator of any real size, +some of what follows will feel familiar — perhaps uncomfortably so.

+
+
+

Where the data comes from

+
+

A simulation is a data generator. As it runs, it produces a steady stream of +things you will want to keep and look at later:

+
+
+
    +
  • +

    Statistics — counters, rates, histograms, and rolled-up metrics that +summarize how the model behaved.

    +
    • +
  • +

    Traces — fine-grained, often per-cycle records of what happened, used for +debugging and detailed analysis.

    +
    • +
  • +

    Checkpoints — snapshots of the model’s state, so a run can be inspected or +replayed after the fact.

    +
    • +
  • +

    Events — notable occurrences worth logging as they happen.

    +
    • +
  • +

    User-interface updates — data feeding a live dashboard or a +post-run visualization tool.

    +
    • +
+
+
+

Individually, none of these is hard to write down. The difficulty is one of +scale and speed: a fast simulator can produce this data faster than a naive +output path can absorb it. The quicker your model runs, the more the plumbing +that carries its data becomes the thing that limits you. It is a common and +demoralizing surprise to discover that a carefully optimized simulator spends +much of its time simply trying to save its own results.

+
+
+
+

Two hard problems at once

+
+

The reason this plumbing is hard is that it forces two difficult requirements +together:

+
+
+
    +
  1. +

    The data handling must not stall the simulation. Even a single-threaded +model runs fast, and it cannot afford to stop and wait on disk (or on +compression, or on a database commit) every time it emits a value. The heavy +data work has to happen off to the side — concurrently — so the simulation +can keep moving.

    +
    2. +
  2. +

    The data must be persisted durably, in a form you can query and share long +after the run is over.

    +
    3. +
+
+
+

Each of these is manageable on its own. Together they are in tension. To keep up +without stalling, the data work must be spread across background threads — but a +file on disk fundamentally wants a single writer. Resolve that tension the +obvious ways and you get one of two bad outcomes: serialize everything through +one lock and the simulation crawls, or let threads race and you get corruption, +lost data, and heisenbugs.

+
+
+ + + + + +
+ + +
+

Your simulator does not need to be multi-threaded for any of this to matter, and +you do not need to write threaded code to fix it. Most hardware simulators are +single-threaded, and that is perfectly fine: SimDB supplies the concurrency for +you, running the data and database work on its own background threads while your +model runs on its own. The tension above is SimDB’s problem to solve, not yours.

+
+
+
+
+
+

The hidden costs of doing it by hand

+
+

Reaching for SQLite directly is a reasonable instinct — it is a real database +in a single file, with no server to run. But used naively to keep up with a fast +simulator, it has sharp edges that are easy to find and hard to remove:

+
+
+
    +
  • +

    Tiny transactions. Committing one row at a time turns every write into a +durability barrier. The overhead dwarfs the actual work, and throughput +collapses.

    +
    • +
  • +

    Implicit file locks. SQLite guards the database file with locks. When +several threads try to write at once, they block, retry, or simply fail with +"database is locked".

    +
    • +
  • +

    Schema-level contention. Coordinating which thread may touch which table, +and when, becomes your responsibility — and getting it wrong is subtle.

    +
    • +
+
+
+

Handle all of this correctly and you have, in effect, taken a second job as a +part-time database concurrency engineer. That is precisely the job SimDB is +meant to spare you.

+
+
+
+

The anti-patterns SimDB replaces

+
+

Faced with these costs, large simulation projects tend to drift into a few +recognizable habits, none of which age well:

+
+
+
    +
  • +

    "Write everything to files." Each subsystem invents its own output file and +its own format. You end up with many formats, many readers, and many bugs — and no single place to ask a question that spans them.

    +
    • +
  • +

    Ad hoc binary formats. Fast to write today, painful forever after: they are +hard to version, hard to evolve, and hard for anyone (including future you) to +read back.

    +
    • +
  • +

    Tangled data paths and unbounded coupling. The simulator’s core sprouts +tendrils into output code until you can no longer change one without breaking +the other.

    +
    • +
+
+
+

These patterns are not the sign of a careless team; they are the natural result +of the underlying problem being genuinely hard. SimDB’s proposition is that the +problem is better solved once, underneath, than re-solved badly in every +project. It replaces the sprawl with one unified, highly optimized path — from +"data produced" to "data stored" — so teams reach a performant solution with +far fewer moving parts.

+
+
+

The obvious question is whether some existing library already does this. That is +what the next chapter examines.

+
+
+
+
+
+

3. The Landscape

+
+
+

Chapter 2 ended with a fair question: surely some existing library already +solves this? There are many excellent libraries in this space, and it is worth +understanding them — both to see why none of them quite fits, and to know when +one of them is actually the better choice than SimDB.

+
+
+

When you go looking, you find two separate shelves. On one are toolkits for +moving work across threads. On the other are conveniences for talking to SQLite. +What you do not find is something that is genuinely both.

+
+
+

Concurrency and dataflow toolkits

+
+

The first shelf holds mature, well-respected libraries for concurrency and +dataflow:

+
+
+
    +
  • +

    Intel TBB (oneTBB) — a flow graph, parallel algorithms, and concurrent +containers.

    +
    • +
  • +

    RaftLib — streaming "kernels" connected by data streams.

    +
    • +
  • +

    Folly — building blocks such as lock-free queues, futures, and executors.

    +
    • +
  • +

    HPX — an asynchronous, many-task runtime.

    +
    • +
  • +

    CAF — the actor model for C++.

    +
    • +
+
+
+

These are very good at what they do: they help you spread work across threads +and pass data between the pieces. What none of them offers is any notion of +durable, queryable storage. Persistence is left entirely to you — which drops +you right back into the SQLite problems from the previous chapter. They solve +the "concurrently" half of the problem and are silent on the "persist it +durably" half.

+
+
+
+

SQLite convenience libraries

+
+

The second shelf holds pleasant C++ wrappers over SQLite:

+
+
+
    +
  • +

    SQLiteCpp — a modern, RAII-style C++ wrapper.

    +
    • +
  • +

    SOCI — a general database access library with a SQLite backend.

    +
    • +
  • +

    sqlite_orm and sqlpp11 — type-safe, ORM- and query-builder-style +interfaces.

    +
    • +
+
+
+

These make SQL access in C++ genuinely nicer to write and safer to get right. +What none of them offers is a concurrency model. They assume you have already +decided who writes when; batching, locking, and contention remain your problem. +They are a fine choice when database access is simple and occasional — and a +poor one for a high-rate simulation data path, because they leave the hardest +part exactly where it was.

+
+
+
+

The gap SimDB fills

+
+

Neither shelf solves the combined problem, and the combined problem is the whole +point. SimDB exists to unify a safe concurrent pipeline with first-class +SQLite, so that:

+
+
+
    +
  • +

    database work is funneled onto a single database thread, with no contention;

    +
    • +
  • +

    operations are batched into implicit transactions and retried until they +succeed, with no BEGIN/COMMIT from you;

    +
    • +
  • +

    worker threads are shared across independent workloads so they scale together;

    +
    • +
  • +

    and the whole thing is approachable enough for a domain expert to use +correctly on the first try.

    +
    • +
+
+
+

That is the meaning of the one-line thesis from Chapter 1 — SQLite
+Concurrency + Flexibility, safe and fast — and it is what no single library on +either shelf provides.

+
+
+ + + + + +
+ + +
+

Could you assemble a concurrency toolkit from the first shelf and a SQLite +wrapper from the second and wire them together yourself? Absolutely — and you +would be taking on precisely the hard integration work that SimDB was built to +do once, carefully, so that you do not have to.

+
+
+
+
+
+

When not to use SimDB

+
+

Being honest about the fit cuts both ways. SimDB is probably not the right tool +when:

+
+
+
    +
  • +

    You do not need persistence at all. If your work is pure in-memory +computation with no data to store, a parallel-algorithms library from the +first shelf is a lighter choice.

    +
    • +
  • +

    You need neither concurrency nor volume. If you write only a handful of rows, +occasionally, a thin SQLite wrapper (or raw SQLite) is simpler and perfectly +adequate.

    +
    • +
  • +

    You need a networked or multi-process database, or streaming across many +machines. Systems like a client/server RDBMS or a distributed log (for +example, Kafka) target a different problem. SimDB is designed for a single +simulation process writing to a single local database file.

    +
    • +
+
+
+

Within its target — one simulation process producing data that must be stored, +queried, replayed, or compared, quickly and safely — SimDB is purpose-built, +and the rest of this book assumes you are in that target.

+
+
+

With the "why" settled, the next chapter gives you the small vocabulary and +mental model that everything else in the book builds on.

+
+
+
+
+
+

4. Core Concepts & Mental Model

+
+
+

The rest of this book uses a small, consistent vocabulary. If you internalize +the handful of ideas in this chapter, everything that follows — schemas, +pipelines, apps, and the Argos case study — will feel like variations on a +theme you already know. There is no code here; this is the map, not the +territory.

+
+
+

One database to rule them all

+
+

The first principle is the simplest: a simulator has one database, and +everything goes into it.

+
+
+

Simulation output tends to sprawl. Statistics land in one file, traces in +another, a debug log somewhere else, and analysis results in a folder nobody can +find six months later. Each format needs its own reader, its own tooling, and +its own tribal knowledge. SimDB rejects that sprawl. All of it — statistics, +trace data, user-interface updates, analysis results — is consolidated into a +single, queryable SQLite database.

+
+
+

That single file is an ordinary SQLite database, which means it is also a +durable, portable artifact. You can archive it, copy it to a colleague, or open +it with any SQLite-compatible tool in any language, long after the simulation +that produced it has finished. The database is the deliverable.

+
+
+
+

The single database thread

+
+

The second principle is the one that makes the rest possible: only one thread +ever talks to the database.

+
+
+

A SQLite database lives on the filesystem, and letting many threads hammer a +filesystem-backed database at once is never fast enough for a high-volume +simulation — you get lock contention, retries, and unpredictable stalls. So +SimDB does the opposite of what a naive design would do: instead of many threads +racing for the database, every database operation is funneled onto one dedicated +database thread (and there is only ever one, or none at all).

+
+
+

Concentrating database work on a single thread is what lets SimDB hand you three +guarantees that would otherwise be your problem to enforce:

+
+
+
    +
  • +

    No user-facing transactions required. You never write BEGIN or COMMIT. +SimDB gathers many operations together and commits them as large, efficient +implicit transactions, which is dramatically faster than many tiny ones.

    +
    • +
  • +

    Retry-on-fail. If the database is momentarily busy or a table is locked, +SimDB keeps retrying the work until it succeeds, rather than failing back to +you.

    +
    • +
  • +

    No manual concurrency management. You do not coordinate access to SQLite, +reason about file locks, or worry about schema-level contention. Safe, +high-throughput execution under heavy parallel load is the default.

    +
    • +
+
+
+ + + + + +
+ + +
+

"Only one database thread" is a rule about writing to the database, not a +limit on your simulation’s parallelism. Your data-processing work can run across +as many threads as you like; it is only the final hand-off to SQLite that is +serialized — on purpose.

+
+
+
+
+
+

The object model

+
+

SimDB gives you four kinds of building blocks. From the outside in:

+
+
+
    +
  • +

    An App is a self-contained unit of functionality that owns a database +schema and, typically, one pipeline. A logger is an app; a statistics +collector is an app; a UI backend is an app. Several producers usually fan in +through multiple pipeline heads on that single pipeline rather than through +multiple pipelines per app. The guiding paradigm is one simulator with many +apps, all writing to the single shared database.

    +
    • +
  • +

    A Pipeline is an ordered arrangement of stages that transforms data on its +way to the database.

    +
    • +
  • +

    A Stage is one step of work. An ordinary stage runs on its own thread. A +special DatabaseStage is the only kind of stage allowed to write to the +database, and every database stage — from every app — runs on that one +shared database thread.

    +
    • +
  • +

    Stages are wired together with ports (a stage’s typed inputs and outputs) +and queues. A queue is a thread-safe, first-in-first-out hand-off that sits +between one stage’s output and the next stage’s input.

    +
    • +
+
+
+

Put together, data flows from your simulation, through one or more apps' +pipelines, and finally onto the single database thread that persists it:

+
+
+
The SimDB mental model: many apps and threads, one database thread, one database
+
+
  Your simulation
+        |
+        |  hand data to an app (moved when possible, copied when needed)
+        v
+  App .......... owns a Schema and one or more Pipelines
+        |        (a simulator may run many apps over ONE database)
+        v
+  Pipeline ..... a chain of Stages connected by thread-safe Queues
+        |
+        |   [Stage] --queue--> [Stage] --queue--> [DatabaseStage]
+        |    own thread         own thread          shared DB thread
+        v
+  Single database thread
+        |        every DatabaseStage, from every app, runs here and
+        |        ONLY here; it batches writes into implicit transactions
+        |        and retries them until they succeed
+        v
+  One SQLite database  (the single source of truth)
+
+
+
+

You will meet each of these pieces in detail in Parts Part III through Part V, and see +them assembled into a real system in Part VII. For now it is enough to hold the +shape in your head: work fans out across many stages and threads, then funnels +back down to one database thread and one database.

+
+
+
+

The threading model

+
+

The durable rule of SimDB’s threading is the one you already know: there is +exactly one database thread, shared by every database stage, and you cannot ask +for a private one. That constraint is not a limitation to work around — it is +the very thing that keeps database access fast and contention-free.

+
+
+

Around that fixed point, the pipeline’s other stages run on ordinary worker +threads. How those worker threads are allocated is an area SimDB is still +evolving: today the thread count grows with the amount of pipeline work you +create, so running many apps at once is not yet as thread-efficient as it will +be. Part V returns to this with the specifics; the takeaway here is simply that +the single database thread is the permanent anchor of the model.

+
+
+
+

Move-only by default

+
+

The last idea is about performance. Data travels through a pipeline’s queues by +move whenever possible: instead of copying a payload from one stage to the +next, SimDB transfers ownership of it, which avoids duplicating potentially +large buffers. When you genuinely need to keep your own copy of the data, you +can copy instead — SimDB supports both — but the fast path, and the default +you should reach for, is to move.

+
+
+

This is why you will often see large payloads (vectors of statistics, compressed +byte buffers) flow through a SimDB pipeline with almost no copying overhead. Keep +"move when you can, copy when you must" in mind and your pipelines will be fast +by construction.

+
+
+
+

A quick glossary

+
+

For reference, here are the core terms as this book uses them:

+
+
+
+
App
+
+

A self-contained unit that owns a schema and one pipeline (often with +multiple pipeline heads). Many apps share one database.

+
+
Schema
+
+

The definition of your database’s tables and columns.

+
+
DatabaseManager
+
+

The object that owns the SQLite connection and applies your +schema.

+
+
Pipeline
+
+

An ordered set of stages that transforms and routes data toward the +database. One app, one pipeline is the usual layout.

+
+
Stage
+
+

One step of work in a pipeline, running on its own thread.

+
+
DatabaseStage
+
+

The special stage that may write to the database; all database +stages run on the single shared database thread.

+
+
Port
+
+

A typed input or output on a stage.

+
+
Queue (ConcurrentQueue)
+
+

The thread-safe FIFO that connects one stage’s output +port to the next stage’s input port.

+
+
Pipeline head
+
+

An unbound input port on the first stage — where producers +emplace work into the pipeline. One pipeline may expose several heads.

+
+
Database thread
+
+

The single dedicated thread through which all database work +flows.

+
+
+
+
+
+
+

Part II: Getting Started

+
+
+Move from concepts to a running program. By the end of this part the reader has +SimDB installed, building in their project, and has written a first database. +
+
+
+

5. Installing SimDB

+
+
+

With the "why" behind us, it is time to get SimDB onto your machine and compile +something. This chapter covers dependencies; the next covers wiring SimDB into +your own build.

+
+
+

What "header-only" means for you

+
+

SimDB is a header-only C++17 library. There is no SimDB .a or .so to build +and link against — you point your compiler at SimDB’s headers and that is the +whole library. What SimDB does need is a small set of ordinary dependencies at +build time:

+
+
+
    +
  • +

    a C++17 compiler (GCC or Clang);

    +
    • +
  • +

    SQLite3 (version 3.19 or newer), the database engine SimDB builds on;

    +
    • +
  • +

    zlib, used for compression;

    +
    • +
  • +

    a threads library (pthreads on Linux), for the pipeline machinery.

    +
    • +
+
+
+

You only need clang-format (specifically clang-format-19) if you intend to +contribute changes back to SimDB; it is not required to use the library.

+
+
+
+

System packages (Ubuntu / Debian)

+
+

On a Debian-based system, install the dependencies with apt:

+
+
+
+
sudo apt-get update
+sudo apt-get install -y \
+    cmake \
+    libsqlite3-dev \
+    zlib1g-dev \
+    build-essential
+
+
+
+

(Add clang-format-19 to that list if you plan to contribute.)

+
+
+
+

Conda environments

+
+

If you work in conda, you can pull the same toolchain into an environment +instead. To add the dependencies to an environment you already have:

+
+
+
+
conda activate <your-env-name>
+conda install -y \
+    cmake \
+    sqlite \
+    zlib \
+    compilers
+
+
+
+

Or to create a fresh environment for SimDB from scratch:

+
+
+
+
conda create -n simdb -y \
+    cmake \
+    sqlite \
+    zlib \
+    compilers
+conda activate simdb
+
+
+
+ + + + + +
+ + +
+

Because SimDB is header-only, "installing" it can mean two different things: +simply making its headers visible to your compiler, or running a formal install +step so it can be found with CMake’s find_package. Both are covered in the +next chapter.

+
+
+
+
+
+
+
+

6. Building & CMake Integration

+
+
+

SimDB uses CMake, and the smoothest way to consume it is from CMake too. This +chapter shows how to install SimDB and then link it into your own project with a +single target.

+
+
+

Installing SimDB so CMake can find it

+
+

From a clone of the SimDB repository, configure a build and install it. For a +system-wide install:

+
+
+
+
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ ..
+sudo make install
+
+
+
+

Or, if you are working inside a conda environment, install into that +environment’s prefix instead:

+
+
+
+
conda activate <your-env-name>
+cmake --install . --prefix $CONDA_PREFIX
+
+
+
+

Installing places SimDB’s headers and a CMake package configuration where +find_package can locate them.

+
+
+
+

Linking SimDB into your project

+
+

SimDB exports a single CMake target, SimDB::simdb. In your project’s +CMakeLists.txt:

+
+
+
+
cmake_minimum_required(VERSION 3.19)
+project(my_simulator LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_package(SimDB REQUIRED)
+
+add_executable(my_simulator main.cpp)
+target_link_libraries(my_simulator PRIVATE SimDB::simdb)
+
+
+
+

That one target_link_libraries line is all you need. Because SimDB::simdb is +an interface target, linking against it automatically pulls in SimDB’s include +directories and its transitive dependencies — SQLite3, zlib, and the threads +library — so you do not list them yourself.

+
+
+ + + + + +
+ + +
+

SimDB requires C++17. Setting CMAKE_CXX_STANDARD to 17 (as above) is the +portable way to guarantee it for your target.

+
+
+
+
+
+

Debug and Release builds

+
+

Choose a build type with CMAKE_BUILD_TYPE, exactly as you would for any CMake +project:

+
+
+
+
# An optimized build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+
+# A debug build (no optimization, full debug info)
+cmake -DCMAKE_BUILD_TYPE=Debug ..
+
+
+
+

Use Release for anything performance-sensitive — which, given why you are +reaching for SimDB, is most of the time. Keep a Debug build around for +development. One thing worth knowing early: SimDB’s own build treats warnings as +errors (-Wall -Wextra -Werror), and Release builds surface issues — such as +unused variables — that a Debug build may let slide. If a file compiles in +Debug but fails in Release, an unused parameter or variable is the usual +culprit.

+
+
+
+
+
+

7. Your First Program

+
+
+

Enough setup — let us write something that runs. Here is a complete SimDB +program that defines a table, creates a database, writes a handful of rows, and +reads them back. It uses only SimDB’s SQLite interface; there is no pipeline +here at all, which is the point: SimDB is useful the moment you want a database, +and the concurrency machinery is something you add later, when you need it.

+
+
+
+
#include "simdb/sqlite/DatabaseManager.hpp"
+
+#include <iostream>
+
+int main()
+{
+    using dt = simdb::SqlDataType;
+
+    // 1. Describe the schema: one table with two columns.
+    simdb::Schema schema;
+    auto& tbl = schema.addTable("Stats");
+    tbl.addColumn("Cycle", dt::uint64_t);
+    tbl.addColumn("Value", dt::double_t);
+
+    // 2. Create the database file and apply the schema.
+    simdb::DatabaseManager db_mgr("first.db", true /* create a new database */);
+    db_mgr.appendSchema(schema);
+
+    // 3. Insert a few rows.
+    for (uint64_t cycle = 0; cycle < 5; ++cycle)
+    {
+        db_mgr.INSERT(
+            SQL_TABLE("Stats"),
+            SQL_COLUMNS("Cycle", "Value"),
+            SQL_VALUES(cycle, cycle * 1.5));
+    }
+
+    // 4. Read them back.
+    auto query = db_mgr.createQuery("Stats");
+
+    uint64_t cycle = 0;
+    double value = 0.0;
+    query->select("Cycle", cycle);
+    query->select("Value", value);
+
+    std::cout << "Stored " << query->count() << " rows:\n";
+    auto results = query->getResultSet();
+    while (results.getNextRecord())
+    {
+        std::cout << "  cycle " << cycle << " -> " << value << "\n";
+    }
+
+    return 0;
+}
+
+
+
+

What it does

+
+

Walk through the program in the order it runs:

+
+
+
    +
  1. +

    Define the schema. Schema and addTable/addColumn build an in-memory description of +your tables and columns; nothing touches disk yet. SqlDataType (aliased here +to dt for brevity) names the column types.

    +
    2. +
  2. +

    Create the database. Constructing a simdb::DatabaseManager on a file path +opens the connection; the second argument true asks for a fresh database. +appendSchema then creates the tables. (You can call appendSchema more than +once to add tables later.)

    +
    3. +
  3. +

    Insert rows. The INSERT macro-based API names a table, its columns, and the +matching values. It is the most direct way to write a row; Part III introduces +prepared statements for high-volume inserts.

    +
    4. +
  4. +

    Read them back. createQuery targets a table; select binds each column to +a variable that is refreshed on every getNextRecord(); count() returns how +many rows match. Constraints, ordering, and limits come in Part III.

    +
    5. +
+
+
+
+

Building and running it

+
+

Pair the program with the consumer CMakeLists.txt from the previous chapter, +then configure, build, and run:

+
+
+
+
cmake -DCMAKE_BUILD_TYPE=Release -B build
+cmake --build build
+./build/my_simulator
+
+
+
+

You should see five rows printed, and a first.db file left behind — an +ordinary SQLite database you can open with any SQLite tool:

+
+
+
+
sqlite3 first.db "SELECT * FROM Stats;"
+
+
+
+ + + + + +
+ + +
+

Run the program from a scratch or build directory rather than from inside a +source tree. SimDB programs create their database file in the working directory, +and you would rather not scatter these files across your source.

+
+
+
+
+

That is the whole SQLite interface in miniature. Part III revisits each of these +steps — schemas, inserts, queries, transactions, and blobs — in depth.

+
+
+
+
+
+

8. Running the Regression Tests

+
+
+

SimDB ships with a regression suite, and it doubles as a library of small, +authoritative worked examples. Building it once up front is the best way to +confirm your toolchain is healthy.

+
+
+

Building the suite

+
+

From a build directory, build the simdb_regress target to compile and run the +tests:

+
+
+
+
# Release
+mkdir release
+cd release
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make simdb_regress
+
+
+
+

The same works for a Debug build (mkdir debug, cmake -DCMAKE_BUILD_TYPE=Debug +..). Under the hood the suite is registered with CTest, so you can also run and +filter individual tests with ctest once they are built.

+
+
+
+

Worked examples for the SQLite interface

+
+

Everything we have covered so far — and everything in Part III — has a small, +complete, runnable counterpart under test/sqlite/. These are the authoritative +worked examples for SimDB’s SQLite interface: each is a self-contained main.cpp +that exercises exactly one area.

+
+
+
    +
  • +

    test/sqlite/Schema/main.cpp — defining tables, columns, indexes, and default +values, and reconnecting to an existing schema.

    +
    • +
  • +

    test/sqlite/Insert/main.cpp — the INSERT API and prepared inserts.

    +
    • +
  • +

    test/sqlite/Query/main.cpp — queries: selecting columns, WHERE +constraints, ordering, limits, and compound clauses.

    +
    • +
  • +

    test/sqlite/Delete/main.cpp — deleting records with constraints.

    +
    • +
  • +

    test/sqlite/UInt64/main.cpp — full-range 64-bit unsigned handling.

    +
    • +
+
+
+

Because they are part of the regression suite, they are guaranteed to compile and +pass, so they never drift from the current API. As you read Part III, keep the +matching program open alongside the chapter. (SimDB’s higher-level pipeline +programs live under examples/; we turn to those in Part IV, once pipelines are +on the table.)

+
+
+
+

A note on where you run things

+
+

SimDB programs write their database file into the current working directory. Run +them from a build directory or a scratch directory, not from the root of a source +tree, so these files do not litter your checkout. This is a small habit that +saves a lot of git status noise later.

+
+
+

With SimDB installed, building, and its test programs at your fingertips, you are +ready for Part III, where we return to the SQLite interface and cover it in +full.

+
+
+
+
+

Part III: The SQLite Interface

+
+
+SimDB’s SQLite layer is useful on its own, with no pipelines involved. This part +is a complete tour of the data layer: schema, inserts, queries, transactions, +blobs, and inspection. +
+
+
+

9. Defining a Schema

+
+
+

Every SimDB database begins with a schema: a description of the tables you want +and the columns each one holds. In SimDB you write that description in ordinary +C++ — there are no SQL CREATE TABLE strings to compose, and no chance of a +typo in a keyword surviving until runtime. A simdb::Schema is a plain in-memory +object; defining one touches no files and opens no connections. It becomes a real +database only when you hand it to a DatabaseManager, which is the subject of the +next chapter.

+
+
+

A first schema

+
+

Here is a small schema with a single table:

+
+
+
+
#include "simdb/sqlite/DatabaseManager.hpp"
+
+simdb::Schema schema;
+using dt = simdb::SqlDataType;
+
+auto& tbl = schema.addTable("Instructions");
+tbl.addColumn("PC", dt::uint64_t);
+tbl.addColumn("Mnemonic", dt::string_t);
+tbl.addColumn("Latency", dt::uint32_t);
+
+
+
+

addTable returns a reference to the new Table, and addColumn names a column +and gives it a type. That is the entire required vocabulary; everything else in +this chapter is optional refinement.

+
+
+
+

Column data types

+
+

Column types come from the simdb::SqlDataType enumeration. Aliasing it to dt +(as above) keeps definitions readable. The available types are:

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeUse it for

int32_t, uint32_t

32-bit signed / unsigned integers

int64_t, uint64_t

64-bit signed / unsigned integers (ticks, cycles, addresses)

double_t

Floating-point values

string_t

Text

blob_t

Arbitrary binary payloads (covered later in this part)

+
+

SimDB maps these onto SQLite’s storage classes for you and preserves full +integer width — including the entire range of uint64_t, which raw SQLite does +not handle gracefully.

+
+
+
+

The implicit Id primary key

+
+

By default, every table you create is given an auto-incrementing 64-bit integer +primary key named Id. You do not declare it, and in fact you cannot add a +column named Id yourself — SimDB manages it. This Id is what lets you fetch +a row directly later (for example with findRecord), and it is why a freshly +inserted record already has a stable identity.

+
+
+

If you want a different column to be the primary key, say so:

+
+
+
+
tbl.setPrimaryKey("PC");   // use PC as the primary key instead of Id
+
+
+
+

And if a table needs no primary key at all, remove it:

+
+
+
+
tbl.unsetPrimaryKey();
+
+
+
+
+

Default values

+
+

A column can carry a default that is used whenever an insert omits it:

+
+
+
+
auto& reports = schema.addTable("Reports");
+reports.addColumn("Name", dt::string_t);
+reports.addColumn("StartTick", dt::uint64_t);
+reports.addColumn("EndTick", dt::uint64_t);
+reports.setColumnDefaultValue("StartTick", 0);
+reports.setColumnDefaultValue("Name", "unnamed");
+
+
+
+

The default’s type must match the column’s type. Blob columns cannot have +defaults — there is no meaningful literal for arbitrary binary data.

+
+
+
+

Indexes

+
+

If you will query a table by a particular column (or combination of columns), add +an index so those lookups stay fast as the table grows:

+
+
+
+
reports.createIndexOn("StartTick");                       // single-column index
+reports.createCompoundIndexOn({"StartTick", "EndTick"});  // multi-column index
+
+
+
+

Indexes speed up reads at a small cost to writes and file size, so add them for +the columns you actually filter or sort on — not reflexively for every column.

+
+
+
+

Uniqueness

+
+

To reject duplicate values in a column, mark it unique:

+
+
+
+
reports.ensureUnique("Name");
+
+
+
+
+

A fluent style

+
+

Because the Table-returning methods all return the table, you can chain a +definition into a single expression when that reads more clearly:

+
+
+
+
schema.addTable("Events")
+    .addColumn("Tick", dt::uint64_t)
+    .addColumn("Kind", dt::string_t)
+    .setColumnDefaultValue("Kind", "generic");
+
+
+
+
+

Composing and growing a schema

+
+

Schemas are composable. appendSchema merges one schema’s tables into another, +which is handy when different modules each contribute their own tables:

+
+
+
+
simdb::Schema combined;
+combined.appendSchema(schema_from_module_a);
+combined.appendSchema(schema_from_module_b);
+
+
+
+

Appending fails if it would create two tables with the same name but different columns. +And, as the next chapter shows, you are not limited to defining everything up front: a +DatabaseManager accepts appendSchema too, so you can add tables to a live +database as your program discovers it needs them.

+
+
+

With a schema in hand, the next step is to turn it into an actual database file.

+
+
+
+
+
+

10. The DatabaseManager

+
+
+

A schema only describes tables; the simdb::DatabaseManager turns that +description into a real database file and is your entry point for everything you +do with it afterward. It owns the underlying database connection, applies +schemas, and is the object you call to insert, query, and manage records. In a +typical program there is one DatabaseManager per database file, and it stays +alive for as long as you need that database open.

+
+
+

Opening a database

+
+

The constructor takes a file name and a flag that controls what happens if the +file already exists:

+
+
+
+
// Create a brand-new database, replacing any file of the same name.
+simdb::DatabaseManager db_mgr("sim.db", true /* force_new_file */);
+
+
+
+

There are three cases worth knowing:

+
+
+
    +
  • +

    Create fresh (force_new_file == true). If the file exists it is replaced, +and you start from an empty database. This is what you want for a program that +produces a database on each run.

    +
    • +
  • +

    Open existing (force_new_file == false, the default). If the file already +exists, the manager reconnects to it and rebuilds the schema from the file +itself. If the file does not exist, a new empty database is created.

    +
    • +
  • +

    Default name. Both arguments are optional; a bare simdb::DatabaseManager +db_mgr; opens (or creates) a file named sim.db.

    +
    • +
+
+
+ + + + + +
+ + +
+

When you open a database file that already existed without forcing a new file, +its schema is fixed: you can read and write records, but you cannot +appendSchema new tables onto it. Schema changes belong to the program that +creates the database. Reach for force_new_file == true when you intend to +define (or redefine) the schema.

+
+
+
+
+
+

Applying a schema

+
+

With a manager in hand, appendSchema creates the tables described by a schema:

+
+
+
+
simdb::Schema schema;
+// ... define tables ...
+
+simdb::DatabaseManager db_mgr("sim.db", true);
+db_mgr.appendSchema(schema);
+
+
+
+

Unlike Schema::appendSchema (which only merges descriptions in memory), +DatabaseManager::appendSchema immediately realizes the tables in the database +file. You may call it more than once on a database you created, which is how you +add tables partway through a run as your program discovers it needs them.

+
+
+
+

The schema is persisted for you

+
+

SimDB records the schema inside the database itself, in a pair of internal +bookkeeping tables. That is why reconnecting to an existing file just works: +the manager reads those tables back and reconstructs an equivalent Schema +automatically, with no need to redeclare it in code:

+
+
+
+
simdb::Schema orig_schema;
+// ... define tables ...
+
+simdb::DatabaseManager orig_db_mgr("orig.db", true);  // Create new DB
+orig_db_mgr.appendSchema(orig_schema);
+
+simdb::DatabaseManager diff_db_mgr("orig.db", false); // Connect to same DB
+assert(orig_schema == diff_db_mgr.getSchema());
+
+
+
+
+

The manager is the hub

+
+

Nearly everything else in this part goes through the DatabaseManager. From it +you:

+
+
+
    +
  • +

    insert rows with INSERT and prepareINSERT (next chapter);

    +
    • +
  • +

    read rows with createQuery, and fetch a single row by its Id with +findRecord (the querying chapter);

    +
    • +
  • +

    group operations atomically with safeTransaction (the transactions chapter).

    +
    • +
+
+
+
+

Lifecycle

+
+

The connection is opened in the constructor and closed automatically when the +DatabaseManager is destroyed — ordinary RAII, so there is no explicit "close" +call to remember. The database file itself, of course, persists on disk after +the program exits, ready to be reopened later or inspected with other tools.

+
+
+

With a live database in hand, we can start putting data into it.

+
+
+
+
+
+

11. Inserting Data

+
+
+

SimDB gives you two ways to write rows: a one-shot INSERT that is convenient +for occasional writes, and a prepared insert that you set up once and reuse for +high-volume writing. Both go through the DatabaseManager, and both are typed — you hand SimDB C++ values, not SQL strings.

+
+
+

One-shot inserts

+
+

The INSERT API names a table, the columns you are setting, and the matching +values:

+
+
+
+
auto record = db_mgr.INSERT(
+    SQL_TABLE("Instructions"),
+    SQL_COLUMNS("PC", "Mnemonic", "Latency"),
+    SQL_VALUES(0x400100ull, "addi", 1u));
+
+
+
+
+

The record handle

+
+

INSERT returns a handle to the row it created — a SqlRecord. From it you can +read back typed properties, and you can modify the row in place; a setProperty +call writes straight through to the database:

+
+
+
+
int32_t id = record->getId();                            // the auto-assigned Id
+uint32_t latency = record->getPropertyUInt32("Latency");
+
+record->setPropertyUInt32("Latency", 2u);                // updates the stored row
+
+
+
+

There is an accessor per column type — getPropertyInt32, getPropertyUInt64, +getPropertyDouble, getPropertyString, and so on — along with a matching +setProperty for each. Blob columns use the templated +getPropertyBlob<T>("col"), covered in the blobs chapter.

+
+
+
+

Insert shorthands

+
+

Two shorter forms of INSERT are handy in the right situation:

+
+
+
+
// Supply every column, in the table's schema order (no SQL_COLUMNS needed).
+db_mgr.INSERT(SQL_TABLE("Instructions"), SQL_VALUES(0x400104ull, "sub", 1u));
+
+// Insert a row that takes each column's default value.
+db_mgr.INSERT(SQL_TABLE("Instructions"));
+
+
+
+

The values-only form is terser but positional — the values must line up with +the columns exactly as the schema declares them — so prefer the explicit +SQL_COLUMNS form when clarity matters, or to future-proof your code against +schema changes e.g. added/reordered columns.

+
+
+
+

Prepared inserts for high volume

+
+

When you are writing many rows — the common case in a simulation — preparing +the statement once and reusing it is far faster than issuing a fresh INSERT +each time. prepareINSERT compiles the statement up front; you then set column +values by position and call createRecord for each row:

+
+
+
+
auto inserter = db_mgr.prepareINSERT(
+    SQL_TABLE("Instructions"),
+    SQL_COLUMNS("PC", "Mnemonic", "Latency"));
+
+for (const auto& inst : instructions)
+{
+    inserter->setColumnValue(0, inst.pc);        // column 0 == "PC"
+    inserter->setColumnValue(1, inst.mnemonic);  // column 1 == "Mnemonic"
+    inserter->setColumnValue(2, inst.latency);   // column 2 == "Latency"
+    inserter->createRecord();
+}
+
+
+
+

setColumnValue takes a zero-based index into the column list you passed to +prepareINSERT, and createRecord returns the new row’s Id. If you prefer to +set every column in one call, createRecordWithColValues does exactly that:

+
+
+
+
int32_t id = inserter->createRecordWithColValues(0x400108ull, "xor", 1u);
+
+
+
+

As with INSERT, there is a prepareINSERT(SQL_TABLE("…​")) overload that binds +every column in schema order when you do not want to spell out SQL_COLUMNS.

+
+
+
+

Choosing between them

+
+

Reach for one-shot INSERT for setup rows, configuration, and any place you are +writing a handful of records. Reach for prepareINSERT in the hot path, where +the same statement runs thousands or millions of times. To push bulk-insert +throughput even higher, wrap the loop in a single transaction — the subject of +the transactions chapter.

+
+
+ + + + + +
+ + +
+

SimDB owns a couple of internal bookkeeping tables (their names begin with +internal$). It deliberately refuses inserts into them, so you cannot corrupt +the schema information it relies on.

+
+
+
+
+
+
+
+

12. Querying Data

+
+
+

Reading data back is where SimDB’s typed, string-free style pays off most. You +build a query against a table, bind C++ variables to the columns you want, add +any constraints, and iterate the results — all without composing a SELECT +statement by hand.

+
+
+

Selecting columns and iterating

+
+

createQuery targets a table. You then call select once per column, binding +each to a variable of the matching type. The trick to the model is that those +variables are bound once and refreshed on every step of the iteration:

+
+
+
+
auto query = db_mgr.createQuery("Instructions");
+
+uint64_t pc = 0;
+query->select("PC", pc);
+
+std::string mnemonic;
+query->select("Mnemonic", mnemonic);
+
+uint32_t latency = 0;
+query->select("Latency", latency);
+
+auto results = query->getResultSet();
+while (results.getNextRecord())
+{
+    // pc, mnemonic, and latency now hold this row's values.
+    std::cout << "0x" << std::hex << pc << ": " << mnemonic
+              << " (latency " << std::dec << latency << ")\n";
+}
+
+
+
+

Each successful getNextRecord() populates the bound variables with the next +row; the loop ends when it returns false. If you only need a tally rather than +the rows themselves, count() returns how many records match:

+
+
+
+
const int n = query->count();
+
+
+
+
+

Constraints (the WHERE clause)

+
+

Narrow a query by adding typed constraints. There is a method per column type — addConstraintForInt, addConstraintForUInt32, addConstraintForUInt64, +addConstraintForDouble, addConstraintForString — each taking a column, a +comparison operator, and a value:

+
+
+
+
query->addConstraintForUInt32("Latency", simdb::Constraints::GREATER, 1);
+query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "addi");
+
+
+
+

The simdb::Constraints operators are EQUAL, NOT_EQUAL, LESS, +LESS_EQUAL, GREATER, and GREATER_EQUAL. Multiple constraints combine with +AND. To clear them and reuse the query, call resetConstraints().

+
+
+
+

Set membership

+
+

To match against a set of values, use simdb::SetConstraints::IN_SET or +NOT_IN_SET with a brace-enclosed list:

+
+
+
+
query->addConstraintForString(
+    "Mnemonic", simdb::SetConstraints::IN_SET, {"addi", "sub", "xor"});
+
+
+
+
+

Matching doubles

+
+

Floating-point equality is treated with care. addConstraintForDouble takes an +extra boolean that enables fuzzy matching, so you can compare against a stored +double without being defeated by machine-precision rounding:

+
+
+
+
auto samples = db_mgr.createQuery("Samples");
+samples->addConstraintForDouble("Value", simdb::Constraints::EQUAL, 3.14, /* fuzzy */ true);
+
+// Query will now return rows where the Value is within machine epsilon of 3.14
+
+
+
+
+

Ordering and limits

+
+

Sort results with orderBy (call it more than once for tie-breakers), and cap +the number of rows with setLimit:

+
+
+
+
query->orderBy("Latency", simdb::QueryOrder::DESC);
+query->orderBy("PC", simdb::QueryOrder::ASC);
+query->setLimit(100);
+
+
+
+

resetOrderBy() and resetLimit() undo these, mirroring resetConstraints().

+
+
+
+

Combining clauses with OR

+
+

Constraints are AND-ed by default. When you need OR, build each group of +constraints, release it as a clause, and combine the clauses explicitly:

+
+
+
+
query->addConstraintForUInt32("Latency", simdb::Constraints::EQUAL, 1);
+query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "addi");
+auto clause_a = query->releaseConstraintClauses();
+
+query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "sub");
+auto clause_b = query->releaseConstraintClauses();
+
+// (Latency == 1 AND Mnemonic == "addi") OR (Mnemonic == "sub")
+query->addCompoundConstraint(clause_a, simdb::QueryOperator::OR, clause_b);
+
+
+
+
+

Fetching a single row by Id

+
+

When you already know a row’s Id — for example one returned by createRecord — you do not need a query at all. Ask the DatabaseManager for it directly:

+
+
+
+
auto record = db_mgr.findRecord("Instructions", id);
+
+
+
+

This hands back the same SqlRecord handle described in the previous chapter, +with its typed getProperty* accessors.

+
+
+
+

Deleting matching rows

+
+

A query can also delete the rows it selects. Constrain it to the rows you want +gone, then call deleteResultSet():

+
+
+
+
auto stale = db_mgr.createQuery("Instructions");
+stale->addConstraintForUInt32("Latency", simdb::Constraints::EQUAL, 0);
+stale->deleteResultSet();
+
+
+
+

With no constraints, that would empty the table — so double-check your +constraints before deleting.

+
+
+
+

Reusing a query

+
+

A query object is reusable. Reset whichever parts you want to change — constraints, ordering, or limit — and run it again; and within a single run, +getResultSet() can be iterated, then reset() and iterated again from the top.

+
+
+
+
+
+

13. Transactions

+
+
+

By default, each write you make to a SQLite database is its own committed +transaction. That is fine for the occasional row, but it is ruinous for +throughput when you are writing thousands or millions of them: the per-commit +overhead dominates. Grouping many operations into a single transaction is the +single most effective thing you can do for write performance, and SimDB makes it +both easy and safe with safeTransaction.

+
+
+

Grouping work with safeTransaction

+
+

safeTransaction takes a function (typically a lambda) and runs everything +inside it within one BEGIN/COMMIT:

+
+
+
+
auto inserter = db_mgr.prepareINSERT(
+    SQL_TABLE("Instructions"),
+    SQL_COLUMNS("PC", "Mnemonic", "Latency"));
+
+db_mgr.safeTransaction(
+    [&]()
+    {
+        for (const auto& inst : instructions)
+        {
+            inserter->setColumnValue(0, inst.pc);
+            inserter->setColumnValue(1, inst.mnemonic);
+            inserter->setColumnValue(2, inst.latency);
+            inserter->createRecord();
+        }
+    });
+
+
+
+

Here every createRecord in the loop is folded into a single commit. The same +loop without a transaction would commit once per row; wrapping it as above can be +orders of magnitude faster for large batches.

+
+
+
+

What makes it "safe"

+
+

The name is not decoration — safeTransaction handles the awkward parts of real +concurrent database access for you:

+
+
+
    +
  • +

    Automatic retry on contention. If the database is momentarily busy or locked +(for instance, another thread is mid-write), SimDB does not fail the call and +hand you a "database is locked" error to cope with. It waits briefly and +retries the whole transaction until it succeeds. Transient contention becomes +invisible.

    +
    • +
  • +

    Safe to nest. Calling safeTransaction while already inside one does not +start a second, illegal nested transaction; the inner call simply joins the +outer one, and the commit happens once when the outermost call finishes. This +means helper functions can each wrap their work in safeTransaction without +worrying about who called whom.

    +
    • +
  • +

    Serialized access. An internal lock guards the connection, so transactions +from different threads are applied one at a time rather than colliding.

    +
    • +
+
+
+

Together these mean you get batching and correctness without writing any retry +loops, lock handling, or nesting bookkeeping yourself — the same +"easy to use, hard to misuse" bargain SimDB makes everywhere.

+
+
+
+

A note on pipelines

+
+

When you move to the concurrent pipeline in Part IV, you will rarely call +safeTransaction by hand on the hot path. SimDB’s database pipeline stages +already batch the rows flowing through them into transactions on the dedicated +database thread, applying exactly this technique for you. safeTransaction +remains the right tool whenever you are writing to the database directly, outside +of a pipeline.

+
+
+
+
+
+

14. Blobs, Compression & Serialization

+
+
+

Not everything worth storing is a scalar. Simulations routinely produce +bulk data — an array of samples, a packed snapshot of some structure, a run of +values captured over a window of time. A blob_t column stores that kind of +payload as raw bytes, and it is how SimDB keeps large, structured data compact +inside an ordinary database file.

+
+
+

Storing and reading a vector

+
+

The simplest blob is a std::vector. Pass one straight into SQL_VALUES, and +SimDB stores its bytes:

+
+
+
+
std::vector<int> samples = collect_samples();
+
+db_mgr.INSERT(
+    SQL_TABLE("Waveforms"),
+    SQL_COLUMNS("Samples"),
+    SQL_VALUES(samples));
+
+
+
+

Reading it back is symmetric: getPropertyBlob<T> returns a std::vector<T> of +the element type you ask for:

+
+
+
+
auto record = db_mgr.findRecord("Waveforms", id);
+std::vector<int> samples = record->getPropertyBlob<int>("Samples");
+
+
+
+

You read with the same element type you wrote. A blob is just bytes; SimDB does +not record what those bytes meant, so asking for the wrong T gives you back +nonsense (or throws, if the byte count does not divide evenly). Store +trivially-copyable data — plain values or POD structs — and never store +pointers, whose addresses are meaningless once reloaded.

+
+
+
+

Raw bytes with SqlBlob

+
+

When your data is not already a vector — a packed struct, or an existing buffer — describe it with a simdb::SqlBlob, which is just a pointer and a byte count:

+
+
+
+
simdb::SqlBlob blob(samples);   // from a vector...
+db_mgr.INSERT(SQL_TABLE("Waveforms"), SQL_COLUMNS("Samples"), SQL_VALUES(blob));
+
+
+
+

The same is available on an existing record through setPropertyBlob, either +from a vector or from a raw pointer and length:

+
+
+
+
record->setPropertyBlob("Samples", other_vector);              // from a vector
+record->setPropertyBlob("Samples", data_ptr, num_bytes);       // from raw bytes
+
+
+
+

Because large payloads are expensive to copy, you can std::move a vector into +SQL_VALUES to hand its buffer to SimDB directly rather than duplicating it:

+
+
+
+
db_mgr.INSERT(SQL_TABLE("Waveforms"), SQL_COLUMNS("Samples"), SQL_VALUES(std::move(samples)));
+
+
+
+
+

Compression

+
+

For genuinely large payloads, SimDB ships zlib-based helpers so you can shrink +the data before it lands on disk. Compress into a std::vector<char>, store that, +and decompress on the way back out:

+
+
+
+
// Writing: compress, then store the compressed bytes.
+std::vector<double> series = collect_series();
+std::vector<char> packed;
+simdb::compressData(series, packed);
+
+db_mgr.INSERT(SQL_TABLE("Series"), SQL_COLUMNS("Data"), SQL_VALUES(packed));
+
+
+
+
+
// Reading: pull the compressed bytes back, then decompress into the original type.
+auto record = db_mgr.findRecord("Series", id);
+std::vector<char> packed = record->getPropertyBlob<char>("Data");
+
+std::vector<double> series;
+simdb::decompressData(packed, series);
+
+
+
+

compressData takes an optional level — simdb::CompressionLevel::FASTEST, +DEFAULT, HIGHEST, or DISABLED — letting you trade CPU time against file +size. Compression is entirely your choice: SimDB provides the tools, and you +decide when a payload is big enough to be worth compressing.

+
+
+

This blob-plus-compression pattern is the foundation for storing high-volume +simulation data efficiently, and it is exactly what SimDB’s higher layers build +on to keep large captures small.

+
+
+
+
+
+

15. Inspecting & Dumping Databases

+
+
+

When something looks wrong — a table that should have rows but does not, a value +that is not what you expected — you want to look inside the database quickly.

+
+
+

Reading the database at the command line

+
+

Because a SimDB database is a standard SQLite file, you are never locked into C++ +for inspection. The sqlite3 command-line tool queries it directly:

+
+
+
+
sqlite3 sim.db "SELECT * FROM Instructions LIMIT 10;"
+
+
+
+

And any language with a SQLite binding — Python included — can open the same +file to analyze or cross-check its contents. This openness is what lets separate +tooling, such as a visualization front-end, read a database that a C++ simulation +produced.

+
+
+
+

Third-party SQLite viewers

+
+

For interactive browsing — clicking through tables, running ad-hoc `SELECT`s, +sorting and filtering by hand — a graphical SQLite viewer is often the best +way to get oriented. These are all third-party tools, not part of SimDB, but +because a SimDB database is a plain SQLite file they open it without any special +handling. A few worth knowing:

+
+
+
    +
  • +

    DB Browser for SQLite (also called "DB4S" or sqlitebrowser) — a free, +open-source, cross-platform desktop application. A good default: browse and +edit tables, run queries, and view schema in one window.

    +
    • +
  • +

    SQLiteStudio — free, open-source, and cross-platform, with a tabbed UI for +working across several tables and databases at once.

    +
    • +
  • +

    DBeaver — a cross-platform database tool (free Community Edition) that +handles SQLite alongside many other databases; handy if you already use it for +other work.

    +
    • +
  • +

    DataGrip — JetBrains' commercial database IDE, with strong query editing and +navigation, if you work in that ecosystem.

    +
    • +
  • +

    Datasette — an open-source tool that serves a SQLite file as a browsable web +application, convenient for read-only exploration and quick sharing.

    +
    • +
+
+
+ + + + + +
+ + +
+

If you just want a quick look and prefer not to install anything, the sqlite3 +command-line tool shown above is already enough. Reach for a GUI when you want to +explore an unfamiliar database interactively.

+
+
+
+
+

This closes our tour of the standalone SQLite interface. You can now define +schemas, open databases, write and read records, batch work into transactions, +store bulk payloads, and inspect the result — all without a single pipeline. In +Part IV we add the other half of SimDB: the concurrent pipeline that moves this +data off the critical path and onto its own threads.

+
+
+
+
+

Part IV: Concurrent Pipelines

+
+
+Here SimDB’s second pillar appears: safe, high-throughput concurrent pipelines +that feed SQLite without contention. This part builds a working pipeline from +first principles. +
+
+
+

16. Pipeline Fundamentals

+
+
+

Part III gave you a database. This part gives you the other half of SimDB: a way +to move data into that database without ever making your simulation wait on it. +A pipeline is a chain of processing stages, each running on its own thread, +passing data along through queues. The simulation hands work to the front of the +pipeline and returns to what it was doing; the stages do the transforming, +compressing, and writing behind the scenes. Before we build one, this chapter +introduces the fundamental unit — the stage — and the small contract that +governs how it runs.

+
+
+

What a stage is

+
+

A stage is a unit of work with typed input and output ports. You create one by +subclassing simdb::pipeline::Stage and implementing a single method, run_. +Every combination of ports is allowed: a stage may have multiple inputs, multiple +outputs, or none at all — a stage with no input port might generate data; one +with no output port might be a sink that only writes.

+
+
+

We cover how ports are declared and wired in the next two chapters. For now, the +important thing is the shape of a stage and what its run_ method promises.

+
+
+
+

The greedy run loop

+
+

SimDB runs each stage on its own thread and calls run_ over and over. Your job +in run_ is to do one unit of work if there is any, and to tell SimDB what +happened by returning a PipelineAction:

+
+
+
+
class MyStage : public simdb::pipeline::Stage
+{
+private:
+    simdb::pipeline::PipelineAction run_(bool force) override
+    {
+        Widget w;
+        if (input_queue_->try_pop(w))
+        {
+            // ... transform w, push results to an output port ...
+            return simdb::pipeline::PipelineAction::PROCEED;  // did work; call me again now
+        }
+
+        return simdb::pipeline::PipelineAction::SLEEP;        // nothing to do; let me nap
+    }
+};
+
+
+
+

The two return values drive everything:

+
+
+
    +
  • +

    PROCEED means "I did something, and there may be more." SimDB immediately +calls run_ again, with no delay. A busy stage is therefore serviced +greedily — it drains its input as fast as it can.

    +
    • +
  • +

    SLEEP means "I had nothing to do." The stage’s thread sleeps briefly (100 ms +by default), then wakes and tries again.

    +
    • +
+
+
+

This is the whole scheduling model. There is no manual thread management, no +condition-variable bookkeeping — you write a run_ that processes one item and +reports back, and SimDB handles the looping, waking, and sleeping.

+
+
+
+

Tuning the polling interval

+
+

That 100 ms nap is the stage’s polling interval, and you can change it through +the Stage constructor when a stage needs to react faster (a shorter interval) +or can afford to idle longer (a longer one). It is a latency-versus-idle-CPU +trade-off: shorter intervals notice new work sooner but wake more often when +there is nothing to do.

+
+
+
+

The force flag and flushing

+
+

run_ receives a bool force. Normally it is false. It becomes true during +a flush — when the pipeline is being drained to a synchronization point, or +when the simulation is over and the pipeline is being torn down.

+
+
+

The flag matters only for stages that buffer data internally. A stage that, say, +accumulates values in a std::vector and forwards them a full batch at a time +should, when force is true, also send whatever partial batch it is holding so +that no data is stranded at shutdown. A stage with no internal buffering — like +the skeleton above, which forwards each item as it arrives — can simply ignore +force.

+
+
+
+

Two kinds of stage: Stage and DatabaseStage

+
+

Every stage you write derives from one of two bases, and the choice determines +which thread it runs on:

+
+
+
    +
  • +

    A plain simdb::pipeline::Stage runs on its own dedicated thread. Use it for +transformation work — compression, filtering, reshaping — that does not touch +the database.

    +
    • +
  • +

    A simdb::pipeline::DatabaseStage runs on the single, shared database thread. +Every database stage, across every pipeline and every app in +the process, is placed on that one thread.

    +
    • +
+
+
+

That second point is the "one database thread to rule them all" principle from +Part I made concrete. SimDB funnels all database writes through a single thread +on purpose: multi-threaded access to a SQLite file never keeps up with a fast +simulation, so instead of fighting for the file, all database work is serialized +onto one thread and batched. A DatabaseStage accordingly gets what it needs to +do that work — getDatabaseManager_() for direct access and +getTableInserter_() for fast prepared inserts — and each call to its run_ +runs inside an implicit batch transaction. We devote a full chapter to database +stages shortly.

+
+
+

With the stage model and its run loop in hand, we can look at how stages are +connected — the ports and queues that carry data between them.

+
+
+
+
+
+

17. Ports & ConcurrentQueue

+
+
+

If stages are the workers, ports are the doors through which data enters and +leaves them, and a queue is the conveyor belt between one stage’s output door +and the next stage’s input door. This chapter covers how you declare those ports +and how you read and write the queues that connect them.

+
+
+

Declaring ports

+
+

A stage declares its ports in its constructor. Each port has a name, a data type, +and a queue pointer that SimDB fills in for you. The pattern is always the same: a +ConcurrentQueue<T>* member initialized to nullptr, registered with +addInPort_ or addOutPort_:

+
+
+
+
class ZlibStage : public simdb::pipeline::Stage
+{
+private:
+    simdb::ConcurrentQueue<StatsValues>*     input_queue_  = nullptr;
+    simdb::ConcurrentQueue<ZlibStatsValues>* output_queue_ = nullptr;
+
+public:
+    ZlibStage()
+    {
+        addInPort_<StatsValues>("uncompressed_input", input_queue_);
+        addOutPort_<ZlibStatsValues>("compressed_output", output_queue_);
+    }
+
+    // ... run_() ...
+};
+
+
+
+

Note what you do not do: you never allocate or free these queues. You declare +the pointer, hand it to addInPort_/addOutPort_, and SimDB points it at the +real queue when the pipeline is wired together (the next chapter). Until then the +pointer is nullptr; by the time run_ executes, it is valid. The type in the +angle brackets is what makes a port typed — an output port carrying +ZlibStatsValues can only be bound to an input port that also expects +ZlibStatsValues, and that match is enforced when you bind stages.

+
+
+
+

The ConcurrentQueue API

+
+

simdb::ConcurrentQueue<T> is a thread-safe FIFO — a queue wrapped with a mutex — and it is the only thing shared directly between two concurrently running +stages. Its interface is deliberately small:

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallMeaning

push(item)

Copy an item onto the back of the queue.

emplace(item)

Move an item onto the back (or construct one in place).

try_pop(out)

Pop the front into out; returns false if the queue was empty.

size()

Number of queued items.

empty()

Whether the queue has no items.

snoop(cb)

Peek at items without removing them (used by snoopers).

+
+

The workhorses are try_pop on the input side and push/emplace on the +output side. try_pop returning false is precisely the "nothing to do" signal +that a stage turns into a SLEEP. snoop — inspecting in-flight items without +consuming them — underpins the snooping feature covered in the advanced part; +you rarely call it directly.

+
+
+
+

Ports in a run_ method

+
+

Putting the pieces together, a typical transforming stage pops from its input, +does its work, pushes to its output, and reports PROCEED; when the input is +dry, it returns SLEEP:

+
+
+
+
simdb::pipeline::PipelineAction run_(bool /*force*/) override
+{
+    StatsValues uncompressed;
+    if (input_queue_->try_pop(uncompressed))
+    {
+        ZlibStatsValues compressed;
+        compressed.uuid = uncompressed.uuid;
+        simdb::compressData(uncompressed.values, compressed.bytes);
+
+        output_queue_->emplace(std::move(compressed));   // hand off to the next stage
+        return simdb::pipeline::PipelineAction::PROCEED;
+    }
+
+    return simdb::pipeline::PipelineAction::SLEEP;
+}
+
+
+
+

Prefer emplace with std::move for anything non-trivial to copy — a stage’s +whole purpose is to pass large payloads along, and moving hands off the buffer +instead of duplicating it. (Part IV closes with a chapter on exactly this kind of +move-friendly, high-throughput style.)

+
+
+
+

Any shape of stage

+
+

Because ports are just declarations, a stage can have as many as it needs:

+
+
+
    +
  • +

    Multiple input ports let a stage act as a mux, combining items that arrive +on different queues — for example, assembling a tuple whose pieces show up at +different times.

    +
    • +
  • +

    Multiple output ports let a stage fan data out to several downstream stages.

    +
    • +
  • +

    Zero input ports make a stage a source that generates data.

    +
    • +
  • +

    Zero output ports make it a sink — most database stages are sinks that +only consume.

    +
    • +
+
+
+

With ports declared on each stage, we are ready to assemble stages into an actual +pipeline and bind their ports together.

+
+
+
+
+
+

18. Building a Pipeline

+
+
+

You have stages, and each stage has ports. Assembling them into a running +pipeline is a short, ordered recipe: create the pipeline, add the stages, bind +the ports, and grab the entry point. SimDB enforces that order — each phase is +sealed off before the next begins — so the wiring is explicit and mistakes +become compile-time or startup errors rather than silent misbehavior.

+
+
+

Pipelines are built through a PipelineManager, which SimDB hands to your +application in a createPipeline hook. The application framework itself is the +subject of Part V; here we focus on the assembly API, which is the same wherever +you call it.

+
+
+

The build protocol

+
+
+
void StatsCollector::createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr)
+{
+    // 1. Create a named pipeline, owned by this app.
+    auto pipeline = pipeline_mgr->createPipeline("stats-collector-pipeline", this);
+
+    // 2. Add stages (each with a name), then seal the stage list.
+    pipeline->addStage<ZlibStage>("compressor");
+    pipeline->addStage<DatabaseStage>("db_writer");
+    pipeline->noMoreStages();
+
+    // 3. Bind one stage's output port to the next stage's input port,
+    //    then seal the bindings.
+    pipeline->bind("compressor.compressed_output", "db_writer.compressed_input");
+    pipeline->noMoreBindings();
+
+    // 4. Grab the still-unbound input port -- the head of the pipeline.
+    pipeline_head_ = pipeline->getInPortQueue<StatsValues>("compressor.uncompressed_input");
+}
+
+
+
+

The four phases, and the rules that bind them:

+
+
+
    +
  1. +

    Create. createPipeline takes a name and the owning app. The name is how +stages and ports are addressed afterward.

    +
    2. +
  2. +

    Add stages. addStage<T>("name") constructs a stage of type T and gives it +a name; it returns a pointer to the stage, which you keep when you later attach +a flusher or snooper to it. Once every stage is added, noMoreStages() seals +the list — you cannot add stages after this.

    +
    3. +
  3. +

    Bind ports. bind connects an output port to an input port, each addressed as +"stagename.portname". The two ports' types must match. noMoreBindings() +then finalizes the wiring and actually creates the ConcurrentQueue objects +behind every port.

    +
    4. +
  4. +

    Grab the head. Only after noMoreBindings() can you ask for a port’s queue. +The one input port you deliberately left unbound — here +compressor.uncompressed_input — is the pipeline’s entry point. Store its +queue; that is where you will push data in.

    +
    5. +
+
+
+

Calling these out of order is an error: SimDB throws if you try to bind before +noMoreStages(), or to fetch a port queue before noMoreBindings(). This +"build, then seal" protocol is what lets SimDB validate the whole graph — every +binding, every type — before a single thread starts running.

+
+
+

The binding above connects the two stages like so:

+
+
+
+
  compressor
+     uncompressed_input   <- pipeline head (left unbound)
+     compressed_output ---+
+                          |  bind
+  db_writer               |
+     compressed_input  <--+
+
+
+
+
+

Feeding data into the pipeline

+
+

The head you captured is just a ConcurrentQueue, so you send work in with the +same push/emplace calls from the previous chapter. An app typically wraps +this in a small method:

+
+
+
+
void StatsCollector::process(StatsValues&& stats)
+{
+    pipeline_head_->emplace(std::move(stats));   // move the payload in
+}
+
+void StatsCollector::process(const StatsValues& stats)
+{
+    pipeline_head_->push(stats);                 // or copy it in
+}
+
+
+
+

Once an item lands in the head queue, it flows through the stages on their own +threads — compressed by compressor, written by db_writer — with no further +involvement from the caller, which returns immediately. That is the entire point: +the simulation deposits data at the head and gets back to simulating, while the +pipeline does the slow work elsewhere.

+
+
+

A pipeline can have more than one unbound input — multiple pipeline heads on +the same pipeline — when a first stage muxes several streams. That is the usual +way to fan in several producers without building a second pipeline (which is +uncommon per app and adds threads). Less commonly, a pipeline may leave outputs +unbound so you read from the tail. The single-input, single-database-output +shape above is the common case, and it is exactly what the SimplePipeline +example builds.

+
+
+

Our example pipeline ends in a DatabaseStage. That stage is where data actually +reaches SQLite, and it plays by special rules — the single database thread and +implicit batch transactions from Part I. The next chapter is devoted to it.

+
+
+
+
+
+

19. Database Stages

+
+
+

A database stage is where pipeline data finally reaches SQLite. It is a stage +like any other — it has input ports and a run_ method — but it is special in +two ways we met back in Part I: it runs on the single, shared database thread, +and its writes are automatically batched into transactions. This chapter shows +how to write one and why those two properties give you high-throughput +persistence for free.

+
+
+

Declaring a database stage

+
+

Instead of simdb::pipeline::Stage, derive from +simdb::pipeline::DatabaseStage<YourApp>. The template argument is your +application type, which is how the stage knows your schema and can hand you +prepared inserters for its tables:

+
+
+
+
class DatabaseStage : public simdb::pipeline::DatabaseStage<StatsCollector>
+{
+private:
+    simdb::ConcurrentQueue<ZlibStatsValues>* input_queue_ = nullptr;
+
+public:
+    DatabaseStage()
+    {
+        addInPort_<ZlibStatsValues>("compressed_input", input_queue_);
+    }
+
+    // ... run_() ...
+};
+
+
+
+

Ports are declared exactly as before. Most database stages are sinks — they +have input ports and no output — because their job is to consume data and write +it out.

+
+
+
+

Writing with prepared inserters

+
+

The fast path for inserting is getTableInserter_, which returns a prepared +statement for one of your app’s tables. You set column values by position and +call createRecord, just like the prepared inserts from Part III:

+
+
+
+
simdb::pipeline::PipelineAction run_(bool /*force*/) override
+{
+    ZlibStatsValues compressed;
+    if (input_queue_->try_pop(compressed))
+    {
+        auto inserter = getTableInserter_("CompressedStats");
+        inserter->setColumnValue(0, compressed.uuid);
+        inserter->setColumnValue(1, compressed.bytes);
+        inserter->createRecord();
+        return simdb::pipeline::PipelineAction::PROCEED;
+    }
+
+    return simdb::pipeline::PipelineAction::SLEEP;
+}
+
+
+
+

This is the recommended way to write from a database stage: prepared statements +are compiled once and reused, which is what you want on a path that runs millions +of times.

+
+
+
+

Direct database access

+
+

A database stage can also reach the DatabaseManager directly with +getDatabaseManager_(). You would not normally use it for plain inserts — the +inserter above is faster — but it is there for everything else a stage might +need to do: run a query, delete records, or issue any other operation. The full +Part III query and record API is available through it.

+
+
+
+
auto db_mgr = getDatabaseManager_();
+auto query = db_mgr->createQuery("CompressedStats");
+// ... constrain, count, iterate, delete ...
+
+
+
+
+

Automatic batch transactions

+
+

Here is the property that makes database stages fast: every call to a database +stage’s run_ executes inside a batched BEGIN/COMMIT TRANSACTION block that +SimDB manages for you. You never call safeTransaction here — the batching is +automatic.

+
+
+

And it is not just your stage. That single transaction is shared across every +database stage on the database thread — other database stages in the same app, +and database stages belonging to other apps running concurrently. All of their +writes accumulate into one transaction and commit together. This is the +throughput technique from the transactions chapter, applied globally and +invisibly: because everyone shares one database thread, everyone can also share +one big, efficient transaction.

+
+
+
+

The single database thread, restated

+
+

This is the concrete payoff of the "one database thread to rule them all" design. +Plain stages each get their own thread and run truly in parallel; all database +stages, in contrast, are multiplexed onto one thread so that nothing ever +contends for the SQLite file. You do not create or manage that thread — adding a +DatabaseStage to any pipeline brings it into existence, and every subsequent +database stage joins it.

+
+
+

One consequence worth stating: a database stage cannot use the asynchronous +database accessor (getAsyncDatabaseAccessor_() throws if called from one). It +does not need it — it already is on the database thread. That accessor exists +for the opposite situation, a non-database stage that occasionally needs to reach +the database, which we return to later.

+
+
+
+

Database stages with outputs

+
+

Although sinks are the common case, a database stage may have output ports when +the design calls for it. A typical example is a cache-and-evict pipeline, where a +database stage signals that a record has been persisted so an upstream cache can +drop its copy:

+
+
+
+
  Sim -> Cache a copy -> Process -> Database -----+
+               ^                                  |
+               |                                  |
+               +------------- Evict --------------+
+
+
+
+

The cache holds originals for fast access and evicts them once the database stage +confirms the write. Whether you need this is a design choice; the mechanism is +just an ordinary output port on the database stage.

+
+
+

With persistence handled, the last piece of the pipeline story is performance: +how to keep data moving with minimal copying. That is the next chapter.

+
+
+
+
+
+

20. Move Semantics & Performance

+
+
+

Most of SimDB’s performance you get for free: contention-free writes, one database +thread, automatic batch transactions. But a pipeline is only as fast as the data +you push through it, and a few good habits keep it moving.

+
+
+

Move data, do not copy it

+
+

The single most important habit is to move payloads through the pipeline rather +than copy them. On a queue, that is the difference between emplace and push:

+
+
+
+
output_queue_->emplace(std::move(payload));  // hand off the buffer -- cheap, O(1)
+output_queue_->push(payload);                // copy the whole payload -- O(n)
+
+
+
+

Pipeline payloads are frequently large — a vector of samples, a compressed byte +buffer. Copying one duplicates every byte; moving one hands the existing buffer +to the next stage in O(1). On a path that runs millions of times, that difference +dominates.

+
+
+

SimDB embraces this: it supports move-only payloads. A payload type may hold a +std::unique_ptr or any other non-copyable member and still flow through a +pipeline, as long as you emplace/std::move it rather than push it. Copying +remains available — push is there when you genuinely need to keep the original +(say, to tee the same payload to two pipeline heads, or duplicate in software +before enqueue) or when the payload is small enough that copying is free. But +move is the default you should reach for.

+
+
+
+

Let SimDB batch the writes

+
+

The other half of throughput is transaction batching, and you have already seen +it: every database stage’s writes are folded into one shared transaction on the +database thread. You do not size those transactions by hand. The main way you +influence them is by how often you flush — forcing a synchronization point +commits the current batch, so flushing on every item would defeat the batching +entirely. Flush when you actually need a consistency point, not reflexively.

+
+
+
+

Threads and stage granularity

+
+

Each stage runs on its own worker thread, and there is only ever one database +thread. Two design choices affect how well a pipeline scales:

+
+
+
    +
  • +

    Stage granularity. Splitting an expensive transform into its own stage lets +it run in parallel with the stages around it — genuine pipeline parallelism. +But every stage costs a thread and a queue hop, so a stage that does almost no +work can cost more in hand-off overhead than it saves. Make a stage when it +does real, separable work; do not shatter a cheap operation into many tiny +stages.

    +
    • +
  • +

    Payload sizing. Each hand-off through a queue takes a lock. For very small +items, that per-item overhead can dwarf the work, so it often pays to batch +many small items into one payload (a vector, say) and move the batch. Do not +overcorrect into enormous payloads, though — oversized batches delay the +moment a downstream stage can start and inflate memory use. Aim for a +middle ground.

    +
    • +
+
+
+

The polling interval from earlier in this part is the third knob: shorten it for +lower latency, lengthen it to waste fewer wakeups when a stage is usually idle.

+
+
+ + + + + +
+ + +
+

SimDB’s throughput assumes it effectively owns the process’s path to disk. If the +surrounding simulator does its own unrelated filesystem work — a separate +logging system writing heavily to disk, for instance — that activity contends +with SimDB’s database thread for the same I/O path and can badly degrade pipeline +performance. Until SimDB offers a first-class way to coordinate such traffic, +treat unrelated heavy disk I/O as a known performance-failure mode: keep it off +the critical path where you can, or route that output through SimDB so it shares +the same batched, single-threaded database access.

+
+
+
+
+

That completes the conceptual core of the pipeline: stages and their run loop, +ports and queues, assembling a pipeline, the database stage, and the performance +habits that keep it fast. The final chapter of this part puts it all in front of +you as runnable code, touring the shipped pipeline examples.

+
+
+
+
+
+

21. Running the Pipeline Examples

+
+
+

Everything in this part now comes together in real, runnable code. SimDB ships a +set of pipeline examples under examples/, and this chapter walks through the +first of them — SimplePipeline — in full, then points you at the rest.

+
+
+

The example catalog

+
+

The examples are meant to be read roughly in this order. Each one isolates a +concept:

+
+
+
    +
  • +

    SimplePipeline — the "hello world" of pipelines: a single-input/ +single-output compression stage feeding a database stage. Shown in full below.

    +
    • +
  • +

    ConcurrentApps — several apps running at once, sharing the one database.

    +
    • +
  • +

    DatabaseWatchdog — a stage with no I/O ports that reaches the database thread +from outside it.

    +
    • +
  • +

    PipelineSnoopers — inspecting data while it is still in flight between +stages.

    +
    • +
  • +

    AppFactory — apps that need non-default constructor arguments.

    +
    • +
+
+
+

SimplePipeline uses only the concepts from this part, so we present its code +here. The others lean on the application framework and more advanced pipeline +features; rather than show that code out of context, we cover each in the part +where it belongs — the application framework in Part V, and snooping, watchdogs, +and multi-app patterns in the advanced part. The catalog in examples/README.md +maps every example to the feature it showcases.

+
+
+
+

SimplePipeline, in full

+
+

SimplePipeline is a two-stage pipeline: a CompressionStage compresses each +incoming vector on its own thread, and a DatabaseStage writes the compressed +bytes to SQLite on the database thread. It is packaged as a simdb::App; the +application framework is Part V's subject, so for now focus on the pipeline parts +you already recognize — the stages, their ports, and the createPipeline +wiring.

+
+
+

The app declares its schema and holds the pipeline’s entry point:

+
+
+
+
class SimplePipeline : public simdb::App
+{
+public:
+    static constexpr auto NAME = "simple-pipeline";
+
+    SimplePipeline(simdb::DatabaseManager* db_mgr) :
+        db_mgr_(db_mgr)
+    {
+    }
+
+    static void defineSchema(simdb::Schema& schema)
+    {
+        using dt = simdb::SqlDataType;
+
+        auto& tbl = schema.addTable("CompressedData");
+        tbl.addColumn("AppInstance", dt::int32_t);
+        tbl.addColumn("DataBlob", dt::blob_t);
+    }
+
+    // ... createPipeline() and process() shown below ...
+
+protected:
+    simdb::DatabaseManager* db_mgr_ = nullptr;
+    simdb::ConcurrentQueue<std::vector<double>>* pipeline_head_ = nullptr;
+};
+
+
+
+

The compression stage is a plain Stage with one input and one output port. Its +run_ is the pop / transform / emplace / PROCEED pattern from earlier, falling +back to SLEEP when there is nothing to compress:

+
+
+
+
class CompressionStage : public simdb::pipeline::Stage
+{
+public:
+    CompressionStage()
+    {
+        addInPort_<std::vector<double>>("input_data", input_queue_);
+        addOutPort_<std::vector<char>>("compressed_data", output_queue_);
+    }
+
+private:
+    simdb::pipeline::PipelineAction run_(bool) override
+    {
+        std::vector<double> data;
+        if (input_queue_->try_pop(data))
+        {
+            std::vector<char> compressed_data;
+            simdb::compressData(data, compressed_data);
+            output_queue_->emplace(std::move(compressed_data));
+            return simdb::pipeline::PROCEED;
+        }
+
+        return simdb::pipeline::SLEEP;
+    }
+
+    simdb::ConcurrentQueue<std::vector<double>>* input_queue_ = nullptr;
+    simdb::ConcurrentQueue<std::vector<char>>* output_queue_ = nullptr;
+};
+
+
+
+

The database stage derives from DatabaseStage<SimplePipeline>, takes the +compressed bytes on its input port, and writes them with a prepared inserter. It +also carries an app-instance number, passed to its constructor, so multiple +instances can tag their rows:

+
+
+
+
class DatabaseStage : public simdb::pipeline::DatabaseStage<SimplePipeline>
+{
+public:
+    DatabaseStage(size_t app_instance_num) :
+        app_instance_num_(app_instance_num)
+    {
+        addInPort_<std::vector<char>>("data_to_write", input_queue_);
+    }
+
+private:
+    simdb::pipeline::PipelineAction run_(bool) override
+    {
+        std::vector<char> data;
+        if (input_queue_->try_pop(data))
+        {
+            auto inserter = getTableInserter_("CompressedData");
+            inserter->setColumnValue(0, (int)app_instance_num_);
+            inserter->setColumnValue(1, data);
+            inserter->createRecord();
+            return simdb::pipeline::PROCEED;
+        }
+
+        return simdb::pipeline::SLEEP;
+    }
+
+    size_t app_instance_num_ = 0;
+    simdb::ConcurrentQueue<std::vector<char>>* input_queue_ = nullptr;
+};
+
+
+
+

Finally, createPipeline wires the two stages together following the build +protocol from earlier — add stages, bind ports, then capture the pipeline +head. process is the app’s front door: it pushes a vector into that head +queue.

+
+
+
+
void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override
+{
+    auto pipeline = pipeline_mgr->createPipeline(NAME, this);
+
+    pipeline->addStage<CompressionStage>("compressor");
+    pipeline->addStage<DatabaseStage>("db_writer", getInstance());
+    pipeline->noMoreStages();
+
+    pipeline->bind("compressor.compressed_data", "db_writer.data_to_write");
+    pipeline->noMoreBindings();
+
+    pipeline_head_ = pipeline->getInPortQueue<std::vector<double>>("compressor.input_data");
+}
+
+void process(const std::vector<double>& data)
+{
+    pipeline_head_->push(data);
+}
+
+
+
+

That is a complete pipeline. Data enters through process, is compressed on the +compressor’s thread, and is written to SQLite on the database thread — and the +caller of process never waits on any of it.

+
+
+ + + + + +
+ + +
+

The shipped examples/SimplePipeline.hpp includes a few test-only assertions and +a stored flusher that we have omitted here for clarity. The pipeline logic above +is otherwise the real example verbatim.

+
+
+
+
+
+

Building and running the examples

+
+

The examples build with the regression target from Part II:

+
+
+
+
cd release
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make simdb_regress
+
+
+
+

Run the resulting binaries from a build or scratch directory so their .db +output does not litter a source tree. examples/README.md is the authoritative +map of which example demonstrates which feature.

+
+
+

This is the end of Part IV. You can now build a concurrent pipeline from stages +and ports, persist through a database stage, and reason about its performance. +What we have deferred — how an app is defined, enabled, and driven through its +lifecycle — is exactly what Part V takes up next.

+
+
+
+
+

Part V: The App Framework

+
+
+Pipelines and schemas are composed inside apps. This part shows how SimDB apps +are defined, registered, enabled, and run — including many apps sharing one +database. +
+
+
+

22. What Is a SimDB App?

+
+
+

In Part IV we built a pipeline, and it lived inside a class that derived from +simdb::App. That wrapper was not incidental. An app is SimDB’s unit of +composition: a self-contained bundle of a schema, one pipeline (the typical +case), and lifecycle logic, all writing into a shared database. When several +simulation threads feed the same app, the usual pattern is multiple pipeline +heads — several unbound input ports on that single pipeline — not multiple +pipelines per app (which is uncommon and costs extra threads). This chapter +explains what an app is and the hooks it provides; the next explains how apps +are registered and run.

+
+
+

One database, many apps

+
+

The guiding paradigm is simple: your simulator has a single output database, +and one or more apps write into it, each with its own tables and its own logic. +One app might collect statistics, another might log events, another might capture +a trace — and all of their data lands in the same SQLite file, each in its own +tables. This is the "one database to rule them all" idea from Part I, expressed +at the level of features: you add a capability to your simulator by adding an +app, not by standing up a new database or a new output format.

+
+
+
+

The anatomy of an app

+
+

You create an app by subclassing simdb::App. A typical app has four parts, all +of which appeared in the SimplePipeline example:

+
+
+
    +
  • +

    A constructor taking a DatabaseManager.* Every app is handed the shared +database manager when it is created, and typically stores it:

    +
    +
    +
    SimplePipeline(simdb::DatabaseManager* db_mgr) : db_mgr_(db_mgr) {}
    +
    +
    +
    • +
  • +

    A static defineSchema. This declares the app’s tables. It is static +because the framework calls it to assemble the combined schema before any app +instance exists:

    +
    +
    +
    static void defineSchema(simdb::Schema& schema)
+{
+    using dt = simdb::SqlDataType;
+    auto& tbl = schema.addTable("CompressedData");
+    tbl.addColumn("AppInstance", dt::int32_t);
+    tbl.addColumn("DataBlob", dt::blob_t);
+}
    +
    +
    +
    • +
  • +

    A createPipeline override. This builds the app’s one pipeline on the +manager it is given, using the Part IV build protocol. Call +createPipeline(name, app) once; fan in multiple producers via multiple +pipeline heads (unbound input ports), not by creating a second pipeline.

    +
    • +
  • +

    Whatever public methods your app needs. SimplePipeline::process is an +example — the app’s own front door for receiving data. These are not framework +hooks; they are simply your app’s interface to the rest of the simulator.

    +
    • +
+
+
+
+

The lifecycle hooks

+
+

Beyond the schema and pipeline, App defines a small set of virtual hooks that +the framework calls at well-defined points. You override the ones you need:

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HookWhen it runs

defineSchema(Schema&) (static)

Up front, to declare the app’s tables.

postInit(argc, argv)

After command-line/config parsing, before the simulation starts. All apps' postInit methods run inside one automatic safeTransaction — the usual place for pre-sim metadata (settings, scaffolding tables, lookup seeds).

createPipeline(PipelineManager*)

To construct the app’s single pipeline (multiple pipeline heads if needed).

preTeardown()

Just before teardown — push any last pending data into your app’s pipeline.

postTeardown()

After the simulation — the place for post-sim metadata writes. All apps' postTeardown methods run inside one automatic safeTransaction.

+
+

The postInit and postTeardown transaction details are worth dwelling on: +because every app’s postInit and postTeardown each run inside a single +safeTransaction that SimDB opens for you, you can write pre-sim and post-sim +metadata directly in those hooks without wrapping INSERTs yourself — the +batching, retry-on-lock, and safety are handled, exactly as they are on the +database thread during the run.

+
+
+

The value of fixed hooks is consistency: however many apps a simulator loads, +each is initialized, run, and torn down in the same order. The AppManager — the next chapter — is what drives them.

+
+
+
+

Instance numbering

+
+

An app can run as more than one instance against the same database — several +copies of the same collector, say, each responsible for a different part of the +model. getInstance() returns an instance number to tell them apart: it is +1-based, with 0 meaning a single-instance app. SimplePipeline used exactly +this to tag each row with the instance that produced it, passing getInstance() +into its database stage so the AppInstance column records the source.

+
+
+
+

What apps are for

+
+

The framework is deliberately open-ended. Apps make good homes for any +self-contained data-producing capability, such as:

+
+
+
    +
  • +

    a logger that records simulation events,

    +
    • +
  • +

    a profiler that tracks performance metrics,

    +
    • +
  • +

    a pipeline collector that instruments CPU blocks in a performance model

    +
    • +
  • +

    a converter (for example, SQLite to another format),

    +
    • +
  • +

    or a backend for a live visualization GUI.

    +
    • +
+
+
+

Each is just a schema, one pipeline, and some lifecycle logic — and each +coexists with the others in one database. An app on its own, though, does +nothing until something registers and enables it. That something is the +AppManager.

+
+
+
+
+
+

23. The AppManager

+
+
+

An app class on its own does nothing. Something has to register it, decide +whether it is enabled for this run, construct it, and then call its lifecycle +hooks in the right order. That something is the app manager, and it is what turns +a set of app types into running apps writing to a database.

+
+
+

Two managers, two jobs

+
+

SimDB splits the responsibility across two classes:

+
+
+
    +
  • +

    simdb::AppManagers (plural) is the top-level coordinator. You register your +app types with it, it owns the per-database managers, and it drives the whole +lifecycle across every app at once.

    +
    • +
  • +

    simdb::AppManager (singular) manages the enabled apps for one database. +Because a program can open more than one database file, there is one +AppManager per file.

    +
    • +
+
+
+

Most simulators have a single output database, so the common shape is one +AppManagers coordinating one AppManager. The plural form is what makes the +"one database, many apps" model — and even "many databases" — fall out +naturally.

+
+
+
+

Registering and enabling apps

+
+

Two distinct steps decide which apps run. First you register an app type so the +framework knows it exists; this happens before any manager is created:

+
+
+
+
simdb::AppManagers app_mgrs;
+app_mgrs.registerApp<SimplePipeline>();
+
+
+
+

Then, for a given database, you enable the apps you actually want for this run:

+
+
+
+
auto& app_mgr = app_mgrs.createAppManager("out.db");
+app_mgr.enableApp(SimplePipeline::NAME);          // enable one instance
+// app_mgr.enableApp(SimplePipeline::NAME, 4);    // or several instances
+
+
+
+

The split matters: registration is a fixed list of everything your simulator +could run, while enabling is the per-run decision driven by your configuration — command-line flags, a config file, whatever your simulator uses. An app that +is registered but not enabled costs nothing.

+
+
+
+

The driven lifecycle

+
+

With apps registered and enabled, AppManagers drives them through the hooks from +the previous chapter in a fixed sequence. Each call fans out across every enabled +app on every database:

+
+
+
+
app_mgrs.createEnabledApps();     // construct the enabled app instances
+app_mgrs.createSchemas();         // call each app's static defineSchema()
+app_mgrs.postInit(argc, argv);    // run each app's postInit() hook
+app_mgrs.initializePipelines();   // build each app's pipeline (createPipeline())
+app_mgrs.openPipelines();         // start the pipeline threads running
+
+// ... run the simulation, feeding data to your app(s) ...
+auto app = app_mgr.getApp<SimplePipeline>();
+for (const auto& data : inputs)
+{
+    app->process(data);
+}
+
+app_mgrs.postSimLoopTeardown();   // preTeardown()/postTeardown(), stop threads
+
+
+
+

Reading top to bottom, that is the entire life of a SimDB program:

+
+
+
    +
  1. +

    createEnabledApps constructs each enabled app (handing it the shared +DatabaseManager).

    +
    2. +
  2. +

    createSchemas invokes every app’s defineSchema, assembling one combined +schema in the database.

    +
    3. +
  3. +

    postInit runs each app’s post-initialization hook.

    +
    4. +
  4. +

    initializePipelines invokes each app’s createPipeline once, building that +app’s pipeline and its threads.

    +
    5. +
  5. +

    openPipelines starts those threads; from here the pipelines are live.

    +
    6. +
  6. +

    Your simulation runs, handing data to apps you retrieve with +getApp<AppT>().

    +
    7. +
  7. +

    postSimLoopTeardown runs the teardown hooks, drains and stops the threads, +and closes things out.

    +
    8. +
+
+
+
+

Reaching your apps and database

+
+

Once things are running, a handful of accessors get you back to a specific app or +database. Some throw when used in a situation that has no single unambiguous +answer — asking for "the" app manager when several databases are open, or for a +single app instance when many were enabled. The table below summarizes them, and +on which class each lives (AppManager is per-database; AppManagers is the +top-level coordinator).

+
+ ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodOnReturns / doesThrows when

enabled<AppT>()

AppManager

Whether AppT is enabled (it may not be instantiated yet).

Does not throw.

getEnabledAppInstances<AppT>()

AppManager

How many instances of AppT are enabled (0 if none).

Does not throw.

getApp<AppT>()

AppManager

The single instance of AppT, or nullptr if it is not enabled.

AppT was enabled with more than one instance (use getAppInstance), or the stored app is not of type AppT.

getAppInstance<AppT>(n)

AppManager

Instance n (zero-based) of a multi-instance app, or nullptr if not enabled/absent.

The stored app is not of type AppT.

getDatabaseManager()

AppManager

The shared DatabaseManager for this database.

Does not throw.

getAppManager()

AppManagers

The sole AppManager.

There is not exactly one AppManager.

getAppManager(db_file)

AppManagers

The AppManager for db_file.

No AppManager exists for db_file.

getDatabaseManager()

AppManagers

The sole DatabaseManager.

There is not exactly one database open.

getDatabaseManager(db_file)

AppManagers

The DatabaseManager for db_file.

No database db_file exists.

getAllManagers()

AppManagers

Every (AppManager*, DatabaseManager*) pair; empty after teardown.

Does not throw.

+
+

In the common single-database, single-instance case, the two you will reach for +are getApp<AppT>() (to call your app’s own methods, like process) and +getDatabaseManager() (to access the database directly; never do this in the hot +path / simulation loop — use the AsyncDatabaseAccessor discussed in Chapter 29).

+
+
+
+

Consistency

+
+

The reason to hand this sequence to a manager rather than wiring it by hand is +consistency: every app, no matter how many you enable, is initialized, run, and +torn down in exactly the same order. Adding a second app does not change the +driver above at all — you simply enable one more app, which is precisely the +subject of the next chapter.

+
+
+
+

Per-thread performance reports

+
+

When the manager tears the pipelines down, each pipeline worker thread prints a +short performance report. It lists which stage (or stages) the thread was +running, how many times it ran, and — most usefully — how it split its time +between doing work and sleeping:

+
+
+
+
Thread containing:
+    compressor
+
+    Performance report:
+        Num times run:      1000
+        Pct time sleeping:  4.2%
+        Pct time working:   95.8%
+
+
+
+

These percentages are your primary guide to tuning stages, and they connect +directly to the performance levers from Part IV:

+
+
+
    +
  • +

    A thread that is mostly working (near 100%) is saturated — it is a +bottleneck holding back the pipeline. That stage is a candidate to split into +finer stages that can run in parallel, to optimize, or to feed with larger +batched payloads so it spends less time per item.

    +
    • +
  • +

    A thread that is mostly sleeping is under-utilized — it wakes on its +polling interval, finds nothing, and goes back to sleep. Either it is starved +downstream of a bottleneck, or its polling interval is shorter than it needs to +be and can be lengthened to waste fewer wakeups.

    +
    • +
+
+
+

Balancing these numbers across stages — no single stage pinned at 100% while +others idle — is how you turn the abstract advice about stage granularity and +polling intervals into concrete adjustments for your pipeline.

+
+
+

To make this concrete, take the compressor stage from our running example. Its +per-item cost is the zlib compression, which you dial with the +compression_level argument to simdb::compressData (from CompressionLevel:: +FASTEST through HIGHEST). Its report tells you which way to turn that dial:

+
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + +
Active %Sleep %Design decision

Very high

~0% (never sleeps)

The compressor is saturated and gating the pipeline. Lower the compression + ratio — pass a faster level such as CompressionLevel::FASTEST — so each + vector costs less CPU and the stage keeps up.

Very low

~100% (always sleeps)

The compressor has capacity to spare. Raise the compression ratio (for + example CompressionLevel::HIGHEST) to shrink the database while the headroom + exists; or keep the ratio and merge this stage into the previous or next one + to save a thread and a queue hop.

Balanced

Balanced

The stage is well tuned — leave it as is.

+
+
+
+
+

24. Running Multiple Apps Concurrently

+
+
+

The payoff of the manager-driven lifecycle is that running many things at once is +nearly free. Because the driver sequence from the previous chapter never changes, +"one app" and "many apps" are the same code — you just enable more. Concurrency +comes in two forms: several instances of one app, and several different apps. +Either way, they all share the one database.

+
+
+

Many instances of one app

+
+

To run several copies of the same app, pass an instance count to enableApp. +Each instance is a full, independent app with its own pipeline and threads; they +differ only in their instance number:

+
+
+
+
simdb::AppManagers app_mgrs;
+app_mgrs.registerApp<SimplePipeline>();
+
+auto& app_mgr = app_mgrs.createAppManager("out.db");
+app_mgr.enableApp(SimplePipeline::NAME, 4);   // four instances
+
+app_mgrs.createEnabledApps();
+
+// Reach each instance by its zero-based index:
+auto app0 = app_mgr.getAppInstance<SimplePipeline>(0);
+auto app1 = app_mgr.getAppInstance<SimplePipeline>(1);
+// ...
+
+
+
+

With more than one instance configured, getApp<AppT>() throws (there is no +single app to return) — use getAppInstance<AppT>(n) instead. Inside each app, +getInstance() returns its number, which is how the instances keep their data +apart: SimplePipeline tags every row it writes with its instance, so all four +instances can share the CompressedData table and you can still tell whose rows +are whose.

+
+
+
+

Many different apps

+
+

Running different apps together is the same story: register each type, enable +each one, and the lifecycle calls fan out to all of them. A statistics app, a +logging app, and a trace collector can all be enabled on the same database, and +the driver code is identical to the single-app case — createEnabledApps, +createSchemas, and the rest simply visit every enabled app. Each app +contributes its own tables via its own defineSchema, so they coexist in one +file without stepping on one another.

+
+
+
+

The threading model, and multi-app scalability

+
+

The one firm rule you already know holds no matter how many apps you run: there +is exactly one database thread, shared by every database stage in the process. +That is the permanent anchor of the model, and it is what keeps database access +fast and contention-free regardless of how many apps are enabled.

+
+
+

The non-database side is simpler than it will eventually be. Today, each pipeline +stage runs on its own worker thread, and there is no sharing of those threads +across apps. Enabling a second app adds that app’s stages — and therefore its +threads — on top of the first. Thread count grows roughly with the total number +of stages across all enabled apps.

+
+
+ + + + + +
+ + +
+

Running many apps at once is not currently thread-scalable. Each additional app +adds its stages' worker threads with no pooling or sharing, so a process with a +large number of apps will spin up a correspondingly large number of threads. In +practice this is yet to be a problem: today a simulation typically runs one +app at a time (or none), which stays comfortably within budget. Keep the +limitation in mind before using many apps in a single simulation.

+
+
+
+
+

This is a known, temporary limitation. The planned direction is for SimDB to +manage an unbounded worker-thread pool under the hood, automatically creating, +merging, and destroying threads as needed to spread stages across the available +threads and keep each thread’s active/sleep balance healthy (the same balance you +read off the per-thread performance reports in Chapter 23). When that lands, +multi-app systems will scale on threads without any change to your app code — the +single database thread will remain the one fixed point.

+
+
+
+

Coordinated teardown

+
+

Just as startup is driven across all apps, so is shutdown. A single +postSimLoopTeardown runs preTeardown and postTeardown for every app, +drains and stops all the threads, and — as noted earlier — wraps all apps' +postTeardown work in one safeTransaction, so their final metadata writes +commit together efficiently. (postInit received the same automatic +safeTransaction treatment at startup.) If the order in which apps are torn down +(or initialized) matters, the manager lets you specify a hook ordering across apps; +by default they run in a consistent internal order.

+
+
+ + + + + +
+ + +
+

postSimLoopTeardown is what makes a database complete and readable — so it +must run even when the simulation ends badly. If your simulator throws an +exception, calls std::abort/std::terminate, or otherwise exits without +reaching teardown, skipping it leaves you with:

+
+
+
    +
  • +

    a partial database — the final batched transactions were never committed;

    +
    • +
  • +

    unreadable or unparseable blobs — compressed/serialized payloads that were +cut off mid-write; and

    +
    • +
  • +

    likely a crash in std::thread::~thread — the pipeline’s worker threads were +never joined, which terminates the process.

    +
    • +
+
+
+

Make sure postSimLoopTeardown is called on every exit path. Wrap your +simulation loop so that it runs during unwinding (for example from a scope guard, +a try/catch that rethrows after tearing down, or an equivalent +last-resort handler) rather than relying on the happy path alone.

+
+
+
+
+ + + + + +
+ + +
+

Today, postSimLoopTeardown does not flush a TinyStrings map if your app +uses one — that flush is currently a separate step. A future design will fold it +into teardown automatically, so that a single call to postSimLoopTeardown is +the one and only API you need to guarantee a good database. Until then, flush any +TinyStrings yourself before (or as part of) teardown.

+
+
+
+
+

Running many apps, then, adds no new machinery — only more enableApp calls. +The last wrinkle is apps whose constructors need more than the DatabaseManager, +which is where app factories come in.

+
+
+
+
+
+

25. App Factories & Non-Default Constructors

+
+
+

Every app you have seen so far has been constructed the same way: the manager +hands it a DatabaseManager* and nothing else. That is the default, and it is +handled for you by a factory you never had to write. This final chapter is about +the case where that is not enough — an app that needs additional constructor +arguments — and the small amount of machinery SimDB provides to support it.

+
+
+

The default factory

+
+

When you register an app with registerApp<MyApp>(), the manager needs two +things from the type: a way to declare its schema, and a way to build an +instance. For an ordinary app it gets both from a built-in template, +simdb::AppFactory<MyApp>:

+
+
+
+
template <typename AppT> class AppFactory : public AppFactoryBase
+{
+public:
+    App* createApp(DatabaseManager* db_mgr) override { return new AppT(db_mgr); }
+    void defineSchema(Schema& schema) const override { AppT::defineSchema(schema); }
+};
+
+
+
+

That is the whole story for a default app: createApp calls new +AppT(db_mgr), and defineSchema forwards to the app’s static +defineSchema. You never mention this class — it is installed automatically at +registration. The only requirement it places on your app is the one you have +followed all along: a constructor taking a single DatabaseManager*, and a +static defineSchema(Schema&).

+
+
+
+

When the default constructor is not enough

+
+

Sometimes an app genuinely needs more than the database manager to do its job — a configuration struct, an ID, a tuning parameter, a handle to something in the +host simulator. Its constructor then looks like:

+
+
+
+
MyApp(simdb::DatabaseManager* db_mgr, int x, float y);
+
+
+
+

The default factory cannot build this — it only knows how to call new +MyApp(db_mgr). To bridge the gap, you give the app its own nested factory: a +public class literally named AppFactory, inheriting from +simdb::AppFactoryBase, that knows how to store the extra arguments and pass +them along.

+
+
+
+

Writing a nested AppFactory

+
+

A nested factory implements the two AppFactoryBase methods (createApp and +defineSchema) and adds one method of its own — parameterize — whose +signature is entirely up to you: give it exactly the arguments your constructor +needs. It stashes them, and later createApp forwards them into the real +constructor:

+
+
+
+
class MyApp : public simdb::App
+{
+public:
+    static constexpr auto NAME = "my-app";
+
+    MyApp(simdb::DatabaseManager* db_mgr, int x, float y) :
+        db_mgr_(db_mgr), x_(x), y_(y)
+    {
+    }
+
+    static void defineSchema(simdb::Schema& schema) { /* ... */ }
+
+    class AppFactory : public simdb::AppFactoryBase
+    {
+    public:
+        // Signature is yours to choose -- match the app's constructor.
+        void parameterize(int x, float y)
+        {
+            ctor_args_ = std::make_pair(x, y);
+        }
+
+        simdb::App* createApp(simdb::DatabaseManager* db_mgr) override
+        {
+            const auto& [x, y] = ctor_args_;
+            return new MyApp(db_mgr, x, y);
+        }
+
+        void defineSchema(simdb::Schema& schema) const override
+        {
+            MyApp::defineSchema(schema);
+        }
+
+    private:
+        std::pair<int, float> ctor_args_;
+    };
+
+private:
+    simdb::DatabaseManager* db_mgr_ = nullptr;
+    const int x_;
+    const float y_;
+};
+
+
+
+

Nothing else about the app changes: it is still registered, enabled, and driven +through its lifecycle exactly as in the previous two chapters. The factory only +changes how the instance is built.

+
+
+
+

How the manager finds your factory

+
+

You never tell the manager which kind of factory to use — it detects it at +compile time. A small type trait checks whether your app has a nested type named +AppFactory:

+
+
+
+
template <typename, typename = void> struct has_nested_factory : std::false_type {};
+template <typename T> struct has_nested_factory<T, std::void_t<typename T::AppFactory>> : std::true_type {};
+
+
+
+

At registration the manager branches on that trait. If your app has no nested +factory, it installs the default AppFactory<AppT> immediately — the app is +ready to build with no further action from you. If your app does have a nested +factory, the manager installs nothing yet: it waits for you to parameterize the +factory (that is when the extra arguments become available), and the factory +object is created there. The practical consequence is a rule to remember:

+
+
+ + + + + +
+ + +
+

If your app defines a nested AppFactory, you must parameterize it after +enabling the app and before createEnabledApps(). Because the manager defers +factory creation for custom-factory apps, forgetting to parameterize means the +factory never exists, and app creation will fail with an error naming the app.

+
+
+
+
+
+

Parameterizing: all instances, or one at a time

+
+

You configure the factory through the manager, after enableApp and before +createEnabledApps. There are two flavors:

+
+
+
    +
  • +

    parameterizeAppFactory<MyApp>(x, y) — configure every instance of the app +with the same arguments.

    +
    • +
  • +

    parameterizeAppFactoryInstance<MyApp>(instance_num, x, y) — configure a +single instance, using the same instance number you will later pass to +getAppInstance. Call it once per instance to give each its own arguments.

    +
    • +
+
+
+

The two interact in one way worth flagging: calling the "all instances" +parameterizeAppFactory discards any per-instance configurations you set +earlier and replaces them with one shared factory (the manager only warns about +this). Pick one strategy per app rather than mixing them. Both flavors require +that the app already be enabled — calling them first throws.

+
+
+
+

Putting it together

+
+

The full sequence for a parameterized, multi-instance app is just the driver from +Chapter 23 with parameterize calls slotted in before creation:

+
+
+
+
simdb::AppManagers app_mgrs;
+app_mgrs.registerApp<MyApp>();
+auto& app_mgr = app_mgrs.createAppManager("sim.db");
+
+app_mgr.enableApp(MyApp::NAME, 2);   // two instances
+
+// One parameterize call per instance, each with its own arguments.
+app_mgr.parameterizeAppFactoryInstance<MyApp>(0, 45, 7.77f);
+app_mgr.parameterizeAppFactoryInstance<MyApp>(1, 98, 3.14f);
+
+app_mgrs.createEnabledApps();        // constructors run here
+
+auto* a0 = app_mgr.getAppInstance<MyApp>(0);   // built with (45, 7.77)
+auto* a1 = app_mgr.getAppInstance<MyApp>(1);   // built with (98, 3.14)
+
+
+
+

Because the factory lives on the manager and construction is deferred until +createEnabledApps, the same pattern extends naturally across databases: hand +each AppManager (one per database) its own parameterized instances, then let a +single createEnabledApps build them all. examples/AppFactory/main.cpp shows +exactly this — two MyApp instances in one database, and then one MyApp +routed to each of two separate databases — and is the authoritative worked +example for everything in this chapter.

+
+
+

That closes Part V. You can now define an app, register and enable it, drive it +through its lifecycle with the manager, tear it down safely, and — when its +constructor needs more than a DatabaseManager — give it a factory to build it. +Part VI returns to the pipeline layer for the advanced patterns we deferred: +snoopers, watchdog stages, multi-port and elastic pipelines, and the multi-app +techniques that build on the framework you have just learned.

+
+
+
+
+

Part VI: Advanced Pipeline Techniques

+
+
+Once the basics work, these techniques handle synchronization, low-latency +retrieval of in-flight data, determinism, cross-thread database access, and how +stages are scheduled onto threads. +
+
+
+

26. Flushers

+
+
+

A pipeline is asynchronous by design: data enters through process, moves stage +by stage on the stages' own threads, and the caller never waits for it to land. +That is exactly what you want almost all of the time. But every so often you need +the opposite — a moment where you can say "stop, finish everything in flight, +and make it real in the database right now." That moment is a flush, and the +object that provides it is a Flusher.

+
+
+

The two occasions that call for one are worth naming up front:

+
+
+
    +
  • +

    Before you query your own data. If you want to count() the rows written so +far, or read back a record your pipeline just produced, the data may still be +sitting in a queue between stages. A flush guarantees it has reached — and been +committed by — the database stage first.

    +
    • +
  • +

    At a controlled stopping point. During teardown, or before taking a snapshot +or checkpoint, you want the queues drained rather than abandoned.

    +
    • +
+
+
+

What a flush actually does

+
+

A flusher holds an ordered list of stages. Calling flush() runs each stage’s +run_() exhaustively — processAll with force = true — stage by stage, in +the order given, cycling round-robin until every stage reports it has nothing +left to do (they all return SLEEP). The effect is that all in-flight data is +pushed forward through the pipeline and out its far end, rather than being +processed lazily on the usual polling cadence.

+
+
+

Because the stages run to completion in order, order matters: an upstream stage +should flush before the downstream stage it feeds, so its output has landed in +the next stage’s input queue before that stage takes its turn. The round-robin +loop then repeats until the whole chain is quiet.

+
+
+
+

Creating a flusher

+
+

You create a flusher inside createPipeline and store it on the app, so it is +available later whenever you need a synchronization point. Naming the stages sets +the flush order explicitly:

+
+
+
+
void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override
+{
+    auto pipeline = pipeline_mgr->createPipeline(NAME, this);
+
+    pipeline->addStage<CompressionStage>("compressor");
+    pipeline->addStage<DatabaseStage>("db_writer", getInstance());
+    pipeline->noMoreStages();
+
+    pipeline->bind("compressor.compressed_data", "db_writer.data_to_write");
+    pipeline->noMoreBindings();
+
+    pipeline_head_ = pipeline->getInPortQueue<std::vector<double>>("compressor.input_data");
+
+    // Flush "compressor", then "db_writer".
+    pipeline_flusher_ = pipeline->createFlusher({"compressor", "db_writer"});
+}
+
+
+
+

If you omit the stage list, the flusher flushes every stage in the order you +added them with addStage — which is usually the order you want anyway, so the +common case is simply:

+
+
+
+
pipeline_flusher_ = pipeline->createFlusher();
+
+
+
+

createFlusher returns a std::unique_ptr<Flusher> that you own; store it in a +member such as pipeline_flusher_. Naming a stage that does not exist throws a +DBException.

+
+
+
+

The single-transaction guarantee

+
+

Here is the detail that makes flushing cheap enough to rely on. If any of the +flushed stages is a DatabaseStage, createFlusher does not hand you a plain +Flusher — it automatically gives you a FlusherWithTransaction, which runs the +entire flush inside one safeTransaction:

+
+
+
+
// Inside FlusherWithTransaction::flush()
+db_mgr_->safeTransaction([&]() { outcome = Flusher::flush(); });
+
+
+
+

So the whole drain — however many records the database stage writes — commits +as a single BEGIN/COMMIT rather than one transaction per record. This is the +same batching principle from Part IV and Chapter 13, applied automatically at the +flush boundary: a database stage is assumed to be doing real work, so its flush +is wrapped for you. You do not choose between the two flusher types; the pipeline +picks the right one based on whether a database stage is present.

+
+
+
+

Using a flush before a query

+
+

The canonical use is the flush-then-read pattern. Because a flush guarantees all +prior data is committed, a query that follows it sees a consistent, complete +picture:

+
+
+
+
size_t getNumCollected()
+{
+    // Drain the compressor, then the db_writer -- all in one transaction.
+    pipeline_flusher_->flush();
+
+    // Everything produced so far is now in the database; count it.
+    auto query = db_mgr_->createQuery("CompressedData");
+    return query->count();
+}
+
+
+
+

Without the flush, this count could miss records still queued between stages. With +it, the answer is exact as of the moment of the call.

+
+
+ + + + + +
+ + +
+

A flush is a hard synchronization point. It does real work on the calling +thread and forces the current batch to commit, so flushing on every item defeats +the batching that makes the pipeline fast in the first place. Flush when you +genuinely need a consistency point — before a query or count, at a checkpoint, or +during teardown — not reflexively after each push.

+
+
+
+
+

When the data you are after has not yet reached the database, a flush followed +by a query is also the reliable fallback for retrieving it. There is a +lower-latency alternative that inspects data while it is still moving between +stages — snooping — which is the subject of the next chapter.

+
+
+
+
+
+

27. Snoopers

+
+
+

A flush answers the question "make everything real so I can query it." But it is +a heavy answer: it drains the whole pipeline to disk and commits. If what you +actually need is a single item — "give me the data for UUID abc123, wherever it +happens to be right now" — and you need it often, flushing every time is +wasteful. Snooping is the lightweight alternative: it peeks into the queues +between stages and returns the item without draining anything or accessing the disk.

+
+
+

The idea: a key and a snooped type

+
+

Snooping is built around two types you choose:

+
+
+
    +
  • +

    a key type that uniquely identifies an item (some kind of UUID), and

    +
    • +
  • +

    a snooped type that every stage can produce for that key.

    +
    • +
+
+
+

The subtlety is that each stage stores data in its own in-flight form — the +compressor’s input holds raw values, its output holds compressed bytes — yet a +snoop must return one common type regardless of which stage happens to hold the +item. So each stage’s snoop is responsible for converting its internal +representation back into the shared snooped type. For snooping to work at all, +the payloads flowing through the queues must carry the key; the examples below +assume small structs that pair a UUID with the data.

+
+
+
+

Peeking into a queue

+
+

The primitive underneath everything is ConcurrentQueue::snoop. It takes a +callback, locks the queue, and invokes the callback on each item in turn until +the callback returns true (found it) or the queue is exhausted:

+
+
+
+
bool snoop(const std::function<bool(const T& queue_item)>& cb) const;
+
+
+
+

Because it holds the queue’s lock while iterating, a snoop sees a consistent +snapshot of what is sitting in that queue at that instant.

+
+
+
+

A stage’s snoop() method

+
+

Each stage that wants to be snoopable implements a method with a fixed signature — bool snoop(const KeyType&, SnoopedType&) — returning true and filling the +out-parameter when it holds the key. Typically it just forwards to its input +queue’s snoop. Here the two stages from our running example: the compressor +holds uncompressed values, so a hit is a direct copy (fast); the database stage +holds compressed bytes, so a hit must be decompressed first (slower):

+
+
+
+
using uuid_t   = std::string;
+using values_t = std::vector<double>;
+
+// CompressionStage: input queue holds { uuid, values }
+bool snoop(const uuid_t& uuid, values_t& values)
+{
+    return input_queue_->snoop([&](const Sample& in)
+    {
+        if (in.uuid == uuid)
+        {
+            values = in.values;   // direct copy -- fast
+            return true;          // found; stop looking
+        }
+        return false;             // keep scanning this queue
+    });
+}
+
+// DatabaseStage: input queue holds { uuid, compressed bytes }
+bool snoop(const uuid_t& uuid, values_t& values)
+{
+    return input_queue_->snoop([&](const CompressedSample& in)
+    {
+        if (in.uuid == uuid)
+        {
+            simdb::decompressData(in.bytes, values);  // undo zlib -- slower
+            return true;
+        }
+        return false;
+    });
+}
+
+
+
+

This is the reason a snooper prefers to find data early: the further down the +pipeline an item has travelled, the more transforms a stage must reverse to +reconstruct the original.

+
+
+
+

Wiring the snooper

+
+

You create a PipelineSnooper<KeyType, SnoopedType> from the pipeline manager and +register each snoopable stage with it, using the stage handles returned by +addStage:

+
+
+
+
void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override
+{
+    auto pipeline    = pipeline_mgr->createPipeline(NAME, this);
+    auto compressor  = pipeline->addStage<CompressionStage>("compressor");
+    auto db_writer   = pipeline->addStage<DatabaseStage>("db_writer", getInstance());
+    pipeline->noMoreStages();
+    // ... bindings, flusher, pipeline head ...
+
+    pipeline_snooper_ = pipeline_mgr->createSnooper<uuid_t, values_t>();
+    pipeline_snooper_->addStage(compressor);
+    pipeline_snooper_->addStage(db_writer);
+}
+
+
+
+

Store the returned std::unique_ptr<PipelineSnooper<…​>> on the app. Register +stages in pipeline order — the snooper tries them in the order you add them, so +adding upstream stages first finds the item in its cheapest form.

+
+
+
+

Retrieving an item

+
+

snoopAllStages(key, out) walks the registered stages, calling each one’s +snoop until one returns true. If none do, the item is not currently sitting +in any queue — either it never existed, or it has already been written to the +database (or is being processed inside a stage’s run_ and so is momentarily +invisible to a queue snoop). The robust pattern therefore falls back to a flush +and a query, exactly the mechanism from Chapter 26:

+
+
+
+
bool snoopPipeline(const uuid_t& uuid, values_t& values)
+{
+    // Fast path: find it in-flight between stages.
+    if (pipeline_snooper_->snoopAllStages(uuid, values))
+    {
+        return true;
+    }
+
+    // Slow path: force everything to disk, then query.
+    pipeline_flusher_->flush();
+
+    auto query = db_mgr_->createQuery("CompressedData");
+    std::vector<char> bytes;
+    query->select("DataBlob", bytes);
+    query->addConstraintForString("UUID", simdb::Constraints::EQUAL, uuid);
+
+    auto results = query->getResultSet();
+    if (results.getNextRecord())
+    {
+        simdb::decompressData(bytes, values);
+        return true;
+    }
+    return false;   // not found anywhere
+}
+
+
+
+
+

Why snooping pauses the pipeline

+
+

snoopAllStages takes an optional third argument, disable_pipeline, that +defaults to true. Before it scans, it quietly pauses all the pipeline’s +stages and threads for the duration of the snoop. There is a concrete reason for +this. If the stages kept running while you searched, an item could slip out of +the compressor’s queue just after you scanned it (a miss), then out of the +database stage’s queue just as you got there (another miss) — and you would +"chase" it down the entire pipeline, never catching it in a queue. Pausing first +freezes the picture, so the item is found in the earliest stage that holds it.

+
+
+

That pause is a small, deliberate cost, and it is your main lever in the +trade-off snoopers present: pipeline throughput versus retrieval latency. A +snoop that pauses the pipeline briefly slows overall throughput a little but +returns in-flight data quickly and without disk I/O; leaning on flush-and-query +instead keeps the pipeline running at full speed but makes each retrieval +expensive. The pause mechanism itself — and the other uses of temporarily +disabling a pipeline — is the subject of the next chapter.

+
+
+

examples/PipelineSnoopers/main.cpp is the complete, authoritative example: a +four-stage pipeline (buffer, serializer, compressor, database writer) where every +stage implements snoop, showing exactly how a single snooped type is +reconstructed from four different in-flight representations.

+
+
+
+
+
+

28. Disabling Pipelines

+
+
+

The previous chapter left a promise: snoopAllStages freezes the pipeline before +it scans, and this chapter explains the mechanism behind that. Occasionally you +need a running pipeline to hold still — so you can inspect it, snapshot it, or +debug it without data moving out from under you. SimDB provides a single RAII tool +for that: the scoped disabler.

+
+
+

Enable, disable, and the polling thread

+
+

Every unit of work on a pipeline thread is a Runnable (a Stage is one), and +every runnable carries an enabled flag. The polling thread simply skips any +runnable whose flag is off. Disabling a stage therefore does not tear anything +down — the stage, its queues, and their contents stay exactly as they are; the +thread just stops calling into it until it is re-enabled.

+
+
+
+

The scoped disabler

+
+

You rarely toggle that flag by hand. Instead you ask the pipeline manager to +disable everything for the duration of a scope:

+
+
+
+
{
+    auto disabler = pipeline_mgr_->scopedDisableAll();
+    // The entire pipeline is frozen here: no stage runs, and (by default)
+    // the polling threads are paused too. Inspect state, snoop queues, or
+    // take a consistent snapshot -- nothing moves.
+}   // disabler goes out of scope -> everything is re-enabled and resumes
+
+
+
+

scopedDisableAll returns a std::unique_ptr<ScopedRunnableDisabler>. On +construction it disables all the pipeline’s runnables (and pauses their polling +threads); on destruction it re-enables and resumes them. Because it is tied to a +scope, you cannot forget to turn the pipeline back on — leaving the block does it +for you, even if an exception unwinds through it.

+
+
+
+

Threads too, or just the work?

+
+

The one parameter, disable_threads_too, defaults to true:

+
+
+
    +
  • +

    true — the polling threads are paused as well, so the process is genuinely +quiet. This is what you want for a clean snapshot or for deterministic +debugging.

    +
    • +
  • +

    false — only the runnables are disabled; the polling threads keep spinning +but find nothing enabled to do. Useful when you want work to halt but the +threads to remain responsive.

    +
    • +
+
+
+
+

Nesting is safe

+
+

If a disabler is already active and something requests another, scopedDisableAll +returns nullptr — the nested request is a harmless no-op, because the pipeline +is already frozen by the outer disabler. This is exactly why the snooper from +Chapter 27 can hold a possibly-null disabler without any special handling:

+
+
+
+
// Inside snoopAllStages(...)
+std::unique_ptr<ScopedRunnableDisabler> disabler =
+    disable_pipeline ? pipeline_mgr_->scopedDisableAll() : nullptr;
+// Scan the stages. If an outer disabler already froze the pipeline,
+// 'disabler' is null and we simply rely on that outer scope.
+
+
+
+
+

When to reach for it

+
+

Two situations justify freezing a pipeline:

+
+
+
    +
  • +

    Determinism while debugging. Concurrency bugs are hard to observe precisely +when every stage is racing ahead. Disabling the pipeline gives you a still, +reproducible picture to inspect.

    +
    • +
  • +

    Snapshot accuracy and performance. This is the snooper’s "chasing" problem +from the last chapter: if stages keep running while you search, an item can +slip from one queue to the next just ahead of you. Freezing first guarantees +you find it in the earliest stage that holds it, and avoids re-scanning a +moving target.

    +
    • +
+
+
+ + + + + +
+ + +
+

Disabling the pipeline stops throughput for as long as the disabler is alive. +It is a deliberate, temporary tool — keep the scope as short as possible, and do +not treat it as a normal operating mode. The default snoop already applies it for +you in exactly the right, minimal window.

+
+
+
+
+
+
+
+

29. Async Database Access from Non-DB Threads

+
+
+

The permanent rule of SimDB’s threading — one dedicated database thread, and only +that thread touches the database — creates a puzzle the moment a non-database +stage needs to read the database. A watchdog that wants to count rows, a stage +that needs to look something up mid-flight: neither may grab the +DatabaseManager and query it directly, because that would mean two threads +racing on the same connection. The AsyncDatabaseAccessor is the sanctioned +bridge. You hand it a callback; it runs that callback on the database thread and +blocks until it finishes.

+
+
+

The callback and eval

+
+

The callback has one fixed signature — it receives the DatabaseManager it will +run against:

+
+
+
+
using AsyncDbAccessFunc = std::function<void(simdb::DatabaseManager*)>;
+
+
+
+

You submit it with eval, which blocks the calling thread until the callback has +run on the database thread (or a timeout fires):

+
+
+
+
void eval(const AsyncDbAccessFunc& func, double timeout_seconds = 0);
+
+
+
+

Because eval blocks, you can capture results by reference in the lambda and read +them out after eval returns — by then the callback has definitely run.

+
+
+
+

Getting an accessor

+
+

There are two places to obtain an accessor, depending on who needs it:

+
+
+
    +
  • +

    From inside a non-database stage, use the protected getAsyncDatabaseAccessor_():

    +
    +
    +
    // Inside a non-DB Stage's run_()
+auto async_db_accessor = getAsyncDatabaseAccessor_();
+async_db_accessor->eval(callback);
    +
    +
    +
    • +
  • +

    From the app itself (its main thread), cache the accessor from the pipeline +manager during createPipeline — it is valid once the pipeline is open:

    +
    +
    +
    void MyApp::createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override
+{
+    // ... build the pipeline ...
+    async_db_accessor_ = pipeline_mgr->getAsyncDatabaseAccessor();
+}
    +
    +
    +
    • +
+
+
+ + + + + +
+ + +
+

Do not call getAsyncDatabaseAccessor_() from a DatabaseStage. A database +stage already runs on the database thread and has direct access through +getDatabaseManager_() and getTableInserter_() — routing through the async +accessor from there is nonsensical, and SimDB throws if you try. The accessor +exists specifically for code that is not on the database thread.

+
+
+
+
+
+

Immediate commit, and the timeout

+
+

Here is the behavior that makes this fast enough to use from a polling loop. The +database thread is usually busy batching writes inside a long-running +transaction. When an async request arrives, SimDB does not make you wait for that +transaction to run its natural course — it immediately commits whatever the +database thread was doing, then runs your callback (itself inside a +safeTransaction). So even if the in-progress batch would otherwise have taken +seconds, you typically wait only for that one commit — a fraction of a second.

+
+
+

The optional timeout_seconds is a safety valve, not the normal wait. It guards +against a genuinely stuck database thread (a bug, a runaway producer); if the +callback has not completed within the timeout, eval throws rather than blocking +forever. In healthy operation you never hit it.

+
+
+
+

Exceptions propagate

+
+

eval is built on std::promise/std::future, and it propagates failure +faithfully: any exception thrown by your callback on the database thread is +re-thrown from eval on the calling thread. You handle database errors from the +async path exactly as you would from a synchronous call — wrap the eval in a +try/catch if you need to.

+
+
+
+

Worked example: the database watchdog

+
+

examples/DatabaseWatchdog/main.cpp is the canonical use. It runs two apps at +once against one database:

+
+
+
    +
  • +

    WatchedPipeline — the familiar compress-and-write pipeline, producing rows +with no upper bound, running on the usual 100 ms polling cadence.

    +
    • +
  • +

    DatabaseWatchdog — a second app whose single stage has no I/O ports at all. +It exists only to poll the database. It runs on a slower 500 ms timer and, each +time it wakes, posts a count() query to the database thread. When the count +crosses a threshold, it tells the first app to stop.

    +
    • +
+
+
+

The watchdog’s stage is the whole point — a non-DB stage reaching the database +safely from its own thread:

+
+
+
+
simdb::pipeline::PipelineAction run_(bool) override
+{
+    if (finished_)
+    {
+        return simdb::pipeline::SLEEP;
+    }
+
+    size_t num_records = 0;
+    auto async_query = [&](simdb::DatabaseManager* db_mgr)
+    {
+        auto query = db_mgr->createQuery("CompressedData");
+        num_records = query->count();
+        if (num_records > TARGET_NUM_RECORDS)
+        {
+            finished_ = true;
+        }
+    };
+
+    // Post the query to the DB thread; 5s timeout is only a runaway guard.
+    auto async_db_accessor = getAsyncDatabaseAccessor_();
+    async_db_accessor->eval(async_query, 5);
+
+    // 'num_records' and 'finished_' are now safe to read: eval() has returned.
+    if (finished_)
+    {
+        watchdog_app_->thresholdReached_();
+    }
+
+    return simdb::pipeline::SLEEP;
+}
+
+
+
+

Notice the shape: the lambda captures num_records by reference, does its work on +the database thread, and by the time eval returns the value is populated and the +finished_ flag is set. The watchdog never touches the database directly — it +only ever posts work to the thread that owns it.

+
+
+

This is also the natural counterpart to the flusher: a watchdog reads the database +as it fills without stopping the producer, whereas a flush would force a +synchronization point. Together with snoopers and the scoped disabler, you now +have the full toolkit for observing and coordinating a live pipeline.

+
+
+
+
+
+

30. Threads & Scheduling

+
+
+

Two chapters back we looked at what runs (stages, snoopers, disablers); this one +is about when and on what thread it runs. Most of it you never have to touch, +but one dial — a stage’s polling cadence — is worth understanding, and the way +threads are allocated is worth knowing because it is actively evolving.

+
+
+

The polling loop

+
+

Recall from Chapter 24 that, today, each stage runs on its own PollingThread, +and there is exactly one database thread. A polling thread does something very +simple in a loop: it asks its stage to process all the work it can, and then — only if there was nothing to do — it sleeps for a fixed interval before asking +again. When there is a steady supply of work, the thread never sleeps; it runs +flat out. This is precisely the behavior the per-thread performance report in +Chapter 23 measures as "working" versus "sleeping."

+
+
+
+

The cadence dial

+
+

That fixed sleep interval is the one scheduling parameter you set directly. It is +a property of the stage, passed to the Stage constructor and defaulting to +100 ms:

+
+
+
+
class Watchdog : public simdb::pipeline::Stage
+{
+public:
+    Watchdog(DatabaseWatchdog* app) :
+        Stage(500),          // poll every 500 ms when idle
+        watchdog_app_(app)
+    {
+    }
+    // ...
+};
+
+
+
+

The watchdog from the previous chapter uses 500 ms deliberately: it only needs to +check a row count periodically, so waking just twice a second keeps it cheap. The +interval is a direct latency-versus-waste trade-off:

+
+
+
    +
  • +

    A shorter interval means the stage notices new work sooner — lower latency — but if work arrives only occasionally, it wakes, finds nothing, and sleeps +again, spending cycles on empty polls (a high "sleeping" percentage).

    +
    • +
  • +

    A longer interval means fewer wasted wakeups, but data already waiting in the +queue can sit up to one interval before the stage picks it up.

    +
    • +
+
+
+

There is nothing to tune for a stage that is saturated — a thread that always has +work never reaches the sleep at all. The cadence matters for the intermittent +stages: watchdogs, low-rate producers, and anything that mostly waits.

+
+
+ + + + + +
+ + +
+

The interval also has a structural role: if two non-database stages are ever +placed on the same thread, they must share one interval. Today that rarely comes +up because each stage gets its own thread — but it is the reason the interval +lives on the stage, and it matters for the thread management described next.

+
+
+
+
+
+

How threads are allocated — today and tomorrow

+
+

The current allocation is the simple one from Chapter 24: one worker thread per +stage, plus the single database thread. It is easy to reason about, but it does +not share threads across stages or apps, so the thread count climbs with the +total number of stages — the multi-app scalability limitation flagged in Part V.

+
+
+

The planned direction is to make this automatic. Rather than fixing one thread per +stage, SimDB will manage an unbounded worker-thread pool and create, merge, and +destroy threads on its own — co-scheduling compatible stages onto shared threads +and splitting them back apart — using exactly the working/sleeping balance from +the Chapter 23 reports as its signal. Saturated stages get their own threads; +mostly-idle stages get merged together so they stop costing a thread each. The +cadence dial stays a per-stage property, and the single database thread stays the +one fixed point. Crucially, none of this will require changes to your app code: +the same pipeline you write today will simply be scheduled more efficiently.

+
+
+

Until that lands, the guidance is the guidance from Part V: prefer few apps per +process, set generous polling intervals on stages that only need to wake +occasionally, and read the per-thread reports to spot stages that are either +saturated or spinning idle.

+
+
+

This closes Part VI. You now have the full advanced toolkit — flushers for +synchronization, snoopers for in-flight retrieval, the scoped disabler for +freezing the pipeline, async database access for reaching the database thread +from anywhere, and a working model of how stages are scheduled. Part VII puts all +of it to work in a single sustained case study: Argos, SimDB’s flagship +collection-and-visualization application.

+
+
+
+
+

Part VII: Case Study — Argos

+
+
+Argos is SimDB’s flagship application: a simulation data collector plus a +Python viewer. It exercises nearly every SimDB feature and is the best way to +see the pieces working together end-to-end. +
+
+
+

31. What Argos Is

+
+
+

Everything so far in this book has been a capability: a way to define a schema, +build a pipeline, persist safely, observe a running system. Argos is what those +capabilities look like when assembled into a real, shipping product. It is +SimDB’s flagship application, and it is the clearest answer to the question "what +is all of this actually for?"

+
+
+

Argos is a collect-then-visualize system. One half runs inside your simulator +and captures its state as it runs; the other half is a desktop application that +opens the captured data and lets you replay the simulation, scrubbing back and +forth through simulated time to see how every collected object evolved.

+
+
+
+The Argos viewer replaying a collected run +
+
Figure 1. The Argos viewer replaying a collected run — where this case study is headed
+
+
+

Two halves: the collector and the viewer

+
+

Argos has exactly two pieces, and they meet at a single file:

+
+
+
    +
  • +

    The collector (simdb::argos::ArgosCollector) is the write side. It is an +ordinary simdb::App — the same kind of app you learned to build in Part V — that you enable inside your simulator. As the simulation advances, it captures +the state of the objects you have marked for collection and streams that data, +through a SimDB pipeline, into one SQLite database.

    +
    • +
  • +

    The viewer is the read side: a standalone desktop GUI. It opens that database +and reconstructs the state of every collected object at any point in simulated +time, presenting it through a set of composable visualization widgets.

    +
    • +
+
+
+

The collector answers "how do I get simulation data out, fast and safely?" The +viewer answers "now that it is captured, how do I look at it?" The only thing +that passes between them is the database file:

+
+
+
+
    your simulator
+   (ArgosCollector, a simdb::App)
+            |
+            |  streams collected state through a SimDB pipeline
+            v
+     collected.db   <-- one SQLite file: the entire contract
+            |
+            |  opened read-only, replayed over simulated time
+            v
+     Argos viewer (desktop GUI)
+
+
+
+ + + + + +
+ + +
+

That single .db file is the whole handoff. Argos does not emit sidecar files, +trace logs, or index files alongside it — the SQLite database is the complete, +self-contained artifact, exactly as promised back in Part II. Copy it, archive +it, or hand it to a colleague, and the viewer can open it anywhere.

+
+
+
+
+
+

How Argos maps onto SimDB

+
+

Argos is worth studying precisely because it exercises nearly every concept from +Parts Part IV through Part VI at once. Nothing in the collector is special-cased; it is +built from the same parts you already know:

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SimDB concept (where it was introduced)How Argos uses it

App + lifecycle (Part V)

The collector is a simdb::App, enabled through an AppManager, with its + schema declared in defineSchema and its pipeline built in createPipeline.

Concurrent pipeline (Part IV)

Collected state is encoded and compressed on worker threads, then written by a + database stage — so the simulator itself is never blocked on disk I/O.

Single database thread (Parts Part IV--Part VI)

All of Argos’s writes funnel onto the one database thread, keeping collection + fast and contention-free no matter how much the simulator produces.

Blobs & compression (Part III)

Collected snapshots and deltas are stored as compressed blobs — the collector + is a heavy, real-world user of the blob and compression APIs.

Flush at teardown (Part VI)

When the run ends, the collector flushes any pending data so the database is + complete before the process exits.

+
+

If you have read this far, you already understand the collector’s machinery. What +remains specific to Argos is what it collects and how it encodes that state +for efficient replay — the subjects of the next two chapters.

+
+
+
+

The data contract

+
+

Because the database is the entire interface between collector and viewer, its +contents form a contract. At a high level it captures four things:

+
+
+
    +
  • +

    A collectable tree — the hierarchy of objects that were collected, mirroring +the structure of the simulated system; it is the set of objects you can choose +to visualize in the viewer.

    +
    • +
  • +

    One or more clocks — the time bases that drove the run. Each collected object +updates on the edges of the clock it was collected against, and a database may +contain a single clock or many.

    +
    • +
  • +

    Timestamps — the points in simulated time at which state was captured, which +become the range you scrub through in the viewer.

    +
    • +
  • +

    Encoded state blobs — periodic full snapshots plus small deltas in between, +from which the viewer reconstructs any object’s exact value at any requested +tick.

    +
    • +
+
+
+

We will unpack each of these — especially the snapshot-plus-delta encoding, which +is the heart of Argos’s efficiency — in "The Argos Collector" and "On-Disk Data +Model & Encodings." For now the shape is enough: a tree of objects, the clocks +that move them, the times they were sampled, and the compact encoding of their +values.

+
+
+
+

Why Argos is the case study

+
+

Two reasons. First, breadth: a single application that touches the schema, +pipeline, app framework, blobs, compression, and flushing is the best possible +demonstration that these features compose into something greater than their +parts. Second, tangibility: the viewer turns all of that invisible machinery into +something you can see and click, which makes it the ideal lens for understanding +what SimDB delivers.

+
+
+

The rest of Part VII follows the data. We start where the data is born — the +collector — then examine how it is laid out on disk, then cross over to the viewer +that brings it back to life, and finally walk a complete run end to end.

+
+
+
+
+
+

32. The Argos Collector

+
+
+

The collector’s job is narrow and demanding: turn the state of a running +simulation into rows and blobs in a database, continuously, without slowing the +simulation down. This chapter is about what it captures and how it encodes +that capture efficiently — the features, not the internals. Everything here is +built from the SimDB machinery of Parts Part IV through Part VI; where that machinery shows +through, we point back to it rather than re-explaining it.

+
+
+

Setting up: clocks, collectables, and time

+
+

Before a run begins, you tell the collector three things.

+
+
+

Clocks. A simulation advances on one or more clocks, and the collector needs to +know about them — each has a name and a period (and, for derived clocks, a +numerator/denominator ratio). Clocks are the time bases everything else is +measured against.

+
+
+

Collectables. Each object you want to capture is registered as a collectable +with three attributes: a path (its dotted position in the system hierarchy, e.g. +top.core0.rob), the clock it is collected against, and its data type. The +collector assigns each collectable a small integer id — a CID — and records the +whole set as a tree of collectables. That set is exactly what you choose from when +building views in the viewer.

+
+
+

A source of simulated time. Finally, you give the collector a way to read the +current simulation time — a back-pointer to a counter, or a function it can call. +Time must advance monotonically; the collector rejects time that moves +backward.

+
+
+
+

Collecting as the simulation runs

+
+

Once the run is live, you collect each object’s current value whenever it +changes (or on whatever cadence your instrumentation chooses). The collector +gathers everything staged at a single point in simulated time into one unit of +work — internally a "ledger" for that time point. When simulated time advances, +the collector seals the current ledger, hands it off to its pipeline, and starts a +fresh one for the new time.

+
+
+ + + + + +
+ + +
+

Collection cadence is entirely up to you and your instrumentation. The +heartbeat introduced below does not decide when you may collect, how often, or +which ticks are captured. It affects only how the already-collected data is +encoded on disk. Collect as much or as little as you like; the heartbeat is a +separate, purely internal performance concern.

+
+
+
+
+
+

Inside the collector: the pipeline

+
+

Handing the ledger off is where Parts IV and V come back into view. The collector +is a simdb::App, and its createPipeline builds a three-stage pipeline. The +simulation thread only ever pushes a ledger into the head of that pipeline — it +never compresses, encodes, or touches the database itself:

+
+
+
+
  simulation thread
+        |  push sealed ledger (one point in sim time)
+        v
+  [ Stager ]      per-CID checkpoint chains; decides FULL snapshot vs DELTA
+        |
+        v
+  [ Compressor ]  zlib-compresses the encoded window
+        |
+        v
+  [ Writer ]      runs on the single database thread; writes blobs + timestamps
+        |
+        v
+     collected.db
+
+
+
+

Three stages, three threads, plus the one database thread — the exact shape from +Part IV. The stager does the interesting work (turning raw snapshots into +snapshots-plus-deltas); the compressor is the same zlib compression from Part III; +and the writer is a DatabaseStage, so all persistence funnels onto the single +database thread. Because the hand-off is a queue push, the simulation is never +blocked waiting on encoding or disk.

+
+
+
+

Snapshots and deltas: what the heartbeat controls

+
+

Storing the full state of every object at every time point would be enormous and +mostly redundant — from one tick to the next, most values do not change. So the +stager keeps, for each collectable, a checkpoint chain and encodes most time +points as a small delta against the previous checkpoint. Periodically it writes +a full snapshot instead, "rebasing" the chain so that reconstruction never has +to walk back too far.

+
+
+

The heartbeat is the knob that governs that rebase cadence: it bounds how many +deltas may accumulate (and how long, in simulated time) before the next full +snapshot is forced. Its default is 10. This is a pure space-versus-speed +trade-off:

+
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + +
HeartbeatEffect on the databaseEffect on the viewer

Larger (more deltas between snapshots)

Smaller database — fewer full snapshots written.

Slower reconstruction — to show a given tick, the viewer must replay more + deltas back to the last snapshot.

Smaller (more frequent snapshots)

Larger database — full snapshots cost more space.

Faster reconstruction — a snapshot is always close to any requested tick.

Default (10)

A balanced starting point for most runs.

Reasonable replay latency without excessive size.

+
+

The key point, restated because it matters: the heartbeat trades simulation-time +database size against viewer responsiveness. It never changes which data is +collected — only how compactly that data is laid down and how much work the viewer +does to replay it.

+
+
+
+

Scalars, containers, and lifecycle

+
+

Collectables come in two shapes. A scalar is a single value (a counter, a state +enum, a struct). A container is a collection of values — a queue or buffer — and comes in two flavors: contiguous (front-packed, like a ring buffer) and +sparse (indexed slots that may have gaps). Containers declare a fixed +capacity which the viewer uses to render queue-utilization views (%-full widgets).

+
+
+

Collectables also have a lifecycle: an entry can be closed to mark that a record +ended at a particular time. The encoder understands these close events and, when +needed, primes them with a fresh snapshot so the viewer can always reconstruct the +state leading up to a close.

+
+
+
+

Multi-clock collection

+
+

A database may contain a single clock or many. When there are several, each +collectable participates only in the windows belonging to its own clock, and the +collector records which clocks were active at each captured timestamp. The viewer +uses that to update each object only on the edges of the clock it was collected +against — so a fast clock and a slow clock coexist correctly in one database.

+
+
+
+

What gets written, and when

+
+

The collector leans on the app lifecycle from Part V to write the right things at +the right moments:

+
+
+
    +
  • +

    Before the run (postInit): the global settings (including the heartbeat), the +clock definitions, and the collectable tree — the static scaffolding the viewer +reads first.

    +
    • +
  • +

    During the run: compressed state blobs and their timestamps, written by the +database stage as ledgers flow through the pipeline.

    +
    • +
  • +

    Just before teardown (preTeardown): the last in-flight ledger is flushed into +the pipeline so no final time point is lost.

    +
    • +
  • +

    After the run (postTeardown): closing metadata — which collectables actually +produced data (only those are flagged to show in the UI; ones that never +collected anything trigger a friendly warning), observed queue maximums, and the +string/enum lookup tables.

    +
    • +
+
+
+ + + + + +
+ + +
+

Two lifecycle details connect directly to earlier chapters. First, every app’s +postInit and postTeardown each run inside one automatic safeTransaction +(Part V) — postInit for pre-sim scaffolding metadata, postTeardown for +closing metadata — so those writes commit together without manual transaction +wrapping. Second, the collector’s string tables are flushed in postTeardown +explicitly — the same TinyStrings flush called out in Chapter 24 as a step +that teardown does not yet do for you automatically.

+
+
+
+
+

That is the collector end to end: register clocks and collectables, feed values as +time advances, and let a three-stage pipeline encode, compress, and persist them +onto the single database thread. The next chapter opens up the file it produces +and reads its schema as the contract the viewer depends on.

+
+
+
+
+
+

33. On-Disk Data Model & Encodings

+
+
+

Because the database file is the entire interface between collector and viewer, +its schema is a contract. The collector promises to write these tables in this +shape; the viewer relies on exactly that shape to reconstruct the run. This +chapter reads that contract — not table by table as a reference, but grouped by +the role each part plays — and then explains the one idea that underpins all of +it: everything is stored in binary.

+
+
+

The schema, by role

+
+

The tables fall into a few natural groups.

+
+
+

Static scaffolding — written before the run, describing its shape:

+
+
+
    +
  • +

    CollectionGlobals — run-wide settings, including the heartbeat.

    +
    • +
  • +

    Clocks — each clock’s name and timing (period, and an optional ratio).

    +
    • +
  • +

    CollectableTreeNodes — one row per collectable: its CID, its dotted +FullPath, the ClockID it belongs to, its TypeName, and a ShowInUI flag. +This table captures the collectable tree — the set of objects you can choose to +visualize in the viewer.

    +
    • +
  • +

    DataTypeSchemas / DataTypeNodes — per-type layout and display metadata (field +names, type names, and format strings) used to render structured values.

    +
    • +
+
+
+

Time and data — the bulk of the file, written as the run proceeds:

+
+
+
    +
  • +

    Timestamps — the distinct points in simulated time that were captured; these +become the range you scrub through.

    +
    • +
  • +

    CollectionRecords — the actual payload: one compressed blob per captured time +point, keyed by TimestampID.

    +
    • +
  • +

    TimestampClocks — which clocks were active at each timestamp, so multi-clock +runs reconstruct correctly.

    +
    • +
+
+
+

Encoding side tables — the lookups that make the binary payload legible:

+
+
+
    +
  • +

    TinyStringIDs — the string-to-integer dictionary (below).

    +
    • +
  • +

    CollectedEnums / EnumMembers — the enum-to-name maps (below).

    +
    • +
+
+
+

Extras:

+
+
+
    +
  • +

    QueueMaxSizes — the peak occupancy each container reached, for +queue-utilization views.

    +
    • +
  • +

    Notifications — sporadic messages surfaced in the viewer’s Logs tab.

    +
    • +
+
+
+
+An Argos database opened in a SQLite browser +
+
Figure 2. An Argos database opened in a SQLite browser, showing the collection tables described here
+
+
+
+

Everything is collected in binary

+
+

The single most important thing to understand about Argos data is that the +collector stores everything in binary. Collected values are not written as +human-readable columns — they are packed as raw bytes into the compressed +CollectionRecords blobs. This is what keeps collection fast and the database +small, and it is why the viewer, not a plain SELECT, is the tool for reading a +run. Two data types deserve special mention because they are not stored the way +you might expect.

+
+
+

Strings become integers. A string is never written as its characters in the +data blobs. Instead the collector interns it through TinyStrings: the first +time a given string is seen it is assigned a small integer id, and only that +integer is written into the blob. The id-to-string dictionary is saved once, at +teardown, into the TinyStringIDs table. The viewer joins against that table to +recover the text. (The empty string is a reserved id of zero.) A repeated label +that might appear millions of times across a run therefore costs a few bytes each +time instead of its full length — a large space win for essentially free.

+
+
+

Enums use their underlying integer. Enum values are always serialized in the +blob as their underlying integer type — never as text. To make those integers +meaningful in the UI, the collector also records a value-to-name map for each enum +type that supports it, in CollectedEnums (the enum’s name and integer type) and +EnumMembers (each name and its raw integer value). The viewer uses that map to +display, say, ISSUED instead of 2. An enum that has no string conversion is +simply shown as its raw integer value — still correct, just not labeled.

+
+
+

The pattern generalizes: the blob is a compact, uniform byte stream, and a handful +of side tables (TinyStringIDs, CollectedEnums, EnumMembers) plus the type +metadata are the key that turns those bytes back into readable values. Nothing is +lost; it is just encoded for size and speed rather than for direct human reading.

+
+
+
+

Full snapshots and deltas on disk

+
+

Chapter 32 explained why the collector writes periodic full snapshots with small +deltas in between, and how the heartbeat bounds that. On disk, this is what fills +CollectionRecords: each collectable’s value at a given time point is encoded +either as a full snapshot or as a delta against its previous checkpoint. To +reconstruct an object’s exact value at a requested tick, the viewer finds the most +recent full snapshot at or before that tick and replays the deltas forward — the +"lookback" whose depth the heartbeat caps. That is the whole replay model; the +mechanics of building the delta chains live in the collector’s implementation and +are not something the on-disk format asks you to understand.

+
+
+
+

Contiguous vs. sparse containers

+
+

Containers record their kind and capacity in their encoded type: a contiguous +container is front-packed (occupied slots first, like a ring buffer’s live +region), while a sparse container keeps values at explicit indices that may have +gaps. Both declare a fixed capacity, and the collector tracks the maximum +occupancy each one actually reached over the run in QueueMaxSizes. The viewer +uses the kind to lay a container out correctly and the max size to scale its +utilization displays.

+
+
+
+

The contract, from the viewer’s side

+
+

Two fields in CollectableTreeNodes are worth calling out because the viewer +leans on them directly. ClockID binds each collectable to the clock it updates +on, so the viewer advances each object only on the right edges. And ShowInUI is +the filter that keeps the object list honest: only collectables that actually +produced data during the run are flagged for display, so an +instrumented-but-never-fired object does not clutter the objects you can choose to +visualize (the collector even emits a notification listing anything it dropped for +this reason).

+
+
+

With the contract in hand — a tree of collectables on their clocks, a timeline of +timestamps, compressed binary blobs of snapshots and deltas, and the small tables +that decode them — we can finally cross to the read side and see how the viewer +turns all of this back into something you can explore.

+
+
+
+
+
+

34. The Argos Viewer

+
+
+

The viewer is the read side of Argos: a desktop application that opens a collected +database and lets you replay the simulation, scrubbing back and forth through +simulated time to watch collected objects change. Where the collector chapter was +about features and encodings, this one is about what you see and do — it is +deliberately a user’s tour of the interface, not a look under the hood.

+
+
+

Opening a database

+
+

You point the viewer at a collected database, and optionally at a saved layout. +When you open a run for the first time, you are met with an almost-empty +workspace: a single canvas tab filling most of the window, and the playback +controls along the bottom. The canvas does not sit blank, though — an empty cell +shows a small Quick Links menu (described next) that is your starting point for +building a view. Nothing is visualized yet because you have not chosen anything to +look at.

+
+
+
+Argos on first open: an empty canvas showing the Quick Links menu +
+
Figure 3. Argos on first open: an empty canvas showing the Quick Links menu, with the playback bar at the bottom
+
+
+

If you have looked at this data before, the viewer reopens your last layout +automatically; you can also hand it a saved view file to jump straight to a +prepared arrangement (more on those below).

+
+
+
+

Quick Links: building a view from an empty cell

+
+

Every empty canvas cell displays a Quick Links menu — a short bulleted list of +hyperlinks — and this is how you populate the viewer. It has two groups:

+
+
+
    +
  • +

    Widgets — one link per kind of widget you can create in that cell: Queue +Inspector, Queue Utilizations, Scheduling Lines, and Watchlist. Clicking +one opens a small dialog to choose which collected objects it should show (for +example, which queue to inspect, or which elements to draw scheduling lines +for), and then places the widget in that cell.

    +
    • +
  • +

    Canvas — links to Split left/right and Split top/bottom, which divide the +cell in two. The freshly created empty half again shows its own Quick Links, so +you build up a multi-widget layout by splitting and filling, cell by cell.

    +
    • +
+
+
+

This is the mental model that replaces any single "browse everything" panel: +choosing data is part of creating each widget, right where that widget will live.

+
+
+
+

Tabs and the splittable canvas

+
+

The workspace is organized as a data inspector of tabs, each holding one +canvas. A trailing + ("Add Tab") page creates a new tab for a different +arrangement, and you can right-click a tab to rename or delete it. When the run +recorded notifications worth surfacing (the sporadic messages from the collector’s +Notifications table), a fixed Logs tab appears as well.

+
+
+

Within a tab, the canvas is recursively splittable — every split leaves two +cells, each of which can hold a widget or be split again — so you can arrange many +widgets side by side and watch related objects together. Splitting is available +both from the Quick Links of an empty cell and from a populated widget itself.

+
+
+
+

Widgets

+
+

Widgets are how collected objects are actually visualized. Each is created from a +Quick Links entry and then configured through its own selection dialog and +settings. The viewer ships several kinds, each suited to a different shape of +data.

+
+
+

Queue Inspector — a tabular view of a container object such as a queue or +buffer. You choose which columns are visible, and per-column auto-colorization +tints values so patterns jump out as you scrub through time.

+
+
+
+A Queue Inspector widget with colorized columns +
+
Figure 4. A Queue Inspector widget with configurable columns and per-column auto-colorization
+
+
+

Queue Utilizations — how full container objects are over time, scaled against +the queue’s max capacity given to the ArgosCollector.

+
+
+
+A queue-utilizations widget +
+
Figure 5. A Queue Utilizations widget showing container occupancy over time
+
+
+

Scheduling Lines — a timeline-style view of scheduled activity across the +elements you select, over a configurable window of ticks before and after the +current position.

+
+
+
+A scheduling-lines widget +
+
Figure 6. A Scheduling Lines widget showing scheduled activity across a window of ticks
+
+
+

Watchlist — an aggregate/summary view gathering several selected objects +together.

+
+
+
+A watchlist / summary widget +
+
Figure 7. A Watchlist widget aggregating several selected objects
+
+
+
+

The playback bar: moving through time

+
+

The bottom of the window is where Argos becomes a replay tool rather than a +static viewer. The playback bar has a clock selector (for multi-clock runs), a +readout of the current cycle/tick, step buttons for jumping by fixed amounts in +either direction, and a scrubber that spans the whole run from start to end.

+
+
+

As you move the playback position, every widget updates to show its objects' +state at that instant — reconstructed behind the scenes from the nearest full +snapshot plus the deltas up to that tick, exactly the encoding from the previous +two chapters. You never think about snapshots or deltas; you just move the slider +and watch the system evolve. In a multi-clock database, each object updates only +on the edges of the clock it was collected against.

+
+
+
+

A thin client, by design

+
+

It is worth being explicit about what the viewer does not do, because it shapes +how Argos scales. The viewer is a deliberately lightweight, thin client: it +never loads a whole run into memory. When you land on a tick, it reads only the +blobs it needs to reconstruct exactly what the visible widgets are showing, right +then. Move to another tick and it does the work again for the new position.

+
+
+

It does not even cache blobs it has already decoded — revisiting a tick re-reads +and re-replays it from scratch. (A cache might be a reasonable future +optimization; today there simply isn’t one.) What makes this practical is that the +viewer is fast at deserializing and replaying the compressed snapshot-and-delta +blobs, so on-demand reconstruction stays responsive even though nothing is kept +around.

+
+
+

The upshot is a viewer whose memory footprint is tied to what is on screen, not +to the size of the database. A multi-gigabyte run opens as readily as a small one, +because Argos only ever pulls the slice of data the current view and tick require.

+
+
+
+

View files: saving and sharing a layout

+
+

A carefully arranged view is worth keeping, so the viewer can save your layout to a +view file. It captures the whole arrangement — the tabs, how each canvas is +split, which widgets sit where, the columns and colorization chosen for each +object, and the selected clock — so you can reopen it later or hand it to a +colleague to reproduce your exact setup. The window title marks when there are +unsaved changes, and the viewer remembers your most recent layout and restores it +automatically the next time you open the same data without naming a view file.

+
+
+

One deliberate exception: the playback position — the specific tick you happen to +be viewing — is treated as a per-session preference and is not stored in the +shared view file. A view file describes how to look at a run, not where in time you +last paused, so sharing one does not drag your colleague to your current tick.

+
+
+ + + + + +
+ + +
+

The viewer is intentionally described here at the level of features and workflow. +Its implementation is being revised, so this chapter avoids tying anything to +specific internals — what a user does with the canvas, Quick Links, widgets, +playback bar, and view files is what matters, and that is stable.

+
+
+
+
+

That is the viewer as an instrument: open a run, create widgets from an empty +cell’s Quick Links (choosing their data as you go), compose them on a splittable +canvas, and scrub through simulated time while everything updates in step. All +that remains is to see the two halves working together in a single, concrete run — the end-to-end walkthrough that closes the case study.

+
+
+
+
+
+

35. End-to-End

+
+
+

We have followed the data from one side to the other: born in the collector, +encoded into a database, brought back to life in the viewer. This closing chapter +walks the whole loop once as a single workflow, and shows how Argos proves to +itself that the round trip is lossless.

+
+
+

The three steps

+
+

Using Argos end to end is always the same three moves:

+
+
+
    +
  1. +

    Instrument and collect. Enable the ArgosCollector app in your simulator, +register its clocks and collectables, and collect values as simulated time +advances. This is ordinary SimDB app usage — the lifecycle from Part V +(registerApp, enableApp, createEnabledApps, createSchemas, postInit, +initializePipelines, openPipelines, and postSimLoopTeardown) driving the +collector’s three-stage pipeline underneath.

    +
    2. +
  2. +

    Produce one database. When the run ends, teardown flushes the pipeline and you +are left with a single .db file — the whole run, and the only artifact.

    +
    3. +
  3. +

    Open it in the viewer. Point the viewer at that file and explore: create +widgets from the Quick Links, arrange them on the canvas, and scrub through +simulated time, as in the previous chapter.

    +
    4. +
+
+
+
+

Instrumenting a run

+
+

The collector-specific setup is small and sits inside the app lifecycle. Before +the run you tell the collector how to read time (a pointer or function it can +sample), register one or more clocks, set the heartbeat if you want something +other than the default, and declare your collectables (scalars and containers, +each on a clock). Then, as the simulation steps, you stage each object’s current +value; the collector seals a ledger whenever time advances and the pipeline does +the rest. There is no separate "flush every tick" step and nothing on the hot path +touches the database.

+
+
+ + + + + +
+ + +
+

As emphasized in Chapter 32, the heartbeat here is only a performance choice — how many deltas accumulate before a full snapshot rebases the chain. It changes +the database’s size and the viewer’s replay effort, never which data you collect. +The next section leans on exactly that property.

+
+
+
+
+
+

Testing the checkpointer logic and Python deserializers

+
+

The authoritative end-to-end exercise lives in test/argos/, and it is more than +a demo — it is a regression test that runs as part of the suite, so the whole +collect-encode-replay loop is continuously verified. It works by replaying the +collected blobs the same way the viewer does — walking the timeline, applying each +snapshot and delta — and reconstructing every object’s value at every tick. It +then compares those reconstructed values against the simulator’s own record of +what it collected. If the bytes on disk did not faithfully capture the run, the +comparison fails.

+
+
+

The most instructive part is how it treats the heartbeat. The test collects the +same simulation three times — at heartbeats of 1, 3, and 10 — producing three +databases of different sizes, and then confirms that the reconstructed data is +identical across all three. That is the Chapter 32 claim made concrete and +enforced: the heartbeat changes only how compactly the run is stored, not the +information it contains. A smaller heartbeat writes more full snapshots and a +larger file; a larger heartbeat leans on more deltas and a smaller file; the +replayed result is the same either way.

+
+
+
+
+

Part VIII: Patterns, Best Practices & Cookbook

+
+
+
+

You now know how SimDB works part by part. This part is the glue: how to design +with it, how to tune and debug pipelines, which application shapes fit +common goals (without pretending there is only one architecture), and how to +test that the .db file your app produces is correct. No new API surface — just the habits and decision frames that keep real projects from fighting the +library.

+
+
+

This part walks through six topics in order:

+
+
+
    +
  • +

    Schema and pipeline together — co-design storage and dataflow before you +write stages.

    +
    • +
  • +

    Stage boundaries and payload types — split, merge, move-only payloads, and +forced-flush behavior.

    +
    • +
  • +

    Performance tuning — measure first, turn one knob at a time, respect the +single database-thread funnel.

    +
    • +
  • +

    Determinism and debugging — freeze, flush, log, and diagnose a concurrent +pipeline without guessing.

    +
    • +
  • +

    Application shapes — stats engines, live UI backends, replay backends, and +farm-scale analysis as decision spaces, not single canonical layouts.

    +
    • +
  • +

    Testing SimDB apps — SimDBTester, flush/teardown discipline, and +verifying the database with a reader that did not write it.

    +
    • +
+
+
+
+
+

36. Designing Schema and Pipeline Together

+
+
+

The recurring mistake with SimDB is to design the two halves separately — to lay +out tables first, then bolt on a pipeline to fill them (or the reverse). The +schema and the pipeline are two views of one design: the pipeline exists to +produce exactly the rows and blobs the schema defines, and the schema exists to +capture exactly what the pipeline can efficiently emit. Design them together.

+
+
+

Start from the reads

+
+

Begin with the question "what will I ask this database later?" Your queries +determine your schema, and your schema determines what the pipeline’s terminal +database stage must write. If you will filter runs by a status column, that status +must be a real column, not buried inside a blob. If you will only ever read a +record back whole, it can be a single blob. Working backward from the reads keeps +you from persisting data you never query and from burying data you do.

+
+
+
+

Columns or blobs?

+
+

The central storage decision is per field: a queryable column, or a byte in a +blob.

+
+
+
    +
  • +

    Columns are for values you will filter, sort, join, or aggregate on — ids, +names, timestamps, small scalars, enums. They are visible to the Query API from +Part III.

    +
    • +
  • +

    Blobs are for bulk or opaque payloads you will read back as a unit — large +vectors, serialized structures, encoded state. They are compact and cheap to +write, but SQLite cannot see inside them.

    +
    • +
+
+
+

Argos is the worked example: its metadata — clocks, the collectable tree, +timestamps — lives in ordinary columns you can query, while the bulk state lives +in compressed blobs in CollectionRecords. The rule of thumb: if you would ever +put it in a WHERE clause, it is a column; if you only ever reconstruct it, it is +a blob.

+
+
+
+

Where compression belongs

+
+

Compression is a pipeline decision, not a schema one. Compress late and off the +hot path: a dedicated worker stage just before the database stage, never on the +thread producing the data. Compress bulk blobs, not small columns — there is +nothing to gain from compressing an integer. This is exactly the shape of the +Argos pipeline (stage, then compress, then write) and of SimplePipeline from +Part IV.

+
+
+
+

Persist versus derive

+
+

Not everything needs to be stored. Persist what is expensive to recompute or +needed to answer a query; derive the cheap and the rare on read. The +snapshot-and-delta encoding from the Argos case study is the aggressive end of +this spectrum — it stores a compact encoding and reconstructs full state on read, +trading a little read-time work for a much smaller database. Most designs sit +somewhere in the middle: store the raw facts, compute summaries at read time.

+
+
+
+

Normalized tables versus a blob-of-record

+
+

Two shapes recur, and the right one depends on access pattern:

+
+
+
    +
  • +

    Normalized tables — data spread across columns and related tables, joinable +and queryable, with repeated strings interned to integer ids (the TinyStrings +pattern from the Argos schema). Choose this when you will slice the data many +ways.

    +
    • +
  • +

    Blob-of-record — one blob per logical record, written and read as a unit. +Choose this when a record is only ever consumed whole and write speed matters +more than queryability.

    +
    • +
+
+
+

Neither is universally right; many real databases use both, as Argos does. The +point of this chapter is simply that the choice is made once, for schema and +pipeline together — the database stage that writes a blob and the table that +stores it are two ends of the same decision.

+
+
+
+
+
+

37. Choosing Stage Boundaries & Data Types

+
+
+

Once you know what the pipeline must produce, you have to decide how to carve it +into stages and what to hand between them. Both choices have a direct cost: +every stage is a thread and every boundary is a queue hop. Getting the +granularity right is most of what separates a pipeline that scales from one that +spends its time shuffling data.

+
+
+

One responsibility per stage — but not too little

+
+

The unit of a stage is a responsibility: transform, compress, write. Keeping +each stage to one job makes the pipeline easy to reason about, easy to reorder, +and easy to tune, because the per-thread performance report (Part V) tells you +exactly which responsibility is the bottleneck.

+
+
+

The failure mode in the other direction is tiny stages. A stage that does almost +no work still costs a thread, a queue, and a handoff on every item. When the +handoff costs more than the work, the stage is a net loss — you have added +latency and a thread to accomplish nothing. If two adjacent stages are both mostly +idle, they belong together.

+
+
+

The practical test is the active/sleep balance from Part V:

+
+
+
    +
  • +

    A stage saturated (near 100% active) and doing parallelizable work is a +candidate to split.

    +
    • +
  • +

    A stage mostly asleep is a candidate to merge into its neighbor.

    +
    • +
  • +

    A stage with a balanced active/sleep ratio is correctly sized — leave it +alone.

    +
    • +
+
+
+
+

Move-only payloads

+
+

The data you pass between stages should move, not copy. Prefer payload types +that own their storage and move cheaply — std::vector, std::unique_ptr, or +your own move-only structs — and hand them across queues with std::move / +emplace. Types that force a copy on every hop (or, worse, that copy large +buffers) turn every stage boundary into a hidden allocation-and-copy. SimDB’s +queues are built around move semantics precisely so that a payload can travel the +length of the pipeline without its bytes ever being duplicated.

+
+
+
+

Buffering and partial flushes

+
+

Some stages naturally batch — a windowing stage, a compressor that prefers +larger inputs, a stage that groups records before a transaction. Batching is good +for throughput, but a batching stage must answer one question correctly: what +happens on a forced flush?

+
+
+

When a flusher runs (Part VI), it drives every stage with the force flag set. A +batching stage must, on force, emit whatever it currently holds — even a partial +batch — rather than waiting for its window to fill. A stage that ignores force +will strand data in memory and defeat the flush-then-query guarantee. Whenever you +introduce internal buffering, handle the forced-flush path explicitly.

+
+
+
+

Sizing the payload

+
+

Payload size is a latency-versus-throughput dial:

+
+
+
    +
  • +

    Larger payloads amortize the per-item overhead — fewer queue hops, fewer +transactions, better compression ratios — at the cost of higher latency and +more memory in flight.

    +
    • +
  • +

    Smaller payloads lower latency and memory but pay the per-hop cost more often.

    +
    • +
+
+
+

There is no universal answer; size the payload to the stage’s job and measure. The +one constant is structural: keep the database stage last and singular, since it +is the single-threaded funnel every payload must pass through (Part IV).

+
+
+
+
+
+

38. Performance Tuning

+
+
+

SimDB gives you several knobs — batch size, compression level, polling cadence, +stage boundaries, heartbeat interval — and the temptation is to turn all of them +at once. Resist it. Performance tuning is a loop: measure, change one thing, +measure again. This chapter is a guide to the knobs and to the instruments that +tell you which one to turn.

+
+
+

Know the funnel

+
+

Every payload in a SimDB pipeline eventually passes through one database thread. +That single thread is the funnel, and it is almost always the thing you are +tuning around. The goal is to keep it fed but not thrashing: enough work batched +per commit that it is not paying transaction overhead on every row, but not so +much held back that memory balloons or data appears late. Most of the knobs below +are really about how work arrives at that funnel.

+
+
+
+

The knobs

+
+

Batch / transaction sizing. Committing many rows in one transaction is far +cheaper than one transaction per row (Part III). Larger batches mean fewer commits +and higher throughput, at the cost of data becoming visible later and more memory +in flight. Flush only at genuine synchronization points (Part VI) — flushing per +item throws the batching advantage away.

+
+
+

Compression level. Compression shrinks the database and the disk I/O the funnel +must do, but it costs CPU on a worker stage. Choose the level by the compressor +stage’s active/sleep balance, exactly as tabulated in Part V: a compressor that +never sleeps wants a lower ratio (or to be split); one that always sleeps can +afford a higher ratio (or to be merged into a neighbor).

+
+
+

Polling cadence. Each stage’s polling interval (default 100 ms, Part VI) +trades latency against wasted wakeups. Shorter intervals lower latency but wake +idle threads more often; longer intervals do the reverse. Intermittent, latency- +tolerant stages want longer intervals; a stage on the critical path wants a +shorter one.

+
+
+

Stage boundaries. Splitting a saturated, parallelizable stage or merging two +idle ones (Chapter 37) is often the highest-leverage change of all, because it +rebalances where the threads spend their time.

+
+
+

Heartbeat cadence (Argos). For collectors, the heartbeat interval is the +size-versus-speed dial from the case study (Part VII): more frequent snapshots +cost simulation speed and database size; less frequent ones cost read-time +reconstruction work. It is a performance knob, not a correctness one.

+
+
+ + + + + +
+ + +Thread count today grows with the number of apps — there is no shared +pool yet (Part V). Keep the app count low; multi-app systems are not yet +thread-scalable. This is on the roadmap, not available now. +
+
+
+ + + + + +
+ + +Unrelated heavy disk I/O from your simulator — its own logging system, +trace dumps, or checkpoint files — contends with SimDB’s dedicated database +thread on the same filesystem path and can collapse throughput (Part IV). Treat +that as an environmental tuning problem: redirect or throttle non-SimDB writes +during collection. +
+
+
+
+

The instruments

+
+

You cannot tune what you have not measured. SimDB gives you three levels of +instrument, from coarse to fine:

+
+
+

Per-thread performance reports are the primary guide. The working-versus- +sleeping percentages (Part V) tell you which stage is the bottleneck and which is +idle — and therefore which knob to turn. Start here every time.

+
+
+

The self-profiler times specific methods or blocks. Drop PROFILE_METHOD at the +top of a function, or PROFILE_BLOCK("label") around a region, and the profiler +prints a per-label average-time report, sorted slowest-first, when the program +exits:

+
+
+
+
void MyStage::process_(Payload&& p)
+{
+    PROFILE_METHOD;                 // times this whole method
+    {
+        PROFILE_BLOCK("compress");  // times just this region
+        compress_(p);
+    }
+}
+
+
+
+

Use it to confirm which line of a hot stage is actually expensive, once the +per-thread report has told you which stage to look at.

+
+
+

RunningMean keeps a running average of any metric you feed it, using Welford’s +method, without storing the samples:

+
+
+
+
simdb::RunningMean batch_sizes;
+batch_sizes.add(records.size());
+// later:
+std::cout << "avg batch: " << batch_sizes.mean() << "\n";
+
+
+
+

Use it to track average payload bytes, average queue depth — rather than time a code +path.

+
+
+
+

The tuning loop

+
+

Put the instruments and knobs together into a discipline:

+
+
+
    +
  1. +

    Run a representative workload and read the per-thread performance report.

    +
    2. +
  2. +

    Find the saturated stage (the funnel, or an upstream bottleneck feeding it) and +the idle ones.

    +
    3. +
  3. +

    Turn one knob — split/merge a stage, change the compression level, resize the +batch, adjust a polling interval.

    +
    4. +
  4. +

    Re-run and compare the reports.

    +
    5. +
+
+
+

Changing one knob at a time is what makes the comparison meaningful. Tuning +everything at once tells you the result but never the cause.

+
+
+
+
+
+

39. Determinism & Debugging Concurrent Pipelines

+
+
+

A concurrent pipeline is nondeterministic by construction: stages run on their own +threads, wake on their own cadence, and interleave differently on every run. That +is exactly what makes it fast — and exactly what makes it hard to debug, because +the state you want to inspect is a moving target. This chapter is about buying back +enough determinism to reason about a running pipeline, and about the symptoms you +will actually encounter.

+
+
+

Freeze before you look

+
+

The first rule of debugging a pipeline is: stop it moving before you inspect it. +If you read a queue, a counter, or the database while stages are still running, +you are reading a value that was already stale by the time you looked. Part VI's +ScopedRunnableDisabler exists for precisely this — it pauses runnables for the +duration of a scope (RAII), so you can take a consistent snapshot:

+
+
+
+
{
+    simdb::ScopedRunnableDisabler disable_all(pipeline_manager);
+    // Nothing advances here. Inspect queues, counters, in-flight state
+    // knowing they will not change out from under you.
+}
+// Stages resume when the disabler goes out of scope.
+
+
+
+

Nested disablers are safe — inner scopes are no-ops — so you can reach for one +without worrying about whether an outer one is already active.

+
+
+
+

Freeze, then flush, then query

+
+

Disabling stops the pipeline in place, but data mid-flight has not yet reached the +database. When you want a consistent on-disk state to inspect with the Query API +or an external SQLite tool, pair disabling with a flush (Part VI). A flusher +drives every stage with the force flag set, draining buffered data all the way to +the single-transaction database commit. The flush-then-query pattern is the only +way to guarantee that what you read from the database reflects everything produced +so far — without it, batched transactions mean rows you expect simply are not +there yet. (See also the FAQ’s Why don’t I see my rows yet?)

+
+
+
+

Inspect in flight with snoopers

+
+

Sometimes you do not want to freeze or flush at all — you want to peek at what is +currently moving through a stage. That is what snoopers (Part VI) are for: +key-based retrieval of in-flight payloads without draining the pipeline. They are +the right tool for a live dashboard or a "what is stage 3 holding right now?" +diagnostic, with the tradeoffs already covered in the advanced chapter (snooping +briefly pauses the pipeline so the snapshot is coherent).

+
+
+
+

Logging without interleaved lines

+
+

std::cout from several stage threads produces garbled, interleaved lines — half of one stage’s message spliced into another’s. ThreadSafeLogger fixes this +by buffering a whole line and writing it under a lock only when the Guard +returned by protect() is destroyed:

+
+
+
+
simdb::ThreadSafeLogger log("[compressor] ");
+log.protect() << "batch " << n << " -> " << nbytes << " bytes\n";
+
+
+
+

Each line is atomic and prefixed by the string you constructed +the logger with. Give each stage its own prefix and every line in the merged +output is instantly attributable to a thread — which is most of what you need to +reconstruct an interleaving after the fact. ThreadSafeFileLogger is the same +thing pointed at a file when you want the log out of the console.

+
+
+
+

"Why is my pipeline sleeping / not draining?" — a checklist

+
+

The most common confusion is a pipeline that appears stuck. Work through this in +order:

+
+
+
    +
  1. +

    Read the per-thread performance report first (Part V). A stage sleeping at +~100% is simply not being fed — the problem is upstream, not here. A stage +active at ~100% is the bottleneck everything else is waiting behind.

    +
    2. +
  2. +

    Check the polling cadence. A long polling interval (Part VI) looks like a +stall — the stage is just waiting for its next wakeup. Latency-sensitive stages +want shorter intervals.

    +
    3. +
  3. +

    Did you forget to flush? Rows you produced but do not see are almost always +still batched in flight. Flush at the synchronization point (above).

    +
    4. +
  4. +

    Is a disabler still in scope? An outstanding ScopedRunnableDisabler — for +example, one held longer than intended — keeps everything paused. Confirm the +scope has exited.

    +
    5. +
  5. +

    Is the database thread the funnel? Everything commits through one thread +(Part IV). If that thread is saturated, upstream stages back up and sleep +waiting for room. Tune batch size and compression (Chapter 38) rather than the +upstream stages.

    +
    6. +
  6. +

    Did teardown run? If the pipeline never drained at shutdown, confirm +postSimLoopTeardown was called on every exit path (Part V) — an app that +aborts without it leaves data stranded and threads unjoined.

    +
    7. +
+
+
+

The theme throughout: don’t guess about a concurrent system. Freeze it, flush it, +log it, and read the perf report — in that order — and the nondeterminism stops +being in your way.

+
+
+
+
+
+

40. Application shapes

+
+
+

The README lists four things people build with SimDB: statistics engines, live +UI backends, replay backends, and aggregate analysis across many runs. Each of +those is a product problem with many valid architectures. SimDB does not pick +one for you — it gives you a consistent way to persist data concurrently and +to reach that data from other threads. This chapter is a set of application +shapes: the decisions you must make, the SimDB pieces that map to each part, and +several workable designs per shape so you are not locked into a single layout.

+
+
+ + + + + +
+ + +These are not copy-paste skeletons and not references to runnable +demos. A half-dozen quality UI backends exist for the same collector; a stats +engine can be as simple as "flat table, query after the run" or as rich as a +multi-clock snapshot/delta system. The book shows mechanics elsewhere (Parts +III—​VII); here we show where the hard design choices live. +
+
+
+

A simulation statistics engine

+
+

Outcome. During simulation, metrics leave the hot path quickly and land in a +.db file you can query, export, or hand to a viewer later.

+
+
+

The spectrum. At one end is the minimal engine: a flat schema — timestamp, +metric id, scalar value (or a small blob) — and a short pipeline (optional +compress, then database stage). After the run, SQL or a script produces CSVs or +summary tables. That is genuinely useful and genuinely easy relative to everything +else on this page. At the other end is a rich collector: hierarchical +collectables, multiple clocks, typed containers, snapshot/delta encoding, and a +schema that is itself a contract for a replay tool. Argos (Part VII) is the +worked example of that end; most teams start at the minimal end and grow toward +rich encoding only when post-hoc flat tables stop answering their questions.

+
+
+

Decisions to make together (schema + pipeline, Chapter 36):

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + +
QuestionConsequence

What will you filter or group on later?

Those fields must be columns, not only bytes inside a blob.

How often do you write?

High frequency favors batching, larger payloads, and fewer commits; low +frequency can afford row-at-a-time simplicity.

Scalars only, or vectors/structures too?

Scalars fit normalized tables; bulk state fits blobs (often compressed).

One time base or several?

One clock keeps the schema simple; multiple clocks need explicit clock ids on +every time-stamped row (as in the Argos data model).

+
+

SimDB pieces. An simdb::App with defineSchema and createPipeline; move- +only payloads between stages; optional compression before the terminal database +stage; createFlusher at the points where you need visibility or teardown safety; +postSimLoopTeardown on every exit path so the file is whole.

+
+
+

Pitfalls. Designing the pipeline before you know your queries (you will bury +filterable fields in blobs). Flushing every sample (you lose batching and stall +the funnel). Growing into Argos-scale encoding before you need it (complexity +without payoff).

+
+
+
+

A live UI backend

+
+

Outcome. Something on screen updates while the simulation runs — or within a +bounded delay — without stalling the simulator.

+
+
+

Split the problem. SimDB owns how data gets out of the sim thread safely and +how it becomes durable. Your UI framework owns windows, layout, plotting, and +networking. No single recipe spans both; the shapes below are only the SimDB-side +patterns. Combine them.

+
+
+

Shape A — Post-run or flush-bound UI (simplest). The collector writes the +database; the UI opens the .db after a flush or after the run. The UI reads +committed rows and blobs only. Latency equals your flush cadence (or end of run). +Design: schema with queryable timestamps and ids; periodic flush() at +heartbeats if you want near-live updates without snoop machinery. Fits tools that +tolerate hundreds of milliseconds to seconds of lag.

+
+
+

Shape B — Thin client over the database (Argos-like). The collector checkpoints +on a heartbeat; the viewer loads on demand — no full preload, no requirement +to cache what was already shown (Part VII). Live enough for many workflows because +deserialization/replay is fast and the UI only pulls what the current widgets need. +Design: rich blob encoding + metadata columns the UI can index; heartbeat as a +performance dial, not a gate on collection.

+
+
+

Shape C — Snoop the pipeline tail (lowest latency for hot metrics). Wire a +snooper on a stage that carries the payloads your dashboard cares about; read by +key without waiting for SQLite (Part VI). Use for a small set of hot signals +(gauge, last N queue depths), not for full history. Accept the tradeoff: snooping +coordinates with the pipeline briefly so the snapshot is coherent.

+
+
+

Shape D — Metadata from the UI thread via async DB access. When the UI thread +needs counts, last timestamp, or schema discovery, post work to the dedicated +database thread with AsyncDatabaseAccessor::eval (Part VI) — never call SQLite +from the UI thread directly, and never from inside a DatabaseStage. Bulk curves +still come from Shape A, B, or C; this shape is for light queries.

+
+
+

Shape E — Dual path (advanced). Pipeline persists everything to the database +(cold, authoritative path) while a separate fan-out — fed by the same stage +output, a snooper, or a side queue you own — pushes decimated samples to the UI +(hot path). Two consumers, one producer; you must define which path is source of +truth when they disagree (almost always the database).

+
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ShapeBest whenSimDB cost

A — flush-bound

Lag OK; simplest mental model

Flusher + schema

B — thin DB client

Rich state; many widgets; replay

Encoding + heartbeat design

C — snoop

Sub-flush latency for few metrics

Snooper wiring + key design

D — async eval

UI thread needs safe metadata queries

Accessor discipline

E — dual path

High rate + live plot + full archive

Two paths to keep consistent

+
+

Pitfalls. Treating snoopers as a generic message bus (they are keyed peeks, not +a stream API). Querying SQLite from the UI thread (locks, wrong thread). Expecting +the UI to mirror the entire simulation state in memory (defeats SimDB’s pipeline +and bloats RAM). Choosing Shape E when Shape B plus a sensible heartbeat would + suffice.

+
+
+
+

A simulation state replayer backend

+
+

Outcome. Given a time (or tick), reconstruct what the simulation would have +looked like at that moment — for a debugger, a regression diff, or a standalone +replay tool.

+
+
+

The contract is the schema. The replayer does not guess. Every blob column, enum +table, and timestamp column is part of a versioned contract: what is snapshotted +vs delta-encoded, how containers are laid out in binary, which clock id applies. +Part VII's on-disk model is one full instance of that contract; your engine can be +smaller, but the same rule applies — document the bytes.

+
+
+

Write path (collector side). Producers emit state on a schedule you define: +full snapshots at rebases, deltas between them, or append-only event rows if your +replay logic prefers an event log. Compression and the database stage stay at the +end of the pipeline; flush at rebases so a crash mid-run still leaves a coherent +prefix.

+
+
+

Read path (replayer side). Load the nearest snapshot at or before the target +time, apply deltas forward (or walk events), deserialize blobs using the same +rules the collector used to serialize them. The replayer may live in another process +(Python, Qt, a batch verifier); it only needs the .db and the contract.

+
+
+

Encoding choices (pick one primary strategy).

+
+
+
    +
  • +

    Snapshot + delta — smaller database, more read work; good when state is +large and changes incrementally (Argos).

    +
    • +
  • +

    Full snapshot only — trivial replay, heavy writes and disk; good when state +is small or runs are short.

    +
    • +
  • +

    Event log — append rows (time, event_type, payload); replay is apply-until-T; +good when state is derivable from events and snapshots are awkward.

    +
    • +
+
+
+

SimDB pieces. Blobs and side tables (TinyStrings, enum maps); flushers at +rebase boundaries; optional ScopedRunnableDisabler plus flush when the replayer +and collector share a process and you need a quiescent point (Chapter 39).

+
+
+

Pitfalls. Changing blob layout without a schema/version column (old databases +become unreadable). Mixing clocks without tagging rows (replay at the wrong +"time"). Skipping flush at rebases (partial last snapshot after crash).

+
+
+
+

Aggregate analysis across many simulations

+
+

Outcome. Compare or summarize results from hundreds or thousands of runs — parameter sweeps, regression suites, nightly farms.

+
+
+

Two phases. (1) Produce one .db per run with the same app schema so every +file has identical tables. (2) Analyze offline — merge, attach, or query across +files. Phase 2 often needs no SimDB pipeline at all; SQLite, Python, or R on the +files is enough. SimDB’s role in phase 1 is making each run’s artifact consistent; +in phase 2 it is optional unless you build a merge app as another simdb::App.

+
+
+

Warehouse shapes.

+
+
+
    +
  • +

    Many files, query with ATTACH — keep runs isolated; good for parallel sim +and simple tooling.

    +
    • +
  • +

    ETL into one warehouse database — copy normalized summary columns into a +runs(id, params…) plus metrics(run_id, …) layout; good for heavy cross-run +SQL.

    +
    • +
  • +

    Blob-per-run with summary columns extracted at end — full state stays in per- +run files; a lightweight post-pass inserts only scalars you care about for sweeps.

    +
    • +
+
+
+

Schema discipline across runs. Store run parameters as columns (or a dedicated +params table) at collection time if you know them upfront; inferring params later +from filenames is fragile. Add a schema_version or tool version column early so +a farm does not mix incompatible blob layouts silently.

+
+
+

SimDB pieces (production phase). Same defineSchema for every run; batch writes +and compression so per-run overhead stays low; reliable teardown so farm workers +do not leave corrupt files.

+
+
+

Pitfalls. One giant pipeline-fed database fed by all sim processes concurrently +(fights the single-writer model — prefer one DB per process). Storing only blobs +with no summary columns (every analysis pass deserializes everything). Schema drift +across tool versions without version metadata.

+
+
+
+

How to use this chapter

+
+

Pick the shape closest to your goal, then go back to the parts that teach the +mechanics: schema and INSERT (Part III), pipelines (Part IV), apps and lifecycle +(Part V), flush/snoop/async (Part VI), and Argos if you need the rich collector/ +viewer split (Part VII). Tuning and debugging loops live in Chapters Chapter 38 and Chapter 39. +None of the shapes above is mandatory; mixing them — flush-bound persistence plus +snoop for one live gauge, flat tables for farm summary columns — is normal +engineering, not a violation of SimDB.

+
+
+
+
+
+

41. Testing SimDB Apps

+
+
+

Regression tests are how SimDB keeps its API honest, and they are the model for +how you should test your own apps. The goal is not to re-run your simulator inside +every test — it is to prove that the artifact your app produces (the .db +file, its schema, and the bytes in its blobs) matches what you expect, independent +of how the app got there. This chapter covers the test harness SimDB uses, the +flush-and-teardown habits pipeline apps require, and how to verify persisted data +without trusting the app that wrote it.

+
+
+

What to test at each layer

+
+

SimDB apps stack three separable concerns. Test them at the boundary where bugs +actually show up:

+
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + +
LayerWhat can go wrongTypical test

SQLite interface

Schema mistakes, query constraints, blob round-trip

Direct +DatabaseManager tests (Part II, test/sqlite/**)

App + pipeline

Stages never drain, wrong flush points, lifecycle skipped

Drive +AppManagers through simulate → teardown; assert on the file

Encoding / contract

Blob layout drift, replay mismatch

Deserialize the .db +with a separate reader (C++ helper, Python script, or the viewer’s replay path)

+
+

You do not need every test to exercise all three. A schema test can skip the +pipeline entirely; an end-to-end collector test should hit the file and an +independent reader, not only in-app getters.

+
+
+
+

The SimDBTester harness

+
+

SimDB’s regression programs use a lightweight macro harness in test/SimDBTester.hpp — not GoogleTest, but the same shape: accumulate failures, print a summary, return +a non-zero exit code for CI.

+
+
+

Every test translation unit starts with TEST_INIT (once, at file scope). Inside +main, use EXPECT_* macros and finish with REPORT_ERROR followed by +return ERROR_CODE:

+
+
+
+
TEST_INIT;
+
+int main()
+{
+    EXPECT_EQUAL(2 + 2, 4);
+    EXPECT_TRUE(some_condition);
+    EXPECT_WITHIN_EPSILON(actual, expected);   // floating-point
+    EXPECT_THROW(bad_call());
+
+    REPORT_ERROR;
+    return ERROR_CODE;
+}
+
+
+
+

The macros worth knowing:

+
+
+
    +
  • +

    EXPECT_EQUAL / EXPECT_NOTEQUAL — equality with values printed on failure.

    +
    • +
  • +

    EXPECT_WITHIN_EPSILON / EXPECT_WITHIN_TOLERANCE — floating-point compares +(the SQLite regression tests use these heavily).

    +
    • +
  • +

    EXPECT_THROW / EXPECT_NOTHROW — exception discipline.

    +
    • +
  • +

    EXPECT_FILES_EQUAL — byte-compare two files (lines starting with # are +ignored, useful for golden text output).

    +
    • +
  • +

    EXPECT_REACHED / ENSURE_ALL_REACHED(N) — mark that a callback or hook fired; +useful when testing that lifecycle methods or stage paths actually ran.

    +
    • +
+
+
+ + + + + +
+ + +Call REPORT_ERROR before teardown that might crash if earlier +assertions left the app in a bad state (dangling pointers, half-built pipelines). +Return ERROR_CODE after teardown completes. The harness separates reporting from +exit so you still get a clean failure summary when cleanup is fragile. +
+
+
+

Build the full regression suite with the simdb_regress target (Part II). Your +own app tests can follow the same pattern and plug into CTest the same way.

+
+
+
+

Flush, teardown, then assert

+
+

Pipeline apps add one rule SQLite-only tests never need: do not query what is still +in flight. Batched stages and the single database thread mean rows and blobs you +just `process()’d may not exist on disk yet. Before any assertion that reads the +database:

+
+
+
    +
  1. +

    Flush if you need mid-run visibility (flusher_→flush(), Part VI).

    +
    2. +
  2. +

    Always call postSimLoopTeardown() on every exit path before you treat the file +as final (Part V) — normal finish, caught exception, or abort handler you +install.

    +
    3. +
+
+
+

A minimal integration-test skeleton for an simdb::App:

+
+
+
+
// Arrange
+simdb::AppManagers app_mgrs;
+app_mgrs.enableApp<MyCollector>();
+app_mgrs.createSchemas();
+app_mgrs.postInit(0, nullptr);
+app_mgrs.initializePipelines();
+app_mgrs.openPipelines();
+
+// Act -- drive your simulator or inject test inputs
+runSimulationOrInjectTestData();
+
+// Assert -- drain and finalize FIRST
+app_mgrs.postSimLoopTeardown();
+
+simdb::DatabaseManager verifier("output.db");  // reopen independently
+auto q = verifier.createQuery("MyMetrics");
+EXPECT_EQUAL(q->count(), expected_rows);
+// ... read columns/blobs and compare ...
+
+REPORT_ERROR;
+return ERROR_CODE;
+
+
+
+

Notice the verifier opens the file with a new DatabaseManager. That is +intentional: you are testing the persisted artifact, not whether your app’s in- +memory caches happen to agree with themselves.

+
+
+
+

Verify the database independent of the app

+
+

The strongest tests treat the .db as the contract and use a reader that does not +share code with the writer.

+
+
+

Reopen and query. After teardown, connect with a fresh DatabaseManager (or the +Query API patterns from Part III). Check row counts, column values, and blob bytes +with EXPECT_EQUAL. This catches schema drift and wrong INSERT paths even when +the app’s own "get last value" helper still works.

+
+
+

Deserialize blobs on their own. When your app stores encoded state, add a small +test-side deserializer (or reuse your replay logic) that reads blobs from +CollectionRecords — or your equivalent table — and compares to an in-test +ground truth you maintained while driving the sim. The Argos regression harness +does exactly this: a simulation engine holds expected values per tick; after +teardown, a blob iterator walks the file and asserts each decoded value matches. +You do not need the viewer for that — only the same decoding rules the viewer +would use.

+
+
+

External scripts. For complex encodings, a Python (or other) script that opens +the .db with sqlite3, validates scaffolding tables, and replays blobs is +often clearer than C++ assertion code. The Argos tree ships test/argos/compare.py, +which compares two databases tick-by-tick using the viewer’s DataRetriever unpack +path — proving two runs produce identical replayed values, not just identical +SQL rows. That pattern generalizes: your script encodes what "correct bytes mean" +and fails CI when decoding diverges.

+
+
+

Baseline comparison. Keep a known-good .db (or exported golden query results) +and diff against test output — EXPECT_FILES_EQUAL for text, or a script for +binary databases. Version the baseline when you intentionally change the encoding; +store schema_version in the file so mismatches are explainable.

+
+
+
+

What belongs in the test vs. what belongs outside

+
+

Keep tests focused on SimDB’s guarantees:

+
+
+
    +
  • +

    Schema matches defineSchema.

    +
    • +
  • +

    Pipeline + teardown produce a readable file.

    +
    • +
  • +

    Queryable columns and blob decodings match ground truth.

    +
    • +
+
+
+

Push these out of the core regression loop when they bloat or flake:

+
+
+
    +
  • +

    Full GUI automation (test the data contract; let humans or separate UI tests +cover widgets).

    +
    • +
  • +

    Performance thresholds unless you have a stable benchmark harness (use +PollingThread::printPerfReport manually or in a dedicated perf job, not every +CI run).

    +
    • +
  • +

    Non-deterministic sim workloads without a fixed seed — reproduce failures first, +then assert.

    +
    • +
+
+
+

For collectors with randomness, fix the RNG seed in tests (the Argos harness uses +a constant seed so two runs are comparable). For concurrent pipelines, prefer +teardown-then-verify over asserting mid-flight ordering unless you are explicitly +testing flush/disable behavior (Chapter 39).

+
+
+
+

A practical checklist

+
+

Before you merge a SimDB app change, ask:

+
+
+
    +
  1. +

    Does every test path call postSimLoopTeardown() (including error paths you +simulate)?

    +
    2. +
  2. +

    Are database assertions made after flush/teardown, on a freshly opened manager +or external reader?

    +
    3. +
  3. +

    Do blob tests decode through the same rules production replay uses?

    +
    4. +
  4. +

    Did you run a Release build? Unused parameters, members, and includes that Debug +ignores can fail CI under -Werror.

    +
    5. +
  5. +

    If you changed encoding or schema, did you update baselines and bump a version +column/metadata so old files fail loudly?

    +
    6. +
+
+
+

Testing SimDB apps is mostly discipline: treat the .db as the product, finalize +it correctly, and verify it with code that did not write it. The harness macros +handle reporting; flush and teardown handle visibility; an independent reader +handles trust.

+
+
+

Part IX collects quick-reference material — utilities, API cheat sheets, FAQ, +troubleshooting, and a migration guide — for when you need a lookup rather than +the narrative through Parts Part I--Part VIII.

+
+
+
+
+

Part IX: Reference & Appendices

+
+
+Quick-reference material for day-to-day lookups. The narrative parts of this book +explain why and when; this part is what and where — headers, macros, +lifecycle calls, glossary terms, common failure modes, and migration paths. +
+
+
+

Appendix A: Utilities Reference

+
+
+

General-purpose helpers in include/simdb/utils/. Most are header-only and usable +outside pipelines.

+
+
+

Compression (Compress.hpp)

+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + +
APIPurpose

compressData(ptr, nbytes, out, level)

Zlib-compress raw bytes into std::vector<char>.

compressData(vector<T>, out, level)

Compress a typed vector’s bytes.

decompressData(in, out)

Decompress into std::vector<T>.

CompressionLevel

DISABLED, DEFAULT, FASTEST, HIGHEST (maps to zlib levels).

+
+

Empty inputs produce a valid minimal zlib wrapper (compatible with Python +zlib.decompress).

+
+
+
+

Queues (ConcurrentQueue.hpp)

+
+

Thread-safe FIFO built on std::deque.

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + +
MethodPurpose

push / emplace

Enqueue (copy, move, or in-place construct).

try_pop(T&)

Dequeue if non-empty; returns false when empty.

size / empty

Queue depth under lock.

snoop(callback)

Scan items without removing; stop when callback returns true. Used by pipeline snoopers (Part VI).

+
+
+

Metrics and timing

+
+

RunningMean (RunningMean.hpp) — Welford streaming average: add(value), +mean(), count(). No sample storage.

+
+
+

SelfProfiler (TickTock.hpp) — Singleton method/block timer. PROFILE_METHOD +or PROFILE_BLOCK("label") at scope start; prints sorted average times on process +exit. Despite the filename, there is no class named TickTock.

+
+
+
+

Logging (ThreadSafeLogger.hpp)

+ ++++ + + + + + + + + + + + + + + + + + + + + +
APIPurpose

ThreadSafeLogger(prefix)

Writes prefixed lines to stdout.

ThreadSafeFileLogger(path)

Same, to a file.

protect()

Returns a Guard; stream into it; one atomic line on destruction.

+
+
+

String interning (TinyStrings.hpp)

+
+

Maps strings to uint32_t ids for compact binary encoding. insert(s) returns +(id, is_new); getStringID(s) returns id (0 = bad/empty). serialize(db_mgr) +writes new mappings to the database (call at teardown when you use this pattern — today not automatic via postSimLoopTeardown; see Part V). Template parameter +MutexProtect enables optional locking for multi-threaded producers.

+
+
+
+

Values and comparison

+
+

ValidValue\<T\> (ValidValue.hpp) — Optional-like holder with explicit validity; +getValue() throws if unset.

+
+
+

approximatelyEqual (FloatCompare.hpp) — Relative tolerance compare for +floating-point types; default tolerance is machine epsilon. Used by +EXPECT_WITHIN_EPSILON in tests.

+
+
+
+

Symbol and type helpers

+
+

demangle / demangle_type\<T\> (Demangle.hpp) — Itanium ABI demangling via +__cxa_demangle; falls back to input on failure. Buffer size +DEMANGLE_BUF_LENGTH (4096).

+
+
+

simdb::type_traits (TypeTraits.hpp) — Template metaprogramming helpers +(enable_if_t, pointer/string detection, container traits, etc.) used throughout +SimDB headers.

+
+
+

ios_format_saver (StreamFormatters.hpp) — RAII restore of std::ios format +flags after temporary manipulators.

+
+
+
+

Lock helpers

+
+

ConditionalLock\<Mutex\> (ConditionalLock.hpp) — Locks only when the bool +constructor argument is true.

+
+
+

DeferredLock\<Mutex\> (DeferredLock.hpp) — Call lock() manually; unlocks in +destructor. Used by TinyStrings when MutexProtect is enabled.

+
+
+
+

Test utilities

+
+

generateRandomData (Random.hpp, namespace simdb::utils) — Random arithmetic +vectors for unit tests.

+
+
+

utf16 (utf16.hpp) — UTF-16 conversion helpers for narrow/wide string +interoperability where needed.

+
+
+
+
+
+

Appendix B: API Quick Reference & Glossary

+
+
+

One-page cheat sheets. For worked examples and rationale, see Parts Part III--Part VI.

+
+
+

SQLite interface

+
+

Schema (simdb/sqlite/Schema.hpp, Table.hpp)

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConstructNotes

schema.addTable("Name")

Returns Table&. Implicit auto-increment Id PK unless changed.

tbl.addColumn("Col", dt::…)

Types: int32_t, uint32_t, int64_t, uint64_t, double_t, string_t, blob_t.

tbl.setPrimaryKey("Col")

Replace default Id PK.

tbl.unsetPrimaryKey()

No primary key.

tbl.setDefaultValue("Col", val)

Column default on INSERT omit.

tbl.addIndex(…)

Query acceleration (Part III).

+
+

DatabaseManager (DatabaseManager.hpp)

+
+ ++++ + + + + + + + + + + + + + + + + + + + + +
CallPurpose

DatabaseManager(path, create_if_missing)

Open or create .db.

appendSchema(schema)

Apply schema to file (create/alter tables).

getDatabaseFilePath()

Path string for reopening elsewhere.

+
+

Insert macros (Table.hpp)

+
+
+
+
db_mgr.INSERT(
+    SQL_TABLE("MyTable"),
+    SQL_COLUMNS("A", "B"),      // optional if all columns in order
+    SQL_VALUES(a_val, b_val));
+
+
+
+

Returns a record handle with typed getters/setters. Omit SQL_COLUMNS when +supplying every column in schema order.

+
+
+

Query (Query.hpp)

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PatternPurpose

createQuery("Table")

Start a SELECT builder.

select("Col", var)

Bind output column to variable (or std::optional for NULL).

addConstraintForInt/UInt/Double/String(…)

WHERE clauses; SetConstraints::IN_SET, etc.

orderBy, setLimit, resetConstraints

Ordering and pagination.

count()

Row count for current query.

getResultSet() + getNextRecord()

Iterate rows.

findRecord(table, id)

Direct row fetch by primary key.

+
+

Transactions (Transaction.hpp)

+
+ ++++ + + + + + + + + + + + + +
APIPurpose

safeTransaction([&]{ … })

Runs lambda inside a transaction; retries on lock/contention until success. Reentrant-safe.

+
+

Inspection — simdb/sqlite/Dump.hpp prints human-readable schema/table dumps for debugging.

+
+
+
+

Pipeline interface

+
+

Core types (Stage.hpp, Pipeline.hpp, PipelineManager.hpp)

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConceptAPI / behavior

Stage

Subclass; implement run_(bool force); return PROCEED or SLEEP.

Ports

addInPort_<T>("name", queue_ptr) / addOutPort_<T>(…) in constructor.

PipelineManager

Owns pipelines for one or more apps.

createPipeline(name, app)

Register one pipeline for an app (typical: one call per app). Multiple apps ⇒ multiple pipelines on the shared manager.

addStage<MyStage>("name", …)

Register stage; order matters for flush.

bind("src.port", "dst.port")

Connect output queue to input.

noMoreStages() / noMoreBindings()

Seal pipeline definition.

Head queue

pipeline→getInPortQueue<T>("stage.port") for producer injection. Several heads = several unbound inputs on the same pipeline.

DatabaseStage<AppT>

Terminal stage; writes via app’s DatabaseManager.

+
+

Synchronization and inspection

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ToolPurpose

pipeline→createFlusher({stage_names…})

Ordered flush; DB stages commit in one safeTransaction when present.

flusher→flush()

Drain all listed stages with force=true.

PipelineManager::createSnooper<Key,Snooped>()

Key-based in-flight reads.

snooper→addStage(…) / snoopAllStages(key, out)

Wire and query without draining.

ScopedRunnableDisabler

RAII pause of pipeline runnables (Part VI).

AsyncDatabaseAccessor::eval(func, timeout)

Run func(DatabaseManager*) on DB thread from any other thread.

+
+

Scheduling — Stage(interval_ms) sets polling sleep when idle (default 100 ms). +PipelineAction::PROCEED keeps the stage hot; SLEEP yields until next poll.

+
+
+
+

App framework

+
+

App lifecycle (App.hpp, AppManager.hpp) — call via AppManagers in order:

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + +
PhaseCalls

Register

enableApp<MyApp>(), optional parameterize(…) for custom factories (Part V).

Startup

createSchemas() → postInit(argc, argv) → initializePipelines() → openPipelines().

Run

Your simulator loop; process() on pipeline heads, collect via app API.

Shutdown

postSimLoopTeardown() on every exit path.

+
+

App hooks (override on your simdb::App subclass)

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + +
HookWhen

defineSchema

Declare tables before DB creation.

postInit

After schema exists; pre-sim metadata and configuration. Runs inside automatic safeTransaction (with all apps' hooks in one transaction).

createPipeline

Wire one pipeline; store one or more pipeline head queues.

preTeardown / postTeardown

App-local cleanup around sim-loop end. postTeardown runs inside automatic safeTransaction (with all apps' hooks in one transaction).

+
+

getInstance() / setInstance(n) — access the active app instance from stages +and static contexts (Part V).

+
+
+
+

Glossary

+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TermMeaning

Stage

A Runnable unit of pipeline work with input/output ports and a run_(force) loop.

Port

Named attachment point on a stage; typed by the queue’s payload.

Queue

ConcurrentQueue<T> connecting an output port to an input port.

Pipeline head

The input queue where the producer (sim thread) injects work. One pipeline may have several heads; one app typically has one pipeline.

DB stage

Terminal DatabaseStage that performs SQLite writes on the dedicated DB thread.

Flusher

Ordered, forced drain of stages through to the database commit point.

Snooper

Key-based peek at in-flight queue items without dequeuing.

Heartbeat

(Argos) Interval between full snapshots; deltas between heartbeats. Performance dial only — does not gate collection.

Collectable

(Argos) A registered simulation object whose state can be collected over time.

CID

Collectable ID — stable integer identity for a collectable in the Argos schema.

Tick

One step of the simulation clock; timestamps in the DB refer to tick (or cycle) values per clock.

Flush-then-query

flush() (and/or teardown) before reading the database so batched data is visible.

Funnel

The single dedicated database thread through which all commits pass.

+
+
+
+
+

Appendix C: FAQ & Troubleshooting

+
+
+

"Database is locked" / SQLite contention

+
+

What SimDB handles for you. All SQLite access from pipeline database stages goes +through one dedicated thread. Writes are batched inside safeTransaction, which +retries until the transaction succeeds if SQLite reports lock contention. You do +not manually retry INSERTs in normal pipeline use.

+
+
+

What you must still avoid. Calling SQLite (or opening a second writer) from + arbitrary threads without AsyncDatabaseAccessor::eval. Opening the same .db + from another process while the sim is writing. Heavy unrelated disk I/O from + your simulator competing with the DB thread (Part IV, Part VIII).

+
+
+

If you see lock errors from async eval. Another tool may hold the file; ensure + only one writer; use read-only opens for external viewers where possible.

+
+
+
+

"Why is my pipeline sleeping?"

+
+

A stage returns SLEEP when run_(force) finds no work (empty input queue and +nothing left to do on force). That is normal idle behavior.

+
+
+

Upstream not feeding. Perf report shows ~100% sleep on a stage whose input +should be busy — check the producer and bindings.

+
+
+

Downstream bottleneck. Upstream stages sleep because the DB stage or a saturated +middle stage is not draining — tune batching/compression (Part VIII) or split hot +stages.

+
+
+

Polling cadence. A stage wakes every interval_ms (default 100). Long intervals +look like "sleeping" between bursts — shorten for latency-sensitive paths.

+
+
+

Disabler left active. ScopedRunnableDisabler still in scope pauses everything.

+
+
+

Wrong action return. Forgetting PROCEED after successful work makes the stage +go idle even when data remains (until the next poll).

+
+
+
+

"Why don’t I see my rows yet?"

+
+

Almost always: data is still in flight.

+
+
+
    +
  • +

    Batched transactions have not committed.

    +
    • +
  • +

    Rows sit in an upstream queue.

    +
    • +
  • +

    You queried before flush() or postSimLoopTeardown().

    +
    • +
+
+
+

Fix: flush at the synchronization point, or teardown before assert; then reopen the +file with a fresh DatabaseManager (Part VIII, Chapter 41).

+
+
+
+

"My database is partial or corrupt after a crash"

+
+

Did postSimLoopTeardown() run? Without it, queues may not drain, blobs may be +truncated, and std::thread destructors may throw on unjoined pipeline threads +(Part V). Install teardown on exception/abort paths you control.

+
+
+

If you use TinyStrings, mappings may not be flushed at teardown today — call +serialize explicitly until automatic flush is added.

+
+
+
+

"Why did CI fail in Release but not Debug?"

+
+

Release builds with -Werror treat unused parameters, members, and includes as +errors. After editing headers or tests, build Release locally:

+
+
+
+
cmake --build build --target YourTest -DCMAKE_BUILD_TYPE=Release
+
+
+
+

Remove dead code rather than silencing warnings unless intentionally stubbing an +interface.

+
+
+
+

"Snooper never finds my key"

+
+

Snoopers search in-flight queue contents on wired stages, not the database. Keys +must match what the stage stores; the snooped type must match the queue payload (or +a registered transform). If data already passed to the DB stage, use flush-then-query +instead.

+
+
+
+

"AsyncDatabaseAccessor eval hangs or times out"

+
+

The DB thread may be blocked on a long transaction or crashed. Do not call eval +from inside a DatabaseStage (deadlock). Set a non-zero timeout_seconds during +development to surface hangs (Part VI).

+
+
+
+
+
+

Appendix D: Migration Guide

+
+
+

Paths for teams replacing ad-hoc data plumbing with SimDB. Each is incremental — you do not need all three at once.

+
+
+

From "write everything to files" to one SQLite database

+
+

Problem. Per-metric log files, ad-hoc directories, post-processing scripts that +merge and repair formats after every run.

+
+
+

Approach.

+
+
+
    +
  1. +

    Inventory outputs — list every file type, who reads it, and what queries +they run (filter by time? by component name?).

    +
    2. +
  2. +

    Design schema first (Part III, Part VIII Chapter 36) — columns for anything you +filter on; blobs for bulk payloads you only read whole.

    +
    3. +
  3. +

    Replace writes incrementally — one metric family at a time; keep legacy +files behind a flag until the DB path is verified.

    +
    4. +
  4. +

    Add a pipeline when the sim thread must not block — producer pushes to the +head queue; compress + DB stage at the tail (Part IV).

    +
    5. +
  5. +

    Wrap in simdb::App when multiple subsystems share one database and lifecycle +(Part V).

    +
    6. +
  6. +

    Retire file writers when tests prove query/export from .db replaces them.

    +
    7. +
+
+
+

Win. One artifact per run, queryable with SQL, Python, or your viewer; no merge +step; SimDB handles concurrent writes safely.

+
+
+
+

From raw SQLite (or a thin wrapper) to the SimDB SQLite interface

+
+

Problem. Hand-written CREATE TABLE, manual prepared statements, custom retry +logic, uint64_t and blob bugs.

+
+
+

Approach.

+
+
+
    +
  1. +

    Express tables as simdb::Schema instead of SQL DDL strings.

    +
    2. +
  2. +

    Replace raw INSERTs with SQL_TABLE / SQL_VALUES and typed record handles.

    +
    3. +
  3. +

    Replace ad-hoc SELECT strings with createQuery and bound select() variables.

    +
    4. +
  4. +

    Wrap multi-row writes in safeTransaction (or let database stages batch for you).

    +
    5. +
  5. +

    Run the test/sqlite/** regressions alongside your port to validate parity (Part II).

    +
    6. +
+
+
+

Keep. External tools that already read SQLite — the file format remains standard +SQLite. SimDB adds type safety and retry semantics, not a proprietary container.

+
+
+
+

From a hand-rolled thread pipeline to SimDB stages

+
+

Problem. Custom worker threads, queues, condition variables, and "one thread owns +SQLite" conventions copied into every project.

+
+
+

Approach.

+
+
+
    +
  1. +

    Map each worker to a stage — one responsibility per stage; move-only payloads +between queues (Part IV, Part VIII Ch 37).

    +
    2. +
  2. +

    Replace your DB thread with DatabaseStage + Flusher — do not keep a +second SQLite writer.

    +
    3. +
  3. +

    Replace cross-thread DB calls with AsyncDatabaseAccessor::eval (Part VI).

    +
    4. +
  4. +

    Replace debug freezes with ScopedRunnableDisabler + flush.

    +
    5. +
  5. +

    Delete home-grown retry/transaction code where safeTransaction and +FlusherWithTransaction cover the same ground.

    +
    6. +
+
+
+

Pitfall. Porting thread-for-thread without rethinking stage boundaries recreates +the old queue graph with extra overhead. Merge idle stages; split saturated ones +using perf reports (Part V, Part VIII Chapter 38).

+
+
+
+

A clean break vs. gradual adoption

+
+

SimDB is designed as a replacement architecture for tangled data paths (README: +"A Clean Break for Complex Codebases"), not a shim around every legacy writer. +Gradual migration works best per metric or subsystem, not by running two parallel +pipelines inside one app forever. Pick a boundary — one database, one pipeline +per app, multiple heads if you need several producers — and converge writers +there until legacy paths can be deleted.

+
+
+

When migration is done, your simulator talks to SimDB at the edges (collect / +process / teardown); everything else is SQL, analysis scripts, or a viewer on the +.db file.

+
+
+
+
+
+ + + + + \ No newline at end of file diff --git a/docs/book/book.pdf b/docs/book/book.pdf new file mode 100644 index 00000000..3aa82c1f Binary files /dev/null and b/docs/book/book.pdf differ diff --git a/docs/book/images/argos-first-launch.png b/docs/book/images/argos-first-launch.png new file mode 100644 index 00000000..5266650a Binary files /dev/null and b/docs/book/images/argos-first-launch.png differ diff --git a/docs/book/images/argos-hero.png b/docs/book/images/argos-hero.png new file mode 100644 index 00000000..91929b5d Binary files /dev/null and b/docs/book/images/argos-hero.png differ diff --git a/docs/book/images/argos-queue-inspector.png b/docs/book/images/argos-queue-inspector.png new file mode 100644 index 00000000..840ee15e Binary files /dev/null and b/docs/book/images/argos-queue-inspector.png differ diff --git a/docs/book/images/argos-queue-utilizations.png b/docs/book/images/argos-queue-utilizations.png new file mode 100644 index 00000000..3409f06f Binary files /dev/null and b/docs/book/images/argos-queue-utilizations.png differ diff --git a/docs/book/images/argos-scheduling-lines.png b/docs/book/images/argos-scheduling-lines.png new file mode 100644 index 00000000..d7a84cf7 Binary files /dev/null and b/docs/book/images/argos-scheduling-lines.png differ diff --git a/docs/book/images/argos-schema-db-browser.png b/docs/book/images/argos-schema-db-browser.png new file mode 100644 index 00000000..cab6d70e Binary files /dev/null and b/docs/book/images/argos-schema-db-browser.png differ diff --git a/docs/book/images/argos-watchlist.png b/docs/book/images/argos-watchlist.png new file mode 100644 index 00000000..d6f2965e Binary files /dev/null and b/docs/book/images/argos-watchlist.png differ diff --git a/docs/book/parts/part1-why-simdb.adoc b/docs/book/parts/part1-why-simdb.adoc new file mode 100644 index 00000000..a2498ca1 --- /dev/null +++ b/docs/book/parts/part1-why-simdb.adoc @@ -0,0 +1,496 @@ +[#part-i] += Part I: Why SimDB + +This part answers the question a brand-new user asks first: _"What is SimDB, and +why would I choose it over the tools I already know?"_ No code yet -- just the +problem, the landscape, and the mental model you will carry through the rest of +the book. + +[#ch-01-introduction] +== Introduction + +Every simulation produces data, and sooner or later that data becomes the +project. Statistics pile up. Traces balloon. Someone wants a live view of what +the model is doing, someone else wants to replay a run from an hour ago, and a +third person wants to compare a thousand runs at once. The simulator was the +interesting part; moving its data around safely and quickly turns into a +second, unplanned engineering project. + +SimDB exists to make that second project disappear. + +=== The one-sentence version + +SimDB is a header-only C++17 module that unifies *concurrent data pipelines* +with *SQLite* to power scalable simulation data engines, analysis tooling, and +user-interface backends. + +"Header-only" means there is no library to build and link against separately: +you include SimDB's headers and compile. "Concurrent data pipelines" means you +can process data on multiple threads without hand-writing the locking and +coordination that usually comes with that. And "SQLite" means everything your +simulation emits can land in a single, ordinary database file that any tool, +language, or teammate can open later. + +The thesis of the whole project fits on one line: + +[.text-center] +*SQLite + Concurrency + Flexibility -- safe and fast.* + +Plenty of libraries give you concurrency. A different set of libraries give you +convenient access to SQLite. SimDB's reason to exist is that it gives you *both +at once*, wired together so the fast path is also the safe path. We will make +that claim concrete when we survey the alternatives later in this part. + +=== Who this book is for + +This book is written for a specific reader: the *domain expert*. You might be a +hardware architect, a performance modeler, or a researcher. You know your +problem domain deeply, you have a real imagination for what you want to build, +and you write code to get your work done -- but you do not write software for a +living, and you would rather not become a concurrency specialist just to save +your results. + +That reader shaped SimDB's single most important design imperative: it should be +*easy to use and hard to misuse*. SimDB was deliberately not built "for software +engineers, by software engineers". The sharp edges that usually come with +threading and databases -- race conditions, lock contention, half-written files, +transactions that are too small to be fast or too large to be safe -- are +handled for you by default. You are meant to spend your attention on _what_ to +capture and _how_ to look at it, not on the machinery that moves it. + +[NOTE] +==== +You do not need to be a database or threading expert to use this book. Where a +concept from either world matters, we introduce it in plain terms the first time +it comes up. +==== + +If you _are_ a career software engineer, you are very welcome here too -- SimDB +will not get in your way, and the later parts expose enough control for advanced +use. But when a tradeoff had to be made between raw flexibility and a design +that is hard to get wrong, SimDB chose the latter, and this book takes the same +stance. + +=== The three pillars + +SimDB's value rests on three ideas that recur throughout the book. + +*Unified pipelines and SQLite.* SimDB pairs a concurrent processing pipeline +with native SQLite integration, so database work is batched and routed +automatically onto a single database thread. You get the throughput of a +purpose-built pipeline and the convenience of a real, queryable database, +without the two fighting each other for access. + +*High performance.* SimDB is built for the kind of throughput that fast +simulations demand -- and it gets there by doing the heavy data work on its own +threads, so your simulator itself does not have to be multi-threaded to benefit. +It minimizes transaction overhead, eliminates database access contention, and +moves data through the pipeline by move (rather than copy) whenever you let it. + +*A clean break for complex codebases.* Large simulation projects tend to +accumulate tangled data paths and "just write it to a file" habits that are hard +to unwind. SimDB replaces that sprawl with one unified, optimized path from +"data produced" to "data stored", which lets teams reach a performant solution +with fewer moving parts. + +=== What you can build + +SimDB is a foundation, not a single-purpose tool. The same building blocks +support a wide range of workflows: + +* Fast, scalable simulation statistics engines. +* Backends for live dashboards or post-run analysis user interfaces. +* Simulation state replayers that reconstruct what the model looked like at any + point in time. +* Aggregate analysis across hundreds or thousands of simulation runs. + +If your work involves producing simulation data and then needing to store, +inspect, replay, or compare it, it very likely fits one of these shapes -- and +<> walks through a complete, real example (Argos) that does several of +them at once. + +=== How to read this book + +The book is built in layers, and the parts are ordered so each one leans only on +what came before. + +* *<>* (this part) makes the case for SimDB: the problem it solves, how it + compares to the alternatives, and the mental model you will carry everywhere + else. +* *Parts <> and <>* get you productive with the standalone SQLite interface -- + installing SimDB, building a first program, and working with schemas, inserts, + queries, and transactions. This is where code snippets begin. +* *Parts <> through <>* introduce the concurrent pipeline and the application + framework that composes pipelines and schemas into cohesive units. +* *<>* is an end-to-end case study of Argos, SimDB's flagship + collection-and-visualization system. +* *Parts <> and <>* collect patterns, recipes, and reference material for when + you are building in earnest. + +[TIP] +==== +This part is intentionally concept-first: there is no code in it. If you learn +best by running something, you can jump to <> now and refer back here when +you want the "why" behind what you are doing. +==== + +Wherever you start, the next chapter is the place to understand the problem +SimDB was built to solve -- because the shape of the solution follows directly +from it. + +[#ch-02-simulation-data-problem] +== The Simulation Data Problem + +Before we can appreciate what SimDB does, it helps to look squarely at the +problem it was built to solve. If you have written a simulator of any real size, +some of what follows will feel familiar -- perhaps uncomfortably so. + +=== Where the data comes from + +A simulation is a data generator. As it runs, it produces a steady stream of +things you will want to keep and look at later: + +* *Statistics* -- counters, rates, histograms, and rolled-up metrics that + summarize how the model behaved. +* *Traces* -- fine-grained, often per-cycle records of what happened, used for + debugging and detailed analysis. +* *Checkpoints* -- snapshots of the model's state, so a run can be inspected or + replayed after the fact. +* *Events* -- notable occurrences worth logging as they happen. +* *User-interface updates* -- data feeding a live dashboard or a + post-run visualization tool. + +Individually, none of these is hard to write down. The difficulty is one of +_scale and speed_: a fast simulator can produce this data faster than a naive +output path can absorb it. The quicker your model runs, the more the plumbing +that carries its data becomes the thing that limits you. It is a common and +demoralizing surprise to discover that a carefully optimized simulator spends +much of its time simply trying to save its own results. + +=== Two hard problems at once + +The reason this plumbing is hard is that it forces two difficult requirements +together: + +. The data handling must not *stall the simulation*. Even a single-threaded + model runs fast, and it cannot afford to stop and wait on disk (or on + compression, or on a database commit) every time it emits a value. The heavy + data work has to happen off to the side -- concurrently -- so the simulation + can keep moving. +. The data must be *persisted durably*, in a form you can query and share long + after the run is over. + +Each of these is manageable on its own. Together they are in tension. To keep up +without stalling, the data work must be spread across background threads -- but a +file on disk fundamentally wants a single writer. Resolve that tension the +obvious ways and you get one of two bad outcomes: serialize everything through +one lock and the simulation crawls, or let threads race and you get corruption, +lost data, and heisenbugs. + +[NOTE] +==== +Your simulator does not need to be multi-threaded for any of this to matter, and +you do not need to write threaded code to fix it. Most hardware simulators are +single-threaded, and that is perfectly fine: SimDB supplies the concurrency for +you, running the data and database work on its own background threads while your +model runs on its own. The tension above is SimDB's problem to solve, not yours. +==== + +=== The hidden costs of doing it by hand + +Reaching for SQLite directly is a reasonable instinct -- it is a real database +in a single file, with no server to run. But used naively to keep up with a fast +simulator, it has sharp edges that are easy to find and hard to remove: + +* *Tiny transactions.* Committing one row at a time turns every write into a + durability barrier. The overhead dwarfs the actual work, and throughput + collapses. +* *Implicit file locks.* SQLite guards the database file with locks. When + several threads try to write at once, they block, retry, or simply fail with + "database is locked". +* *Schema-level contention.* Coordinating which thread may touch which table, + and when, becomes your responsibility -- and getting it wrong is subtle. + +Handle all of this correctly and you have, in effect, taken a second job as a +part-time database concurrency engineer. That is precisely the job SimDB is +meant to spare you. + +=== The anti-patterns SimDB replaces + +Faced with these costs, large simulation projects tend to drift into a few +recognizable habits, none of which age well: + +* *"Write everything to files."* Each subsystem invents its own output file and + its own format. You end up with many formats, many readers, and many bugs -- + and no single place to ask a question that spans them. +* *Ad hoc binary formats.* Fast to write today, painful forever after: they are + hard to version, hard to evolve, and hard for anyone (including future you) to + read back. +* *Tangled data paths and unbounded coupling.* The simulator's core sprouts + tendrils into output code until you can no longer change one without breaking + the other. + +These patterns are not the sign of a careless team; they are the natural result +of the underlying problem being genuinely hard. SimDB's proposition is that the +problem is better solved once, underneath, than re-solved badly in every +project. It replaces the sprawl with one unified, highly optimized path -- from +"data produced" to "data stored" -- so teams reach a performant solution with +far fewer moving parts. + +The obvious question is whether some existing library already does this. That is +what the next chapter examines. + +[#ch-03-landscape] +== The Landscape + +<> ended with a fair question: surely some existing library already +solves this? There are many excellent libraries in this space, and it is worth +understanding them -- both to see why none of them quite fits, and to know when +one of them is actually the better choice than SimDB. + +When you go looking, you find two separate shelves. On one are toolkits for +moving work across threads. On the other are conveniences for talking to SQLite. +What you do not find is something that is genuinely both. + +=== Concurrency and dataflow toolkits + +The first shelf holds mature, well-respected libraries for concurrency and +dataflow: + +* *Intel TBB (oneTBB)* -- a flow graph, parallel algorithms, and concurrent + containers. +* *RaftLib* -- streaming "kernels" connected by data streams. +* *Folly* -- building blocks such as lock-free queues, futures, and executors. +* *HPX* -- an asynchronous, many-task runtime. +* *CAF* -- the actor model for C++. + +These are very good at what they do: they help you spread work across threads +and pass data between the pieces. What none of them offers is any notion of +_durable, queryable storage_. Persistence is left entirely to you -- which drops +you right back into the SQLite problems from the previous chapter. They solve +the "concurrently" half of the problem and are silent on the "persist it +durably" half. + +=== SQLite convenience libraries + +The second shelf holds pleasant C++ wrappers over SQLite: + +* *SQLiteCpp* -- a modern, RAII-style C++ wrapper. +* *SOCI* -- a general database access library with a SQLite backend. +* *sqlite_orm* and *sqlpp11* -- type-safe, ORM- and query-builder-style + interfaces. + +These make SQL access in C++ genuinely nicer to write and safer to get right. +What none of them offers is a _concurrency model_. They assume you have already +decided who writes when; batching, locking, and contention remain your problem. +They are a fine choice when database access is simple and occasional -- and a +poor one for a high-rate simulation data path, because they leave the hardest +part exactly where it was. + +=== The gap SimDB fills + +Neither shelf solves the combined problem, and the combined problem is the whole +point. SimDB exists to unify a *safe concurrent pipeline* with *first-class +SQLite*, so that: + +* database work is funneled onto a single database thread, with no contention; +* operations are batched into implicit transactions and retried until they + succeed, with no `BEGIN`/`COMMIT` from you; +* worker threads are shared across independent workloads so they scale together; +* and the whole thing is approachable enough for a domain expert to use + correctly on the first try. + +That is the meaning of the one-line thesis from <> -- *SQLite + +Concurrency + Flexibility, safe and fast* -- and it is what no single library on +either shelf provides. + +[NOTE] +==== +Could you assemble a concurrency toolkit from the first shelf and a SQLite +wrapper from the second and wire them together yourself? Absolutely -- and you +would be taking on precisely the hard integration work that SimDB was built to +do once, carefully, so that you do not have to. +==== + +=== When not to use SimDB + +Being honest about the fit cuts both ways. SimDB is probably not the right tool +when: + +* *You do not need persistence at all.* If your work is pure in-memory + computation with no data to store, a parallel-algorithms library from the + first shelf is a lighter choice. +* *You need neither concurrency nor volume.* If you write only a handful of rows, + occasionally, a thin SQLite wrapper (or raw SQLite) is simpler and perfectly + adequate. +* *You need a networked or multi-process database, or streaming across many + machines.* Systems like a client/server RDBMS or a distributed log (for + example, Kafka) target a different problem. SimDB is designed for a single + simulation process writing to a single local database file. + +Within its target -- one simulation process producing data that must be stored, +queried, replayed, or compared, quickly and safely -- SimDB is purpose-built, +and the rest of this book assumes you are in that target. + +With the "why" settled, the next chapter gives you the small vocabulary and +mental model that everything else in the book builds on. + +[#ch-04-core-concepts] +== Core Concepts & Mental Model + +The rest of this book uses a small, consistent vocabulary. If you internalize +the handful of ideas in this chapter, everything that follows -- schemas, +pipelines, apps, and the Argos case study -- will feel like variations on a +theme you already know. There is no code here; this is the map, not the +territory. + +=== One database to rule them all + +The first principle is the simplest: *a simulator has one database, and +everything goes into it*. + +Simulation output tends to sprawl. Statistics land in one file, traces in +another, a debug log somewhere else, and analysis results in a folder nobody can +find six months later. Each format needs its own reader, its own tooling, and +its own tribal knowledge. SimDB rejects that sprawl. All of it -- statistics, +trace data, user-interface updates, analysis results -- is consolidated into a +single, queryable SQLite database. + +That single file is an ordinary SQLite database, which means it is also a +durable, portable artifact. You can archive it, copy it to a colleague, or open +it with any SQLite-compatible tool in any language, long after the simulation +that produced it has finished. The database _is_ the deliverable. + +=== The single database thread + +The second principle is the one that makes the rest possible: *only one thread +ever talks to the database*. + +A SQLite database lives on the filesystem, and letting many threads hammer a +filesystem-backed database at once is never fast enough for a high-volume +simulation -- you get lock contention, retries, and unpredictable stalls. So +SimDB does the opposite of what a naive design would do: instead of many threads +racing for the database, every database operation is funneled onto one dedicated +database thread (and there is only ever one, or none at all). + +Concentrating database work on a single thread is what lets SimDB hand you three +guarantees that would otherwise be your problem to enforce: + +* *No user-facing transactions required.* You never write `BEGIN` or `COMMIT`. + SimDB gathers many operations together and commits them as large, efficient + implicit transactions, which is dramatically faster than many tiny ones. +* *Retry-on-fail.* If the database is momentarily busy or a table is locked, + SimDB keeps retrying the work until it succeeds, rather than failing back to + you. +* *No manual concurrency management.* You do not coordinate access to SQLite, + reason about file locks, or worry about schema-level contention. Safe, + high-throughput execution under heavy parallel load is the default. + +[NOTE] +==== +"Only one database thread" is a rule about _writing to the database_, not a +limit on your simulation's parallelism. Your data-processing work can run across +as many threads as you like; it is only the final hand-off to SQLite that is +serialized -- on purpose. +==== + +=== The object model + +SimDB gives you four kinds of building blocks. From the outside in: + +* An *App* is a self-contained unit of functionality that owns a database + *schema* and, typically, *one pipeline*. A logger is an app; a statistics + collector is an app; a UI backend is an app. Several producers usually fan in + through *multiple pipeline heads* on that single pipeline rather than through + multiple pipelines per app. The guiding paradigm is one simulator with many + apps, all writing to the single shared database. +* A *Pipeline* is an ordered arrangement of stages that transforms data on its + way to the database. +* A *Stage* is one step of work. An ordinary stage runs on its own thread. A + special *DatabaseStage* is the only kind of stage allowed to write to the + database, and every database stage -- from every app -- runs on that one + shared database thread. +* Stages are wired together with *ports* (a stage's typed inputs and outputs) + and *queues*. A queue is a thread-safe, first-in-first-out hand-off that sits + between one stage's output and the next stage's input. + +Put together, data flows from your simulation, through one or more apps' +pipelines, and finally onto the single database thread that persists it: + +.The SimDB mental model: many apps and threads, one database thread, one database +[listing] +---- + Your simulation + | + | hand data to an app (moved when possible, copied when needed) + v + App .......... owns a Schema and one or more Pipelines + | (a simulator may run many apps over ONE database) + v + Pipeline ..... a chain of Stages connected by thread-safe Queues + | + | [Stage] --queue--> [Stage] --queue--> [DatabaseStage] + | own thread own thread shared DB thread + v + Single database thread + | every DatabaseStage, from every app, runs here and + | ONLY here; it batches writes into implicit transactions + | and retries them until they succeed + v + One SQLite database (the single source of truth) +---- + +You will meet each of these pieces in detail in Parts <> through <>, and see +them assembled into a real system in <>. For now it is enough to hold the +shape in your head: work fans out across many stages and threads, then funnels +back down to one database thread and one database. + +=== The threading model + +The durable rule of SimDB's threading is the one you already know: there is +exactly one database thread, shared by every database stage, and you cannot ask +for a private one. That constraint is not a limitation to work around -- it is +the very thing that keeps database access fast and contention-free. + +Around that fixed point, the pipeline's other stages run on ordinary worker +threads. How those worker threads are allocated is an area SimDB is still +evolving: today the thread count grows with the amount of pipeline work you +create, so running many apps at once is not yet as thread-efficient as it will +be. <> returns to this with the specifics; the takeaway here is simply that +the single database thread is the permanent anchor of the model. + +=== Move-only by default + +The last idea is about performance. Data travels through a pipeline's queues by +*move* whenever possible: instead of copying a payload from one stage to the +next, SimDB transfers ownership of it, which avoids duplicating potentially +large buffers. When you genuinely need to keep your own copy of the data, you +can copy instead -- SimDB supports both -- but the fast path, and the default +you should reach for, is to move. + +This is why you will often see large payloads (vectors of statistics, compressed +byte buffers) flow through a SimDB pipeline with almost no copying overhead. Keep +"move when you can, copy when you must" in mind and your pipelines will be fast +by construction. + +=== A quick glossary + +For reference, here are the core terms as this book uses them: + +App:: A self-contained unit that owns a schema and one pipeline (often with +multiple pipeline heads). Many apps share one database. +Schema:: The definition of your database's tables and columns. +DatabaseManager:: The object that owns the SQLite connection and applies your +schema. +Pipeline:: An ordered set of stages that transforms and routes data toward the +database. One app, one pipeline is the usual layout. +Stage:: One step of work in a pipeline, running on its own thread. +DatabaseStage:: The special stage that may write to the database; all database +stages run on the single shared database thread. +Port:: A typed input or output on a stage. +Queue (ConcurrentQueue):: The thread-safe FIFO that connects one stage's output +port to the next stage's input port. +Pipeline head:: An unbound input port on the first stage -- where producers +`emplace` work into the pipeline. One pipeline may expose several heads. +Database thread:: The single dedicated thread through which all database work +flows. diff --git a/docs/book/parts/part2-getting-started.adoc b/docs/book/parts/part2-getting-started.adoc new file mode 100644 index 00000000..e10abeb7 --- /dev/null +++ b/docs/book/parts/part2-getting-started.adoc @@ -0,0 +1,322 @@ +[#part-ii] += Part II: Getting Started + +Move from concepts to a running program. By the end of this part the reader has +SimDB installed, building in their project, and has written a first database. + +[#ch-05-installing-simdb] +== Installing SimDB + +With the "why" behind us, it is time to get SimDB onto your machine and compile +something. This chapter covers dependencies; the next covers wiring SimDB into +your own build. + +=== What "header-only" means for you + +SimDB is a header-only C++17 library. There is no SimDB `.a` or `.so` to build +and link against -- you point your compiler at SimDB's headers and that is the +whole library. What SimDB _does_ need is a small set of ordinary dependencies at +build time: + +* a C++17 compiler (GCC or Clang); +* *SQLite3* (version 3.19 or newer), the database engine SimDB builds on; +* *zlib*, used for compression; +* a threads library (pthreads on Linux), for the pipeline machinery. + +You only need `clang-format` (specifically `clang-format-19`) if you intend to +contribute changes back to SimDB; it is not required to use the library. + +=== System packages (Ubuntu / Debian) + +On a Debian-based system, install the dependencies with `apt`: + +[source,bash] +---- +sudo apt-get update +sudo apt-get install -y \ + cmake \ + libsqlite3-dev \ + zlib1g-dev \ + build-essential +---- + +(Add `clang-format-19` to that list if you plan to contribute.) + +=== Conda environments + +If you work in conda, you can pull the same toolchain into an environment +instead. To add the dependencies to an environment you already have: + +[source,bash] +---- +conda activate +conda install -y \ + cmake \ + sqlite \ + zlib \ + compilers +---- + +Or to create a fresh environment for SimDB from scratch: + +[source,bash] +---- +conda create -n simdb -y \ + cmake \ + sqlite \ + zlib \ + compilers +conda activate simdb +---- + +[NOTE] +==== +Because SimDB is header-only, "installing" it can mean two different things: +simply making its headers visible to your compiler, or running a formal install +step so it can be found with CMake's `find_package`. Both are covered in the +next chapter. +==== + +[#ch-06-building-cmake] +== Building & CMake Integration + +SimDB uses CMake, and the smoothest way to consume it is from CMake too. This +chapter shows how to install SimDB and then link it into your own project with a +single target. + +=== Installing SimDB so CMake can find it + +From a clone of the SimDB repository, configure a build and install it. For a +system-wide install: + +[source,bash] +---- +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ .. +sudo make install +---- + +Or, if you are working inside a conda environment, install into that +environment's prefix instead: + +[source,bash] +---- +conda activate +cmake --install . --prefix $CONDA_PREFIX +---- + +Installing places SimDB's headers and a CMake package configuration where +`find_package` can locate them. + +=== Linking SimDB into your project + +SimDB exports a single CMake target, `SimDB::simdb`. In your project's +`CMakeLists.txt`: + +[source,cmake] +---- +cmake_minimum_required(VERSION 3.19) +project(my_simulator LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(SimDB REQUIRED) + +add_executable(my_simulator main.cpp) +target_link_libraries(my_simulator PRIVATE SimDB::simdb) +---- + +That one `target_link_libraries` line is all you need. Because `SimDB::simdb` is +an interface target, linking against it automatically pulls in SimDB's include +directories and its transitive dependencies -- SQLite3, zlib, and the threads +library -- so you do not list them yourself. + +[TIP] +==== +SimDB requires C++17. Setting `CMAKE_CXX_STANDARD` to `17` (as above) is the +portable way to guarantee it for your target. +==== + +=== Debug and Release builds + +Choose a build type with `CMAKE_BUILD_TYPE`, exactly as you would for any CMake +project: + +[source,bash] +---- +# An optimized build +cmake -DCMAKE_BUILD_TYPE=Release .. + +# A debug build (no optimization, full debug info) +cmake -DCMAKE_BUILD_TYPE=Debug .. +---- + +Use Release for anything performance-sensitive -- which, given why you are +reaching for SimDB, is most of the time. Keep a Debug build around for +development. One thing worth knowing early: SimDB's own build treats warnings as +errors (`-Wall -Wextra -Werror`), and Release builds surface issues -- such as +unused variables -- that a Debug build may let slide. If a file compiles in +Debug but fails in Release, an unused parameter or variable is the usual +culprit. + +[#ch-07-first-program] +== Your First Program + +Enough setup -- let us write something that runs. Here is a complete SimDB +program that defines a table, creates a database, writes a handful of rows, and +reads them back. It uses only SimDB's SQLite interface; there is no pipeline +here at all, which is the point: SimDB is useful the moment you want a database, +and the concurrency machinery is something you add later, when you need it. + +[source,cpp] +---- +#include "simdb/sqlite/DatabaseManager.hpp" + +#include + +int main() +{ + using dt = simdb::SqlDataType; + + // 1. Describe the schema: one table with two columns. + simdb::Schema schema; + auto& tbl = schema.addTable("Stats"); + tbl.addColumn("Cycle", dt::uint64_t); + tbl.addColumn("Value", dt::double_t); + + // 2. Create the database file and apply the schema. + simdb::DatabaseManager db_mgr("first.db", true /* create a new database */); + db_mgr.appendSchema(schema); + + // 3. Insert a few rows. + for (uint64_t cycle = 0; cycle < 5; ++cycle) + { + db_mgr.INSERT( + SQL_TABLE("Stats"), + SQL_COLUMNS("Cycle", "Value"), + SQL_VALUES(cycle, cycle * 1.5)); + } + + // 4. Read them back. + auto query = db_mgr.createQuery("Stats"); + + uint64_t cycle = 0; + double value = 0.0; + query->select("Cycle", cycle); + query->select("Value", value); + + std::cout << "Stored " << query->count() << " rows:\n"; + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + std::cout << " cycle " << cycle << " -> " << value << "\n"; + } + + return 0; +} +---- + +=== What it does + +Walk through the program in the order it runs: + +. *Define the schema.* `Schema` and `addTable`/`addColumn` build an in-memory description of + your tables and columns; nothing touches disk yet. `SqlDataType` (aliased here + to `dt` for brevity) names the column types. +. *Create the database.* Constructing a `simdb::DatabaseManager` on a file path + opens the connection; the second argument `true` asks for a fresh database. + `appendSchema` then creates the tables. (You can call `appendSchema` more than + once to add tables later.) +. *Insert rows.* The `INSERT` macro-based API names a table, its columns, and the + matching values. It is the most direct way to write a row; <> introduces + prepared statements for high-volume inserts. +. *Read them back.* `createQuery` targets a table; `select` binds each column to + a variable that is refreshed on every `getNextRecord()`; `count()` returns how + many rows match. Constraints, ordering, and limits come in <>. + +=== Building and running it + +Pair the program with the consumer `CMakeLists.txt` from the previous chapter, +then configure, build, and run: + +[source,bash] +---- +cmake -DCMAKE_BUILD_TYPE=Release -B build +cmake --build build +./build/my_simulator +---- + +You should see five rows printed, and a `first.db` file left behind -- an +ordinary SQLite database you can open with any SQLite tool: + +[source,bash] +---- +sqlite3 first.db "SELECT * FROM Stats;" +---- + +[TIP] +==== +Run the program from a scratch or build directory rather than from inside a +source tree. SimDB programs create their database file in the working directory, +and you would rather not scatter these files across your source. +==== + +That is the whole SQLite interface in miniature. <> revisits each of these +steps -- schemas, inserts, queries, transactions, and blobs -- in depth. + +[#ch-08-regression-tests] +== Running the Regression Tests + +SimDB ships with a regression suite, and it doubles as a library of small, +authoritative worked examples. Building it once up front is the best way to +confirm your toolchain is healthy. + +=== Building the suite + +From a build directory, build the `simdb_regress` target to compile and run the +tests: + +[source,bash] +---- +# Release +mkdir release +cd release +cmake -DCMAKE_BUILD_TYPE=Release .. +make simdb_regress +---- + +The same works for a Debug build (`mkdir debug`, `cmake -DCMAKE_BUILD_TYPE=Debug +..`). Under the hood the suite is registered with CTest, so you can also run and +filter individual tests with `ctest` once they are built. + +=== Worked examples for the SQLite interface + +Everything we have covered so far -- and everything in <> -- has a small, +complete, runnable counterpart under `test/sqlite/`. These are the authoritative +worked examples for SimDB's SQLite interface: each is a self-contained `main.cpp` +that exercises exactly one area. + +* `test/sqlite/Schema/main.cpp` -- defining tables, columns, indexes, and default + values, and reconnecting to an existing schema. +* `test/sqlite/Insert/main.cpp` -- the `INSERT` API and prepared inserts. +* `test/sqlite/Query/main.cpp` -- queries: selecting columns, `WHERE` + constraints, ordering, limits, and compound clauses. +* `test/sqlite/Delete/main.cpp` -- deleting records with constraints. +* `test/sqlite/UInt64/main.cpp` -- full-range 64-bit unsigned handling. + +Because they are part of the regression suite, they are guaranteed to compile and +pass, so they never drift from the current API. As you read <>, keep the +matching program open alongside the chapter. (SimDB's higher-level pipeline +programs live under `examples/`; we turn to those in <>, once pipelines are +on the table.) + +=== A note on where you run things + +SimDB programs write their database file into the current working directory. Run +them from a build directory or a scratch directory, not from the root of a source +tree, so these files do not litter your checkout. This is a small habit that +saves a lot of `git status` noise later. + +With SimDB installed, building, and its test programs at your fingertips, you are +ready for <>, where we return to the SQLite interface and cover it in +full. diff --git a/docs/book/parts/part3-sqlite-interface.adoc b/docs/book/parts/part3-sqlite-interface.adoc new file mode 100644 index 00000000..95ba7758 --- /dev/null +++ b/docs/book/parts/part3-sqlite-interface.adoc @@ -0,0 +1,735 @@ +[#part-iii] += Part III: The SQLite Interface + +SimDB's SQLite layer is useful on its own, with no pipelines involved. This part +is a complete tour of the data layer: schema, inserts, queries, transactions, +blobs, and inspection. + +[#ch-09-defining-schema] +== Defining a Schema + +Every SimDB database begins with a schema: a description of the tables you want +and the columns each one holds. In SimDB you write that description in ordinary +C++ -- there are no SQL `CREATE TABLE` strings to compose, and no chance of a +typo in a keyword surviving until runtime. A `simdb::Schema` is a plain in-memory +object; defining one touches no files and opens no connections. It becomes a real +database only when you hand it to a `DatabaseManager`, which is the subject of the +next chapter. + +=== A first schema + +Here is a small schema with a single table: + +[source,cpp] +---- +#include "simdb/sqlite/DatabaseManager.hpp" + +simdb::Schema schema; +using dt = simdb::SqlDataType; + +auto& tbl = schema.addTable("Instructions"); +tbl.addColumn("PC", dt::uint64_t); +tbl.addColumn("Mnemonic", dt::string_t); +tbl.addColumn("Latency", dt::uint32_t); +---- + +`addTable` returns a reference to the new `Table`, and `addColumn` names a column +and gives it a type. That is the entire required vocabulary; everything else in +this chapter is optional refinement. + +=== Column data types + +Column types come from the `simdb::SqlDataType` enumeration. Aliasing it to `dt` +(as above) keeps definitions readable. The available types are: + +[cols="1,3"] +|=== +| Type | Use it for + +| `int32_t`, `uint32_t` | 32-bit signed / unsigned integers +| `int64_t`, `uint64_t` | 64-bit signed / unsigned integers (ticks, cycles, addresses) +| `double_t` | Floating-point values +| `string_t` | Text +| `blob_t` | Arbitrary binary payloads (covered later in this part) +|=== + +SimDB maps these onto SQLite's storage classes for you and preserves full +integer width -- including the entire range of `uint64_t`, which raw SQLite does +not handle gracefully. + +=== The implicit `Id` primary key + +By default, every table you create is given an auto-incrementing 64-bit integer +primary key named `Id`. You do not declare it, and in fact you cannot add a +column named `Id` yourself -- SimDB manages it. This `Id` is what lets you fetch +a row directly later (for example with `findRecord`), and it is why a freshly +inserted record already has a stable identity. + +If you want a different column to be the primary key, say so: + +[source,cpp] +---- +tbl.setPrimaryKey("PC"); // use PC as the primary key instead of Id +---- + +And if a table needs no primary key at all, remove it: + +[source,cpp] +---- +tbl.unsetPrimaryKey(); +---- + +=== Default values + +A column can carry a default that is used whenever an insert omits it: + +[source,cpp] +---- +auto& reports = schema.addTable("Reports"); +reports.addColumn("Name", dt::string_t); +reports.addColumn("StartTick", dt::uint64_t); +reports.addColumn("EndTick", dt::uint64_t); +reports.setColumnDefaultValue("StartTick", 0); +reports.setColumnDefaultValue("Name", "unnamed"); +---- + +The default's type must match the column's type. Blob columns cannot have +defaults -- there is no meaningful literal for arbitrary binary data. + +=== Indexes + +If you will query a table by a particular column (or combination of columns), add +an index so those lookups stay fast as the table grows: + +[source,cpp] +---- +reports.createIndexOn("StartTick"); // single-column index +reports.createCompoundIndexOn({"StartTick", "EndTick"}); // multi-column index +---- + +Indexes speed up reads at a small cost to writes and file size, so add them for +the columns you actually filter or sort on -- not reflexively for every column. + +=== Uniqueness + +To reject duplicate values in a column, mark it unique: + +[source,cpp] +---- +reports.ensureUnique("Name"); +---- + +=== A fluent style + +Because the `Table`-returning methods all return the table, you can chain a +definition into a single expression when that reads more clearly: + +[source,cpp] +---- +schema.addTable("Events") + .addColumn("Tick", dt::uint64_t) + .addColumn("Kind", dt::string_t) + .setColumnDefaultValue("Kind", "generic"); +---- + +=== Composing and growing a schema + +Schemas are composable. `appendSchema` merges one schema's tables into another, +which is handy when different modules each contribute their own tables: + +[source,cpp] +---- +simdb::Schema combined; +combined.appendSchema(schema_from_module_a); +combined.appendSchema(schema_from_module_b); +---- + +Appending fails if it would create two tables with the same name but different columns. +And, as the next chapter shows, you are not limited to defining everything up front: a +`DatabaseManager` accepts `appendSchema` too, so you can add tables to a live +database as your program discovers it needs them. + +With a schema in hand, the next step is to turn it into an actual database file. + +[#ch-10-database-manager] +== The DatabaseManager + +A schema only describes tables; the `simdb::DatabaseManager` turns that +description into a real database file and is your entry point for everything you +do with it afterward. It owns the underlying database connection, applies +schemas, and is the object you call to insert, query, and manage records. In a +typical program there is one `DatabaseManager` per database file, and it stays +alive for as long as you need that database open. + +=== Opening a database + +The constructor takes a file name and a flag that controls what happens if the +file already exists: + +[source,cpp] +---- +// Create a brand-new database, replacing any file of the same name. +simdb::DatabaseManager db_mgr("sim.db", true /* force_new_file */); +---- + +There are three cases worth knowing: + +* *Create fresh (`force_new_file == true`).* If the file exists it is replaced, + and you start from an empty database. This is what you want for a program that + produces a database on each run. +* *Open existing (`force_new_file == false`, the default).* If the file already + exists, the manager reconnects to it and rebuilds the schema from the file + itself. If the file does _not_ exist, a new empty database is created. +* *Default name.* Both arguments are optional; a bare `simdb::DatabaseManager + db_mgr;` opens (or creates) a file named `sim.db`. + +[IMPORTANT] +==== +When you open a database file that *already existed* without forcing a new file, +its schema is *fixed*: you can read and write records, but you cannot +`appendSchema` new tables onto it. Schema changes belong to the program that +creates the database. Reach for `force_new_file == true` when you intend to +define (or redefine) the schema. +==== + +=== Applying a schema + +With a manager in hand, `appendSchema` creates the tables described by a schema: + +[source,cpp] +---- +simdb::Schema schema; +// ... define tables ... + +simdb::DatabaseManager db_mgr("sim.db", true); +db_mgr.appendSchema(schema); +---- + +Unlike `Schema::appendSchema` (which only merges descriptions in memory), +`DatabaseManager::appendSchema` immediately realizes the tables in the database +file. You may call it more than once on a database you created, which is how you +add tables partway through a run as your program discovers it needs them. + +=== The schema is persisted for you + +SimDB records the schema inside the database itself, in a pair of internal +bookkeeping tables. That is why *reconnecting to an existing file just works*: +the manager reads those tables back and reconstructs an equivalent `Schema` +automatically, with no need to redeclare it in code: + +[source,cpp] +---- +simdb::Schema orig_schema; +// ... define tables ... + +simdb::DatabaseManager orig_db_mgr("orig.db", true); // Create new DB +orig_db_mgr.appendSchema(orig_schema); + +simdb::DatabaseManager diff_db_mgr("orig.db", false); // Connect to same DB +assert(orig_schema == diff_db_mgr.getSchema()); +---- + +=== The manager is the hub + +Nearly everything else in this part goes through the `DatabaseManager`. From it +you: + +* insert rows with `INSERT` and `prepareINSERT` (next chapter); +* read rows with `createQuery`, and fetch a single row by its `Id` with + `findRecord` (the querying chapter); +* group operations atomically with `safeTransaction` (the transactions chapter). + +=== Lifecycle + +The connection is opened in the constructor and closed automatically when the +`DatabaseManager` is destroyed -- ordinary RAII, so there is no explicit "close" +call to remember. The database file itself, of course, persists on disk after +the program exits, ready to be reopened later or inspected with other tools. + +With a live database in hand, we can start putting data into it. + +[#ch-11-inserting-data] +== Inserting Data + +SimDB gives you two ways to write rows: a one-shot `INSERT` that is convenient +for occasional writes, and a _prepared_ insert that you set up once and reuse for +high-volume writing. Both go through the `DatabaseManager`, and both are typed -- +you hand SimDB C++ values, not SQL strings. + +=== One-shot inserts + +The `INSERT` API names a table, the columns you are setting, and the matching +values: + +[source,cpp] +---- +auto record = db_mgr.INSERT( + SQL_TABLE("Instructions"), + SQL_COLUMNS("PC", "Mnemonic", "Latency"), + SQL_VALUES(0x400100ull, "addi", 1u)); +---- + +=== The record handle + +`INSERT` returns a handle to the row it created -- a `SqlRecord`. From it you can +read back typed properties, and you can modify the row in place; a `setProperty` +call writes straight through to the database: + +[source,cpp] +---- +int32_t id = record->getId(); // the auto-assigned Id +uint32_t latency = record->getPropertyUInt32("Latency"); + +record->setPropertyUInt32("Latency", 2u); // updates the stored row +---- + +There is an accessor per column type -- `getPropertyInt32`, `getPropertyUInt64`, +`getPropertyDouble`, `getPropertyString`, and so on -- along with a matching +`setProperty` for each. Blob columns use the templated +`getPropertyBlob("col")`, covered in the blobs chapter. + +=== Insert shorthands + +Two shorter forms of `INSERT` are handy in the right situation: + +[source,cpp] +---- +// Supply every column, in the table's schema order (no SQL_COLUMNS needed). +db_mgr.INSERT(SQL_TABLE("Instructions"), SQL_VALUES(0x400104ull, "sub", 1u)); + +// Insert a row that takes each column's default value. +db_mgr.INSERT(SQL_TABLE("Instructions")); +---- + +The values-only form is terser but positional -- the values must line up with +the columns exactly as the schema declares them -- so prefer the explicit +`SQL_COLUMNS` form when clarity matters, or to future-proof your code against +schema changes e.g. added/reordered columns. + +=== Prepared inserts for high volume + +When you are writing many rows -- the common case in a simulation -- preparing +the statement once and reusing it is far faster than issuing a fresh `INSERT` +each time. `prepareINSERT` compiles the statement up front; you then set column +values by position and call `createRecord` for each row: + +[source,cpp] +---- +auto inserter = db_mgr.prepareINSERT( + SQL_TABLE("Instructions"), + SQL_COLUMNS("PC", "Mnemonic", "Latency")); + +for (const auto& inst : instructions) +{ + inserter->setColumnValue(0, inst.pc); // column 0 == "PC" + inserter->setColumnValue(1, inst.mnemonic); // column 1 == "Mnemonic" + inserter->setColumnValue(2, inst.latency); // column 2 == "Latency" + inserter->createRecord(); +} +---- + +`setColumnValue` takes a zero-based index into the column list you passed to +`prepareINSERT`, and `createRecord` returns the new row's `Id`. If you prefer to +set every column in one call, `createRecordWithColValues` does exactly that: + +[source,cpp] +---- +int32_t id = inserter->createRecordWithColValues(0x400108ull, "xor", 1u); +---- + +As with `INSERT`, there is a `prepareINSERT(SQL_TABLE("..."))` overload that binds +every column in schema order when you do not want to spell out `SQL_COLUMNS`. + +=== Choosing between them + +Reach for one-shot `INSERT` for setup rows, configuration, and any place you are +writing a handful of records. Reach for `prepareINSERT` in the hot path, where +the same statement runs thousands or millions of times. To push bulk-insert +throughput even higher, wrap the loop in a single transaction -- the subject of +the transactions chapter. + +[NOTE] +==== +SimDB owns a couple of internal bookkeeping tables (their names begin with +`internal$`). It deliberately refuses inserts into them, so you cannot corrupt +the schema information it relies on. +==== + +[#ch-12-querying-data] +== Querying Data + +Reading data back is where SimDB's typed, string-free style pays off most. You +build a query against a table, bind C++ variables to the columns you want, add +any constraints, and iterate the results -- all without composing a `SELECT` +statement by hand. + +=== Selecting columns and iterating + +`createQuery` targets a table. You then call `select` once per column, binding +each to a variable of the matching type. The trick to the model is that those +variables are _bound once_ and refreshed on every step of the iteration: + +[source,cpp] +---- +auto query = db_mgr.createQuery("Instructions"); + +uint64_t pc = 0; +query->select("PC", pc); + +std::string mnemonic; +query->select("Mnemonic", mnemonic); + +uint32_t latency = 0; +query->select("Latency", latency); + +auto results = query->getResultSet(); +while (results.getNextRecord()) +{ + // pc, mnemonic, and latency now hold this row's values. + std::cout << "0x" << std::hex << pc << ": " << mnemonic + << " (latency " << std::dec << latency << ")\n"; +} +---- + +Each successful `getNextRecord()` populates the bound variables with the next +row; the loop ends when it returns `false`. If you only need a tally rather than +the rows themselves, `count()` returns how many records match: + +[source,cpp] +---- +const int n = query->count(); +---- + +=== Constraints (the `WHERE` clause) + +Narrow a query by adding typed constraints. There is a method per column type -- +`addConstraintForInt`, `addConstraintForUInt32`, `addConstraintForUInt64`, +`addConstraintForDouble`, `addConstraintForString` -- each taking a column, a +comparison operator, and a value: + +[source,cpp] +---- +query->addConstraintForUInt32("Latency", simdb::Constraints::GREATER, 1); +query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "addi"); +---- + +The `simdb::Constraints` operators are `EQUAL`, `NOT_EQUAL`, `LESS`, +`LESS_EQUAL`, `GREATER`, and `GREATER_EQUAL`. Multiple constraints combine with +`AND`. To clear them and reuse the query, call `resetConstraints()`. + +=== Set membership + +To match against a set of values, use `simdb::SetConstraints::IN_SET` or +`NOT_IN_SET` with a brace-enclosed list: + +[source,cpp] +---- +query->addConstraintForString( + "Mnemonic", simdb::SetConstraints::IN_SET, {"addi", "sub", "xor"}); +---- + +=== Matching doubles + +Floating-point equality is treated with care. `addConstraintForDouble` takes an +extra boolean that enables _fuzzy_ matching, so you can compare against a stored +`double` without being defeated by machine-precision rounding: + +[source,cpp] +---- +auto samples = db_mgr.createQuery("Samples"); +samples->addConstraintForDouble("Value", simdb::Constraints::EQUAL, 3.14, /* fuzzy */ true); + +// Query will now return rows where the Value is within machine epsilon of 3.14 +---- + +=== Ordering and limits + +Sort results with `orderBy` (call it more than once for tie-breakers), and cap +the number of rows with `setLimit`: + +[source,cpp] +---- +query->orderBy("Latency", simdb::QueryOrder::DESC); +query->orderBy("PC", simdb::QueryOrder::ASC); +query->setLimit(100); +---- + +`resetOrderBy()` and `resetLimit()` undo these, mirroring `resetConstraints()`. + +=== Combining clauses with OR + +Constraints are `AND`-ed by default. When you need `OR`, build each group of +constraints, release it as a clause, and combine the clauses explicitly: + +[source,cpp] +---- +query->addConstraintForUInt32("Latency", simdb::Constraints::EQUAL, 1); +query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "addi"); +auto clause_a = query->releaseConstraintClauses(); + +query->addConstraintForString("Mnemonic", simdb::Constraints::EQUAL, "sub"); +auto clause_b = query->releaseConstraintClauses(); + +// (Latency == 1 AND Mnemonic == "addi") OR (Mnemonic == "sub") +query->addCompoundConstraint(clause_a, simdb::QueryOperator::OR, clause_b); +---- + +=== Fetching a single row by Id + +When you already know a row's `Id` -- for example one returned by `createRecord` +-- you do not need a query at all. Ask the `DatabaseManager` for it directly: + +[source,cpp] +---- +auto record = db_mgr.findRecord("Instructions", id); +---- + +This hands back the same `SqlRecord` handle described in the previous chapter, +with its typed `getProperty*` accessors. + +=== Deleting matching rows + +A query can also delete the rows it selects. Constrain it to the rows you want +gone, then call `deleteResultSet()`: + +[source,cpp] +---- +auto stale = db_mgr.createQuery("Instructions"); +stale->addConstraintForUInt32("Latency", simdb::Constraints::EQUAL, 0); +stale->deleteResultSet(); +---- + +With no constraints, that would empty the table -- so double-check your +constraints before deleting. + +=== Reusing a query + +A query object is reusable. Reset whichever parts you want to change -- +constraints, ordering, or limit -- and run it again; and within a single run, +`getResultSet()` can be iterated, then `reset()` and iterated again from the top. + +[#ch-13-transactions] +== Transactions + +By default, each write you make to a SQLite database is its own committed +transaction. That is fine for the occasional row, but it is ruinous for +throughput when you are writing thousands or millions of them: the per-commit +overhead dominates. Grouping many operations into a single transaction is the +single most effective thing you can do for write performance, and SimDB makes it +both easy and safe with `safeTransaction`. + +=== Grouping work with safeTransaction + +`safeTransaction` takes a function (typically a lambda) and runs everything +inside it within one `BEGIN`/`COMMIT`: + +[source,cpp] +---- +auto inserter = db_mgr.prepareINSERT( + SQL_TABLE("Instructions"), + SQL_COLUMNS("PC", "Mnemonic", "Latency")); + +db_mgr.safeTransaction( + [&]() + { + for (const auto& inst : instructions) + { + inserter->setColumnValue(0, inst.pc); + inserter->setColumnValue(1, inst.mnemonic); + inserter->setColumnValue(2, inst.latency); + inserter->createRecord(); + } + }); +---- + +Here every `createRecord` in the loop is folded into a single commit. The same +loop without a transaction would commit once per row; wrapping it as above can be +orders of magnitude faster for large batches. + +=== What makes it "safe" + +The name is not decoration -- `safeTransaction` handles the awkward parts of real +concurrent database access for you: + +* *Automatic retry on contention.* If the database is momentarily busy or locked + (for instance, another thread is mid-write), SimDB does not fail the call and + hand you a "database is locked" error to cope with. It waits briefly and + retries the whole transaction until it succeeds. Transient contention becomes + invisible. +* *Safe to nest.* Calling `safeTransaction` while already inside one does _not_ + start a second, illegal nested transaction; the inner call simply joins the + outer one, and the commit happens once when the outermost call finishes. This + means helper functions can each wrap their work in `safeTransaction` without + worrying about who called whom. +* *Serialized access.* An internal lock guards the connection, so transactions + from different threads are applied one at a time rather than colliding. + +Together these mean you get batching and correctness without writing any retry +loops, lock handling, or nesting bookkeeping yourself -- the same +"easy to use, hard to misuse" bargain SimDB makes everywhere. + +=== A note on pipelines + +When you move to the concurrent pipeline in <>, you will rarely call +`safeTransaction` by hand on the hot path. SimDB's database pipeline stages +already batch the rows flowing through them into transactions on the dedicated +database thread, applying exactly this technique for you. `safeTransaction` +remains the right tool whenever you are writing to the database directly, outside +of a pipeline. + +[#ch-14-blobs-compression] +== Blobs, Compression & Serialization + +Not everything worth storing is a scalar. Simulations routinely produce +_bulk_ data -- an array of samples, a packed snapshot of some structure, a run of +values captured over a window of time. A `blob_t` column stores that kind of +payload as raw bytes, and it is how SimDB keeps large, structured data compact +inside an ordinary database file. + +=== Storing and reading a vector + +The simplest blob is a `std::vector`. Pass one straight into `SQL_VALUES`, and +SimDB stores its bytes: + +[source,cpp] +---- +std::vector samples = collect_samples(); + +db_mgr.INSERT( + SQL_TABLE("Waveforms"), + SQL_COLUMNS("Samples"), + SQL_VALUES(samples)); +---- + +Reading it back is symmetric: `getPropertyBlob` returns a `std::vector` of +the element type you ask for: + +[source,cpp] +---- +auto record = db_mgr.findRecord("Waveforms", id); +std::vector samples = record->getPropertyBlob("Samples"); +---- + +You read with the *same element type* you wrote. A blob is just bytes; SimDB does +not record what those bytes meant, so asking for the wrong `T` gives you back +nonsense (or throws, if the byte count does not divide evenly). Store +trivially-copyable data -- plain values or POD structs -- and never store +pointers, whose addresses are meaningless once reloaded. + +=== Raw bytes with SqlBlob + +When your data is not already a vector -- a packed struct, or an existing buffer +-- describe it with a `simdb::SqlBlob`, which is just a pointer and a byte count: + +[source,cpp] +---- +simdb::SqlBlob blob(samples); // from a vector... +db_mgr.INSERT(SQL_TABLE("Waveforms"), SQL_COLUMNS("Samples"), SQL_VALUES(blob)); +---- + +The same is available on an existing record through `setPropertyBlob`, either +from a vector or from a raw pointer and length: + +[source,cpp] +---- +record->setPropertyBlob("Samples", other_vector); // from a vector +record->setPropertyBlob("Samples", data_ptr, num_bytes); // from raw bytes +---- + +Because large payloads are expensive to copy, you can `std::move` a vector into +`SQL_VALUES` to hand its buffer to SimDB directly rather than duplicating it: + +[source,cpp] +---- +db_mgr.INSERT(SQL_TABLE("Waveforms"), SQL_COLUMNS("Samples"), SQL_VALUES(std::move(samples))); +---- + +=== Compression + +For genuinely large payloads, SimDB ships zlib-based helpers so you can shrink +the data before it lands on disk. Compress into a `std::vector`, store that, +and decompress on the way back out: + +[source,cpp] +---- +// Writing: compress, then store the compressed bytes. +std::vector series = collect_series(); +std::vector packed; +simdb::compressData(series, packed); + +db_mgr.INSERT(SQL_TABLE("Series"), SQL_COLUMNS("Data"), SQL_VALUES(packed)); +---- + +[source,cpp] +---- +// Reading: pull the compressed bytes back, then decompress into the original type. +auto record = db_mgr.findRecord("Series", id); +std::vector packed = record->getPropertyBlob("Data"); + +std::vector series; +simdb::decompressData(packed, series); +---- + +`compressData` takes an optional level -- `simdb::CompressionLevel::FASTEST`, +`DEFAULT`, `HIGHEST`, or `DISABLED` -- letting you trade CPU time against file +size. Compression is entirely your choice: SimDB provides the tools, and you +decide when a payload is big enough to be worth compressing. + +This blob-plus-compression pattern is the foundation for storing high-volume +simulation data efficiently, and it is exactly what SimDB's higher layers build +on to keep large captures small. + +[#ch-15-inspecting-dumping] +== Inspecting & Dumping Databases + +When something looks wrong -- a table that should have rows but does not, a value +that is not what you expected -- you want to look inside the database quickly. + +=== Reading the database at the command line + +Because a SimDB database is a standard SQLite file, you are never locked into C++ +for inspection. The `sqlite3` command-line tool queries it directly: + +[source,bash] +---- +sqlite3 sim.db "SELECT * FROM Instructions LIMIT 10;" +---- + +And any language with a SQLite binding -- Python included -- can open the same +file to analyze or cross-check its contents. This openness is what lets separate +tooling, such as a visualization front-end, read a database that a C++ simulation +produced. + +=== Third-party SQLite viewers + +For interactive browsing -- clicking through tables, running ad-hoc `SELECT`s, +sorting and filtering by hand -- a graphical SQLite viewer is often the best +way to get oriented. These are all third-party tools, not part of SimDB, but +because a SimDB database is a plain SQLite file they open it without any special +handling. A few worth knowing: + +* *DB Browser for SQLite* (also called "DB4S" or `sqlitebrowser`) -- a free, + open-source, cross-platform desktop application. A good default: browse and + edit tables, run queries, and view schema in one window. +* *SQLiteStudio* -- free, open-source, and cross-platform, with a tabbed UI for + working across several tables and databases at once. +* *DBeaver* -- a cross-platform database tool (free Community Edition) that + handles SQLite alongside many other databases; handy if you already use it for + other work. +* *DataGrip* -- JetBrains' commercial database IDE, with strong query editing and + navigation, if you work in that ecosystem. +* *Datasette* -- an open-source tool that serves a SQLite file as a browsable web + application, convenient for read-only exploration and quick sharing. + +[TIP] +==== +If you just want a quick look and prefer not to install anything, the `sqlite3` +command-line tool shown above is already enough. Reach for a GUI when you want to +explore an unfamiliar database interactively. +==== + +This closes our tour of the standalone SQLite interface. You can now define +schemas, open databases, write and read records, batch work into transactions, +store bulk payloads, and inspect the result -- all without a single pipeline. In +<> we add the other half of SimDB: the concurrent pipeline that moves this +data off the critical path and onto its own threads. diff --git a/docs/book/parts/part4-concurrent-pipelines.adoc b/docs/book/parts/part4-concurrent-pipelines.adoc new file mode 100644 index 00000000..c2a4ac7a --- /dev/null +++ b/docs/book/parts/part4-concurrent-pipelines.adoc @@ -0,0 +1,739 @@ +[#part-iv] += Part IV: Concurrent Pipelines + +Here SimDB's second pillar appears: safe, high-throughput concurrent pipelines +that feed SQLite without contention. This part builds a working pipeline from +first principles. + +[#ch-16-pipeline-fundamentals] +== Pipeline Fundamentals + +<> gave you a database. This part gives you the other half of SimDB: a way +to move data _into_ that database without ever making your simulation wait on it. +A *pipeline* is a chain of processing *stages*, each running on its own thread, +passing data along through queues. The simulation hands work to the front of the +pipeline and returns to what it was doing; the stages do the transforming, +compressing, and writing behind the scenes. Before we build one, this chapter +introduces the fundamental unit -- the stage -- and the small contract that +governs how it runs. + +=== What a stage is + +A stage is a unit of work with typed input and output *ports*. You create one by +subclassing `simdb::pipeline::Stage` and implementing a single method, `run_`. +Every combination of ports is allowed: a stage may have multiple inputs, multiple +outputs, or none at all -- a stage with no input port might generate data; one +with no output port might be a sink that only writes. + +We cover how ports are declared and wired in the next two chapters. For now, the +important thing is the shape of a stage and what its `run_` method promises. + +=== The greedy run loop + +SimDB runs each stage on its own thread and calls `run_` over and over. Your job +in `run_` is to do _one unit_ of work if there is any, and to tell SimDB what +happened by returning a `PipelineAction`: + +[source,cpp] +---- +class MyStage : public simdb::pipeline::Stage +{ +private: + simdb::pipeline::PipelineAction run_(bool force) override + { + Widget w; + if (input_queue_->try_pop(w)) + { + // ... transform w, push results to an output port ... + return simdb::pipeline::PipelineAction::PROCEED; // did work; call me again now + } + + return simdb::pipeline::PipelineAction::SLEEP; // nothing to do; let me nap + } +}; +---- + +The two return values drive everything: + +* `PROCEED` means "I did something, and there may be more." SimDB immediately + calls `run_` again, with no delay. A busy stage is therefore serviced + _greedily_ -- it drains its input as fast as it can. +* `SLEEP` means "I had nothing to do." The stage's thread sleeps briefly (100 ms + by default), then wakes and tries again. + +This is the whole scheduling model. There is no manual thread management, no +condition-variable bookkeeping -- you write a `run_` that processes one item and +reports back, and SimDB handles the looping, waking, and sleeping. + +=== Tuning the polling interval + +That 100 ms nap is the stage's _polling interval_, and you can change it through +the `Stage` constructor when a stage needs to react faster (a shorter interval) +or can afford to idle longer (a longer one). It is a latency-versus-idle-CPU +trade-off: shorter intervals notice new work sooner but wake more often when +there is nothing to do. + +=== The `force` flag and flushing + +`run_` receives a `bool force`. Normally it is `false`. It becomes `true` during +a *flush* -- when the pipeline is being drained to a synchronization point, or +when the simulation is over and the pipeline is being torn down. + +The flag matters only for stages that buffer data internally. A stage that, say, +accumulates values in a `std::vector` and forwards them a full batch at a time +should, when `force` is `true`, also send whatever partial batch it is holding so +that no data is stranded at shutdown. A stage with no internal buffering -- like +the skeleton above, which forwards each item as it arrives -- can simply ignore +`force`. + +=== Two kinds of stage: `Stage` and `DatabaseStage` + +Every stage you write derives from one of two bases, and the choice determines +which thread it runs on: + +* A plain `simdb::pipeline::Stage` runs on *its own dedicated thread*. Use it for + transformation work -- compression, filtering, reshaping -- that does not touch + the database. +* A `simdb::pipeline::DatabaseStage` runs on the *single, shared database thread*. + Every database stage, across every pipeline and every app in + the process, is placed on that one thread. + +That second point is the "one database thread to rule them all" principle from +<> made concrete. SimDB funnels all database writes through a single thread +on purpose: multi-threaded access to a SQLite file never keeps up with a fast +simulation, so instead of fighting for the file, all database work is serialized +onto one thread and batched. A `DatabaseStage` accordingly gets what it needs to +do that work -- `getDatabaseManager_()` for direct access and +`getTableInserter_()` for fast prepared inserts -- and each call to its `run_` +runs inside an implicit batch transaction. We devote a full chapter to database +stages shortly. + +With the stage model and its run loop in hand, we can look at how stages are +connected -- the ports and queues that carry data between them. + +[#ch-17-ports-concurrent-queue] +== Ports & ConcurrentQueue + +If stages are the workers, *ports* are the doors through which data enters and +leaves them, and a *queue* is the conveyor belt between one stage's output door +and the next stage's input door. This chapter covers how you declare those ports +and how you read and write the queues that connect them. + +=== Declaring ports + +A stage declares its ports in its constructor. Each port has a name, a data type, +and a queue pointer that SimDB fills in for you. The pattern is always the same: a +`ConcurrentQueue*` member initialized to `nullptr`, registered with +`addInPort_` or `addOutPort_`: + +[source,cpp] +---- +class ZlibStage : public simdb::pipeline::Stage +{ +private: + simdb::ConcurrentQueue* input_queue_ = nullptr; + simdb::ConcurrentQueue* output_queue_ = nullptr; + +public: + ZlibStage() + { + addInPort_("uncompressed_input", input_queue_); + addOutPort_("compressed_output", output_queue_); + } + + // ... run_() ... +}; +---- + +Note what you do _not_ do: you never allocate or free these queues. You declare +the pointer, hand it to `addInPort_`/`addOutPort_`, and SimDB points it at the +real queue when the pipeline is wired together (the next chapter). Until then the +pointer is `nullptr`; by the time `run_` executes, it is valid. The type in the +angle brackets is what makes a port typed -- an output port carrying +`ZlibStatsValues` can only be bound to an input port that also expects +`ZlibStatsValues`, and that match is enforced when you bind stages. + +=== The ConcurrentQueue API + +`simdb::ConcurrentQueue` is a thread-safe FIFO -- a queue wrapped with a mutex +-- and it is the only thing shared directly between two concurrently running +stages. Its interface is deliberately small: + +[cols="1,3"] +|=== +| Call | Meaning + +| `push(item)` | Copy an item onto the back of the queue. +| `emplace(item)` | Move an item onto the back (or construct one in place). +| `try_pop(out)` | Pop the front into `out`; returns `false` if the queue was empty. +| `size()` | Number of queued items. +| `empty()` | Whether the queue has no items. +| `snoop(cb)` | Peek at items without removing them (used by snoopers). +|=== + +The workhorses are `try_pop` on the input side and `push`/`emplace` on the +output side. `try_pop` returning `false` is precisely the "nothing to do" signal +that a stage turns into a `SLEEP`. `snoop` -- inspecting in-flight items without +consuming them -- underpins the snooping feature covered in the advanced part; +you rarely call it directly. + +=== Ports in a run_ method + +Putting the pieces together, a typical transforming stage pops from its input, +does its work, pushes to its output, and reports `PROCEED`; when the input is +dry, it returns `SLEEP`: + +[source,cpp] +---- +simdb::pipeline::PipelineAction run_(bool /*force*/) override +{ + StatsValues uncompressed; + if (input_queue_->try_pop(uncompressed)) + { + ZlibStatsValues compressed; + compressed.uuid = uncompressed.uuid; + simdb::compressData(uncompressed.values, compressed.bytes); + + output_queue_->emplace(std::move(compressed)); // hand off to the next stage + return simdb::pipeline::PipelineAction::PROCEED; + } + + return simdb::pipeline::PipelineAction::SLEEP; +} +---- + +Prefer `emplace` with `std::move` for anything non-trivial to copy -- a stage's +whole purpose is to pass large payloads along, and moving hands off the buffer +instead of duplicating it. (<> closes with a chapter on exactly this kind of +move-friendly, high-throughput style.) + +=== Any shape of stage + +Because ports are just declarations, a stage can have as many as it needs: + +* *Multiple input ports* let a stage act as a *mux*, combining items that arrive + on different queues -- for example, assembling a tuple whose pieces show up at + different times. +* *Multiple output ports* let a stage fan data out to several downstream stages. +* *Zero input ports* make a stage a *source* that generates data. +* *Zero output ports* make it a *sink* -- most database stages are sinks that + only consume. + +With ports declared on each stage, we are ready to assemble stages into an actual +pipeline and bind their ports together. + +[#ch-18-building-pipeline] +== Building a Pipeline + +You have stages, and each stage has ports. Assembling them into a running +pipeline is a short, ordered recipe: create the pipeline, add the stages, bind +the ports, and grab the entry point. SimDB enforces that order -- each phase is +sealed off before the next begins -- so the wiring is explicit and mistakes +become compile-time or startup errors rather than silent misbehavior. + +Pipelines are built through a `PipelineManager`, which SimDB hands to your +application in a `createPipeline` hook. The application framework itself is the +subject of <>; here we focus on the assembly API, which is the same wherever +you call it. + +=== The build protocol + +[source,cpp] +---- +void StatsCollector::createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) +{ + // 1. Create a named pipeline, owned by this app. + auto pipeline = pipeline_mgr->createPipeline("stats-collector-pipeline", this); + + // 2. Add stages (each with a name), then seal the stage list. + pipeline->addStage("compressor"); + pipeline->addStage("db_writer"); + pipeline->noMoreStages(); + + // 3. Bind one stage's output port to the next stage's input port, + // then seal the bindings. + pipeline->bind("compressor.compressed_output", "db_writer.compressed_input"); + pipeline->noMoreBindings(); + + // 4. Grab the still-unbound input port -- the head of the pipeline. + pipeline_head_ = pipeline->getInPortQueue("compressor.uncompressed_input"); +} +---- + +The four phases, and the rules that bind them: + +. *Create.* `createPipeline` takes a name and the owning app. The name is how + stages and ports are addressed afterward. +. *Add stages.* `addStage("name")` constructs a stage of type `T` and gives it + a name; it returns a pointer to the stage, which you keep when you later attach + a flusher or snooper to it. Once every stage is added, `noMoreStages()` seals + the list -- you cannot add stages after this. +. *Bind ports.* `bind` connects an output port to an input port, each addressed as + `"stagename.portname"`. The two ports' types must match. `noMoreBindings()` + then finalizes the wiring and actually creates the `ConcurrentQueue` objects + behind every port. +. *Grab the head.* Only after `noMoreBindings()` can you ask for a port's queue. + The one input port you deliberately left _unbound_ -- here + `compressor.uncompressed_input` -- is the pipeline's entry point. Store its + queue; that is where you will push data in. + +Calling these out of order is an error: SimDB throws if you try to `bind` before +`noMoreStages()`, or to fetch a port queue before `noMoreBindings()`. This +"build, then seal" protocol is what lets SimDB validate the whole graph -- every +binding, every type -- before a single thread starts running. + +The binding above connects the two stages like so: + +.... + compressor + uncompressed_input <- pipeline head (left unbound) + compressed_output ---+ + | bind + db_writer | + compressed_input <--+ +.... + +=== Feeding data into the pipeline + +The head you captured is just a `ConcurrentQueue`, so you send work in with the +same `push`/`emplace` calls from the previous chapter. An app typically wraps +this in a small method: + +[source,cpp] +---- +void StatsCollector::process(StatsValues&& stats) +{ + pipeline_head_->emplace(std::move(stats)); // move the payload in +} + +void StatsCollector::process(const StatsValues& stats) +{ + pipeline_head_->push(stats); // or copy it in +} +---- + +Once an item lands in the head queue, it flows through the stages on their own +threads -- compressed by `compressor`, written by `db_writer` -- with no further +involvement from the caller, which returns immediately. That is the entire point: +the simulation deposits data at the head and gets back to simulating, while the +pipeline does the slow work elsewhere. + +A pipeline can have more than one unbound input -- *multiple pipeline heads* on +the same pipeline -- when a first stage muxes several streams. That is the usual +way to fan in several producers without building a second pipeline (which is +uncommon per app and adds threads). Less commonly, a pipeline may leave outputs +unbound so you read from the tail. The single-input, single-database-output +shape above is the common case, and it is exactly what the `SimplePipeline` +example builds. + +Our example pipeline ends in a `DatabaseStage`. That stage is where data actually +reaches SQLite, and it plays by special rules -- the single database thread and +implicit batch transactions from <>. The next chapter is devoted to it. + +[#ch-19-database-stages] +== Database Stages + +A database stage is where pipeline data finally reaches SQLite. It is a stage +like any other -- it has input ports and a `run_` method -- but it is special in +two ways we met back in <>: it runs on the *single, shared database thread*, +and its writes are automatically *batched into transactions*. This chapter shows +how to write one and why those two properties give you high-throughput +persistence for free. + +=== Declaring a database stage + +Instead of `simdb::pipeline::Stage`, derive from +`simdb::pipeline::DatabaseStage`. The template argument is your +application type, which is how the stage knows your schema and can hand you +prepared inserters for its tables: + +[source,cpp] +---- +class DatabaseStage : public simdb::pipeline::DatabaseStage +{ +private: + simdb::ConcurrentQueue* input_queue_ = nullptr; + +public: + DatabaseStage() + { + addInPort_("compressed_input", input_queue_); + } + + // ... run_() ... +}; +---- + +Ports are declared exactly as before. Most database stages are *sinks* -- they +have input ports and no output -- because their job is to consume data and write +it out. + +=== Writing with prepared inserters + +The fast path for inserting is `getTableInserter_`, which returns a prepared +statement for one of your app's tables. You set column values by position and +call `createRecord`, just like the prepared inserts from <>: + +[source,cpp] +---- +simdb::pipeline::PipelineAction run_(bool /*force*/) override +{ + ZlibStatsValues compressed; + if (input_queue_->try_pop(compressed)) + { + auto inserter = getTableInserter_("CompressedStats"); + inserter->setColumnValue(0, compressed.uuid); + inserter->setColumnValue(1, compressed.bytes); + inserter->createRecord(); + return simdb::pipeline::PipelineAction::PROCEED; + } + + return simdb::pipeline::PipelineAction::SLEEP; +} +---- + +This is the recommended way to write from a database stage: prepared statements +are compiled once and reused, which is what you want on a path that runs millions +of times. + +=== Direct database access + +A database stage can also reach the `DatabaseManager` directly with +`getDatabaseManager_()`. You would not normally use it for plain inserts -- the +inserter above is faster -- but it is there for everything else a stage might +need to do: run a query, delete records, or issue any other operation. The full +<> query and record API is available through it. + +[source,cpp] +---- +auto db_mgr = getDatabaseManager_(); +auto query = db_mgr->createQuery("CompressedStats"); +// ... constrain, count, iterate, delete ... +---- + +=== Automatic batch transactions + +Here is the property that makes database stages fast: *every* call to a database +stage's `run_` executes inside a batched `BEGIN`/`COMMIT TRANSACTION` block that +SimDB manages for you. You never call `safeTransaction` here -- the batching is +automatic. + +And it is not just your stage. That single transaction is *shared across every +database stage on the database thread* -- other database stages in the same app, +and database stages belonging to other apps running concurrently. All of their +writes accumulate into one transaction and commit together. This is the +throughput technique from the transactions chapter, applied globally and +invisibly: because everyone shares one database thread, everyone can also share +one big, efficient transaction. + +=== The single database thread, restated + +This is the concrete payoff of the "one database thread to rule them all" design. +Plain stages each get their own thread and run truly in parallel; all database +stages, in contrast, are multiplexed onto one thread so that nothing ever +contends for the SQLite file. You do not create or manage that thread -- adding a +`DatabaseStage` to any pipeline brings it into existence, and every subsequent +database stage joins it. + +One consequence worth stating: a database stage cannot use the asynchronous +database accessor (`getAsyncDatabaseAccessor_()` throws if called from one). It +does not need it -- it already _is_ on the database thread. That accessor exists +for the opposite situation, a non-database stage that occasionally needs to reach +the database, which we return to later. + +=== Database stages with outputs + +Although sinks are the common case, a database stage may have output ports when +the design calls for it. A typical example is a cache-and-evict pipeline, where a +database stage signals that a record has been persisted so an upstream cache can +drop its copy: + +.... + Sim -> Cache a copy -> Process -> Database -----+ + ^ | + | | + +------------- Evict --------------+ +.... + +The cache holds originals for fast access and evicts them once the database stage +confirms the write. Whether you need this is a design choice; the mechanism is +just an ordinary output port on the database stage. + +With persistence handled, the last piece of the pipeline story is performance: +how to keep data moving with minimal copying. That is the next chapter. + +[#ch-20-move-semantics] +== Move Semantics & Performance + +Most of SimDB's performance you get for free: contention-free writes, one database +thread, automatic batch transactions. But a pipeline is only as fast as the data +you push through it, and a few good habits keep it moving. + +=== Move data, do not copy it + +The single most important habit is to *move* payloads through the pipeline rather +than copy them. On a queue, that is the difference between `emplace` and `push`: + +[source,cpp] +---- +output_queue_->emplace(std::move(payload)); // hand off the buffer -- cheap, O(1) +output_queue_->push(payload); // copy the whole payload -- O(n) +---- + +Pipeline payloads are frequently large -- a vector of samples, a compressed byte +buffer. Copying one duplicates every byte; moving one hands the existing buffer +to the next stage in O(1). On a path that runs millions of times, that difference +dominates. + +SimDB embraces this: it *supports move-only payloads*. A payload type may hold a +`std::unique_ptr` or any other non-copyable member and still flow through a +pipeline, as long as you `emplace`/`std::move` it rather than `push` it. Copying +remains available -- `push` is there when you genuinely need to keep the original +(say, to tee the same payload to two pipeline heads, or duplicate in software +before enqueue) or when the payload is small enough that copying is free. But +move is the default you should reach for. + +=== Let SimDB batch the writes + +The other half of throughput is transaction batching, and you have already seen +it: every database stage's writes are folded into one shared transaction on the +database thread. You do not size those transactions by hand. The main way you +influence them is by *how often you flush* -- forcing a synchronization point +commits the current batch, so flushing on every item would defeat the batching +entirely. Flush when you actually need a consistency point, not reflexively. + +=== Threads and stage granularity + +Each stage runs on its own worker thread, and there is only ever one database +thread. Two design choices affect how well a pipeline scales: + +* *Stage granularity.* Splitting an expensive transform into its own stage lets + it run in parallel with the stages around it -- genuine pipeline parallelism. + But every stage costs a thread and a queue hop, so a stage that does almost no + work can cost more in hand-off overhead than it saves. Make a stage when it + does real, separable work; do not shatter a cheap operation into many tiny + stages. +* *Payload sizing.* Each hand-off through a queue takes a lock. For very small + items, that per-item overhead can dwarf the work, so it often pays to batch + many small items into one payload (a vector, say) and move the batch. Do not + overcorrect into enormous payloads, though -- oversized batches delay the + moment a downstream stage can start and inflate memory use. Aim for a + middle ground. + +The polling interval from earlier in this part is the third knob: shorten it for +lower latency, lengthen it to waste fewer wakeups when a stage is usually idle. + +[WARNING] +==== +SimDB's throughput assumes it effectively owns the process's path to disk. If the +surrounding simulator does its *own* unrelated filesystem work -- a separate +logging system writing heavily to disk, for instance -- that activity contends +with SimDB's database thread for the same I/O path and can badly degrade pipeline +performance. Until SimDB offers a first-class way to coordinate such traffic, +treat unrelated heavy disk I/O as a known performance-failure mode: keep it off +the critical path where you can, or route that output through SimDB so it shares +the same batched, single-threaded database access. +==== + +That completes the conceptual core of the pipeline: stages and their run loop, +ports and queues, assembling a pipeline, the database stage, and the performance +habits that keep it fast. The final chapter of this part puts it all in front of +you as runnable code, touring the shipped pipeline examples. + +[#ch-21-pipeline-examples] +== Running the Pipeline Examples + +Everything in this part now comes together in real, runnable code. SimDB ships a +set of pipeline examples under `examples/`, and this chapter walks through the +first of them -- `SimplePipeline` -- in full, then points you at the rest. + +=== The example catalog + +The examples are meant to be read roughly in this order. Each one isolates a +concept: + +* *SimplePipeline* -- the "hello world" of pipelines: a single-input/ + single-output compression stage feeding a database stage. Shown in full below. +* *ConcurrentApps* -- several apps running at once, sharing the one database. +* *DatabaseWatchdog* -- a stage with no I/O ports that reaches the database thread + from outside it. +* *PipelineSnoopers* -- inspecting data while it is still in flight between + stages. +* *AppFactory* -- apps that need non-default constructor arguments. + +`SimplePipeline` uses only the concepts from this part, so we present its code +here. The others lean on the application framework and more advanced pipeline +features; rather than show that code out of context, we cover each in the part +where it belongs -- the application framework in <>, and snooping, watchdogs, +and multi-app patterns in the advanced part. The catalog in `examples/README.md` +maps every example to the feature it showcases. + +=== SimplePipeline, in full + +`SimplePipeline` is a two-stage pipeline: a `CompressionStage` compresses each +incoming vector on its own thread, and a `DatabaseStage` writes the compressed +bytes to SQLite on the database thread. It is packaged as a `simdb::App`; the +application framework is <>'s subject, so for now focus on the pipeline parts +you already recognize -- the stages, their ports, and the `createPipeline` +wiring. + +The app declares its schema and holds the pipeline's entry point: + +[source,cpp] +---- +class SimplePipeline : public simdb::App +{ +public: + static constexpr auto NAME = "simple-pipeline"; + + SimplePipeline(simdb::DatabaseManager* db_mgr) : + db_mgr_(db_mgr) + { + } + + static void defineSchema(simdb::Schema& schema) + { + using dt = simdb::SqlDataType; + + auto& tbl = schema.addTable("CompressedData"); + tbl.addColumn("AppInstance", dt::int32_t); + tbl.addColumn("DataBlob", dt::blob_t); + } + + // ... createPipeline() and process() shown below ... + +protected: + simdb::DatabaseManager* db_mgr_ = nullptr; + simdb::ConcurrentQueue>* pipeline_head_ = nullptr; +}; +---- + +The compression stage is a plain `Stage` with one input and one output port. Its +`run_` is the pop / transform / emplace / `PROCEED` pattern from earlier, falling +back to `SLEEP` when there is nothing to compress: + +[source,cpp] +---- +class CompressionStage : public simdb::pipeline::Stage +{ +public: + CompressionStage() + { + addInPort_>("input_data", input_queue_); + addOutPort_>("compressed_data", output_queue_); + } + +private: + simdb::pipeline::PipelineAction run_(bool) override + { + std::vector data; + if (input_queue_->try_pop(data)) + { + std::vector compressed_data; + simdb::compressData(data, compressed_data); + output_queue_->emplace(std::move(compressed_data)); + return simdb::pipeline::PROCEED; + } + + return simdb::pipeline::SLEEP; + } + + simdb::ConcurrentQueue>* input_queue_ = nullptr; + simdb::ConcurrentQueue>* output_queue_ = nullptr; +}; +---- + +The database stage derives from `DatabaseStage`, takes the +compressed bytes on its input port, and writes them with a prepared inserter. It +also carries an app-instance number, passed to its constructor, so multiple +instances can tag their rows: + +[source,cpp] +---- +class DatabaseStage : public simdb::pipeline::DatabaseStage +{ +public: + DatabaseStage(size_t app_instance_num) : + app_instance_num_(app_instance_num) + { + addInPort_>("data_to_write", input_queue_); + } + +private: + simdb::pipeline::PipelineAction run_(bool) override + { + std::vector data; + if (input_queue_->try_pop(data)) + { + auto inserter = getTableInserter_("CompressedData"); + inserter->setColumnValue(0, (int)app_instance_num_); + inserter->setColumnValue(1, data); + inserter->createRecord(); + return simdb::pipeline::PROCEED; + } + + return simdb::pipeline::SLEEP; + } + + size_t app_instance_num_ = 0; + simdb::ConcurrentQueue>* input_queue_ = nullptr; +}; +---- + +Finally, `createPipeline` wires the two stages together following the build +protocol from earlier -- add stages, bind ports, then capture the pipeline +head. `process` is the app's front door: it pushes a vector into that head +queue. + +[source,cpp] +---- +void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override +{ + auto pipeline = pipeline_mgr->createPipeline(NAME, this); + + pipeline->addStage("compressor"); + pipeline->addStage("db_writer", getInstance()); + pipeline->noMoreStages(); + + pipeline->bind("compressor.compressed_data", "db_writer.data_to_write"); + pipeline->noMoreBindings(); + + pipeline_head_ = pipeline->getInPortQueue>("compressor.input_data"); +} + +void process(const std::vector& data) +{ + pipeline_head_->push(data); +} +---- + +That is a complete pipeline. Data enters through `process`, is compressed on the +compressor's thread, and is written to SQLite on the database thread -- and the +caller of `process` never waits on any of it. + +[NOTE] +==== +The shipped `examples/SimplePipeline.hpp` includes a few test-only assertions and +a stored flusher that we have omitted here for clarity. The pipeline logic above +is otherwise the real example verbatim. +==== + +=== Building and running the examples + +The examples build with the regression target from <>: + +[source,bash] +---- +cd release +cmake -DCMAKE_BUILD_TYPE=Release .. +make simdb_regress +---- + +Run the resulting binaries from a build or scratch directory so their `.db` +output does not litter a source tree. `examples/README.md` is the authoritative +map of which example demonstrates which feature. + +This is the end of <>. You can now build a concurrent pipeline from stages +and ports, persist through a database stage, and reason about its performance. +What we have deferred -- how an app is defined, enabled, and driven through its +lifecycle -- is exactly what <> takes up next. diff --git a/docs/book/parts/part5-app-framework.adoc b/docs/book/parts/part5-app-framework.adoc new file mode 100644 index 00000000..c46ca5b9 --- /dev/null +++ b/docs/book/parts/part5-app-framework.adoc @@ -0,0 +1,651 @@ +[#part-v] += Part V: The App Framework + +Pipelines and schemas are composed inside _apps_. This part shows how SimDB apps +are defined, registered, enabled, and run -- including many apps sharing one +database. + +[#ch-22-simdb-app] +== What Is a SimDB App? + +In <> we built a pipeline, and it lived inside a class that derived from +`simdb::App`. That wrapper was not incidental. An *app* is SimDB's unit of +composition: a self-contained bundle of a schema, *one pipeline* (the typical +case), and lifecycle logic, all writing into a shared database. When several +simulation threads feed the same app, the usual pattern is *multiple pipeline +heads* -- several unbound input ports on that single pipeline -- not multiple +pipelines per app (which is uncommon and costs extra threads). This chapter +explains what an app is and the hooks it provides; the next explains how apps +are registered and run. + +=== One database, many apps + +The guiding paradigm is simple: your simulator has a *single output database*, +and *one or more apps* write into it, each with its own tables and its own logic. +One app might collect statistics, another might log events, another might capture +a trace -- and all of their data lands in the same SQLite file, each in its own +tables. This is the "one database to rule them all" idea from <>, expressed +at the level of features: you add a capability to your simulator by adding an +app, not by standing up a new database or a new output format. + +=== The anatomy of an app + +You create an app by subclassing `simdb::App`. A typical app has four parts, all +of which appeared in the `SimplePipeline` example: + +* *A constructor taking a `DatabaseManager*`.* Every app is handed the shared + database manager when it is created, and typically stores it: ++ +[source,cpp] +---- +SimplePipeline(simdb::DatabaseManager* db_mgr) : db_mgr_(db_mgr) {} +---- + +* *A static `defineSchema`.* This declares the app's tables. It is `static` + because the framework calls it to assemble the combined schema _before_ any app + instance exists: ++ +[source,cpp] +---- +static void defineSchema(simdb::Schema& schema) +{ + using dt = simdb::SqlDataType; + auto& tbl = schema.addTable("CompressedData"); + tbl.addColumn("AppInstance", dt::int32_t); + tbl.addColumn("DataBlob", dt::blob_t); +} +---- + +* *A `createPipeline` override.* This builds the app's *one* pipeline on the + manager it is given, using the <> build protocol. Call + `createPipeline(name, app)` once; fan in multiple producers via multiple + pipeline heads (unbound input ports), not by creating a second pipeline. + +* *Whatever public methods your app needs.* `SimplePipeline::process` is an + example -- the app's own front door for receiving data. These are not framework + hooks; they are simply your app's interface to the rest of the simulator. + +=== The lifecycle hooks + +Beyond the schema and pipeline, `App` defines a small set of virtual hooks that +the framework calls at well-defined points. You override the ones you need: + +[cols="1,3"] +|=== +| Hook | When it runs + +| `defineSchema(Schema&)` (static) | Up front, to declare the app's tables. +| `postInit(argc, argv)` | After command-line/config parsing, before the simulation starts. All apps' `postInit` methods run inside one automatic `safeTransaction` -- the usual place for pre-sim metadata (settings, scaffolding tables, lookup seeds). +| `createPipeline(PipelineManager*)` | To construct the app's single pipeline (multiple pipeline heads if needed). +| `preTeardown()` | Just before teardown -- push any last pending data into your app's pipeline. +| `postTeardown()` | After the simulation -- the place for post-sim metadata writes. All apps' `postTeardown` methods run inside one automatic `safeTransaction`. +|=== + +The `postInit` and `postTeardown` transaction details are worth dwelling on: +because every app's `postInit` and `postTeardown` each run inside a single +`safeTransaction` that SimDB opens for you, you can write pre-sim and post-sim +metadata directly in those hooks without wrapping INSERTs yourself -- the +batching, retry-on-lock, and safety are handled, exactly as they are on the +database thread during the run. + +The value of fixed hooks is consistency: however many apps a simulator loads, +each is initialized, run, and torn down in the same order. The `AppManager` -- +the next chapter -- is what drives them. + +=== Instance numbering + +An app can run as more than one *instance* against the same database -- several +copies of the same collector, say, each responsible for a different part of the +model. `getInstance()` returns an instance number to tell them apart: it is +1-based, with `0` meaning a single-instance app. `SimplePipeline` used exactly +this to tag each row with the instance that produced it, passing `getInstance()` +into its database stage so the `AppInstance` column records the source. + +=== What apps are for + +The framework is deliberately open-ended. Apps make good homes for any +self-contained data-producing capability, such as: + +* a *logger* that records simulation events, +* a *profiler* that tracks performance metrics, +* a *pipeline collector* that instruments CPU blocks in a performance model +* a *converter* (for example, SQLite to another format), +* or a *backend* for a live visualization GUI. + +Each is just a schema, one pipeline, and some lifecycle logic -- and each +coexists with the others in one database. An app on its own, though, does +nothing until something registers and enables it. That something is the +`AppManager`. + +[#ch-23-app-manager] +== The AppManager + +An app class on its own does nothing. Something has to register it, decide +whether it is enabled for this run, construct it, and then call its lifecycle +hooks in the right order. That something is the app manager, and it is what turns +a set of app _types_ into running apps writing to a database. + +=== Two managers, two jobs + +SimDB splits the responsibility across two classes: + +* `simdb::AppManagers` (plural) is the top-level coordinator. You register your + app types with it, it owns the per-database managers, and it drives the whole + lifecycle across every app at once. +* `simdb::AppManager` (singular) manages the enabled apps for *one database*. + Because a program can open more than one database file, there is one + `AppManager` per file. + +Most simulators have a single output database, so the common shape is one +`AppManagers` coordinating one `AppManager`. The plural form is what makes the +"one database, many apps" model -- and even "many databases" -- fall out +naturally. + +=== Registering and enabling apps + +Two distinct steps decide which apps run. First you *register* an app type so the +framework knows it exists; this happens before any manager is created: + +[source,cpp] +---- +simdb::AppManagers app_mgrs; +app_mgrs.registerApp(); +---- + +Then, for a given database, you *enable* the apps you actually want for this run: + +[source,cpp] +---- +auto& app_mgr = app_mgrs.createAppManager("out.db"); +app_mgr.enableApp(SimplePipeline::NAME); // enable one instance +// app_mgr.enableApp(SimplePipeline::NAME, 4); // or several instances +---- + +The split matters: registration is a fixed list of everything your simulator +_could_ run, while enabling is the per-run decision driven by your configuration +-- command-line flags, a config file, whatever your simulator uses. An app that +is registered but not enabled costs nothing. + +=== The driven lifecycle + +With apps registered and enabled, `AppManagers` drives them through the hooks from +the previous chapter in a fixed sequence. Each call fans out across every enabled +app on every database: + +[source,cpp] +---- +app_mgrs.createEnabledApps(); // construct the enabled app instances +app_mgrs.createSchemas(); // call each app's static defineSchema() +app_mgrs.postInit(argc, argv); // run each app's postInit() hook +app_mgrs.initializePipelines(); // build each app's pipeline (createPipeline()) +app_mgrs.openPipelines(); // start the pipeline threads running + +// ... run the simulation, feeding data to your app(s) ... +auto app = app_mgr.getApp(); +for (const auto& data : inputs) +{ + app->process(data); +} + +app_mgrs.postSimLoopTeardown(); // preTeardown()/postTeardown(), stop threads +---- + +Reading top to bottom, that is the entire life of a SimDB program: + +. `createEnabledApps` constructs each enabled app (handing it the shared + `DatabaseManager`). +. `createSchemas` invokes every app's `defineSchema`, assembling one combined + schema in the database. +. `postInit` runs each app's post-initialization hook. +. `initializePipelines` invokes each app's `createPipeline` once, building that + app's pipeline and its threads. +. `openPipelines` starts those threads; from here the pipelines are live. +. Your simulation runs, handing data to apps you retrieve with + `getApp()`. +. `postSimLoopTeardown` runs the teardown hooks, drains and stops the threads, + and closes things out. + +=== Reaching your apps and database + +Once things are running, a handful of accessors get you back to a specific app or +database. Some throw when used in a situation that has no single unambiguous +answer -- asking for "the" app manager when several databases are open, or for a +single app instance when many were enabled. The table below summarizes them, and +on which class each lives (`AppManager` is per-database; `AppManagers` is the +top-level coordinator). + +[cols="2,1,2,2"] +|=== +| Method | On | Returns / does | Throws when + +| `enabled()` +| `AppManager` +| Whether `AppT` is enabled (it may not be instantiated yet). +| Does not throw. + +| `getEnabledAppInstances()` +| `AppManager` +| How many instances of `AppT` are enabled (`0` if none). +| Does not throw. + +| `getApp()` +| `AppManager` +| The single instance of `AppT`, or `nullptr` if it is not enabled. +| `AppT` was enabled with more than one instance (use `getAppInstance`), or the stored app is not of type `AppT`. + +| `getAppInstance(n)` +| `AppManager` +| Instance `n` (zero-based) of a multi-instance app, or `nullptr` if not enabled/absent. +| The stored app is not of type `AppT`. + +| `getDatabaseManager()` +| `AppManager` +| The shared `DatabaseManager` for this database. +| Does not throw. + +| `getAppManager()` +| `AppManagers` +| The sole `AppManager`. +| There is not exactly one `AppManager`. + +| `getAppManager(db_file)` +| `AppManagers` +| The `AppManager` for `db_file`. +| No `AppManager` exists for `db_file`. + +| `getDatabaseManager()` +| `AppManagers` +| The sole `DatabaseManager`. +| There is not exactly one database open. + +| `getDatabaseManager(db_file)` +| `AppManagers` +| The `DatabaseManager` for `db_file`. +| No database `db_file` exists. + +| `getAllManagers()` +| `AppManagers` +| Every `(AppManager*, DatabaseManager*)` pair; empty after teardown. +| Does not throw. +|=== + +In the common single-database, single-instance case, the two you will reach for +are `getApp()` (to call your app's own methods, like `process`) and +`getDatabaseManager()` (to access the database directly; never do this in the hot +path / simulation loop -- use the `AsyncDatabaseAccessor` discussed in <>). + +=== Consistency + +The reason to hand this sequence to a manager rather than wiring it by hand is +consistency: every app, no matter how many you enable, is initialized, run, and +torn down in exactly the same order. Adding a second app does not change the +driver above at all -- you simply enable one more app, which is precisely the +subject of the next chapter. + +=== Per-thread performance reports + +When the manager tears the pipelines down, each pipeline worker thread prints a +short performance report. It lists which stage (or stages) the thread was +running, how many times it ran, and -- most usefully -- how it split its time +between doing work and sleeping: + +.... +Thread containing: + compressor + + Performance report: + Num times run: 1000 + Pct time sleeping: 4.2% + Pct time working: 95.8% +.... + +These percentages are your primary guide to *tuning stages*, and they connect +directly to the performance levers from <>: + +* A thread that is *mostly working* (near 100%) is *saturated* -- it is a + bottleneck holding back the pipeline. That stage is a candidate to split into + finer stages that can run in parallel, to optimize, or to feed with larger + batched payloads so it spends less time per item. +* A thread that is *mostly sleeping* is *under-utilized* -- it wakes on its + polling interval, finds nothing, and goes back to sleep. Either it is starved + downstream of a bottleneck, or its polling interval is shorter than it needs to + be and can be lengthened to waste fewer wakeups. + +Balancing these numbers across stages -- no single stage pinned at 100% while +others idle -- is how you turn the abstract advice about stage granularity and +polling intervals into concrete adjustments for _your_ pipeline. + +To make this concrete, take the `compressor` stage from our running example. Its +per-item cost is the zlib compression, which you dial with the +`compression_level` argument to `simdb::compressData` (from `CompressionLevel:: +FASTEST` through `HIGHEST`). Its report tells you which way to turn that dial: + +[cols="1,1,3"] +|=== +| Active % | Sleep % | Design decision + +| Very high | ~0% (never sleeps) +| The compressor is saturated and gating the pipeline. *Lower the compression + ratio* -- pass a faster level such as `CompressionLevel::FASTEST` -- so each + vector costs less CPU and the stage keeps up. + +| Very low | ~100% (always sleeps) +| The compressor has capacity to spare. *Raise the compression ratio* (for + example `CompressionLevel::HIGHEST`) to shrink the database while the headroom + exists; or keep the ratio and *merge this stage into the previous or next one* + to save a thread and a queue hop. + +| Balanced | Balanced +| The stage is well tuned -- leave it as is. +|=== + +[#ch-24-multiple-apps] +== Running Multiple Apps Concurrently + +The payoff of the manager-driven lifecycle is that running many things at once is +nearly free. Because the driver sequence from the previous chapter never changes, +"one app" and "many apps" are the same code -- you just enable more. Concurrency +comes in two forms: several _instances_ of one app, and several _different_ apps. +Either way, they all share the one database. + +=== Many instances of one app + +To run several copies of the same app, pass an instance count to `enableApp`. +Each instance is a full, independent app with its own pipeline and threads; they +differ only in their instance number: + +[source,cpp] +---- +simdb::AppManagers app_mgrs; +app_mgrs.registerApp(); + +auto& app_mgr = app_mgrs.createAppManager("out.db"); +app_mgr.enableApp(SimplePipeline::NAME, 4); // four instances + +app_mgrs.createEnabledApps(); + +// Reach each instance by its zero-based index: +auto app0 = app_mgr.getAppInstance(0); +auto app1 = app_mgr.getAppInstance(1); +// ... +---- + +With more than one instance configured, `getApp()` throws (there is no +single app to return) -- use `getAppInstance(n)` instead. Inside each app, +`getInstance()` returns its number, which is how the instances keep their data +apart: `SimplePipeline` tags every row it writes with its instance, so all four +instances can share the `CompressedData` table and you can still tell whose rows +are whose. + +=== Many different apps + +Running different apps together is the same story: register each type, enable +each one, and the lifecycle calls fan out to all of them. A statistics app, a +logging app, and a trace collector can all be enabled on the same database, and +the driver code is identical to the single-app case -- `createEnabledApps`, +`createSchemas`, and the rest simply visit every enabled app. Each app +contributes its own tables via its own `defineSchema`, so they coexist in one +file without stepping on one another. + +=== The threading model, and multi-app scalability + +The one firm rule you already know holds no matter how many apps you run: there +is *exactly one database thread*, shared by every database stage in the process. +That is the permanent anchor of the model, and it is what keeps database access +fast and contention-free regardless of how many apps are enabled. + +The non-database side is simpler than it will eventually be. Today, each pipeline +stage runs on its own worker thread, and there is no sharing of those threads +across apps. Enabling a second app adds that app's stages -- and therefore its +threads -- on top of the first. Thread count grows roughly with the total number +of stages across all enabled apps. + +[WARNING] +==== +Running many apps at once is *not currently thread-scalable*. Each additional app +adds its stages' worker threads with no pooling or sharing, so a process with a +large number of apps will spin up a correspondingly large number of threads. In +practice this is yet to be a problem: today a simulation typically runs one +app at a time (or none), which stays comfortably within budget. Keep the +limitation in mind before using many apps in a single simulation. +==== + +This is a known, temporary limitation. The planned direction is for SimDB to +manage an unbounded worker-thread pool under the hood, automatically *creating, +merging, and destroying* threads as needed to spread stages across the available +threads and keep each thread's active/sleep balance healthy (the same balance you +read off the per-thread performance reports in <>). When that lands, +multi-app systems will scale on threads without any change to your app code -- the +single database thread will remain the one fixed point. + +=== Coordinated teardown + +Just as startup is driven across all apps, so is shutdown. A single +`postSimLoopTeardown` runs `preTeardown` and `postTeardown` for every app, +drains and stops all the threads, and -- as noted earlier -- wraps all apps' +`postTeardown` work in one `safeTransaction`, so their final metadata writes +commit together efficiently. (`postInit` received the same automatic +`safeTransaction` treatment at startup.) If the order in which apps are torn down +(or initialized) matters, the manager lets you specify a hook ordering across apps; +by default they run in a consistent internal order. + +[WARNING] +==== +`postSimLoopTeardown` is what makes a database complete and readable -- so it +*must* run even when the simulation ends badly. If your simulator throws an +exception, calls `std::abort`/`std::terminate`, or otherwise exits without +reaching teardown, skipping it leaves you with: + +* a *partial database* -- the final batched transactions were never committed; +* *unreadable or unparseable blobs* -- compressed/serialized payloads that were + cut off mid-write; and +* likely a *crash in `std::thread::~thread`* -- the pipeline's worker threads were + never joined, which terminates the process. + +Make sure `postSimLoopTeardown` is called on every exit path. Wrap your +simulation loop so that it runs during unwinding (for example from a scope guard, +a `try`/`catch` that rethrows after tearing down, or an equivalent +last-resort handler) rather than relying on the happy path alone. +==== + +[NOTE] +==== +Today, `postSimLoopTeardown` does *not* flush a `TinyStrings` map if your app +uses one -- that flush is currently a separate step. A future design will fold it +into teardown automatically, so that a single call to `postSimLoopTeardown` is +the one and only API you need to guarantee a good database. Until then, flush any +`TinyStrings` yourself before (or as part of) teardown. +==== + +Running many apps, then, adds no new machinery -- only more `enableApp` calls. +The last wrinkle is apps whose constructors need more than the `DatabaseManager`, +which is where app factories come in. + +[#ch-25-app-factories] +== App Factories & Non-Default Constructors + +Every app you have seen so far has been constructed the same way: the manager +hands it a `DatabaseManager*` and nothing else. That is the default, and it is +handled for you by a factory you never had to write. This final chapter is about +the case where that is not enough -- an app that needs additional constructor +arguments -- and the small amount of machinery SimDB provides to support it. + +=== The default factory + +When you register an app with `registerApp()`, the manager needs two +things from the type: a way to declare its schema, and a way to build an +instance. For an ordinary app it gets both from a built-in template, +`simdb::AppFactory`: + +[source,cpp] +---- +template class AppFactory : public AppFactoryBase +{ +public: + App* createApp(DatabaseManager* db_mgr) override { return new AppT(db_mgr); } + void defineSchema(Schema& schema) const override { AppT::defineSchema(schema); } +}; +---- + +That is the whole story for a default app: `createApp` calls `new +AppT(db_mgr)`, and `defineSchema` forwards to the app's static +`defineSchema`. You never mention this class -- it is installed automatically at +registration. The only requirement it places on your app is the one you have +followed all along: a constructor taking a single `DatabaseManager*`, and a +static `defineSchema(Schema&)`. + +=== When the default constructor is not enough + +Sometimes an app genuinely needs more than the database manager to do its job -- +a configuration struct, an ID, a tuning parameter, a handle to something in the +host simulator. Its constructor then looks like: + +[source,cpp] +---- +MyApp(simdb::DatabaseManager* db_mgr, int x, float y); +---- + +The default factory cannot build this -- it only knows how to call `new +MyApp(db_mgr)`. To bridge the gap, you give the app its own *nested* factory: a +public class literally named `AppFactory`, inheriting from +`simdb::AppFactoryBase`, that knows how to store the extra arguments and pass +them along. + +=== Writing a nested AppFactory + +A nested factory implements the two `AppFactoryBase` methods (`createApp` and +`defineSchema`) and adds one method of its own -- `parameterize` -- whose +signature is entirely up to you: give it exactly the arguments your constructor +needs. It stashes them, and later `createApp` forwards them into the real +constructor: + +[source,cpp] +---- +class MyApp : public simdb::App +{ +public: + static constexpr auto NAME = "my-app"; + + MyApp(simdb::DatabaseManager* db_mgr, int x, float y) : + db_mgr_(db_mgr), x_(x), y_(y) + { + } + + static void defineSchema(simdb::Schema& schema) { /* ... */ } + + class AppFactory : public simdb::AppFactoryBase + { + public: + // Signature is yours to choose -- match the app's constructor. + void parameterize(int x, float y) + { + ctor_args_ = std::make_pair(x, y); + } + + simdb::App* createApp(simdb::DatabaseManager* db_mgr) override + { + const auto& [x, y] = ctor_args_; + return new MyApp(db_mgr, x, y); + } + + void defineSchema(simdb::Schema& schema) const override + { + MyApp::defineSchema(schema); + } + + private: + std::pair ctor_args_; + }; + +private: + simdb::DatabaseManager* db_mgr_ = nullptr; + const int x_; + const float y_; +}; +---- + +Nothing else about the app changes: it is still registered, enabled, and driven +through its lifecycle exactly as in the previous two chapters. The factory only +changes *how the instance is built*. + +=== How the manager finds your factory + +You never tell the manager which kind of factory to use -- it detects it at +compile time. A small type trait checks whether your app has a nested type named +`AppFactory`: + +[source,cpp] +---- +template struct has_nested_factory : std::false_type {}; +template struct has_nested_factory> : std::true_type {}; +---- + +At registration the manager branches on that trait. If your app has *no* nested +factory, it installs the default `AppFactory` immediately -- the app is +ready to build with no further action from you. If your app *does* have a nested +factory, the manager installs nothing yet: it waits for you to *parameterize* the +factory (that is when the extra arguments become available), and the factory +object is created there. The practical consequence is a rule to remember: + +[IMPORTANT] +==== +If your app defines a nested `AppFactory`, you *must* parameterize it after +enabling the app and before `createEnabledApps()`. Because the manager defers +factory creation for custom-factory apps, forgetting to parameterize means the +factory never exists, and app creation will fail with an error naming the app. +==== + +=== Parameterizing: all instances, or one at a time + +You configure the factory through the manager, after `enableApp` and before +`createEnabledApps`. There are two flavors: + +* `parameterizeAppFactory(x, y)` -- configure *every* instance of the app + with the same arguments. +* `parameterizeAppFactoryInstance(instance_num, x, y)` -- configure a + *single* instance, using the same instance number you will later pass to + `getAppInstance`. Call it once per instance to give each its own arguments. + +The two interact in one way worth flagging: calling the "all instances" +`parameterizeAppFactory` *discards* any per-instance configurations you set +earlier and replaces them with one shared factory (the manager only warns about +this). Pick one strategy per app rather than mixing them. Both flavors require +that the app already be enabled -- calling them first throws. + +=== Putting it together + +The full sequence for a parameterized, multi-instance app is just the driver from +<> with parameterize calls slotted in before creation: + +[source,cpp] +---- +simdb::AppManagers app_mgrs; +app_mgrs.registerApp(); +auto& app_mgr = app_mgrs.createAppManager("sim.db"); + +app_mgr.enableApp(MyApp::NAME, 2); // two instances + +// One parameterize call per instance, each with its own arguments. +app_mgr.parameterizeAppFactoryInstance(0, 45, 7.77f); +app_mgr.parameterizeAppFactoryInstance(1, 98, 3.14f); + +app_mgrs.createEnabledApps(); // constructors run here + +auto* a0 = app_mgr.getAppInstance(0); // built with (45, 7.77) +auto* a1 = app_mgr.getAppInstance(1); // built with (98, 3.14) +---- + +Because the factory lives on the manager and construction is deferred until +`createEnabledApps`, the same pattern extends naturally across databases: hand +each `AppManager` (one per database) its own parameterized instances, then let a +single `createEnabledApps` build them all. `examples/AppFactory/main.cpp` shows +exactly this -- two `MyApp` instances in one database, and then one `MyApp` +routed to each of two separate databases -- and is the authoritative worked +example for everything in this chapter. + +That closes <>. You can now define an app, register and enable it, drive it +through its lifecycle with the manager, tear it down safely, and -- when its +constructor needs more than a `DatabaseManager` -- give it a factory to build it. +<> returns to the pipeline layer for the advanced patterns we deferred: +snoopers, watchdog stages, multi-port and elastic pipelines, and the multi-app +techniques that build on the framework you have just learned. diff --git a/docs/book/parts/part6-advanced-pipelines.adoc b/docs/book/parts/part6-advanced-pipelines.adoc new file mode 100644 index 00000000..66c69a12 --- /dev/null +++ b/docs/book/parts/part6-advanced-pipelines.adoc @@ -0,0 +1,631 @@ +[#part-vi] += Part VI: Advanced Pipeline Techniques + +Once the basics work, these techniques handle synchronization, low-latency +retrieval of in-flight data, determinism, cross-thread database access, and how +stages are scheduled onto threads. + +[#ch-26-flushers] +== Flushers + +A pipeline is asynchronous by design: data enters through `process`, moves stage +by stage on the stages' own threads, and the caller never waits for it to land. +That is exactly what you want almost all of the time. But every so often you need +the opposite -- a moment where you can say *"stop, finish everything in flight, +and make it real in the database right now."* That moment is a *flush*, and the +object that provides it is a `Flusher`. + +The two occasions that call for one are worth naming up front: + +* *Before you query your own data.* If you want to `count()` the rows written so + far, or read back a record your pipeline just produced, the data may still be + sitting in a queue between stages. A flush guarantees it has reached -- and been + committed by -- the database stage first. +* *At a controlled stopping point.* During teardown, or before taking a snapshot + or checkpoint, you want the queues drained rather than abandoned. + +=== What a flush actually does + +A flusher holds an ordered list of stages. Calling `flush()` runs each stage's +`run_()` exhaustively -- `processAll` with `force = true` -- stage by stage, in +the order given, cycling round-robin until *every* stage reports it has nothing +left to do (they all return `SLEEP`). The effect is that all in-flight data is +pushed forward through the pipeline and out its far end, rather than being +processed lazily on the usual polling cadence. + +Because the stages run to completion in order, *order matters*: an upstream stage +should flush before the downstream stage it feeds, so its output has landed in +the next stage's input queue before that stage takes its turn. The round-robin +loop then repeats until the whole chain is quiet. + +=== Creating a flusher + +You create a flusher inside `createPipeline` and store it on the app, so it is +available later whenever you need a synchronization point. Naming the stages sets +the flush order explicitly: + +[source,cpp] +---- +void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override +{ + auto pipeline = pipeline_mgr->createPipeline(NAME, this); + + pipeline->addStage("compressor"); + pipeline->addStage("db_writer", getInstance()); + pipeline->noMoreStages(); + + pipeline->bind("compressor.compressed_data", "db_writer.data_to_write"); + pipeline->noMoreBindings(); + + pipeline_head_ = pipeline->getInPortQueue>("compressor.input_data"); + + // Flush "compressor", then "db_writer". + pipeline_flusher_ = pipeline->createFlusher({"compressor", "db_writer"}); +} +---- + +If you omit the stage list, the flusher flushes every stage in the order you +added them with `addStage` -- which is usually the order you want anyway, so the +common case is simply: + +[source,cpp] +---- +pipeline_flusher_ = pipeline->createFlusher(); +---- + +`createFlusher` returns a `std::unique_ptr` that you own; store it in a +member such as `pipeline_flusher_`. Naming a stage that does not exist throws a +`DBException`. + +=== The single-transaction guarantee + +Here is the detail that makes flushing cheap enough to rely on. If *any* of the +flushed stages is a `DatabaseStage`, `createFlusher` does not hand you a plain +`Flusher` -- it automatically gives you a `FlusherWithTransaction`, which runs the +entire flush inside one `safeTransaction`: + +[source,cpp] +---- +// Inside FlusherWithTransaction::flush() +db_mgr_->safeTransaction([&]() { outcome = Flusher::flush(); }); +---- + +So the whole drain -- however many records the database stage writes -- commits +as a single `BEGIN`/`COMMIT` rather than one transaction per record. This is the +same batching principle from <> and <>, applied automatically at the +flush boundary: a database stage is assumed to be doing real work, so its flush +is wrapped for you. You do not choose between the two flusher types; the pipeline +picks the right one based on whether a database stage is present. + +=== Using a flush before a query + +The canonical use is the flush-then-read pattern. Because a flush guarantees all +prior data is committed, a query that follows it sees a consistent, complete +picture: + +[source,cpp] +---- +size_t getNumCollected() +{ + // Drain the compressor, then the db_writer -- all in one transaction. + pipeline_flusher_->flush(); + + // Everything produced so far is now in the database; count it. + auto query = db_mgr_->createQuery("CompressedData"); + return query->count(); +} +---- + +Without the flush, this count could miss records still queued between stages. With +it, the answer is exact as of the moment of the call. + +[WARNING] +==== +A flush is a hard synchronization point. It does real work on the *calling* +thread and forces the current batch to commit, so flushing on every item defeats +the batching that makes the pipeline fast in the first place. Flush when you +genuinely need a consistency point -- before a query or count, at a checkpoint, or +during teardown -- not reflexively after each push. +==== + +When the data you are after has *not* yet reached the database, a flush followed +by a query is also the reliable fallback for retrieving it. There is a +lower-latency alternative that inspects data while it is still moving between +stages -- snooping -- which is the subject of the next chapter. + +[#ch-27-snoopers] +== Snoopers + +A flush answers the question "make everything real so I can query it." But it is +a heavy answer: it drains the whole pipeline to disk and commits. If what you +actually need is a single item -- "give me the data for UUID `abc123`, wherever it +happens to be right now" -- and you need it *often*, flushing every time is +wasteful. *Snooping* is the lightweight alternative: it peeks into the queues +*between* stages and returns the item without draining anything or accessing the disk. + +=== The idea: a key and a snooped type + +Snooping is built around two types you choose: + +* a *key* type that uniquely identifies an item (some kind of UUID), and +* a *snooped* type that every stage can produce for that key. + +The subtlety is that each stage stores data in its own in-flight form -- the +compressor's input holds raw values, its output holds compressed bytes -- yet a +snoop must return *one* common type regardless of which stage happens to hold the +item. So each stage's snoop is responsible for converting its internal +representation back into the shared snooped type. For snooping to work at all, +the payloads flowing through the queues must carry the key; the examples below +assume small structs that pair a UUID with the data. + +=== Peeking into a queue + +The primitive underneath everything is `ConcurrentQueue::snoop`. It takes a +callback, locks the queue, and invokes the callback on each item in turn until +the callback returns `true` (found it) or the queue is exhausted: + +[source,cpp] +---- +bool snoop(const std::function& cb) const; +---- + +Because it holds the queue's lock while iterating, a snoop sees a consistent +snapshot of what is sitting in that queue at that instant. + +=== A stage's snoop() method + +Each stage that wants to be snoopable implements a method with a fixed signature +-- `bool snoop(const KeyType&, SnoopedType&)` -- returning `true` and filling the +out-parameter when it holds the key. Typically it just forwards to its input +queue's `snoop`. Here the two stages from our running example: the compressor +holds uncompressed values, so a hit is a direct copy (fast); the database stage +holds compressed bytes, so a hit must be decompressed first (slower): + +[source,cpp] +---- +using uuid_t = std::string; +using values_t = std::vector; + +// CompressionStage: input queue holds { uuid, values } +bool snoop(const uuid_t& uuid, values_t& values) +{ + return input_queue_->snoop([&](const Sample& in) + { + if (in.uuid == uuid) + { + values = in.values; // direct copy -- fast + return true; // found; stop looking + } + return false; // keep scanning this queue + }); +} + +// DatabaseStage: input queue holds { uuid, compressed bytes } +bool snoop(const uuid_t& uuid, values_t& values) +{ + return input_queue_->snoop([&](const CompressedSample& in) + { + if (in.uuid == uuid) + { + simdb::decompressData(in.bytes, values); // undo zlib -- slower + return true; + } + return false; + }); +} +---- + +This is the reason a snooper prefers to find data *early*: the further down the +pipeline an item has travelled, the more transforms a stage must reverse to +reconstruct the original. + +=== Wiring the snooper + +You create a `PipelineSnooper` from the pipeline manager and +register each snoopable stage with it, using the stage handles returned by +`addStage`: + +[source,cpp] +---- +void createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override +{ + auto pipeline = pipeline_mgr->createPipeline(NAME, this); + auto compressor = pipeline->addStage("compressor"); + auto db_writer = pipeline->addStage("db_writer", getInstance()); + pipeline->noMoreStages(); + // ... bindings, flusher, pipeline head ... + + pipeline_snooper_ = pipeline_mgr->createSnooper(); + pipeline_snooper_->addStage(compressor); + pipeline_snooper_->addStage(db_writer); +} +---- + +Store the returned `std::unique_ptr>` on the app. Register +stages in pipeline order -- the snooper tries them in the order you add them, so +adding upstream stages first finds the item in its cheapest form. + +=== Retrieving an item + +`snoopAllStages(key, out)` walks the registered stages, calling each one's +`snoop` until one returns `true`. If none do, the item is not currently sitting +in any queue -- either it never existed, or it has already been written to the +database (or is being processed *inside* a stage's `run_` and so is momentarily +invisible to a queue snoop). The robust pattern therefore falls back to a flush +and a query, exactly the mechanism from <>: + +[source,cpp] +---- +bool snoopPipeline(const uuid_t& uuid, values_t& values) +{ + // Fast path: find it in-flight between stages. + if (pipeline_snooper_->snoopAllStages(uuid, values)) + { + return true; + } + + // Slow path: force everything to disk, then query. + pipeline_flusher_->flush(); + + auto query = db_mgr_->createQuery("CompressedData"); + std::vector bytes; + query->select("DataBlob", bytes); + query->addConstraintForString("UUID", simdb::Constraints::EQUAL, uuid); + + auto results = query->getResultSet(); + if (results.getNextRecord()) + { + simdb::decompressData(bytes, values); + return true; + } + return false; // not found anywhere +} +---- + +=== Why snooping pauses the pipeline + +`snoopAllStages` takes an optional third argument, `disable_pipeline`, that +*defaults to `true`*. Before it scans, it quietly pauses all the pipeline's +stages and threads for the duration of the snoop. There is a concrete reason for +this. If the stages kept running while you searched, an item could slip out of +the compressor's queue just after you scanned it (a miss), then out of the +database stage's queue just as you got there (another miss) -- and you would +"chase" it down the entire pipeline, never catching it in a queue. Pausing first +freezes the picture, so the item is found in the earliest stage that holds it. + +That pause is a small, deliberate cost, and it is your main lever in the +trade-off snoopers present: *pipeline throughput versus retrieval latency*. A +snoop that pauses the pipeline briefly slows overall throughput a little but +returns in-flight data quickly and without disk I/O; leaning on flush-and-query +instead keeps the pipeline running at full speed but makes each retrieval +expensive. The pause mechanism itself -- and the other uses of temporarily +disabling a pipeline -- is the subject of the next chapter. + +`examples/PipelineSnoopers/main.cpp` is the complete, authoritative example: a +four-stage pipeline (buffer, serializer, compressor, database writer) where every +stage implements `snoop`, showing exactly how a single snooped type is +reconstructed from four different in-flight representations. + +[#ch-28-disabling-pipelines] +== Disabling Pipelines + +The previous chapter left a promise: `snoopAllStages` freezes the pipeline before +it scans, and this chapter explains the mechanism behind that. Occasionally you +need a running pipeline to *hold still* -- so you can inspect it, snapshot it, or +debug it without data moving out from under you. SimDB provides a single RAII tool +for that: the *scoped disabler*. + +=== Enable, disable, and the polling thread + +Every unit of work on a pipeline thread is a `Runnable` (a `Stage` is one), and +every runnable carries an enabled flag. The polling thread simply *skips* any +runnable whose flag is off. Disabling a stage therefore does not tear anything +down -- the stage, its queues, and their contents stay exactly as they are; the +thread just stops calling into it until it is re-enabled. + +=== The scoped disabler + +You rarely toggle that flag by hand. Instead you ask the pipeline manager to +disable everything for the duration of a scope: + +[source,cpp] +---- +{ + auto disabler = pipeline_mgr_->scopedDisableAll(); + // The entire pipeline is frozen here: no stage runs, and (by default) + // the polling threads are paused too. Inspect state, snoop queues, or + // take a consistent snapshot -- nothing moves. +} // disabler goes out of scope -> everything is re-enabled and resumes +---- + +`scopedDisableAll` returns a `std::unique_ptr`. On +construction it disables all the pipeline's runnables (and pauses their polling +threads); on destruction it re-enables and resumes them. Because it is tied to a +scope, you cannot forget to turn the pipeline back on -- leaving the block does it +for you, even if an exception unwinds through it. + +=== Threads too, or just the work? + +The one parameter, `disable_threads_too`, defaults to `true`: + +* `true` -- the polling threads are paused as well, so the process is genuinely + quiet. This is what you want for a clean snapshot or for deterministic + debugging. +* `false` -- only the runnables are disabled; the polling threads keep spinning + but find nothing enabled to do. Useful when you want work to halt but the + threads to remain responsive. + +=== Nesting is safe + +If a disabler is already active and something requests another, `scopedDisableAll` +returns `nullptr` -- the nested request is a harmless no-op, because the pipeline +is already frozen by the outer disabler. This is exactly why the snooper from +<> can hold a possibly-null disabler without any special handling: + +[source,cpp] +---- +// Inside snoopAllStages(...) +std::unique_ptr disabler = + disable_pipeline ? pipeline_mgr_->scopedDisableAll() : nullptr; +// Scan the stages. If an outer disabler already froze the pipeline, +// 'disabler' is null and we simply rely on that outer scope. +---- + +=== When to reach for it + +Two situations justify freezing a pipeline: + +* *Determinism while debugging.* Concurrency bugs are hard to observe precisely + when every stage is racing ahead. Disabling the pipeline gives you a still, + reproducible picture to inspect. +* *Snapshot accuracy and performance.* This is the snooper's "chasing" problem + from the last chapter: if stages keep running while you search, an item can + slip from one queue to the next just ahead of you. Freezing first guarantees + you find it in the earliest stage that holds it, and avoids re-scanning a + moving target. + +[WARNING] +==== +Disabling the pipeline stops throughput for as long as the disabler is alive. +It is a deliberate, temporary tool -- keep the scope as short as possible, and do +not treat it as a normal operating mode. The default snoop already applies it for +you in exactly the right, minimal window. +==== + +[#ch-29-async-db-access] +== Async Database Access from Non-DB Threads + +The permanent rule of SimDB's threading -- one dedicated database thread, and only +that thread touches the database -- creates a puzzle the moment a *non*-database +stage needs to read the database. A watchdog that wants to count rows, a stage +that needs to look something up mid-flight: neither may grab the +`DatabaseManager` and query it directly, because that would mean two threads +racing on the same connection. The `AsyncDatabaseAccessor` is the sanctioned +bridge. You hand it a callback; it runs that callback *on the database thread* and +blocks until it finishes. + +=== The callback and eval + +The callback has one fixed signature -- it receives the `DatabaseManager` it will +run against: + +[source,cpp] +---- +using AsyncDbAccessFunc = std::function; +---- + +You submit it with `eval`, which blocks the calling thread until the callback has +run on the database thread (or a timeout fires): + +[source,cpp] +---- +void eval(const AsyncDbAccessFunc& func, double timeout_seconds = 0); +---- + +Because `eval` blocks, you can capture results by reference in the lambda and read +them out after `eval` returns -- by then the callback has definitely run. + +=== Getting an accessor + +There are two places to obtain an accessor, depending on who needs it: + +* *From inside a non-database stage*, use the protected `getAsyncDatabaseAccessor_()`: ++ +[source,cpp] +---- +// Inside a non-DB Stage's run_() +auto async_db_accessor = getAsyncDatabaseAccessor_(); +async_db_accessor->eval(callback); +---- + +* *From the app itself* (its main thread), cache the accessor from the pipeline + manager during `createPipeline` -- it is valid once the pipeline is open: ++ +[source,cpp] +---- +void MyApp::createPipeline(simdb::pipeline::PipelineManager* pipeline_mgr) override +{ + // ... build the pipeline ... + async_db_accessor_ = pipeline_mgr->getAsyncDatabaseAccessor(); +} +---- + +[IMPORTANT] +==== +Do *not* call `getAsyncDatabaseAccessor_()` from a `DatabaseStage`. A database +stage already runs *on* the database thread and has direct access through +`getDatabaseManager_()` and `getTableInserter_()` -- routing through the async +accessor from there is nonsensical, and SimDB throws if you try. The accessor +exists specifically for code that is *not* on the database thread. +==== + +=== Immediate commit, and the timeout + +Here is the behavior that makes this fast enough to use from a polling loop. The +database thread is usually busy batching writes inside a long-running +transaction. When an async request arrives, SimDB does not make you wait for that +transaction to run its natural course -- it *immediately commits* whatever the +database thread was doing, then runs your callback (itself inside a +`safeTransaction`). So even if the in-progress batch would otherwise have taken +seconds, you typically wait only for that one commit -- a fraction of a second. + +The optional `timeout_seconds` is a safety valve, not the normal wait. It guards +against a genuinely stuck database thread (a bug, a runaway producer); if the +callback has not completed within the timeout, `eval` throws rather than blocking +forever. In healthy operation you never hit it. + +=== Exceptions propagate + +`eval` is built on `std::promise`/`std::future`, and it propagates failure +faithfully: any exception thrown by your callback on the database thread is +re-thrown from `eval` on the calling thread. You handle database errors from the +async path exactly as you would from a synchronous call -- wrap the `eval` in a +`try`/`catch` if you need to. + +=== Worked example: the database watchdog + +`examples/DatabaseWatchdog/main.cpp` is the canonical use. It runs two apps at +once against one database: + +* *WatchedPipeline* -- the familiar compress-and-write pipeline, producing rows + with no upper bound, running on the usual 100 ms polling cadence. +* *DatabaseWatchdog* -- a second app whose single stage has *no I/O ports at all*. + It exists only to poll the database. It runs on a slower 500 ms timer and, each + time it wakes, posts a `count()` query to the database thread. When the count + crosses a threshold, it tells the first app to stop. + +The watchdog's stage is the whole point -- a non-DB stage reaching the database +safely from its own thread: + +[source,cpp] +---- +simdb::pipeline::PipelineAction run_(bool) override +{ + if (finished_) + { + return simdb::pipeline::SLEEP; + } + + size_t num_records = 0; + auto async_query = [&](simdb::DatabaseManager* db_mgr) + { + auto query = db_mgr->createQuery("CompressedData"); + num_records = query->count(); + if (num_records > TARGET_NUM_RECORDS) + { + finished_ = true; + } + }; + + // Post the query to the DB thread; 5s timeout is only a runaway guard. + auto async_db_accessor = getAsyncDatabaseAccessor_(); + async_db_accessor->eval(async_query, 5); + + // 'num_records' and 'finished_' are now safe to read: eval() has returned. + if (finished_) + { + watchdog_app_->thresholdReached_(); + } + + return simdb::pipeline::SLEEP; +} +---- + +Notice the shape: the lambda captures `num_records` by reference, does its work on +the database thread, and by the time `eval` returns the value is populated and the +`finished_` flag is set. The watchdog never touches the database directly -- it +only ever posts work to the thread that owns it. + +This is also the natural counterpart to the flusher: a watchdog reads the database +*as it fills* without stopping the producer, whereas a flush would force a +synchronization point. Together with snoopers and the scoped disabler, you now +have the full toolkit for observing and coordinating a live pipeline. + +[#ch-30-threads-scheduling] +== Threads & Scheduling + +Two chapters back we looked at *what* runs (stages, snoopers, disablers); this one +is about *when* and *on what thread* it runs. Most of it you never have to touch, +but one dial -- a stage's polling cadence -- is worth understanding, and the way +threads are allocated is worth knowing because it is actively evolving. + +=== The polling loop + +Recall from <> that, today, each stage runs on its own `PollingThread`, +and there is exactly one database thread. A polling thread does something very +simple in a loop: it asks its stage to process all the work it can, and then -- +*only if there was nothing to do* -- it sleeps for a fixed interval before asking +again. When there is a steady supply of work, the thread never sleeps; it runs +flat out. This is precisely the behavior the per-thread performance report in +<> measures as "working" versus "sleeping." + +=== The cadence dial + +That fixed sleep interval is the one scheduling parameter you set directly. It is +a property of the stage, passed to the `Stage` constructor and defaulting to +100 ms: + +[source,cpp] +---- +class Watchdog : public simdb::pipeline::Stage +{ +public: + Watchdog(DatabaseWatchdog* app) : + Stage(500), // poll every 500 ms when idle + watchdog_app_(app) + { + } + // ... +}; +---- + +The watchdog from the previous chapter uses 500 ms deliberately: it only needs to +check a row count periodically, so waking just twice a second keeps it cheap. The +interval is a direct *latency-versus-waste* trade-off: + +* A *shorter* interval means the stage notices new work sooner -- lower latency -- + but if work arrives only occasionally, it wakes, finds nothing, and sleeps + again, spending cycles on empty polls (a high "sleeping" percentage). +* A *longer* interval means fewer wasted wakeups, but data already waiting in the + queue can sit up to one interval before the stage picks it up. + +There is nothing to tune for a stage that is saturated -- a thread that always has +work never reaches the sleep at all. The cadence matters for the *intermittent* +stages: watchdogs, low-rate producers, and anything that mostly waits. + +[NOTE] +==== +The interval also has a structural role: if two non-database stages are ever +placed on the *same* thread, they must share one interval. Today that rarely comes +up because each stage gets its own thread -- but it is the reason the interval +lives on the stage, and it matters for the thread management described next. +==== + +=== How threads are allocated -- today and tomorrow + +The current allocation is the simple one from <>: one worker thread per +stage, plus the single database thread. It is easy to reason about, but it does +not share threads across stages or apps, so the thread count climbs with the +total number of stages -- the multi-app scalability limitation flagged in <>. + +The planned direction is to make this automatic. Rather than fixing one thread per +stage, SimDB will manage an unbounded worker-thread pool and *create, merge, and +destroy* threads on its own -- co-scheduling compatible stages onto shared threads +and splitting them back apart -- using exactly the working/sleeping balance from +the <> reports as its signal. Saturated stages get their own threads; +mostly-idle stages get merged together so they stop costing a thread each. The +cadence dial stays a per-stage property, and the single database thread stays the +one fixed point. Crucially, none of this will require changes to your app code: +the same pipeline you write today will simply be scheduled more efficiently. + +Until that lands, the guidance is the guidance from <>: prefer few apps per +process, set generous polling intervals on stages that only need to wake +occasionally, and read the per-thread reports to spot stages that are either +saturated or spinning idle. + +This closes <>. You now have the full advanced toolkit -- flushers for +synchronization, snoopers for in-flight retrieval, the scoped disabler for +freezing the pipeline, async database access for reaching the database thread +from anywhere, and a working model of how stages are scheduled. <> puts all +of it to work in a single sustained case study: Argos, SimDB's flagship +collection-and-visualization application. diff --git a/docs/book/parts/part7-argos-case-study.adoc b/docs/book/parts/part7-argos-case-study.adoc new file mode 100644 index 00000000..3328140e --- /dev/null +++ b/docs/book/parts/part7-argos-case-study.adoc @@ -0,0 +1,634 @@ +[#part-vii] += Part VII: Case Study -- Argos + +Argos is SimDB's flagship application: a simulation data _collector_ plus a +Python _viewer_. It exercises nearly every SimDB feature and is the best way to +see the pieces working together end-to-end. + +[#ch-31-argos-is] +== What Argos Is + +Everything so far in this book has been a *capability*: a way to define a schema, +build a pipeline, persist safely, observe a running system. Argos is what those +capabilities look like when assembled into a real, shipping product. It is +SimDB's flagship application, and it is the clearest answer to the question "what +is all of this actually *for*?" + +Argos is a *collect-then-visualize* system. One half runs inside your simulator +and captures its state as it runs; the other half is a desktop application that +opens the captured data and lets you replay the simulation, scrubbing back and +forth through simulated time to see how every collected object evolved. + +.The Argos viewer replaying a collected run -- where this case study is headed +image::argos-hero.png[The Argos viewer replaying a collected run,role=text-center] + +=== Two halves: the collector and the viewer + +Argos has exactly two pieces, and they meet at a single file: + +* The *collector* (`simdb::argos::ArgosCollector`) is the write side. It is an + ordinary `simdb::App` -- the same kind of app you learned to build in <> -- + that you enable inside your simulator. As the simulation advances, it captures + the state of the objects you have marked for collection and streams that data, + through a SimDB pipeline, into *one* SQLite database. +* The *viewer* is the read side: a standalone desktop GUI. It opens that database + and reconstructs the state of every collected object at any point in simulated + time, presenting it through a set of composable visualization widgets. + +The collector answers "how do I get simulation data out, fast and safely?" The +viewer answers "now that it is captured, how do I look at it?" The only thing +that passes between them is the database file: + +.... + your simulator + (ArgosCollector, a simdb::App) + | + | streams collected state through a SimDB pipeline + v + collected.db <-- one SQLite file: the entire contract + | + | opened read-only, replayed over simulated time + v + Argos viewer (desktop GUI) +.... + +[NOTE] +==== +That single `.db` file is the *whole* handoff. Argos does not emit sidecar files, +trace logs, or index files alongside it -- the SQLite database is the complete, +self-contained artifact, exactly as promised back in <>. Copy it, archive +it, or hand it to a colleague, and the viewer can open it anywhere. +==== + +=== How Argos maps onto SimDB + +Argos is worth studying precisely because it exercises nearly every concept from +Parts <> through <> at once. Nothing in the collector is special-cased; it is +built from the same parts you already know: + +[cols="1,2"] +|=== +| SimDB concept (where it was introduced) | How Argos uses it + +| *App + lifecycle* (<>) +| The collector is a `simdb::App`, enabled through an `AppManager`, with its + schema declared in `defineSchema` and its pipeline built in `createPipeline`. + +| *Concurrent pipeline* (<>) +| Collected state is encoded and compressed on worker threads, then written by a + database stage -- so the simulator itself is never blocked on disk I/O. + +| *Single database thread* (Parts <>--<>) +| All of Argos's writes funnel onto the one database thread, keeping collection + fast and contention-free no matter how much the simulator produces. + +| *Blobs & compression* (<>) +| Collected snapshots and deltas are stored as compressed blobs -- the collector + is a heavy, real-world user of the blob and compression APIs. + +| *Flush at teardown* (<>) +| When the run ends, the collector flushes any pending data so the database is + complete before the process exits. +|=== + +If you have read this far, you already understand the collector's machinery. What +remains specific to Argos is *what* it collects and *how it encodes* that state +for efficient replay -- the subjects of the next two chapters. + +=== The data contract + +Because the database is the entire interface between collector and viewer, its +contents form a contract. At a high level it captures four things: + +* A *collectable tree* -- the hierarchy of objects that were collected, mirroring + the structure of the simulated system; it is the set of objects you can choose + to visualize in the viewer. +* One or more *clocks* -- the time bases that drove the run. Each collected object + updates on the edges of the clock it was collected against, and a database may + contain a single clock or many. +* *Timestamps* -- the points in simulated time at which state was captured, which + become the range you scrub through in the viewer. +* *Encoded state blobs* -- periodic full snapshots plus small deltas in between, + from which the viewer reconstructs any object's exact value at any requested + tick. + +We will unpack each of these -- especially the snapshot-plus-delta encoding, which +is the heart of Argos's efficiency -- in "The Argos Collector" and "On-Disk Data +Model & Encodings." For now the shape is enough: a tree of objects, the clocks +that move them, the times they were sampled, and the compact encoding of their +values. + +=== Why Argos is the case study + +Two reasons. First, breadth: a single application that touches the schema, +pipeline, app framework, blobs, compression, and flushing is the best possible +demonstration that these features compose into something greater than their +parts. Second, tangibility: the viewer turns all of that invisible machinery into +something you can *see and click*, which makes it the ideal lens for understanding +what SimDB delivers. + +The rest of <> follows the data. We start where the data is born -- the +collector -- then examine how it is laid out on disk, then cross over to the viewer +that brings it back to life, and finally walk a complete run end to end. + +[#ch-32-argos-collector] +== The Argos Collector + +The collector's job is narrow and demanding: turn the state of a running +simulation into rows and blobs in a database, continuously, without slowing the +simulation down. This chapter is about *what it captures* and *how it encodes* +that capture efficiently -- the features, not the internals. Everything here is +built from the SimDB machinery of Parts <> through <>; where that machinery shows +through, we point back to it rather than re-explaining it. + +=== Setting up: clocks, collectables, and time + +Before a run begins, you tell the collector three things. + +*Clocks.* A simulation advances on one or more clocks, and the collector needs to +know about them -- each has a name and a period (and, for derived clocks, a +numerator/denominator ratio). Clocks are the time bases everything else is +measured against. + +*Collectables.* Each object you want to capture is registered as a *collectable* +with three attributes: a *path* (its dotted position in the system hierarchy, e.g. +`top.core0.rob`), the *clock* it is collected against, and its *data type*. The +collector assigns each collectable a small integer id -- a *CID* -- and records the +whole set as a tree of collectables. That set is exactly what you choose from when +building views in the viewer. + +*A source of simulated time.* Finally, you give the collector a way to read the +current simulation time -- a back-pointer to a counter, or a function it can call. +Time must advance *monotonically*; the collector rejects time that moves +backward. + +=== Collecting as the simulation runs + +Once the run is live, you *collect* each object's current value whenever it +changes (or on whatever cadence your instrumentation chooses). The collector +gathers everything staged at a single point in simulated time into one unit of +work -- internally a "ledger" for that time point. When simulated time advances, +the collector seals the current ledger, hands it off to its pipeline, and starts a +fresh one for the new time. + +[IMPORTANT] +==== +Collection cadence is entirely up to you and your instrumentation. The +*heartbeat* introduced below does *not* decide when you may collect, how often, or +which ticks are captured. It affects only how the already-collected data is +*encoded* on disk. Collect as much or as little as you like; the heartbeat is a +separate, purely internal performance concern. +==== + +=== Inside the collector: the pipeline + +Handing the ledger off is where Parts IV and V come back into view. The collector +is a `simdb::App`, and its `createPipeline` builds a three-stage pipeline. The +simulation thread only ever pushes a ledger into the head of that pipeline -- it +never compresses, encodes, or touches the database itself: + +.... + simulation thread + | push sealed ledger (one point in sim time) + v + [ Stager ] per-CID checkpoint chains; decides FULL snapshot vs DELTA + | + v + [ Compressor ] zlib-compresses the encoded window + | + v + [ Writer ] runs on the single database thread; writes blobs + timestamps + | + v + collected.db +.... + +Three stages, three threads, plus the one database thread -- the exact shape from +<>. The stager does the interesting work (turning raw snapshots into +snapshots-plus-deltas); the compressor is the same zlib compression from <>; +and the writer is a `DatabaseStage`, so all persistence funnels onto the single +database thread. Because the hand-off is a queue push, the simulation is never +blocked waiting on encoding or disk. + +=== Snapshots and deltas: what the heartbeat controls + +Storing the full state of every object at every time point would be enormous and +mostly redundant -- from one tick to the next, most values do not change. So the +stager keeps, for each collectable, a *checkpoint chain* and encodes most time +points as a small *delta* against the previous checkpoint. Periodically it writes +a *full snapshot* instead, "rebasing" the chain so that reconstruction never has +to walk back too far. + +The *heartbeat* is the knob that governs that rebase cadence: it bounds how many +deltas may accumulate (and how long, in simulated time) before the next full +snapshot is forced. Its default is `10`. This is a pure space-versus-speed +trade-off: + +[cols="1,2,2"] +|=== +| Heartbeat | Effect on the database | Effect on the viewer + +| *Larger* (more deltas between snapshots) +| Smaller database -- fewer full snapshots written. +| Slower reconstruction -- to show a given tick, the viewer must replay more + deltas back to the last snapshot. + +| *Smaller* (more frequent snapshots) +| Larger database -- full snapshots cost more space. +| Faster reconstruction -- a snapshot is always close to any requested tick. + +| *Default (10)* +| A balanced starting point for most runs. +| Reasonable replay latency without excessive size. +|=== + +The key point, restated because it matters: the heartbeat trades *simulation-time +database size* against *viewer responsiveness*. It never changes which data is +collected -- only how compactly that data is laid down and how much work the viewer +does to replay it. + +=== Scalars, containers, and lifecycle + +Collectables come in two shapes. A *scalar* is a single value (a counter, a state +enum, a struct). A *container* is a collection of values -- a queue or buffer -- +and comes in two flavors: *contiguous* (front-packed, like a ring buffer) and +*sparse* (indexed slots that may have gaps). Containers declare a fixed +*capacity* which the viewer uses to render queue-utilization views (%-full widgets). + +Collectables also have a lifecycle: an entry can be *closed* to mark that a record +ended at a particular time. The encoder understands these close events and, when +needed, primes them with a fresh snapshot so the viewer can always reconstruct the +state leading up to a close. + +=== Multi-clock collection + +A database may contain a single clock or many. When there are several, each +collectable participates only in the windows belonging to *its own* clock, and the +collector records which clocks were active at each captured timestamp. The viewer +uses that to update each object only on the edges of the clock it was collected +against -- so a fast clock and a slow clock coexist correctly in one database. + +=== What gets written, and when + +The collector leans on the app lifecycle from <> to write the right things at +the right moments: + +* *Before the run* (`postInit`): the global settings (including the heartbeat), the + clock definitions, and the collectable tree -- the static scaffolding the viewer + reads first. +* *During the run*: compressed state blobs and their timestamps, written by the + database stage as ledgers flow through the pipeline. +* *Just before teardown* (`preTeardown`): the last in-flight ledger is flushed into + the pipeline so no final time point is lost. +* *After the run* (`postTeardown`): closing metadata -- which collectables actually + produced data (only those are flagged to *show in the UI*; ones that never + collected anything trigger a friendly warning), observed queue maximums, and the + string/enum lookup tables. + +[NOTE] +==== +Two lifecycle details connect directly to earlier chapters. First, every app's +`postInit` and `postTeardown` each run inside one automatic `safeTransaction` +(<>) -- `postInit` for pre-sim scaffolding metadata, `postTeardown` for +closing metadata -- so those writes commit together without manual transaction +wrapping. Second, the collector's string tables are flushed in `postTeardown` +explicitly -- the same `TinyStrings` flush called out in <> as a step +that teardown does not yet do for you automatically. +==== + +That is the collector end to end: register clocks and collectables, feed values as +time advances, and let a three-stage pipeline encode, compress, and persist them +onto the single database thread. The next chapter opens up the file it produces +and reads its schema as the contract the viewer depends on. + +[#ch-33-on-disk-model] +== On-Disk Data Model & Encodings + +Because the database file is the entire interface between collector and viewer, +its schema is a *contract*. The collector promises to write these tables in this +shape; the viewer relies on exactly that shape to reconstruct the run. This +chapter reads that contract -- not table by table as a reference, but grouped by +the role each part plays -- and then explains the one idea that underpins all of +it: everything is stored in binary. + +=== The schema, by role + +The tables fall into a few natural groups. + +*Static scaffolding* -- written before the run, describing its shape: + +* `CollectionGlobals` -- run-wide settings, including the heartbeat. +* `Clocks` -- each clock's name and timing (period, and an optional ratio). +* `CollectableTreeNodes` -- one row per collectable: its `CID`, its dotted + `FullPath`, the `ClockID` it belongs to, its `TypeName`, and a `ShowInUI` flag. + This table captures the collectable tree -- the set of objects you can choose to + visualize in the viewer. +* `DataTypeSchemas` / `DataTypeNodes` -- per-type layout and display metadata (field + names, type names, and format strings) used to render structured values. + +*Time and data* -- the bulk of the file, written as the run proceeds: + +* `Timestamps` -- the distinct points in simulated time that were captured; these + become the range you scrub through. +* `CollectionRecords` -- the actual payload: one compressed blob per captured time + point, keyed by `TimestampID`. +* `TimestampClocks` -- which clocks were active at each timestamp, so multi-clock + runs reconstruct correctly. + +*Encoding side tables* -- the lookups that make the binary payload legible: + +* `TinyStringIDs` -- the string-to-integer dictionary (below). +* `CollectedEnums` / `EnumMembers` -- the enum-to-name maps (below). + +*Extras*: + +* `QueueMaxSizes` -- the peak occupancy each container reached, for + queue-utilization views. +* `Notifications` -- sporadic messages surfaced in the viewer's Logs tab. + +.An Argos database opened in a SQLite browser, showing the collection tables described here +image::argos-schema-db-browser.png[An Argos database opened in a SQLite browser,role=text-center] + +=== Everything is collected in binary + +The single most important thing to understand about Argos data is that the +collector stores *everything in binary*. Collected values are not written as +human-readable columns -- they are packed as raw bytes into the compressed +`CollectionRecords` blobs. This is what keeps collection fast and the database +small, and it is why the viewer, not a plain `SELECT`, is the tool for reading a +run. Two data types deserve special mention because they are *not* stored the way +you might expect. + +*Strings become integers.* A string is never written as its characters in the +data blobs. Instead the collector interns it through *`TinyStrings`*: the first +time a given string is seen it is assigned a small integer id, and only that +integer is written into the blob. The id-to-string dictionary is saved once, at +teardown, into the `TinyStringIDs` table. The viewer joins against that table to +recover the text. (The empty string is a reserved id of zero.) A repeated label +that might appear millions of times across a run therefore costs a few bytes each +time instead of its full length -- a large space win for essentially free. + +*Enums use their underlying integer.* Enum values are always serialized in the +blob as their underlying integer type -- never as text. To make those integers +meaningful in the UI, the collector also records a value-to-name map for each enum +type that supports it, in `CollectedEnums` (the enum's name and integer type) and +`EnumMembers` (each `name` and its raw integer value). The viewer uses that map to +display, say, `ISSUED` instead of `2`. An enum that has no string conversion is +simply shown as its raw integer value -- still correct, just not labeled. + +The pattern generalizes: the blob is a compact, uniform byte stream, and a handful +of side tables (`TinyStringIDs`, `CollectedEnums`, `EnumMembers`) plus the type +metadata are the key that turns those bytes back into readable values. Nothing is +lost; it is just encoded for size and speed rather than for direct human reading. + +=== Full snapshots and deltas on disk + +<> explained *why* the collector writes periodic full snapshots with small +deltas in between, and how the heartbeat bounds that. On disk, this is what fills +`CollectionRecords`: each collectable's value at a given time point is encoded +either as a *full snapshot* or as a *delta* against its previous checkpoint. To +reconstruct an object's exact value at a requested tick, the viewer finds the most +recent full snapshot at or before that tick and replays the deltas forward -- the +"lookback" whose depth the heartbeat caps. That is the whole replay model; the +mechanics of building the delta chains live in the collector's implementation and +are not something the on-disk format asks you to understand. + +=== Contiguous vs. sparse containers + +Containers record their kind and capacity in their encoded type: a *contiguous* +container is front-packed (occupied slots first, like a ring buffer's live +region), while a *sparse* container keeps values at explicit indices that may have +gaps. Both declare a fixed capacity, and the collector tracks the maximum +occupancy each one actually reached over the run in `QueueMaxSizes`. The viewer +uses the kind to lay a container out correctly and the max size to scale its +utilization displays. + +=== The contract, from the viewer's side + +Two fields in `CollectableTreeNodes` are worth calling out because the viewer +leans on them directly. `ClockID` binds each collectable to the clock it updates +on, so the viewer advances each object only on the right edges. And `ShowInUI` is +the filter that keeps the object list honest: only collectables that actually +produced data during the run are flagged for display, so an +instrumented-but-never-fired object does not clutter the objects you can choose to +visualize (the collector even emits a notification listing anything it dropped for +this reason). + +With the contract in hand -- a tree of collectables on their clocks, a timeline of +timestamps, compressed binary blobs of snapshots and deltas, and the small tables +that decode them -- we can finally cross to the read side and see how the viewer +turns all of this back into something you can explore. + +[#ch-34-argos-viewer] +== The Argos Viewer + +The viewer is the read side of Argos: a desktop application that opens a collected +database and lets you *replay* the simulation, scrubbing back and forth through +simulated time to watch collected objects change. Where the collector chapter was +about features and encodings, this one is about *what you see and do* -- it is +deliberately a user's tour of the interface, not a look under the hood. + +=== Opening a database + +You point the viewer at a collected database, and optionally at a saved layout. +When you open a run for the first time, you are met with an almost-empty +workspace: a single canvas tab filling most of the window, and the playback +controls along the bottom. The canvas does not sit blank, though -- an empty cell +shows a small *Quick Links* menu (described next) that is your starting point for +building a view. Nothing is visualized yet because you have not chosen anything to +look at. + +.Argos on first open: an empty canvas showing the Quick Links menu, with the playback bar at the bottom +image::argos-first-launch.png[Argos on first open: an empty canvas showing the Quick Links menu,role=text-center] + +If you have looked at this data before, the viewer reopens your last layout +automatically; you can also hand it a saved *view file* to jump straight to a +prepared arrangement (more on those below). + +=== Quick Links: building a view from an empty cell + +Every empty canvas cell displays a *Quick Links* menu -- a short bulleted list of +hyperlinks -- and this is how you populate the viewer. It has two groups: + +* *Widgets* -- one link per kind of widget you can create in that cell: *Queue + Inspector*, *Queue Utilizations*, *Scheduling Lines*, and *Watchlist*. Clicking + one opens a small dialog to choose *which* collected objects it should show (for + example, which queue to inspect, or which elements to draw scheduling lines + for), and then places the widget in that cell. +* *Canvas* -- links to *Split left/right* and *Split top/bottom*, which divide the + cell in two. The freshly created empty half again shows its own Quick Links, so + you build up a multi-widget layout by splitting and filling, cell by cell. + +This is the mental model that replaces any single "browse everything" panel: +choosing data is part of creating each widget, right where that widget will live. + +=== Tabs and the splittable canvas + +The workspace is organized as a *data inspector* of tabs, each holding one +*canvas*. A trailing `+` ("Add Tab") page creates a new tab for a different +arrangement, and you can right-click a tab to rename or delete it. When the run +recorded notifications worth surfacing (the sporadic messages from the collector's +`Notifications` table), a fixed *Logs* tab appears as well. + +Within a tab, the canvas is recursively splittable -- every split leaves two +cells, each of which can hold a widget or be split again -- so you can arrange many +widgets side by side and watch related objects together. Splitting is available +both from the Quick Links of an empty cell and from a populated widget itself. + +=== Widgets + +Widgets are how collected objects are actually visualized. Each is created from a +Quick Links entry and then configured through its own selection dialog and +settings. The viewer ships several kinds, each suited to a different shape of +data. + +*Queue Inspector* -- a tabular view of a container object such as a queue or +buffer. You choose which columns are visible, and per-column *auto-colorization* +tints values so patterns jump out as you scrub through time. + +.A Queue Inspector widget with configurable columns and per-column auto-colorization +image::argos-queue-inspector.png[A Queue Inspector widget with colorized columns,role=text-center] + +*Queue Utilizations* -- how full container objects are over time, scaled against +the queue's max capacity given to the `ArgosCollector`. + +.A Queue Utilizations widget showing container occupancy over time +image::argos-queue-utilizations.png[A queue-utilizations widget,role=text-center] + +*Scheduling Lines* -- a timeline-style view of scheduled activity across the +elements you select, over a configurable window of ticks before and after the +current position. + +.A Scheduling Lines widget showing scheduled activity across a window of ticks +image::argos-scheduling-lines.png[A scheduling-lines widget,role=text-center] + +*Watchlist* -- an aggregate/summary view gathering several selected objects +together. + +.A Watchlist widget aggregating several selected objects +image::argos-watchlist.png[A watchlist / summary widget,role=text-center] + +=== The playback bar: moving through time + +The bottom of the window is where Argos becomes a *replay* tool rather than a +static viewer. The playback bar has a clock selector (for multi-clock runs), a +readout of the current cycle/tick, step buttons for jumping by fixed amounts in +either direction, and a scrubber that spans the whole run from start to end. + +As you move the playback position, every widget updates to show its objects' +state *at that instant* -- reconstructed behind the scenes from the nearest full +snapshot plus the deltas up to that tick, exactly the encoding from the previous +two chapters. You never think about snapshots or deltas; you just move the slider +and watch the system evolve. In a multi-clock database, each object updates only +on the edges of the clock it was collected against. + +=== A thin client, by design + +It is worth being explicit about what the viewer does *not* do, because it shapes +how Argos scales. The viewer is a deliberately *lightweight, thin client*: it +never loads a whole run into memory. When you land on a tick, it reads only the +blobs it needs to reconstruct exactly what the visible widgets are showing, right +then. Move to another tick and it does the work again for the new position. + +It does not even cache blobs it has already decoded -- revisiting a tick re-reads +and re-replays it from scratch. (A cache might be a reasonable future +optimization; today there simply isn't one.) What makes this practical is that the +viewer is *fast at deserializing and replaying* the compressed snapshot-and-delta +blobs, so on-demand reconstruction stays responsive even though nothing is kept +around. + +The upshot is a viewer whose memory footprint is tied to *what is on screen*, not +to the size of the database. A multi-gigabyte run opens as readily as a small one, +because Argos only ever pulls the slice of data the current view and tick require. + +=== View files: saving and sharing a layout + +A carefully arranged view is worth keeping, so the viewer can save your layout to a +*view file*. It captures the whole arrangement -- the tabs, how each canvas is +split, which widgets sit where, the columns and colorization chosen for each +object, and the selected clock -- so you can reopen it later or hand it to a +colleague to reproduce your exact setup. The window title marks when there are +unsaved changes, and the viewer remembers your most recent layout and restores it +automatically the next time you open the same data without naming a view file. + +One deliberate exception: the *playback position* -- the specific tick you happen to +be viewing -- is treated as a per-session preference and is *not* stored in the +shared view file. A view file describes how to look at a run, not where in time you +last paused, so sharing one does not drag your colleague to your current tick. + +[NOTE] +==== +The viewer is intentionally described here at the level of features and workflow. +Its implementation is being revised, so this chapter avoids tying anything to +specific internals -- what a user does with the canvas, Quick Links, widgets, +playback bar, and view files is what matters, and that is stable. +==== + +That is the viewer as an instrument: open a run, create widgets from an empty +cell's Quick Links (choosing their data as you go), compose them on a splittable +canvas, and scrub through simulated time while everything updates in step. All +that remains is to see the two halves working together in a single, concrete run +-- the end-to-end walkthrough that closes the case study. + +[#ch-35-end-to-end] +== End-to-End + +We have followed the data from one side to the other: born in the collector, +encoded into a database, brought back to life in the viewer. This closing chapter +walks the whole loop once as a single workflow, and shows how Argos proves to +itself that the round trip is lossless. + +=== The three steps + +Using Argos end to end is always the same three moves: + +. *Instrument and collect.* Enable the `ArgosCollector` app in your simulator, + register its clocks and collectables, and collect values as simulated time + advances. This is ordinary SimDB app usage -- the lifecycle from <> + (`registerApp`, `enableApp`, `createEnabledApps`, `createSchemas`, `postInit`, + `initializePipelines`, `openPipelines`, and `postSimLoopTeardown`) driving the + collector's three-stage pipeline underneath. +. *Produce one database.* When the run ends, teardown flushes the pipeline and you + are left with a single `.db` file -- the whole run, and the only artifact. +. *Open it in the viewer.* Point the viewer at that file and explore: create + widgets from the Quick Links, arrange them on the canvas, and scrub through + simulated time, as in the previous chapter. + +=== Instrumenting a run + +The collector-specific setup is small and sits inside the app lifecycle. Before +the run you tell the collector how to read time (a pointer or function it can +sample), register one or more clocks, set the heartbeat if you want something +other than the default, and declare your collectables (scalars and containers, +each on a clock). Then, as the simulation steps, you stage each object's current +value; the collector seals a ledger whenever time advances and the pipeline does +the rest. There is no separate "flush every tick" step and nothing on the hot path +touches the database. + +[NOTE] +==== +As emphasized in <>, the heartbeat here is *only* a performance choice -- +how many deltas accumulate before a full snapshot rebases the chain. It changes +the database's size and the viewer's replay effort, never which data you collect. +The next section leans on exactly that property. +==== + +=== Testing the checkpointer logic and Python deserializers + +The authoritative end-to-end exercise lives in `test/argos/`, and it is more than +a demo -- it is a regression test that runs as part of the suite, so the whole +collect-encode-replay loop is continuously verified. It works by *replaying* the +collected blobs the same way the viewer does -- walking the timeline, applying each +snapshot and delta -- and reconstructing every object's value at every tick. It +then compares those reconstructed values against the simulator's own record of +what it collected. If the bytes on disk did not faithfully capture the run, the +comparison fails. + +The most instructive part is how it treats the heartbeat. The test collects *the +same simulation three times* -- at heartbeats of 1, 3, and 10 -- producing three +databases of different sizes, and then confirms that the *reconstructed data is +identical* across all three. That is the <> claim made concrete and +enforced: the heartbeat changes only how compactly the run is stored, not the +information it contains. A smaller heartbeat writes more full snapshots and a +larger file; a larger heartbeat leans on more deltas and a smaller file; the +replayed result is the same either way. diff --git a/docs/book/parts/part8-patterns-cookbook.adoc b/docs/book/parts/part8-patterns-cookbook.adoc new file mode 100644 index 00000000..5a7c5b94 --- /dev/null +++ b/docs/book/parts/part8-patterns-cookbook.adoc @@ -0,0 +1,778 @@ +[#part-viii] += Part VIII: Patterns, Best Practices & Cookbook + +You now know how SimDB works part by part. This part is the glue: how to *design* +with it, how to *tune* and *debug* pipelines, which *application shapes* fit +common goals (without pretending there is only one architecture), and how to +*test* that the `.db` file your app produces is correct. No new API surface -- +just the habits and decision frames that keep real projects from fighting the +library. + +This part walks through six topics in order: + +* *Schema and pipeline together* -- co-design storage and dataflow before you + write stages. +* *Stage boundaries and payload types* -- split, merge, move-only payloads, and + forced-flush behavior. +* *Performance tuning* -- measure first, turn one knob at a time, respect the + single database-thread funnel. +* *Determinism and debugging* -- freeze, flush, log, and diagnose a concurrent + pipeline without guessing. +* *Application shapes* -- stats engines, live UI backends, replay backends, and + farm-scale analysis as decision spaces, not single canonical layouts. +* *Testing SimDB apps* -- `SimDBTester`, flush/teardown discipline, and + verifying the database with a reader that did not write it. + +[#ch-36-schema-pipeline-together] +== Designing Schema and Pipeline Together + +The recurring mistake with SimDB is to design the two halves separately -- to lay +out tables first, then bolt on a pipeline to fill them (or the reverse). The +schema and the pipeline are two views of one design: the pipeline exists to +produce exactly the rows and blobs the schema defines, and the schema exists to +capture exactly what the pipeline can efficiently emit. Design them together. + +=== Start from the reads + +Begin with the question *"what will I ask this database later?"* Your queries +determine your schema, and your schema determines what the pipeline's terminal +database stage must write. If you will filter runs by a status column, that status +must be a real column, not buried inside a blob. If you will only ever read a +record back whole, it can be a single blob. Working backward from the reads keeps +you from persisting data you never query and from burying data you do. + +=== Columns or blobs? + +The central storage decision is per field: a queryable column, or a byte in a +blob. + +* *Columns* are for values you will *filter, sort, join, or aggregate on* -- ids, + names, timestamps, small scalars, enums. They are visible to the Query API from + <>. +* *Blobs* are for *bulk or opaque payloads* you will read back as a unit -- large + vectors, serialized structures, encoded state. They are compact and cheap to + write, but SQLite cannot see inside them. + +Argos is the worked example: its *metadata* -- clocks, the collectable tree, +timestamps -- lives in ordinary columns you can query, while the *bulk state* lives +in compressed blobs in `CollectionRecords`. The rule of thumb: if you would ever +put it in a `WHERE` clause, it is a column; if you only ever reconstruct it, it is +a blob. + +=== Where compression belongs + +Compression is a pipeline decision, not a schema one. Compress *late and off the +hot path*: a dedicated worker stage just before the database stage, never on the +thread producing the data. Compress *bulk blobs*, not small columns -- there is +nothing to gain from compressing an integer. This is exactly the shape of the +Argos pipeline (stage, then compress, then write) and of `SimplePipeline` from +<>. + +=== Persist versus derive + +Not everything needs to be stored. Persist what is *expensive to recompute* or +*needed to answer a query*; derive the cheap and the rare on read. The +snapshot-and-delta encoding from the Argos case study is the aggressive end of +this spectrum -- it stores a compact encoding and reconstructs full state on read, +trading a little read-time work for a much smaller database. Most designs sit +somewhere in the middle: store the raw facts, compute summaries at read time. + +=== Normalized tables versus a blob-of-record + +Two shapes recur, and the right one depends on access pattern: + +* *Normalized tables* -- data spread across columns and related tables, joinable + and queryable, with repeated strings interned to integer ids (the `TinyStrings` + pattern from the Argos schema). Choose this when you will slice the data many + ways. +* *Blob-of-record* -- one blob per logical record, written and read as a unit. + Choose this when a record is only ever consumed whole and write speed matters + more than queryability. + +Neither is universally right; many real databases use both, as Argos does. The +point of this chapter is simply that the choice is made *once, for schema and +pipeline together* -- the database stage that writes a blob and the table that +stores it are two ends of the same decision. + +[#ch-37-stage-boundaries] +== Choosing Stage Boundaries & Data Types + +Once you know what the pipeline must produce, you have to decide how to *carve it +into stages* and *what to hand between them*. Both choices have a direct cost: +every stage is a thread and every boundary is a queue hop. Getting the +granularity right is most of what separates a pipeline that scales from one that +spends its time shuffling data. + +=== One responsibility per stage -- but not too little + +The unit of a stage is a *responsibility*: transform, compress, write. Keeping +each stage to one job makes the pipeline easy to reason about, easy to reorder, +and easy to tune, because the per-thread performance report (<>) tells you +exactly which responsibility is the bottleneck. + +The failure mode in the other direction is *tiny stages*. A stage that does almost +no work still costs a thread, a queue, and a handoff on every item. When the +handoff costs more than the work, the stage is a net loss -- you have added +latency and a thread to accomplish nothing. If two adjacent stages are both mostly +idle, they belong together. + +The practical test is the active/sleep balance from <>: + +* A stage *saturated* (near 100% active) and doing parallelizable work is a + candidate to *split*. +* A stage *mostly asleep* is a candidate to *merge* into its neighbor. +* A stage with a *balanced* active/sleep ratio is correctly sized -- leave it + alone. + +=== Move-only payloads + +The data you pass between stages should *move, not copy*. Prefer payload types +that own their storage and move cheaply -- `std::vector`, `std::unique_ptr`, or +your own move-only structs -- and hand them across queues with `std::move` / +`emplace`. Types that force a copy on every hop (or, worse, that copy large +buffers) turn every stage boundary into a hidden allocation-and-copy. SimDB's +queues are built around move semantics precisely so that a payload can travel the +length of the pipeline without its bytes ever being duplicated. + +=== Buffering and partial flushes + +Some stages naturally *batch* -- a windowing stage, a compressor that prefers +larger inputs, a stage that groups records before a transaction. Batching is good +for throughput, but a batching stage must answer one question correctly: *what +happens on a forced flush?* + +When a flusher runs (<>), it drives every stage with the force flag set. A +batching stage must, on force, emit whatever it currently holds -- *even a partial +batch* -- rather than waiting for its window to fill. A stage that ignores force +will strand data in memory and defeat the flush-then-query guarantee. Whenever you +introduce internal buffering, handle the forced-flush path explicitly. + +=== Sizing the payload + +Payload size is a latency-versus-throughput dial: + +* *Larger payloads* amortize the per-item overhead -- fewer queue hops, fewer + transactions, better compression ratios -- at the cost of higher latency and + more memory in flight. +* *Smaller payloads* lower latency and memory but pay the per-hop cost more often. + +There is no universal answer; size the payload to the stage's job and measure. The +one constant is structural: keep the database stage *last and singular*, since it +is the single-threaded funnel every payload must pass through (<>). + +[#ch-38-performance-tuning] +== Performance Tuning + +SimDB gives you several knobs -- batch size, compression level, polling cadence, +stage boundaries, heartbeat interval -- and the temptation is to turn all of them +at once. Resist it. Performance tuning is a loop: *measure, change one thing, +measure again.* This chapter is a guide to the knobs and to the instruments that +tell you which one to turn. + +=== Know the funnel + +Every payload in a SimDB pipeline eventually passes through *one* database thread. +That single thread is the funnel, and it is almost always the thing you are +tuning around. The goal is to keep it *fed but not thrashing*: enough work batched +per commit that it is not paying transaction overhead on every row, but not so +much held back that memory balloons or data appears late. Most of the knobs below +are really about how work arrives at that funnel. + +=== The knobs + +*Batch / transaction sizing.* Committing many rows in one transaction is far +cheaper than one transaction per row (<>). Larger batches mean fewer commits +and higher throughput, at the cost of data becoming visible later and more memory +in flight. Flush only at genuine synchronization points (<>) -- flushing per +item throws the batching advantage away. + +*Compression level.* Compression shrinks the database and the disk I/O the funnel +must do, but it costs CPU on a worker stage. Choose the level by the compressor +stage's active/sleep balance, exactly as tabulated in <>: a compressor that +never sleeps wants a *lower* ratio (or to be split); one that always sleeps can +afford a *higher* ratio (or to be merged into a neighbor). + +*Polling cadence.* Each stage's polling interval (default 100 ms, <>) +trades latency against wasted wakeups. Shorter intervals lower latency but wake +idle threads more often; longer intervals do the reverse. Intermittent, latency- +tolerant stages want *longer* intervals; a stage on the critical path wants a +*shorter* one. + +*Stage boundaries.* Splitting a saturated, parallelizable stage or merging two +idle ones (<>) is often the highest-leverage change of all, because it +rebalances where the threads spend their time. + +*Heartbeat cadence (Argos).* For collectors, the heartbeat interval is the +size-versus-speed dial from the case study (<>): more frequent snapshots +cost simulation speed and database size; less frequent ones cost read-time +reconstruction work. It is a performance knob, not a correctness one. + +WARNING: Thread count today grows with the number of apps -- there is no shared +pool yet (<>). Keep the app count low; multi-app systems are not yet +thread-scalable. This is on the roadmap, not available now. + +WARNING: Unrelated heavy disk I/O from your simulator -- its own logging system, +trace dumps, or checkpoint files -- contends with SimDB's dedicated database +thread on the same filesystem path and can collapse throughput (<>). Treat +that as an environmental tuning problem: redirect or throttle non-SimDB writes +during collection. + +=== The instruments + +You cannot tune what you have not measured. SimDB gives you three levels of +instrument, from coarse to fine: + +*Per-thread performance reports* are the primary guide. The working-versus- +sleeping percentages (<>) tell you which stage is the bottleneck and which is +idle -- and therefore which knob to turn. Start here every time. + +*The self-profiler* times specific methods or blocks. Drop `PROFILE_METHOD` at the +top of a function, or `PROFILE_BLOCK("label")` around a region, and the profiler +prints a per-label average-time report, sorted slowest-first, when the program +exits: + +[,cpp] +---- +void MyStage::process_(Payload&& p) +{ + PROFILE_METHOD; // times this whole method + { + PROFILE_BLOCK("compress"); // times just this region + compress_(p); + } +} +---- + +Use it to confirm *which line* of a hot stage is actually expensive, once the +per-thread report has told you which stage to look at. + +*`RunningMean`* keeps a running average of any metric you feed it, using Welford's +method, without storing the samples: + +[,cpp] +---- +simdb::RunningMean batch_sizes; +batch_sizes.add(records.size()); +// later: +std::cout << "avg batch: " << batch_sizes.mean() << "\n"; +---- + +Use it to track average payload bytes, average queue depth -- rather than time a code +path. + +=== The tuning loop + +Put the instruments and knobs together into a discipline: + +1. Run a representative workload and read the per-thread performance report. +2. Find the saturated stage (the funnel, or an upstream bottleneck feeding it) and + the idle ones. +3. Turn *one* knob -- split/merge a stage, change the compression level, resize the + batch, adjust a polling interval. +4. Re-run and compare the reports. + +Changing one knob at a time is what makes the comparison meaningful. Tuning +everything at once tells you the result but never the cause. + +[#ch-39-determinism-debugging] +== Determinism & Debugging Concurrent Pipelines + +A concurrent pipeline is nondeterministic by construction: stages run on their own +threads, wake on their own cadence, and interleave differently on every run. That +is exactly what makes it fast -- and exactly what makes it hard to debug, because +the state you want to inspect is a moving target. This chapter is about buying back +*enough* determinism to reason about a running pipeline, and about the symptoms you +will actually encounter. + +=== Freeze before you look + +The first rule of debugging a pipeline is: *stop it moving before you inspect it.* +If you read a queue, a counter, or the database while stages are still running, +you are reading a value that was already stale by the time you looked. <>'s +`ScopedRunnableDisabler` exists for precisely this -- it pauses runnables for the +duration of a scope (RAII), so you can take a *consistent* snapshot: + +[,cpp] +---- +{ + simdb::ScopedRunnableDisabler disable_all(pipeline_manager); + // Nothing advances here. Inspect queues, counters, in-flight state + // knowing they will not change out from under you. +} +// Stages resume when the disabler goes out of scope. +---- + +Nested disablers are safe -- inner scopes are no-ops -- so you can reach for one +without worrying about whether an outer one is already active. + +=== Freeze, then flush, then query + +Disabling stops the pipeline in place, but data mid-flight has not yet reached the +database. When you want a *consistent on-disk* state to inspect with the Query API +or an external SQLite tool, pair disabling with a *flush* (<>). A flusher +drives every stage with the force flag set, draining buffered data all the way to +the single-transaction database commit. The flush-then-query pattern is the only +way to guarantee that what you read from the database reflects everything produced +so far -- without it, batched transactions mean rows you expect simply are not +there yet. (See also the FAQ's <>) + +=== Inspect in flight with snoopers + +Sometimes you do not want to freeze or flush at all -- you want to peek at what is +*currently* moving through a stage. That is what snoopers (<>) are for: +key-based retrieval of in-flight payloads without draining the pipeline. They are +the right tool for a live dashboard or a "what is stage 3 holding right now?" +diagnostic, with the tradeoffs already covered in the advanced chapter (snooping +briefly pauses the pipeline so the snapshot is coherent). + +=== Logging without interleaved lines + +`std::cout` from several stage threads produces garbled, interleaved lines -- +half of one stage's message spliced into another's. `ThreadSafeLogger` fixes this +by buffering a whole line and writing it under a lock only when the `Guard` +returned by `protect()` is destroyed: + +[,cpp] +---- +simdb::ThreadSafeLogger log("[compressor] "); +log.protect() << "batch " << n << " -> " << nbytes << " bytes\n"; +---- + +Each line is atomic and prefixed by the string you constructed +the logger with. Give each stage its own prefix and every line in the merged +output is instantly attributable to a thread -- which is most of what you need to +reconstruct an interleaving after the fact. `ThreadSafeFileLogger` is the same +thing pointed at a file when you want the log out of the console. + +=== "Why is my pipeline sleeping / not draining?" -- a checklist + +The most common confusion is a pipeline that appears stuck. Work through this in +order: + +. *Read the per-thread performance report first* (<>). A stage sleeping at + ~100% is simply not being fed -- the problem is upstream, not here. A stage + active at ~100% is the bottleneck everything else is waiting behind. +. *Check the polling cadence.* A long polling interval (<>) looks like a + stall -- the stage is just waiting for its next wakeup. Latency-sensitive stages + want shorter intervals. +. *Did you forget to flush?* Rows you produced but do not see are almost always + still batched in flight. Flush at the synchronization point (above). +. *Is a disabler still in scope?* An outstanding `ScopedRunnableDisabler` -- for + example, one held longer than intended -- keeps everything paused. Confirm the + scope has exited. +. *Is the database thread the funnel?* Everything commits through one thread + (<>). If that thread is saturated, upstream stages back up and sleep + waiting for room. Tune batch size and compression (<>) rather than the + upstream stages. +. *Did teardown run?* If the pipeline never drained at shutdown, confirm + `postSimLoopTeardown` was called on every exit path (<>) -- an app that + aborts without it leaves data stranded and threads unjoined. + +The theme throughout: don't guess about a concurrent system. *Freeze it, flush it, +log it, and read the perf report* -- in that order -- and the nondeterminism stops +being in your way. + +[#ch-40-application-shapes] +== Application shapes + +The README lists four things people build with SimDB: statistics engines, live +UI backends, replay backends, and aggregate analysis across many runs. Each of +those is a *product* problem with many valid architectures. SimDB does not pick +one for you -- it gives you a consistent way to *persist* data concurrently and +to *reach* that data from other threads. This chapter is a set of *application +shapes*: the decisions you must make, the SimDB pieces that map to each part, and +several workable designs per shape so you are not locked into a single layout. + +IMPORTANT: These are not copy-paste skeletons and not references to runnable +demos. A half-dozen quality UI backends exist for the same collector; a stats +engine can be as simple as "flat table, query after the run" or as rich as a +multi-clock snapshot/delta system. The book shows mechanics elsewhere (Parts +III--VII); here we show *where the hard design choices live*. + +=== A simulation statistics engine + +*Outcome.* During simulation, metrics leave the hot path quickly and land in a +`.db` file you can query, export, or hand to a viewer later. + +*The spectrum.* At one end is the *minimal engine*: a flat schema -- timestamp, +metric id, scalar value (or a small blob) -- and a short pipeline (optional +compress, then database stage). After the run, SQL or a script produces CSVs or +summary tables. That is genuinely useful and genuinely easy relative to everything +else on this page. At the other end is a *rich collector*: hierarchical +collectables, multiple clocks, typed containers, snapshot/delta encoding, and a +schema that is itself a contract for a replay tool. Argos (<>) is the +worked example of that end; most teams start at the minimal end and grow toward +rich encoding only when post-hoc flat tables stop answering their questions. + +*Decisions to make together* (schema + pipeline, <>): + +[cols="1,2", options="header"] +|=== +| Question | Consequence + +| What will you filter or group on later? +| Those fields must be *columns*, not only bytes inside a blob. + +| How often do you write? +| High frequency favors batching, larger payloads, and fewer commits; low +frequency can afford row-at-a-time simplicity. + +| Scalars only, or vectors/structures too? +| Scalars fit normalized tables; bulk state fits blobs (often compressed). + +| One time base or several? +| One clock keeps the schema simple; multiple clocks need explicit clock ids on +every time-stamped row (as in the Argos data model). +|=== + +*SimDB pieces.* An `simdb::App` with `defineSchema` and `createPipeline`; move- +only payloads between stages; optional compression before the terminal database +stage; `createFlusher` at the points where you need visibility or teardown safety; +`postSimLoopTeardown` on every exit path so the file is whole. + +*Pitfalls.* Designing the pipeline before you know your queries (you will bury +filterable fields in blobs). Flushing every sample (you lose batching and stall +the funnel). Growing into Argos-scale encoding before you need it (complexity +without payoff). + +=== A live UI backend + +*Outcome.* Something on screen updates while the simulation runs -- or within a +bounded delay -- without stalling the simulator. + +*Split the problem.* SimDB owns *how data gets out of the sim thread safely* and +*how it becomes durable*. Your UI framework owns windows, layout, plotting, and +networking. No single recipe spans both; the shapes below are only the SimDB-side +patterns. Combine them. + +*Shape A -- Post-run or flush-bound UI (simplest).* The collector writes the +database; the UI opens the `.db` after a flush or after the run. The UI reads +committed rows and blobs only. Latency equals your flush cadence (or end of run). +Design: schema with queryable timestamps and ids; periodic `flush()` at +heartbeats if you want near-live updates without snoop machinery. Fits tools that +tolerate hundreds of milliseconds to seconds of lag. + +*Shape B -- Thin client over the database (Argos-like).* The collector checkpoints +on a heartbeat; the viewer loads *on demand* -- no full preload, no requirement +to cache what was already shown (<>). Live enough for many workflows because +deserialization/replay is fast and the UI only pulls what the current widgets need. +Design: rich blob encoding + metadata columns the UI can index; heartbeat as a +performance dial, not a gate on collection. + +*Shape C -- Snoop the pipeline tail (lowest latency for hot metrics).* Wire a +snooper on a stage that carries the payloads your dashboard cares about; read by +key without waiting for SQLite (<>). Use for a *small* set of hot signals +(gauge, last N queue depths), not for full history. Accept the tradeoff: snooping +coordinates with the pipeline briefly so the snapshot is coherent. + +*Shape D -- Metadata from the UI thread via async DB access.* When the UI thread +needs counts, last timestamp, or schema discovery, post work to the dedicated +database thread with `AsyncDatabaseAccessor::eval` (<>) -- never call SQLite +from the UI thread directly, and never from inside a `DatabaseStage`. Bulk curves +still come from Shape A, B, or C; this shape is for *light* queries. + +*Shape E -- Dual path (advanced).* Pipeline persists everything to the database +(cold, authoritative path) while a separate fan-out -- fed by the same stage +output, a snooper, or a side queue you own -- pushes decimated samples to the UI +(hot path). Two consumers, one producer; you must define which path is source of +truth when they disagree (almost always the database). + +[cols="1,1,1", options="header"] +|=== +| Shape | Best when | SimDB cost + +| A -- flush-bound | Lag OK; simplest mental model | Flusher + schema +| B -- thin DB client | Rich state; many widgets; replay | Encoding + heartbeat design +| C -- snoop | Sub-flush latency for few metrics | Snooper wiring + key design +| D -- async eval | UI thread needs safe metadata queries | Accessor discipline +| E -- dual path | High rate + live plot + full archive | Two paths to keep consistent +|=== + +*Pitfalls.* Treating snoopers as a generic message bus (they are keyed peeks, not +a stream API). Querying SQLite from the UI thread (locks, wrong thread). Expecting +the UI to mirror the entire simulation state in memory (defeats SimDB's pipeline +and bloats RAM). Choosing Shape E when Shape B plus a sensible heartbeat would + suffice. + +=== A simulation state replayer backend + +*Outcome.* Given a time (or tick), reconstruct what the simulation *would have +looked like* at that moment -- for a debugger, a regression diff, or a standalone +replay tool. + +*The contract is the schema.* The replayer does not guess. Every blob column, enum +table, and timestamp column is part of a *versioned contract*: what is snapshotted +vs delta-encoded, how containers are laid out in binary, which clock id applies. +<>'s on-disk model is one full instance of that contract; your engine can be +smaller, but the same rule applies -- document the bytes. + +*Write path (collector side).* Producers emit state on a schedule you define: +full snapshots at rebases, deltas between them, or append-only event rows if your +replay logic prefers an event log. Compression and the database stage stay at the +end of the pipeline; flush at rebases so a crash mid-run still leaves a coherent +prefix. + +*Read path (replayer side).* Load the nearest snapshot at or before the target +time, apply deltas forward (or walk events), deserialize blobs using the same +rules the collector used to serialize them. The replayer may live in another process +(Python, Qt, a batch verifier); it only needs the `.db` and the contract. + +*Encoding choices (pick one primary strategy).* + +* *Snapshot + delta* -- smaller database, more read work; good when state is +large and changes incrementally (Argos). +* *Full snapshot only* -- trivial replay, heavy writes and disk; good when state +is small or runs are short. +* *Event log* -- append rows `(time, event_type, payload)`; replay is apply-until-T; +good when state is derivable from events and snapshots are awkward. + +*SimDB pieces.* Blobs and side tables (`TinyStrings`, enum maps); flushers at +rebase boundaries; optional `ScopedRunnableDisabler` plus flush when the replayer +and collector share a process and you need a quiescent point (<>). + +*Pitfalls.* Changing blob layout without a schema/version column (old databases +become unreadable). Mixing clocks without tagging rows (replay at the wrong +"time"). Skipping flush at rebases (partial last snapshot after crash). + +=== Aggregate analysis across many simulations + +*Outcome.* Compare or summarize results from hundreds or thousands of runs -- +parameter sweeps, regression suites, nightly farms. + +*Two phases.* (1) *Produce* one `.db` per run with the *same* app schema so every +file has identical tables. (2) *Analyze* offline -- merge, attach, or query across +files. Phase 2 often needs *no* SimDB pipeline at all; SQLite, Python, or R on the +files is enough. SimDB's role in phase 1 is making each run's artifact consistent; +in phase 2 it is optional unless you build a merge app as another `simdb::App`. + +*Warehouse shapes.* + +* *Many files, query with `ATTACH`* -- keep runs isolated; good for parallel sim +and simple tooling. +* *ETL into one warehouse database* -- copy normalized summary columns into a +`runs(id, params…)` plus `metrics(run_id, …)` layout; good for heavy cross-run +SQL. +* *Blob-per-run with summary columns extracted at end* -- full state stays in per- +run files; a lightweight post-pass inserts only scalars you care about for sweeps. + +*Schema discipline across runs.* Store run parameters as *columns* (or a dedicated +params table) at collection time if you know them upfront; inferring params later +from filenames is fragile. Add a `schema_version` or tool version column early so +a farm does not mix incompatible blob layouts silently. + +*SimDB pieces (production phase).* Same `defineSchema` for every run; batch writes +and compression so per-run overhead stays low; reliable teardown so farm workers +do not leave corrupt files. + +*Pitfalls.* One giant pipeline-fed database fed by all sim processes concurrently +(fights the single-writer model -- prefer one DB per process). Storing only blobs +with no summary columns (every analysis pass deserializes everything). Schema drift +across tool versions without version metadata. + +=== How to use this chapter + +Pick the shape closest to your goal, then go back to the parts that teach the +mechanics: schema and INSERT (<>), pipelines (<>), apps and lifecycle +(<>), flush/snoop/async (<>), and Argos if you need the rich collector/ +viewer split (<>). Tuning and debugging loops live in Chapters <> and <>. +None of the shapes above is mandatory; mixing them -- flush-bound persistence plus +snoop for one live gauge, flat tables for farm summary columns -- is normal +engineering, not a violation of SimDB. + +[#ch-41-testing-simdb-apps] +== Testing SimDB Apps + +Regression tests are how SimDB keeps its API honest, and they are the model for +how you should test your own apps. The goal is not to re-run your simulator inside +every test -- it is to prove that the *artifact* your app produces (the `.db` +file, its schema, and the bytes in its blobs) matches what you expect, independent +of how the app got there. This chapter covers the test harness SimDB uses, the +flush-and-teardown habits pipeline apps require, and how to verify persisted data +without trusting the app that wrote it. + +=== What to test at each layer + +SimDB apps stack three separable concerns. Test them at the boundary where bugs +actually show up: + +[cols="1,2,1", options="header"] +|=== +| Layer | What can go wrong | Typical test + +| SQLite interface | Schema mistakes, query constraints, blob round-trip | Direct +`DatabaseManager` tests (<>, `test/sqlite/**`) +| App + pipeline | Stages never drain, wrong flush points, lifecycle skipped | Drive +`AppManagers` through simulate → teardown; assert on the file +| Encoding / contract | Blob layout drift, replay mismatch | Deserialize the `.db` +with a *separate* reader (C++ helper, Python script, or the viewer's replay path) +|=== + +You do not need every test to exercise all three. A schema test can skip the +pipeline entirely; an end-to-end collector test should hit the file and an +independent reader, not only in-app getters. + +=== The SimDBTester harness + +SimDB's regression programs use a lightweight macro harness in `test/SimDBTester.hpp` +-- not GoogleTest, but the same shape: accumulate failures, print a summary, return +a non-zero exit code for CI. + +Every test translation unit starts with `TEST_INIT` (once, at file scope). Inside +`main`, use `EXPECT_*` macros and finish with `REPORT_ERROR` followed by +`return ERROR_CODE`: + +[,cpp] +---- +TEST_INIT; + +int main() +{ + EXPECT_EQUAL(2 + 2, 4); + EXPECT_TRUE(some_condition); + EXPECT_WITHIN_EPSILON(actual, expected); // floating-point + EXPECT_THROW(bad_call()); + + REPORT_ERROR; + return ERROR_CODE; +} +---- + +The macros worth knowing: + +* `EXPECT_EQUAL` / `EXPECT_NOTEQUAL` -- equality with values printed on failure. +* `EXPECT_WITHIN_EPSILON` / `EXPECT_WITHIN_TOLERANCE` -- floating-point compares + (the SQLite regression tests use these heavily). +* `EXPECT_THROW` / `EXPECT_NOTHROW` -- exception discipline. +* `EXPECT_FILES_EQUAL` -- byte-compare two files (lines starting with `#` are + ignored, useful for golden text output). +* `EXPECT_REACHED` / `ENSURE_ALL_REACHED(N)` -- mark that a callback or hook fired; + useful when testing that lifecycle methods or stage paths actually ran. + +IMPORTANT: Call `REPORT_ERROR` *before* teardown that might crash if earlier +assertions left the app in a bad state (dangling pointers, half-built pipelines). +Return `ERROR_CODE` after teardown completes. The harness separates reporting from +exit so you still get a clean failure summary when cleanup is fragile. + +Build the full regression suite with the `simdb_regress` target (<>). Your +own app tests can follow the same pattern and plug into CTest the same way. + +=== Flush, teardown, then assert + +Pipeline apps add one rule SQLite-only tests never need: *do not query what is still +in flight.* Batched stages and the single database thread mean rows and blobs you +just `process()`'d may not exist on disk yet. Before any assertion that reads the +database: + +1. *Flush* if you need mid-run visibility (`flusher_->flush()`, <>). +2. Always call `postSimLoopTeardown()` on every exit path before you treat the file + as final (<>) -- normal finish, caught exception, or abort handler you + install. + +A minimal integration-test skeleton for an `simdb::App`: + +[,cpp] +---- +// Arrange +simdb::AppManagers app_mgrs; +app_mgrs.enableApp(); +app_mgrs.createSchemas(); +app_mgrs.postInit(0, nullptr); +app_mgrs.initializePipelines(); +app_mgrs.openPipelines(); + +// Act -- drive your simulator or inject test inputs +runSimulationOrInjectTestData(); + +// Assert -- drain and finalize FIRST +app_mgrs.postSimLoopTeardown(); + +simdb::DatabaseManager verifier("output.db"); // reopen independently +auto q = verifier.createQuery("MyMetrics"); +EXPECT_EQUAL(q->count(), expected_rows); +// ... read columns/blobs and compare ... + +REPORT_ERROR; +return ERROR_CODE; +---- + +Notice the verifier opens the file with a *new* `DatabaseManager`. That is +intentional: you are testing the persisted artifact, not whether your app's in- +memory caches happen to agree with themselves. + +=== Verify the database independent of the app + +The strongest tests treat the `.db` as the contract and use a reader that does not +share code with the writer. + +*Reopen and query.* After teardown, connect with a fresh `DatabaseManager` (or the +Query API patterns from <>). Check row counts, column values, and blob bytes +with `EXPECT_EQUAL`. This catches schema drift and wrong INSERT paths even when +the app's own "get last value" helper still works. + +*Deserialize blobs on their own.* When your app stores encoded state, add a small +test-side deserializer (or reuse your replay logic) that reads blobs from +`CollectionRecords` -- or your equivalent table -- and compares to an in-test +*ground truth* you maintained while driving the sim. The Argos regression harness +does exactly this: a simulation engine holds expected values per tick; after +teardown, a blob iterator walks the file and asserts each decoded value matches. +You do not need the viewer for that -- only the same decoding rules the viewer +would use. + +*External scripts.* For complex encodings, a Python (or other) script that opens +the `.db` with `sqlite3`, validates scaffolding tables, and replays blobs is +often clearer than C++ assertion code. The Argos tree ships `test/argos/compare.py`, +which compares two databases tick-by-tick using the viewer's `DataRetriever` unpack +path -- proving two runs produce identical replayed values, not just identical +SQL rows. That pattern generalizes: *your* script encodes what "correct bytes mean" +and fails CI when decoding diverges. + +*Baseline comparison.* Keep a known-good `.db` (or exported golden query results) +and diff against test output -- `EXPECT_FILES_EQUAL` for text, or a script for +binary databases. Version the baseline when you intentionally change the encoding; +store `schema_version` in the file so mismatches are explainable. + +=== What belongs in the test vs. what belongs outside + +Keep tests *focused on SimDB's guarantees*: + +* Schema matches `defineSchema`. +* Pipeline + teardown produce a readable file. +* Queryable columns and blob decodings match ground truth. + +Push these out of the core regression loop when they bloat or flake: + +* Full GUI automation (test the data contract; let humans or separate UI tests + cover widgets). +* Performance thresholds unless you have a stable benchmark harness (use + `PollingThread::printPerfReport` manually or in a dedicated perf job, not every + CI run). +* Non-deterministic sim workloads without a fixed seed -- reproduce failures first, + then assert. + +For collectors with randomness, fix the RNG seed in tests (the Argos harness uses +a constant seed so two runs are comparable). For concurrent pipelines, prefer +teardown-then-verify over asserting mid-flight ordering unless you are explicitly +testing flush/disable behavior (<>). + +=== A practical checklist + +Before you merge a SimDB app change, ask: + +. Does every test path call `postSimLoopTeardown()` (including error paths you + simulate)? +. Are database assertions made *after* flush/teardown, on a freshly opened manager + or external reader? +. Do blob tests decode through the same rules production replay uses? +. Did you run a Release build? Unused parameters, members, and includes that Debug + ignores can fail CI under `-Werror`. +. If you changed encoding or schema, did you update baselines *and* bump a version + column/metadata so old files fail loudly? + +Testing SimDB apps is mostly discipline: treat the `.db` as the product, finalize +it correctly, and verify it with code that did not write it. The harness macros +handle reporting; flush and teardown handle visibility; an independent reader +handles trust. + +<> collects quick-reference material -- utilities, API cheat sheets, FAQ, +troubleshooting, and a migration guide -- for when you need a lookup rather than +the narrative through Parts <>--<>. diff --git a/docs/book/parts/part9-reference.adoc b/docs/book/parts/part9-reference.adoc new file mode 100644 index 00000000..4cbb9def --- /dev/null +++ b/docs/book/parts/part9-reference.adoc @@ -0,0 +1,436 @@ +[#part-ix] += Part IX: Reference & Appendices + +Quick-reference material for day-to-day lookups. The narrative parts of this book +explain *why* and *when*; this part is *what* and *where* -- headers, macros, +lifecycle calls, glossary terms, common failure modes, and migration paths. + +[appendix] +[#appendix-utilities-reference] +== Utilities Reference + +General-purpose helpers in `include/simdb/utils/`. Most are header-only and usable +outside pipelines. + +=== Compression (`Compress.hpp`) + +[cols="1,3", options="header"] +|=== +| API | Purpose + +| `compressData(ptr, nbytes, out, level)` | Zlib-compress raw bytes into `std::vector`. +| `compressData(vector, out, level)` | Compress a typed vector's bytes. +| `decompressData(in, out)` | Decompress into `std::vector`. +| `CompressionLevel` | `DISABLED`, `DEFAULT`, `FASTEST`, `HIGHEST` (maps to zlib levels). +|=== + +Empty inputs produce a valid minimal zlib wrapper (compatible with Python +`zlib.decompress`). + +=== Queues (`ConcurrentQueue.hpp`) + +Thread-safe FIFO built on `std::deque`. + +[cols="1,3", options="header"] +|=== +| Method | Purpose + +| `push` / `emplace` | Enqueue (copy, move, or in-place construct). +| `try_pop(T&)` | Dequeue if non-empty; returns `false` when empty. +| `size` / `empty` | Queue depth under lock. +| `snoop(callback)` | Scan items without removing; stop when callback returns `true`. Used by pipeline snoopers (<>). +|=== + +=== Metrics and timing + +*RunningMean* (`RunningMean.hpp`) -- Welford streaming average: `add(value)`, +`mean()`, `count()`. No sample storage. + +*SelfProfiler* (`TickTock.hpp`) -- Singleton method/block timer. `PROFILE_METHOD` +or `PROFILE_BLOCK("label")` at scope start; prints sorted average times on process +exit. Despite the filename, there is no class named `TickTock`. + +=== Logging (`ThreadSafeLogger.hpp`) + +[cols="1,3", options="header"] +|=== +| API | Purpose + +| `ThreadSafeLogger(prefix)` | Writes prefixed lines to stdout. +| `ThreadSafeFileLogger(path)` | Same, to a file. +| `protect()` | Returns a `Guard`; stream into it; one atomic line on destruction. +|=== + +=== String interning (`TinyStrings.hpp`) + +Maps strings to `uint32_t` ids for compact binary encoding. `insert(s)` returns +`(id, is_new)`; `getStringID(s)` returns id (`0` = bad/empty). `serialize(db_mgr)` +writes new mappings to the database (call at teardown when you use this pattern -- +today not automatic via `postSimLoopTeardown`; see <>). Template parameter +`MutexProtect` enables optional locking for multi-threaded producers. + +=== Values and comparison + +*ValidValue\* (`ValidValue.hpp`) -- Optional-like holder with explicit validity; +`getValue()` throws if unset. + +*approximatelyEqual* (`FloatCompare.hpp`) -- Relative tolerance compare for +floating-point types; default tolerance is machine epsilon. Used by +`EXPECT_WITHIN_EPSILON` in tests. + +=== Symbol and type helpers + +*demangle* / *demangle_type\* (`Demangle.hpp`) -- Itanium ABI demangling via +`__cxa_demangle`; falls back to input on failure. Buffer size +`DEMANGLE_BUF_LENGTH` (4096). + +*simdb::type_traits* (`TypeTraits.hpp`) -- Template metaprogramming helpers +(`enable_if_t`, pointer/string detection, container traits, etc.) used throughout +SimDB headers. + +*ios_format_saver* (`StreamFormatters.hpp`) -- RAII restore of `std::ios` format +flags after temporary manipulators. + +=== Lock helpers + +*ConditionalLock\* (`ConditionalLock.hpp`) -- Locks only when the bool +constructor argument is true. + +*DeferredLock\* (`DeferredLock.hpp`) -- Call `lock()` manually; unlocks in +destructor. Used by `TinyStrings` when `MutexProtect` is enabled. + +=== Test utilities + +*generateRandomData* (`Random.hpp`, namespace `simdb::utils`) -- Random arithmetic +vectors for unit tests. + +*utf16* (`utf16.hpp`) -- UTF-16 conversion helpers for narrow/wide string +interoperability where needed. + +[appendix] +[#appendix-api-glossary] +== API Quick Reference & Glossary + +One-page cheat sheets. For worked examples and rationale, see Parts <>--<>. + +=== SQLite interface + +*Schema* (`simdb/sqlite/Schema.hpp`, `Table.hpp`) + +[cols="1,3", options="header"] +|=== +| Construct | Notes + +| `schema.addTable("Name")` | Returns `Table&`. Implicit auto-increment `Id` PK unless changed. +| `tbl.addColumn("Col", dt::…)` | Types: `int32_t`, `uint32_t`, `int64_t`, `uint64_t`, `double_t`, `string_t`, `blob_t`. +| `tbl.setPrimaryKey("Col")` | Replace default `Id` PK. +| `tbl.unsetPrimaryKey()` | No primary key. +| `tbl.setDefaultValue("Col", val)` | Column default on INSERT omit. +| `tbl.addIndex(…)` | Query acceleration (<>). +|=== + +*DatabaseManager* (`DatabaseManager.hpp`) + +[cols="1,3", options="header"] +|=== +| Call | Purpose + +| `DatabaseManager(path, create_if_missing)` | Open or create `.db`. +| `appendSchema(schema)` | Apply schema to file (create/alter tables). +| `getDatabaseFilePath()` | Path string for reopening elsewhere. +|=== + +*Insert macros* (`Table.hpp`) + +[,cpp] +---- +db_mgr.INSERT( + SQL_TABLE("MyTable"), + SQL_COLUMNS("A", "B"), // optional if all columns in order + SQL_VALUES(a_val, b_val)); +---- + +Returns a record handle with typed getters/setters. Omit `SQL_COLUMNS` when +supplying every column in schema order. + +*Query* (`Query.hpp`) + +[cols="1,3", options="header"] +|=== +| Pattern | Purpose + +| `createQuery("Table")` | Start a SELECT builder. +| `select("Col", var)` | Bind output column to variable (or `std::optional` for NULL). +| `addConstraintForInt/UInt/Double/String(…)` | WHERE clauses; `SetConstraints::IN_SET`, etc. +| `orderBy`, `setLimit`, `resetConstraints` | Ordering and pagination. +| `count()` | Row count for current query. +| `getResultSet()` + `getNextRecord()` | Iterate rows. +| `findRecord(table, id)` | Direct row fetch by primary key. +|=== + +*Transactions* (`Transaction.hpp`) + +[cols="1,3", options="header"] +|=== +| API | Purpose + +| `safeTransaction([&]{ … })` | Runs lambda inside a transaction; retries on lock/contention until success. Reentrant-safe. +|=== + +*Inspection* -- `simdb/sqlite/Dump.hpp` prints human-readable schema/table dumps for debugging. + +=== Pipeline interface + +*Core types* (`Stage.hpp`, `Pipeline.hpp`, `PipelineManager.hpp`) + +[cols="1,3", options="header"] +|=== +| Concept | API / behavior + +| `Stage` | Subclass; implement `run_(bool force)`; return `PROCEED` or `SLEEP`. +| Ports | `addInPort_("name", queue_ptr)` / `addOutPort_(…)` in constructor. +| `PipelineManager` | Owns pipelines for one or more apps. +| `createPipeline(name, app)` | Register one pipeline for an app (typical: one call per app). Multiple apps ⇒ multiple pipelines on the shared manager. +| `addStage("name", …)` | Register stage; order matters for flush. +| `bind("src.port", "dst.port")` | Connect output queue to input. +| `noMoreStages()` / `noMoreBindings()` | Seal pipeline definition. +| Head queue | `pipeline->getInPortQueue("stage.port")` for producer injection. Several heads = several unbound inputs on the same pipeline. +| `DatabaseStage` | Terminal stage; writes via app's `DatabaseManager`. +|=== + +*Synchronization and inspection* + +[cols="1,3", options="header"] +|=== +| Tool | Purpose + +| `pipeline->createFlusher({stage_names…})` | Ordered flush; DB stages commit in one `safeTransaction` when present. +| `flusher->flush()` | Drain all listed stages with `force=true`. +| `PipelineManager::createSnooper()` | Key-based in-flight reads. +| `snooper->addStage(…)` / `snoopAllStages(key, out)` | Wire and query without draining. +| `ScopedRunnableDisabler` | RAII pause of pipeline runnables (<>). +| `AsyncDatabaseAccessor::eval(func, timeout)` | Run `func(DatabaseManager*)` on DB thread from any other thread. +|=== + +*Scheduling* -- `Stage(interval_ms)` sets polling sleep when idle (default 100 ms). +`PipelineAction::PROCEED` keeps the stage hot; `SLEEP` yields until next poll. + +=== App framework + +*App lifecycle* (`App.hpp`, `AppManager.hpp`) -- call via `AppManagers` in order: + +[cols="1,2", options="header"] +|=== +| Phase | Calls + +| Register | `enableApp()`, optional `parameterize(…)` for custom factories (<>). +| Startup | `createSchemas()` → `postInit(argc, argv)` → `initializePipelines()` → `openPipelines()`. +| Run | Your simulator loop; `process()` on pipeline heads, collect via app API. +| Shutdown | `postSimLoopTeardown()` on *every* exit path. +|=== + +*App hooks* (override on your `simdb::App` subclass) + +[cols="1,3", options="header"] +|=== +| Hook | When + +| `defineSchema` | Declare tables before DB creation. +| `postInit` | After schema exists; pre-sim metadata and configuration. Runs inside automatic `safeTransaction` (with all apps' hooks in one transaction). +| `createPipeline` | Wire one pipeline; store one or more pipeline head queues. +| `preTeardown` / `postTeardown` | App-local cleanup around sim-loop end. `postTeardown` runs inside automatic `safeTransaction` (with all apps' hooks in one transaction). +|=== + +`getInstance()` / `setInstance(n)` -- access the active app instance from stages +and static contexts (<>). + +=== Glossary + +[cols="1,3", options="header"] +|=== +| Term | Meaning + +| *Stage* | A `Runnable` unit of pipeline work with input/output ports and a `run_(force)` loop. +| *Port* | Named attachment point on a stage; typed by the queue's payload. +| *Queue* | `ConcurrentQueue` connecting an output port to an input port. +| *Pipeline head* | The input queue where the producer (sim thread) injects work. One pipeline may have several heads; one app typically has one pipeline. +| *DB stage* | Terminal `DatabaseStage` that performs SQLite writes on the dedicated DB thread. +| *Flusher* | Ordered, forced drain of stages through to the database commit point. +| *Snooper* | Key-based peek at in-flight queue items without dequeuing. +| *Heartbeat* | (Argos) Interval between full snapshots; deltas between heartbeats. Performance dial only -- does not gate collection. +| *Collectable* | (Argos) A registered simulation object whose state can be collected over time. +| *CID* | Collectable ID -- stable integer identity for a collectable in the Argos schema. +| *Tick* | One step of the simulation clock; timestamps in the DB refer to tick (or cycle) values per clock. +| *Flush-then-query* | `flush()` (and/or teardown) before reading the database so batched data is visible. +| *Funnel* | The single dedicated database thread through which all commits pass. +|=== + +[appendix] +[#appendix-faq] +== FAQ & Troubleshooting + +[#faq-database-locked] +=== "Database is locked" / SQLite contention + +*What SimDB handles for you.* All SQLite access from pipeline database stages goes +through one dedicated thread. Writes are batched inside `safeTransaction`, which +*retries* until the transaction succeeds if SQLite reports lock contention. You do +not manually retry INSERTs in normal pipeline use. + +*What you must still avoid.* Calling SQLite (or opening a second writer) from + arbitrary threads without `AsyncDatabaseAccessor::eval`. Opening the same `.db` + from another *process* while the sim is writing. Heavy unrelated disk I/O from + your simulator competing with the DB thread (<>, <>). + +*If you see lock errors from async `eval`.* Another tool may hold the file; ensure + only one writer; use read-only opens for external viewers where possible. + +[#faq-pipeline-sleeping] +=== "Why is my pipeline sleeping?" + +A stage returns `SLEEP` when `run_(force)` finds no work (empty input queue and +nothing left to do on force). That is normal idle behavior. + +*Upstream not feeding.* Perf report shows ~100% sleep on a stage whose input +should be busy -- check the producer and bindings. + +*Downstream bottleneck.* Upstream stages sleep because the DB stage or a saturated +middle stage is not draining -- tune batching/compression (<>) or split hot +stages. + +*Polling cadence.* A stage wakes every `interval_ms` (default 100). Long intervals +look like "sleeping" between bursts -- shorten for latency-sensitive paths. + +*Disabler left active.* `ScopedRunnableDisabler` still in scope pauses everything. + +*Wrong action return.* Forgetting `PROCEED` after successful work makes the stage +go idle even when data remains (until the next poll). + +[#faq-rows-not-visible] +=== "Why don't I see my rows yet?" + +Almost always: *data is still in flight.* + +* Batched transactions have not committed. +* Rows sit in an upstream queue. +* You queried before `flush()` or `postSimLoopTeardown()`. + +Fix: flush at the synchronization point, or teardown before assert; then reopen the +file with a fresh `DatabaseManager` (<>, <>). + +[#faq-partial-database] +=== "My database is partial or corrupt after a crash" + +Did `postSimLoopTeardown()` run? Without it, queues may not drain, blobs may be +truncated, and `std::thread` destructors may throw on unjoined pipeline threads +(<>). Install teardown on exception/abort paths you control. + +If you use `TinyStrings`, mappings may not be flushed at teardown today -- call +`serialize` explicitly until automatic flush is added. + +[#faq-release-werror] +=== "Why did CI fail in Release but not Debug?" + +Release builds with `-Werror` treat unused parameters, members, and includes as +errors. After editing headers or tests, build Release locally: + +[source,bash,subs=none] +---- +cmake --build build --target YourTest -DCMAKE_BUILD_TYPE=Release +---- + +Remove dead code rather than silencing warnings unless intentionally stubbing an +interface. + +[#faq-snooper-key] +=== "Snooper never finds my key" + +Snoopers search *in-flight* queue contents on wired stages, not the database. Keys +must match what the stage stores; the snooped type must match the queue payload (or +a registered transform). If data already passed to the DB stage, use flush-then-query +instead. + +[#faq-async-eval-hangs] +=== "AsyncDatabaseAccessor eval hangs or times out" + +The DB thread may be blocked on a long transaction or crashed. Do not call `eval` +from inside a `DatabaseStage` (deadlock). Set a non-zero `timeout_seconds` during +development to surface hangs (<>). + +[appendix] +[#appendix-migration-guide] +== Migration Guide + +Paths for teams replacing ad-hoc data plumbing with SimDB. Each is incremental -- +you do not need all three at once. + +=== From "write everything to files" to one SQLite database + +*Problem.* Per-metric log files, ad-hoc directories, post-processing scripts that +merge and repair formats after every run. + +*Approach.* + +1. *Inventory outputs* -- list every file type, who reads it, and what queries + they run (filter by time? by component name?). +2. *Design schema first* (<>, <> <>) -- columns for anything you + filter on; blobs for bulk payloads you only read whole. +3. *Replace writes incrementally* -- one metric family at a time; keep legacy + files behind a flag until the DB path is verified. +4. *Add a pipeline when the sim thread must not block* -- producer pushes to the + head queue; compress + DB stage at the tail (<>). +5. *Wrap in `simdb::App`* when multiple subsystems share one database and lifecycle + (<>). +6. *Retire file writers* when tests prove query/export from `.db` replaces them. + +*Win.* One artifact per run, queryable with SQL, Python, or your viewer; no merge +step; SimDB handles concurrent writes safely. + +=== From raw SQLite (or a thin wrapper) to the SimDB SQLite interface + +*Problem.* Hand-written `CREATE TABLE`, manual prepared statements, custom retry +logic, `uint64_t` and blob bugs. + +*Approach.* + +1. Express tables as `simdb::Schema` instead of SQL DDL strings. +2. Replace raw INSERTs with `SQL_TABLE` / `SQL_VALUES` and typed record handles. +3. Replace ad-hoc SELECT strings with `createQuery` and bound `select()` variables. +4. Wrap multi-row writes in `safeTransaction` (or let database stages batch for you). +5. Run the `test/sqlite/**` regressions alongside your port to validate parity (<>). + +*Keep.* External tools that already read SQLite -- the file format remains standard +SQLite. SimDB adds type safety and retry semantics, not a proprietary container. + +=== From a hand-rolled thread pipeline to SimDB stages + +*Problem.* Custom worker threads, queues, condition variables, and "one thread owns +SQLite" conventions copied into every project. + +*Approach.* + +1. *Map each worker to a stage* -- one responsibility per stage; move-only payloads + between queues (<>, <> Ch 37). +2. *Replace your DB thread* with `DatabaseStage` + `Flusher` -- do not keep a + second SQLite writer. +3. *Replace cross-thread DB calls* with `AsyncDatabaseAccessor::eval` (<>). +4. *Replace debug freezes* with `ScopedRunnableDisabler` + flush. +5. *Delete home-grown retry/transaction code* where `safeTransaction` and + `FlusherWithTransaction` cover the same ground. + +*Pitfall.* Porting thread-for-thread without rethinking stage boundaries recreates +the old queue graph with extra overhead. Merge idle stages; split saturated ones +using perf reports (<>, <> <>). + +=== A clean break vs. gradual adoption + +SimDB is designed as a *replacement* architecture for tangled data paths (README: +"A Clean Break for Complex Codebases"), not a shim around every legacy writer. +Gradual migration works best *per metric or subsystem*, not by running two parallel +pipelines inside one app forever. Pick a boundary -- one database, one pipeline +per app, multiple heads if you need several producers -- and converge writers +there until legacy paths can be deleted. + +When migration is done, your simulator talks to SimDB at the edges (collect / +process / teardown); everything else is SQL, analysis scripts, or a viewer on the +`.db` file. diff --git a/examples/AppFactory/main.cpp b/examples/AppFactory/main.cpp index d7ad189d..cb70eabc 100644 --- a/examples/AppFactory/main.cpp +++ b/examples/AppFactory/main.cpp @@ -98,6 +98,9 @@ class UnregisteredAppWithFactory : public simdb::App return nullptr; } + // TODO cnyce: we should not make users write this code + // so just have a class simdb::CustomAppFactory + // in between the users' AppFactory and AppFactoryBase. void defineSchema(simdb::Schema&) const override { EXPECT_TRUE(false); // Should never get hit diff --git a/include/simdb/apps/App.hpp b/include/simdb/apps/App.hpp index f1e3959e..95baf8ab 100644 --- a/include/simdb/apps/App.hpp +++ b/include/simdb/apps/App.hpp @@ -10,28 +10,32 @@ /// configuration (e.g. command line options, config file, etc). /// /// - Create your own data/metadata schema tables just for your app -/// - Use SimDB utilities to build async compression/DB pipelines +/// - Use SimDB utilities to build async compression/transform/DB pipelines /// /// Example applications: /// /// - Logger that records simulation events /// - Profiler that tracks performance metrics -/// - Pipeline collector -/// - SQLite-to-HDF5 converter +/// - CPU pipeline instrumentation/collection with front-end viewer (Argos) /// - Backend for a live data visualization GUI / web interface +/// - Backend for post-sim analysis using simulation state replayers /// /// Since SimDB is designed to be simulator-agnostic, apps also provide /// a variety of hooks that allow you to run code at different stages of /// the simulation lifecycle and to ensure that all apps in your simulator /// are initialized and run in a consistent manner. /// -/// - appendSchema: first hook after command line args / config files are -/// parsed -/// - postInit: called before the simulation starts, after command line -/// parsing -/// - postSim: called after the simulation loop ends -/// - teardown: called after the simulation ends, for resource cleanup -/// tasks +/// - defineSchema: (static) declare the app's schema tables. Called via the +/// app's AppFactory before any app instance is created. +/// - postInit: after command-line / config parsing, before the +/// simulation starts. +/// - createPipeline: create and configure the app's pipeline(s). +/// - preTeardown: just before simulation teardown; push any pending data +/// to your app's pipeline +/// - postTeardown: after simulation; post-sim metadata DB writes occur here; +/// all running apps' postTeardown methods are executed in a +/// single safeTransaction automatically (don't worry about +/// using safeTransaction explicitly) /// /// The general paradigm is that your simulator has a single output database, /// with 1-to-many apps that are all writing to it with their own custom schemas @@ -88,6 +92,8 @@ class App protected: void setAppLogger_(ThreadSafeLogger* logger) { app_logger_ = logger; } + bool verbose() const { return verbose_; } + private: /// Instance number for multi-instance apps (1-based). /// If zero, then this is a single-instance app. @@ -95,6 +101,10 @@ class App /// Thread-safe loggers. ThreadSafeLogger* app_logger_ = nullptr; + + /// Verbose flag. + bool verbose_ = false; + friend class AppManager; }; diff --git a/include/simdb/apps/AppManager.hpp b/include/simdb/apps/AppManager.hpp index 0cc596fd..5f4e6d68 100644 --- a/include/simdb/apps/AppManager.hpp +++ b/include/simdb/apps/AppManager.hpp @@ -5,6 +5,7 @@ #include "simdb/apps/App.hpp" #include "simdb/pipeline/PipelineManager.hpp" #include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/utils/StreamFormatters.hpp" #include "simdb/utils/ThreadSafeLogger.hpp" #include @@ -190,6 +191,16 @@ class AppManager } } + /// Enable verbose mode for all apps in this AppManager. + void setVerbose(bool verbose = true) + { + verbose_ = verbose; + for (auto app : getApps_()) + { + app->verbose_ = verbose_; + } + } + /// After parsing command line arguments or configuration files, /// enable an app by its name. This will allow the app to be instantiated /// and run during the simulation lifecycle. @@ -477,6 +488,7 @@ class AppManager App* app = factory->createApp(db_mgr_); app->app_logger_ = app_logger_; app->instance_ = instance_num; + app->verbose_ = verbose_; std::string instance_name = app_name + std::string("-") + std::to_string(instance_num); apps_[instance_name] = std::unique_ptr(app); } @@ -488,6 +500,12 @@ class AppManager { PROFILE_APP_PHASE + // TODO cnyce: We are currently creating the apps first, then creating the schemas. + // There is no reason to do this in that order, and we should create the schemas + // first. The doc (book) was written assuming schemas come first; leave the doc + // that way and change the C++ code to match. Call out that you can freely write + // to your tables in defineSchema right in your App's constructor. Then ensure + // that the createEnabledApps call is inside a safeTransaction. db_mgr_->safeTransaction([&]() { for (const auto& [app_name, app] : apps_) { @@ -500,6 +518,12 @@ class AppManager auto instance_num = static_cast(std::stoull(tmp_substr)); AppFactoryBase* factory = getAppFactory_(user_app_name, instance_num); + // TODO cnyce: We should consider automatically prepending "$" + // before every table in the app schema. With a growing number of apps, + // and SimDB living in open source, this will eventually trip someone + // up from e.g. using a generic table like "SimMetadata" which collides. + // You will have to document that the app name might have restrictions + // that will match whatever restrictions SQLite puts on table names. Schema app_schema; factory->defineSchema(app_schema); db_mgr_->appendSchema(app_schema); @@ -587,6 +611,12 @@ class AppManager { app->postTeardown(); } + + // TODO cnyce: We should associate the DatabaseManager and + // the TinyStrings so that it gets flushed automatically. + // This will let us document that postSimLoopTeardown() + // is the only thing you have to call in the event of + // an exception/abort/terminate during simulation. }); } @@ -750,6 +780,9 @@ class AppManager /// App logger (thread-safe). Owned by AppManagers. ThreadSafeLogger* app_logger_ = nullptr; + /// Verbose flag. + bool verbose_ = false; + /// RAII timer to measure the performance of various app setup/teardown /// phases. class ScopedTimer @@ -765,6 +798,8 @@ class AppManager ~ScopedTimer() { + [[maybe_unused]] ios_format_saver fmt_saver(std::cout); + auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration dur = end - start_; auto us = std::chrono::duration_cast(dur).count(); @@ -850,6 +885,16 @@ class AppManagers app_logger_ = std::make_unique(filename); } + /// Enable verbose mode for all apps in all AppManager's. + void setVerbose(bool verbose = true) + { + verbose_ = verbose; + for (auto& [app_mgr, _] : db_mgrs_by_app_mgr_) + { + app_mgr->setVerbose(verbose_); + } + } + /// Create a new AppManager with a new database. /// /// Pass in new_db=true to overwrite existing database, or new_db=false to use @@ -874,6 +919,8 @@ class AppManagers std::shared_ptr db_mgr(new DatabaseManager(db_file, new_db)); std::shared_ptr app_mgr(new AppManager(db_mgr.get(), app_logger_.get())); + app_mgr->setVerbose(verbose_); + db_mgrs_by_db_file_[db_file] = db_mgr; app_mgrs_by_db_file_[db_file] = app_mgr; @@ -1053,6 +1100,7 @@ class AppManagers std::unique_ptr app_logger_; bool accepting_logger_requests_ = true; + bool verbose_ = false; }; /*! diff --git a/include/simdb/apps/argos/ArgosCollector.hpp b/include/simdb/apps/argos/ArgosCollector.hpp new file mode 100644 index 00000000..e9b41936 --- /dev/null +++ b/include/simdb/apps/argos/ArgosCollector.hpp @@ -0,0 +1,843 @@ +// -*- C++ *-* + +#pragma once + +#include "simdb/apps/App.hpp" +#include "simdb/apps/argos/Checkpointer.hpp" +#include "simdb/apps/argos/EntryPoint.hpp" +#include "simdb/apps/argos/EnumInspector.hpp" +#include "simdb/apps/argos/PipelineDataTypes.hpp" +#include "simdb/apps/argos/PipelineStagerInterface.hpp" +#include "simdb/apps/argos/Timestamps.hpp" +#include "simdb/pipeline/PipelineManager.hpp" +#include "simdb/sqlite/Dump.hpp" +#include "simdb/utils/Compress.hpp" +#include "simdb/utils/TinyStrings.hpp" +#include "simdb/utils/TypeTraits.hpp" + +namespace simdb::argos { + +inline constexpr size_t DEFAULT_HEARTBEAT = 10; + +//! \class ArgosCollector +//! \brief Main entry point into the Argos collection system. +class ArgosCollector : public App, public PipelineStagerInterface +{ +public: + //! Required by all SimDB apps + static constexpr auto NAME = "argos-collector"; + + ArgosCollector(DatabaseManager* db_mgr) : + db_mgr_(db_mgr) + { + } + + static void defineSchema(Schema& schema) + { + using dt = SqlDataType; + + auto& globals_tbl = schema.addTable("CollectionGlobals"); + globals_tbl.addColumn("Heartbeat", dt::int32_t); + + auto& clks_tbl = schema.addTable("Clocks"); + clks_tbl.addColumn("Name", dt::string_t); + clks_tbl.addColumn("Period", dt::uint32_t); + clks_tbl.addColumn("Numer", dt::uint32_t); + clks_tbl.addColumn("Denom", dt::uint32_t); + clks_tbl.setColumnDefaultValue("Numer", 0); + clks_tbl.setColumnDefaultValue("Denom", 0); + + auto& collectable_tns_tbl = schema.addTable("CollectableTreeNodes"); + collectable_tns_tbl.addColumn("CID", dt::int32_t); + collectable_tns_tbl.addColumn("FullPath", dt::string_t); + collectable_tns_tbl.addColumn("ClockID", dt::int32_t); + collectable_tns_tbl.addColumn("TypeName", dt::string_t); + collectable_tns_tbl.addColumn("ShowInUI", dt::int32_t); + collectable_tns_tbl.setColumnDefaultValue("ShowInUI", 0); + collectable_tns_tbl.ensureUnique("CID"); + collectable_tns_tbl.createIndexOn("CID"); + collectable_tns_tbl.unsetPrimaryKey(); + + // TODO cnyce: populate this table in SimDB (Sparta will handle it for now) + auto& dtype_schemas_tbl = schema.addTable("DataTypeSchemas"); + dtype_schemas_tbl.addColumn("RootTypeName", dt::string_t); + + // TODO cnyce: populate this table in SimDB (Sparta will handle it for now) + auto& dtype_nodes_tbl = schema.addTable("DataTypeNodes"); + dtype_nodes_tbl.addColumn("SchemaId", dt::int32_t); + dtype_nodes_tbl.addColumn("Name", dt::string_t); + dtype_nodes_tbl.addColumn("TypeName", dt::string_t); + dtype_nodes_tbl.addColumn("FormatStr", dt::string_t); + + auto& enum_itypes_tbl = schema.addTable("CollectedEnums"); + enum_itypes_tbl.addColumn("EnumName", dt::string_t); + enum_itypes_tbl.addColumn("EnumIntTypeName", dt::string_t); + + auto& enum_members_tbl = schema.addTable("EnumMembers"); + enum_members_tbl.addColumn("EnumID", dt::int32_t); + enum_members_tbl.addColumn("MemberName", dt::string_t); + enum_members_tbl.addColumn("MemberValueStr", dt::string_t); + + auto& timestamps_tbl = schema.addTable("Timestamps"); + timestamps_tbl.addColumn("Timestamp", dt::uint64_t); + timestamps_tbl.ensureUnique("Timestamp"); + + auto& collection_records_tbl = schema.addTable("CollectionRecords"); + collection_records_tbl.addColumn("TimestampID", dt::int32_t); + collection_records_tbl.addColumn("Records", dt::blob_t); + collection_records_tbl.ensureUnique("TimestampID"); + collection_records_tbl.unsetPrimaryKey(); + + auto& timestamp_clocks_tbl = schema.addTable("TimestampClocks"); + timestamp_clocks_tbl.addColumn("TimestampID", dt::int32_t); + timestamp_clocks_tbl.addColumn("ClockID", dt::int32_t); + timestamp_clocks_tbl.createCompoundIndexOn({"TimestampID", "ClockID"}); + timestamp_clocks_tbl.unsetPrimaryKey(); + + auto& queue_max_sizes_tbl = schema.addTable("QueueMaxSizes"); + queue_max_sizes_tbl.addColumn("CID", dt::int32_t); + queue_max_sizes_tbl.addColumn("MaxSize", dt::int32_t); + queue_max_sizes_tbl.ensureUnique("CID"); + queue_max_sizes_tbl.unsetPrimaryKey(); + + auto& notif_tbl = schema.addTable("Notifications"); + notif_tbl.addColumn("Timestamp", dt::uint64_t); + notif_tbl.addColumn("NotifType", dt::int32_t); + notif_tbl.addColumn("NotifStr", dt::string_t); + + auto& tiny_string_ids_tbl = schema.addTable("TinyStringIDs"); + tiny_string_ids_tbl.addColumn("StringValue", dt::string_t); + tiny_string_ids_tbl.addColumn("StringID", dt::uint32_t); + } + + void setHeartbeat(size_t heartbeat) + { + if (heartbeat == 0) + { + throw DBException("Cannot use 0 for Argos collector heartbeat"); + } + heartbeat_ = heartbeat; + } + + void addClock(const std::string& clk_name, size_t period) { addClock(clk_name, period, 0, 0); } + + void addClock(const std::string& clk_name, size_t period, size_t numer, size_t denom) + { + for (const auto& [_clk_name, _period, _numer, _denom] : clocks_) + { + if (clk_name == _clk_name) + { + if (period != _period || numer != _numer || denom != _denom) + { + throw DBException("Clock mismatch - already registered with different params: ") << clk_name; + } + } + } + + auto clk_desc = std::make_tuple(clk_name, period, numer, denom); + clocks_.emplace_back(std::move(clk_desc)); + } + + void timestampWith(const uint64_t* backpointer) + { + if (timestamp_ != nullptr) + { + throw DBException("Cannot change timestamp object once created!"); + } + timestamp_ = std::make_unique(backpointer); + } + + void timestampWith(uint64_t (*fn)()) + { + if (timestamp_ != nullptr) + { + throw DBException("Cannot change timestamp object once created!"); + } + timestamp_ = std::make_unique(fn); + } + + void timestampWith(std::function fn) + { + if (timestamp_ != nullptr) + { + throw DBException("Cannot change timestamp object once created!"); + } + timestamp_ = std::make_unique(fn); + } + + //! TODO cnyce: Once the collection code from Sparta is moved to SimDB, change this + //! to a template method so we can figure out the encoded data type name ourselves. + //! Scalar types are encoded as follows: + //! + //! For scalar PODs: + //! typeid(T).name() + //! "bool" + //! "unsigned long" + //! ... + //! + //! For scalar enums with operator<< (defn held in separate table for string-int map): + //! typeid(T).name() + //! "IssueType" + //! "MMUState" + //! ... + //! Enum values in collection blobs are serialized as std::underlying_type_t. + //! + //! For scalar enums without operator<< (treated just like int PODs): + //! typeid(std::underlying_type_t).name() + //! "int" + //! "unsigned int" + //! ... + //! + //! For scalar struct-like types: + //! typeid(T).name() + //! "Packet" + //! "Inst" + //! ... + //! + //! For scalar string-like types (std::string, const char*): + //! "string" + EntryPoint* createScalarCollector(const std::string& path, const std::string& clk_name, + const std::string& encoded_scalar_type) + { + auto entry_point = std::make_unique(this, &tiny_strings_); + meta_by_cid_[entry_point->getID()] = std::make_tuple(path, clk_name, encoded_scalar_type); + entry_points_.emplace_back(std::move(entry_point)); + return entry_points_.back().get(); + } + + //! TODO cnyce: Once the collection code from Sparta is moved to SimDB, change this + //! to a template method so we can figure out the encoded data type name ourselves. + //! Container types are encoded as follows: + //! + //! __capacity + //! "Inst_sparse_capacity32" + //! "bool_contig_capacity4" + //! ... + EntryPoint* createContainerCollector(const std::string& path, const std::string& clk_name, + const std::string& encoded_container_type) + { + auto entry_point = std::make_unique(this, &tiny_strings_); + meta_by_cid_[entry_point->getID()] = std::make_tuple(path, clk_name, encoded_container_type); + entry_points_.emplace_back(std::move(entry_point)); + return entry_points_.back().get(); + } + + TinyStrings<>* getTinyStrings() { return &tiny_strings_; } + + EnumInspector* getEnumInspector() { return &enum_inspector_; } + + void createPipeline(pipeline::PipelineManager* pipeline_mgr) override + { + auto pipeline = pipeline_mgr->createPipeline(NAME, this); + + pipeline_stager_ = pipeline->addStage("stager", heartbeat_); + pipeline->addStage("compressor"); + pipeline->addStage("writer"); + pipeline->noMoreStages(); + + pipeline->bind("stager.data_output_queue", "compressor.data_input_queue"); + pipeline->bind("compressor.data_output_queue", "writer.data_input_queue"); + pipeline->noMoreBindings(); + + pipeline_head_ = pipeline->getInPortQueue("stager.main_input_queue"); + notif_head_ = pipeline->getInPortQueue("writer.notif_input_queue"); + + for (const auto& [cid, meta] : meta_by_cid_) + { + const auto& encoded_dtype = std::get<2>(meta); + + ContainerMeta container_meta; + if (extractContainerMeta_(encoded_dtype, container_meta)) + { + pipeline_stager_->setContainerDataType(cid, container_meta.sparse, container_meta.capacity); + } else + { + pipeline_stager_->setScalarDataType(cid); + } + } + + NotifEntry notif_entry; + while (pending_notif_entries_.try_pop(notif_entry)) + { + notif_head_->emplace(std::move(notif_entry)); + } + + for (const auto& collector : entry_points_) + { + auto cid = collector->getID(); + const auto& clk_name = std::get<1>(meta_by_cid_.at(cid)); + pipeline_stager_->setCollectableClock(cid, getClockId_(clk_name)); + } + + is_live_ = true; + } + + void postInit(int, char**) override + { + db_mgr_->INSERT(SQL_TABLE("CollectionGlobals"), SQL_VALUES(heartbeat_)); + + std::map clk_ids; + auto clk_inserter = db_mgr_->prepareINSERT(SQL_TABLE("Clocks")); + for (const auto& [_clk_name, _period, _numer, _denom] : clocks_) + { + auto id = clk_inserter->createRecordWithColValues(_clk_name, _period, _numer, _denom); + clk_ids[_clk_name] = id; + } + + auto ctn_inserter = db_mgr_->prepareINSERT(SQL_TABLE("CollectableTreeNodes")); + for (const auto& collector : entry_points_) + { + auto cid = (int)collector->getID(); + const auto& full_path = std::get<0>(meta_by_cid_.at(cid)); + const auto& clk_name = std::get<1>(meta_by_cid_.at(cid)); + const auto& dtype_name = std::get<2>(meta_by_cid_.at(cid)); + const auto clk_id = clk_ids.at(clk_name); + ctn_inserter->createRecordWithColValues(cid, full_path, clk_id, dtype_name); + } + } + + void stage(uint16_t cid, std::vector&& scalar_bytes) override + { + assertLive_(); + checkTimeAdvanced_(); + ledger_->recordScalar(cid, std::move(scalar_bytes)); + } + + void stage(uint16_t cid, std::vector>&& contig_bin_bytes) override + { + assertLive_(); + checkTimeAdvanced_(); + ledger_->recordContig(cid, std::move(contig_bin_bytes)); + } + + void stage(uint16_t cid, std::map>&& sparse_bin_bytes) override + { + assertLive_(); + checkTimeAdvanced_(); + ledger_->recordSparse(cid, std::move(sparse_bin_bytes)); + } + + void closeRecord(uint16_t cid) override + { + assertLive_(); + checkTimeAdvanced_(); + ledger_->closeRecord(cid); + } + + void postNotif(const std::string& notif, NotifType type) override + { + NotifEntry entry(timestamp_.get(), notif, type); + auto notif_entries = is_live_ ? notif_head_ : &pending_notif_entries_; + notif_entries->emplace(std::move(entry)); + } + + void preTeardown() override + { + if (ledger_ && ledger_->hasEntries()) + { + pipeline_head_->emplace(std::move(ledger_)); + } + } + + void postTeardown() override + { + if (pipeline_stager_) + { + pipeline_stager_->writeMetaOnPostTeardown(db_mgr_); + } + tiny_strings_.serialize(db_mgr_); + enum_inspector_.serializeEnumMaps(db_mgr_); + + if (verbose()) + { + std::cout << "[simdb] Collection tables at the end of simulation (except timestamps/blobs):\n\n"; + dumpTable(db_mgr_, "CollectionGlobals"); + dumpTable(db_mgr_, "Clocks"); + dumpTable(db_mgr_, "CollectableTreeNodes"); + dumpTable(db_mgr_, "DataTypeSchemas"); + dumpTable(db_mgr_, "DataTypeNodes"); + dumpTable(db_mgr_, "CollectedEnums"); + dumpTable(db_mgr_, "EnumMembers"); + dumpTable(db_mgr_, "QueueMaxSizes"); + dumpTable(db_mgr_, "Notifications"); + } + } + +private: + /// Entry to the DB pipeline (1st async stage input) + ConcurrentQueue* pipeline_head_ = nullptr; + + /// Entry to the DB writer stage's notifications + ConcurrentQueue* notif_head_ = nullptr; + + /// Queued notifications prior to createPipeline() + ConcurrentQueue pending_notif_entries_; + + /// Ledger of all collection activity at one simulation time point + LedgerPtr ledger_; + + /// \class PipelineStager + /// \brief 1st async stage. Takes collected data from the Ledger and appends checkpoints to the + /// CID's checkpoint chains for snapshot-delta compression. + class PipelineStager : public pipeline::Stage + { + public: + PipelineStager(size_t heartbeat) : + heartbeat_(heartbeat) + { + addInPort_("main_input_queue", main_input_queue_); + addOutPort_("data_output_queue", data_output_queue_); + } + + /// Must be called from main thread + void setCollectableClock(uint16_t cid, uint32_t clock_id) + { + collectable_clock_ids_[cid] = clock_id; + clock_ids_.insert(clock_id); + } + + /// Must be called from main thread + void setScalarDataType(uint16_t cid) + { + checkpointers_[cid] = std::make_unique(heartbeat_); + ++num_scalars_; + } + + /// Must be called from main thread + void setContainerDataType(uint16_t cid, bool sparse, size_t capacity) + { + if (sparse) + { + checkpointers_[cid] = std::make_unique(heartbeat_, capacity); + ++num_sparses_; + } else + { + checkpointers_[cid] = std::make_unique(heartbeat_, capacity); + ++num_contigs_; + } + } + + size_t getNumScalars() const { return num_scalars_; } + + size_t getNumContigs() const { return num_contigs_; } + + size_t getNumSparses() const { return num_sparses_; } + + /// Must be called from main thread + void writeMetaOnPostTeardown(DatabaseManager* db_mgr) + { + writeShowInUI_(db_mgr); + writeQueueMaxSizes_(db_mgr); + } + + private: + pipeline::PipelineAction run_(bool) override + { + auto action = pipeline::PipelineAction::SLEEP; + + LedgerPtr ledger; + while (main_input_queue_->try_pop(ledger)) + { + auto window_id = ledger->getWindowId(); + + auto scalars = ledger->releaseScalarEntries(); + for (auto& scalar : scalars) + { + auto cid = scalar.cid; + auto data = std::move(scalar.scalar_bytes); + checkpointers_.at(cid)->createCheckpoint(window_id, std::move(data)); + } + + auto contigs = ledger->releaseContigEntries(); + for (auto& contig : contigs) + { + auto cid = contig.cid; + auto data = std::move(contig.contig_bin_bytes); + checkpointers_.at(cid)->createCheckpoint(window_id, std::move(data)); + updateContainerMaxSize_(cid, detail::getContainerSize(data)); + } + + auto sparses = ledger->releaseSparseEntries(); + for (auto& sparse : sparses) + { + auto cid = sparse.cid; + auto data = std::move(sparse.sparse_bin_bytes); + checkpointers_.at(cid)->createCheckpoint(window_id, std::move(data)); + updateContainerMaxSize_(cid, detail::getContainerSize(data)); + } + + for (const auto cid : ledger->getClosedCIDs()) + { + checkpointers_.at(cid)->closeRecord(window_id); + } + + QueueCollectionData to_send; + auto sim_time = ledger->getSimTime(); + to_send.sim_time = sim_time; + + const auto multi_clock = clock_ids_.size() > 1; + for (auto& [cid, checkpointer] : checkpointers_) + { + if (multi_clock && !checkpointer->participatedInWindow(window_id)) + { + continue; + } + + auto entries = checkpointer->encodeForPipeline(window_id, sim_time, cid); + for (auto& entry : entries) + { + cids_with_data_.insert(cid); + to_send.entries.emplace_back(std::move(entry)); + if (const auto clk_it = collectable_clock_ids_.find(cid); + clk_it != collectable_clock_ids_.end()) + { + to_send.clock_ids.insert(clk_it->second); + } + } + } + + if (!to_send.entries.empty()) + { + data_output_queue_->emplace(std::move(to_send)); + } + + action = pipeline::PipelineAction::PROCEED; + } + + return action; + } + + void updateContainerMaxSize_(uint16_t cid, uint16_t size) + { + auto it = container_max_sizes_.find(cid); + if (it == container_max_sizes_.end()) + { + container_max_sizes_.emplace(cid, size); + } else + { + it->second = std::max(it->second, size); + } + } + + // Set ShowInUI=1 for all the collectables that actually collected data + void writeShowInUI_(DatabaseManager* db_mgr) const + { + const std::vector valid_cids(cids_with_data_.begin(), cids_with_data_.end()); + if (!valid_cids.empty()) + { + std::ostringstream oss; + oss << "UPDATE CollectableTreeNodes SET ShowInUI=1 WHERE CID IN ("; + + bool comma = false; + for (const auto cid : valid_cids) + { + if (comma) + { + oss << ","; + } + oss << cid; + comma = true; + } + oss << ")"; + db_mgr->EXECUTE(oss.str()); + } + + auto query = db_mgr->createQuery("CollectableTreeNodes"); + query->addConstraintForInt("CID", SetConstraints::NOT_IN_SET, valid_cids); + + struct CID_Info + { + std::string path; + std::string type; + + CID_Info(const std::string& path, const std::string& type) : + path(path), + type(type) + { + } + }; + + std::string path; + query->select("FullPath", path); + + std::string type; + query->select("TypeName", type); + + std::vector cid_infos; + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + cid_infos.emplace_back(path, type); + } + + if (!cid_infos.empty()) + { + std::ostringstream oss; + oss << "No data was ever collected for the following collectables, and will not be shown in Argos:\n"; + size_t leftcol_w = 0; + for (const auto& info : cid_infos) + { + leftcol_w = std::max(leftcol_w, info.path.size()); + } + + leftcol_w += 12; + for (const auto& info : cid_infos) + { + oss << std::left << std::setw(leftcol_w) << info.path; + if (auto idx = info.type.find("_sparse"); idx != std::string::npos) + { + auto base_type = info.type.substr(0, idx); + oss << "(Sparse container of '" << base_type << "')"; + } else if (auto idx = info.type.find("_contig"); idx != std::string::npos) + { + auto base_type = info.type.substr(0, idx); + oss << "(Contig container of '" << base_type << "')"; + } else + { + oss << "(" << info.type << ")"; + } + oss << "\n"; + } + + auto warning = oss.str(); + warning.pop_back(); + + db_mgr->INSERT(SQL_TABLE("Notifications"), SQL_COLUMNS("NotifStr", "NotifType"), + SQL_VALUES(warning, (int)NotifType::WARNING)); + } + } + + void writeQueueMaxSizes_(DatabaseManager* db_mgr) const + { + auto inserter = db_mgr->prepareINSERT(SQL_TABLE("QueueMaxSizes")); + for (const auto& [cid, max_size] : container_max_sizes_) + { + inserter->createRecordWithColValues((int)cid, (int)max_size); + } + } + + const size_t heartbeat_; + + ConcurrentQueue* main_input_queue_ = nullptr; + ConcurrentQueue* data_output_queue_ = nullptr; + + std::unordered_map collectable_clock_ids_; + std::unordered_set clock_ids_; + std::unordered_set cids_with_data_; + + std::unordered_map> checkpointers_; + std::unordered_map container_max_sizes_; + + size_t num_scalars_ = 0; + size_t num_contigs_ = 0; + size_t num_sparses_ = 0; + }; + + /// \class Compressor + /// \brief 2nd async stage. Performs zlib compression. + class Compressor : public pipeline::Stage + { + public: + Compressor() + { + addInPort_("data_input_queue", input_queue_); + addOutPort_("data_output_queue", output_queue_); + } + + private: + pipeline::PipelineAction run_(bool) override + { + QueueCollectionData collection_at_time; + if (input_queue_->try_pop(collection_at_time)) + { + std::vector uncompressed; + for (const auto& src : collection_at_time.entries) + { + const auto& src_data = src->getData(); + uncompressed.insert(uncompressed.end(), src_data.begin(), src_data.end()); + } + + CompressedQueueCollectionData compressed; + compressData(uncompressed, compressed.compressed_collection_data); + compressed.sim_time = collection_at_time.sim_time; + compressed.clock_ids = std::move(collection_at_time.clock_ids); + output_queue_->emplace(std::move(compressed)); + return pipeline::PipelineAction::PROCEED; + } + + return pipeline::PipelineAction::SLEEP; + } + + ConcurrentQueue* input_queue_ = nullptr; + ConcurrentQueue* output_queue_ = nullptr; + }; + + /// \class Writer + /// \brief Final async stage. Writes collected/checkpointed/compressed data and notifications + /// to the database. + class Writer : public pipeline::DatabaseStage + { + public: + Writer() + { + addInPort_("data_input_queue", data_input_queue_); + addInPort_("notif_input_queue", notif_input_queue_); + } + + private: + pipeline::PipelineAction run_(bool) override + { + auto action = pipeline::PipelineAction::SLEEP; + + CompressedQueueCollectionData collection_at_time; + if (data_input_queue_->try_pop(collection_at_time)) + { + auto db_mgr = getDatabaseManager_(); + auto id = Timestamp::createTimestampInDatabase(db_mgr, collection_at_time.sim_time); + + auto inserter = getTableInserter_("CollectionRecords"); + const auto& bytes = collection_at_time.compressed_collection_data; + inserter->createRecordWithColValues(id, bytes); + + if (!collection_at_time.clock_ids.empty()) + { + auto clk_inserter = getTableInserter_("TimestampClocks"); + for (const auto clock_id : collection_at_time.clock_ids) + { + clk_inserter->createRecordWithColValues(id, static_cast(clock_id)); + } + } + + action = pipeline::PipelineAction::PROCEED; + } + + // Notifications are expected to be "sporadic" / one-off. Flush all + // of the notifs now, and do not consider "notification-only" to mean + // "keep greedily processing / keep threads running". This is the + // reason why we use WHILE here instead of IF like above, and the + // returned action is not set to PROCEED based on the availability + // of new notifications. + NotifEntry notif; + while (notif_input_queue_->try_pop(notif)) + { + auto inserter = getTableInserter_("Notifications"); + + const auto& notif_str = notif.notif; + const auto notif_type = notif.type; + + if (notif.sim_time.isValid()) + { + inserter->setColumnValue(0, notif.sim_time.getValue()); + } + inserter->setColumnValue(1, (int)notif_type); + inserter->setColumnValue(2, notif_str); + inserter->createRecord(); + } + + return action; + } + + ConcurrentQueue* data_input_queue_ = nullptr; + ConcurrentQueue* notif_input_queue_ = nullptr; + }; + + void assertLive_() const + { + if (!is_live_ || !timestamp_) + { + throw DBException("API call cannot be made until pipeline is open and " + "timestampWith() was called"); + } + } + + void checkTimeAdvanced_() + { + assertLive_(); + + auto current_time = timestamp_->getTime(); + if (!current_stage_time_.isValid()) + { + current_stage_time_ = current_time; + ledger_ = std::make_unique(current_time, current_window_id_++, pipeline_stager_->getNumScalars(), + pipeline_stager_->getNumContigs(), pipeline_stager_->getNumSparses()); + } else if (current_time < current_stage_time_.getValue()) + { + throw DBException("Time must be monotonically increasing"); + } else if (current_time > current_stage_time_.getValue()) + { + if (ledger_) + { + // Send current ledger to the pipeline and create a new one. + pipeline_head_->emplace(std::move(ledger_)); + + ledger_ = + std::make_unique(current_time, current_window_id_++, pipeline_stager_->getNumScalars(), + pipeline_stager_->getNumContigs(), pipeline_stager_->getNumSparses()); + } + + current_stage_time_ = current_time; + } + } + + struct ContainerMeta + { + ValidValue sparse; + ValidValue capacity; + }; + + static bool extractContainerMeta_(const std::string& encoded_dtype, ContainerMeta& meta) + { + auto extract_capacity = [&](const bool contig) { + const std::string lookfor = contig ? "_contig_capacity" : "_sparse_capacity"; + if (auto idx = encoded_dtype.find(lookfor); idx != std::string::npos) + { + auto capacity = std::stoi(encoded_dtype.substr(idx + lookfor.size())); + meta.sparse = !contig; + meta.capacity = capacity; + return true; + } + return false; + }; + + return extract_capacity(true) || extract_capacity(false); + } + + uint32_t getClockId_(const std::string& clk_name) const + { + for (size_t i = 0; i < clocks_.size(); ++i) + { + if (std::get<0>(clocks_[i]) == clk_name) + { + return static_cast(i + 1); + } + } + throw DBException("Unknown clock: ") << clk_name; + } + + DatabaseManager* const db_mgr_; + size_t heartbeat_ = DEFAULT_HEARTBEAT; + + using ClockDescriptor = std::tuple; + std::vector clocks_; + + using CollectableMeta = std::tuple; + std::map meta_by_cid_; + + std::unique_ptr timestamp_; + std::vector> entry_points_; + PipelineStager* pipeline_stager_ = nullptr; + ValidValue current_stage_time_; + uint64_t current_window_id_ = 1; + TinyStrings<> tiny_strings_; + EnumInspector enum_inspector_; + bool is_live_ = false; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/Checkpoint.hpp b/include/simdb/apps/argos/Checkpoint.hpp new file mode 100644 index 00000000..28cc6262 --- /dev/null +++ b/include/simdb/apps/argos/Checkpoint.hpp @@ -0,0 +1,645 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/Exceptions.hpp" +#include "simdb/apps/argos/CheckpointDeltas.hpp" +#include "simdb/apps/argos/CollectedData.hpp" +#include "simdb/utils/ValidValue.hpp" + +namespace simdb::argos { + +//! Encodings that C++ and python agree on. The python deserializers interpret +//! collected data using the header: +//! +//! [uint16_t cid] // collectable ID +//! [uint8_t action] // encoding +//! +//! See CheckpointEncodings.hpp for full action / byte layout documentation. +enum class Action : uint8_t { + // Tier 1: 0x00–0x0F — lifecycle / common (scalars + all collectables) + CLOSED = 0x00, + FULL = 0x01, + CARRY = 0x02, + // 0x03–0x0F reserved + + // Tier 2: 0x10–0x1F — any-container (identical wire for contig & sparse) + CONTAINER_SWAP = 0x10, + CONTAINER_MULTI_SWAP = 0x11, + // 0x12–0x1F reserved + + // Tier 3: 0x20–0x2F — contig-specific (FIFO queue semantics) + CONTIG_ARRIVE = 0x20, + CONTIG_DEPART = 0x21, + CONTIG_BOOKENDS = 0x22, + CONTIG_MIMO = 0x23, + // 0x24–0x2F reserved + + // Tier 4: 0x30–0x3F — sparse-specific (explicit bin indices) + SPARSE_REMOVE = 0x30, + SPARSE_ADD = 0x31, + SPARSE_MULTI_REMOVE = 0x32, + // 0x33–0x3F reserved +}; + +//! \class CollectableCheckpoint +class CollectableCheckpoint +{ +public: + /// Destroys this checkpoint node. + virtual ~CollectableCheckpoint() = default; + + /// Returns the previous checkpoint in the chain, or nullptr. + CollectableCheckpoint* getPrev() const { return prev_.get(); } + + /// Returns the next checkpoint in the chain, or nullptr. + CollectableCheckpoint* getNext() const { return next_; } + + /// Returns the collection window that created this checkpoint. + uint64_t getWindowID() const { return window_id_; } + + /// Drops the link to the previous checkpoint. + void detachPrev() { prev_.reset(); } + + /// Returns the shared_ptr to the previous checkpoint, if any. + const std::shared_ptr& getPrevShared() const { return prev_; } + +protected: + /// Links \p prev to this node and stores \p window_id. + CollectableCheckpoint(std::shared_ptr prev, uint64_t window_id) : + prev_(prev), + window_id_(window_id) + { + if (prev_) + { + prev_->next_ = this; + } + } + + /// Returns true if a CLOSED event occurred between \p data_prev and \p self. + template + static bool closedSincePrevDataCheckpoint_(const CheckpointT* self, const CheckpointT* data_prev) + { + if (!data_prev) + { + return false; + } + const CheckpointT* checkpoint = self->prev(); + while (checkpoint && checkpoint != data_prev) + { + if (checkpoint->isClosedEvent()) + { + return true; + } + checkpoint = checkpoint->prev(); + } + return false; + } + + std::shared_ptr prev_; + CollectableCheckpoint* next_ = nullptr; + const uint64_t window_id_; +}; + +class ScalarCheckpoint final : public CollectableCheckpoint +{ +public: + /// Creates a data checkpoint holding a full scalar snapshot. + ScalarCheckpoint(std::shared_ptr prev, uint64_t window_id, const std::vector& full_bytes) : + CollectableCheckpoint(prev, window_id), + full_bytes_(full_bytes) + { + } + + /// Creates a lifecycle checkpoint recording a close transition. + ScalarCheckpoint(std::shared_ptr prev, uint64_t window_id, bool switched_to_closed) : + CollectableCheckpoint(prev, window_id), + lifecycle_change_(switched_to_closed) + { + } + + /// Returns the previous scalar checkpoint in the chain. + ScalarCheckpoint* prev() { return static_cast(getPrev()); } + + /// Returns the previous scalar checkpoint in the chain. + const ScalarCheckpoint* prev() const { return static_cast(getPrev()); } + + /// Returns the next scalar checkpoint in the chain. + ScalarCheckpoint* next() { return static_cast(getNext()); } + + /// Returns true when this node holds collected bytes rather than a lifecycle flag. + bool isDataCheckpoint() const { return !lifecycle_change_.isValid(); } + + /// Returns true when this node records a transition to closed. + bool isClosedEvent() const { return lifecycle_change_.isValid() && lifecycle_change_.getValue(); } + + /// Returns true for any lifecycle (close) checkpoint. + bool isLifecycleEvent() const { return lifecycle_change_.isValid(); } + + /// Encodes an unconditional FULL snapshot for heartbeat or absent-CID refresh. + std::unique_ptr encodeSnapshotForPipeline(uint16_t cid) const + { + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + buf.append(full_bytes_); + return encoded; + } + + /// Encodes CLOSED, FULL, or CARRY by diffing against the prior data checkpoint. + std::unique_ptr encodeForPipeline(uint16_t cid, bool force_snapshot) + { + if (lifecycle_change_.isValid()) + { + return encodeLifecycleEvt_(cid); + } + + if (force_snapshot) + { + return encodeFull_(cid); + } + + const auto* data_prev = prevDataCheckpoint_(); + if (!data_prev) + { + return encodeFull_(cid); + } + + if (data_prev->full_bytes_ != full_bytes_) + { + return encodeFull_(cid); + } + + if (closedSincePrevDataCheckpoint_(this, data_prev)) + { + return encodeFull_(cid); + } + + return encodeDelta_(cid); + } + +private: + /// Walks backward to the nearest prior data checkpoint. + const ScalarCheckpoint* prevDataCheckpoint_() const + { + const auto* checkpoint = prev(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Encodes a CLOSED lifecycle wire record. + std::unique_ptr encodeLifecycleEvt_(uint16_t cid) + { + assert(lifecycle_change_.isValid() && lifecycle_change_.getValue()); + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + return encoded; + } + + /// Encodes a FULL snapshot of the current scalar bytes. + std::unique_ptr encodeFull_(uint16_t cid) + { + assert(!full_bytes_.empty()); + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + buf.append(full_bytes_); + return encoded; + } + + /// Encodes CARRY when bytes are unchanged since the prior data checkpoint. + std::unique_ptr encodeDelta_(uint16_t cid) + { + const auto* data_prev = prevDataCheckpoint_(); + assert(data_prev && data_prev->full_bytes_ == full_bytes_ && !full_bytes_.empty()); + (void)data_prev; + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::CARRY); + return encoded; + } + + std::vector full_bytes_; + ValidValue lifecycle_change_; +}; + +class ContigContainerCheckpoint : public CollectableCheckpoint +{ +public: + /// Creates a data checkpoint holding a full contig container snapshot. + ContigContainerCheckpoint(std::shared_ptr prev, uint64_t window_id, + const std::vector>& full_bytes) : + CollectableCheckpoint(prev, window_id), + full_bytes_(full_bytes) + { + } + + /// Creates a lifecycle checkpoint recording a close transition. + ContigContainerCheckpoint(std::shared_ptr prev, uint64_t window_id, + bool switched_to_closed) : + CollectableCheckpoint(prev, window_id), + lifecycle_change_(switched_to_closed) + { + } + + /// Returns the previous contig checkpoint in the chain. + ContigContainerCheckpoint* prev() { return static_cast(getPrev()); } + + /// Returns the previous contig checkpoint in the chain. + const ContigContainerCheckpoint* prev() const { return static_cast(getPrev()); } + + /// Returns the next contig checkpoint in the chain. + ContigContainerCheckpoint* next() { return static_cast(getNext()); } + + /// Returns true when this node holds collected bytes rather than a lifecycle flag. + bool isDataCheckpoint() const { return !lifecycle_change_.isValid(); } + + /// Returns true when this node records a transition to closed. + bool isClosedEvent() const { return lifecycle_change_.isValid() && lifecycle_change_.getValue(); } + + /// Returns true for any lifecycle (close) checkpoint. + bool isLifecycleEvent() const { return lifecycle_change_.isValid(); } + + /// Encodes an unconditional FULL snapshot for heartbeat or absent-CID refresh. + std::unique_ptr encodeSnapshotForPipeline(uint16_t cid) const + { + assert(isDataCheckpoint()); + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + appendContigBins_(buf, full_bytes_); + return encoded; + } + + /// Encodes CLOSED, FULL, or a contig delta by diffing against the prior data checkpoint. + std::unique_ptr encodeForPipeline(uint16_t cid, bool force_snapshot) + { + if (lifecycle_change_.isValid()) + { + return encodeLifecycleEvt_(cid); + } + + if (force_snapshot) + { + return encodeFull_(cid); + } + + auto* data_prev = prevDataCheckpoint_(); + if (!data_prev) + { + return encodeFull_(cid); + } + + if (closedSincePrevDataCheckpoint_(this, data_prev)) + { + return encodeFull_(cid); + } + + const auto classification = classifyContigChange(data_prev->full_bytes_, full_bytes_); + if (classification.kind == ContigDeltaKind::FULL) + { + return encodeFull_(cid); + } + + return encodeDelta_(cid, classification); + } + +private: + /// Walks backward to the nearest prior data checkpoint. + const ContigContainerCheckpoint* prevDataCheckpoint_() const + { + const auto* checkpoint = prev(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Maps a contig delta classification to its wire Action byte. + static Action contigActionFromKind_(ContigDeltaKind kind) + { + switch (kind) + { + case ContigDeltaKind::CARRY: + return Action::CARRY; + case ContigDeltaKind::SWAP: + return Action::CONTAINER_SWAP; + case ContigDeltaKind::MULTI_SWAP: + return Action::CONTAINER_MULTI_SWAP; + case ContigDeltaKind::ARRIVE: + return Action::CONTIG_ARRIVE; + case ContigDeltaKind::DEPART: + return Action::CONTIG_DEPART; + case ContigDeltaKind::BOOKENDS: + return Action::CONTIG_BOOKENDS; + case ContigDeltaKind::MIMO: + return Action::CONTIG_MIMO; + case ContigDeltaKind::FULL: + break; + } + throw DBException("Invalid contig delta kind"); + } + + /// Appends the FULL tail for the current contig snapshot to \p buf. + void appendFullTail_(StreamBuffer& buf) const { appendContigBins_(buf, full_bytes_); } + + /// Appends contig bins as [count][bin bytes]... to \p buf. + static void appendContigBins_(StreamBuffer& buf, const std::vector>& bins) + { + const auto size = countContigElements(bins); + buf.append(size); + for (uint16_t i = 0; i < size; ++i) + { + buf.append(bins[i]); + } + } + + /// Encodes a CLOSED lifecycle wire record. + std::unique_ptr encodeLifecycleEvt_(uint16_t cid) + { + assert(lifecycle_change_.isValid() && lifecycle_change_.getValue()); + (void)cid; + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + return encoded; + } + + /// Encodes a FULL snapshot of the current contig container. + std::unique_ptr encodeFull_(uint16_t cid) + { + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + appendFullTail_(buf); + return encoded; + } + + /// Encodes a classified contig delta (SWAP, MIMO, etc.) to the wire buffer. + std::unique_ptr encodeDelta_(uint16_t cid, const ContigDeltaClassification& classification) + { + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + const auto action = contigActionFromKind_(classification.kind); + buf.append(action); + + switch (action) + { + case Action::CONTAINER_SWAP: + assert(classification.swap_index.isValid()); + assert(!classification.payload.empty()); + buf.append(classification.swap_index.getValue()); + buf.append(classification.payload); + break; + case Action::CONTAINER_MULTI_SWAP: + assert(classification.swap_indices.size() >= 2); + assert(classification.swap_indices.size() == classification.swap_payloads.size()); + buf.append(static_cast(classification.swap_indices.size())); + for (size_t i = 0; i < classification.swap_indices.size(); ++i) + { + buf.append(classification.swap_indices[i]); + buf.append(classification.swap_payloads[i]); + } + break; + case Action::CONTIG_ARRIVE: + case Action::CONTIG_BOOKENDS: + assert(!classification.payload.empty()); + buf.append(classification.payload); + break; + case Action::CONTIG_MIMO: + buf.append(classification.depart_count); + buf.append(classification.arrive_count); + for (const auto& arrive_payload : classification.arrive_payloads) + { + buf.append(arrive_payload); + } + break; + case Action::CONTIG_DEPART: + case Action::CARRY: + break; + default: + throw DBException("Unexpected contig delta action"); + } + + return encoded; + } + + std::vector> full_bytes_; + ValidValue lifecycle_change_; +}; + +class SparseContainerCheckpoint : public CollectableCheckpoint +{ +public: + /// Creates a data checkpoint holding a full sparse container snapshot. + SparseContainerCheckpoint(std::shared_ptr prev, uint64_t window_id, + const std::map>& full_bytes) : + CollectableCheckpoint(prev, window_id), + full_bytes_(full_bytes) + { + } + + /// Creates a lifecycle checkpoint recording a close transition. + SparseContainerCheckpoint(std::shared_ptr prev, uint64_t window_id, + bool switched_to_closed) : + CollectableCheckpoint(prev, window_id), + lifecycle_change_(switched_to_closed) + { + } + + /// Returns the previous sparse checkpoint in the chain. + SparseContainerCheckpoint* prev() { return static_cast(getPrev()); } + + /// Returns the previous sparse checkpoint in the chain. + const SparseContainerCheckpoint* prev() const { return static_cast(getPrev()); } + + /// Returns the next sparse checkpoint in the chain. + SparseContainerCheckpoint* next() { return static_cast(getNext()); } + + /// Returns true when this node holds collected bytes rather than a lifecycle flag. + bool isDataCheckpoint() const { return !lifecycle_change_.isValid(); } + + /// Returns true when this node records a transition to closed. + bool isClosedEvent() const { return lifecycle_change_.isValid() && lifecycle_change_.getValue(); } + + /// Returns true for any lifecycle (close) checkpoint. + bool isLifecycleEvent() const { return lifecycle_change_.isValid(); } + + /// Encodes an unconditional FULL snapshot for heartbeat or absent-CID refresh. + std::unique_ptr encodeSnapshotForPipeline(uint16_t cid) const + { + assert(isDataCheckpoint()); + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + appendFullTail_(buf); + return encoded; + } + + /// Encodes CLOSED, FULL, or a sparse delta by diffing against the prior data checkpoint. + std::unique_ptr encodeForPipeline(uint16_t cid, bool force_snapshot) + { + if (lifecycle_change_.isValid()) + { + return encodeLifecycleEvt_(cid); + } + + if (force_snapshot) + { + return encodeFull_(cid); + } + + const auto* data_prev = prevDataCheckpoint_(); + if (!data_prev) + { + return encodeFull_(cid); + } + + if (closedSincePrevDataCheckpoint_(this, data_prev)) + { + return encodeFull_(cid); + } + + const auto classification = classifySparseChange(data_prev->full_bytes_, full_bytes_); + if (classification.kind == SparseDeltaKind::FULL) + { + return encodeFull_(cid); + } + + return encodeDelta_(cid, classification); + } + +private: + /// Walks backward to the nearest prior data checkpoint. + const SparseContainerCheckpoint* prevDataCheckpoint_() const + { + const auto* checkpoint = prev(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Maps a sparse delta classification to its wire Action byte. + static Action sparseActionFromKind_(SparseDeltaKind kind) + { + switch (kind) + { + case SparseDeltaKind::CARRY: + return Action::CARRY; + case SparseDeltaKind::SWAP: + return Action::CONTAINER_SWAP; + case SparseDeltaKind::MULTI_SWAP: + return Action::CONTAINER_MULTI_SWAP; + case SparseDeltaKind::REMOVE: + return Action::SPARSE_REMOVE; + case SparseDeltaKind::ADD: + return Action::SPARSE_ADD; + case SparseDeltaKind::MULTI_REMOVE: + return Action::SPARSE_MULTI_REMOVE; + case SparseDeltaKind::FULL: + break; + } + throw DBException("Invalid sparse delta kind"); + } + + /// Appends the FULL tail for the current sparse snapshot to \p buf. + void appendFullTail_(StreamBuffer& buf) const { appendSparseBins_(buf, full_bytes_); } + + /// Appends sparse bins as [count][idx][bin bytes]... to \p buf. + static void appendSparseBins_(StreamBuffer& buf, const std::map>& bins) + { + const auto size = countSparseElements(bins); + buf.append(size); + for (const auto& [bin_idx, bin_bytes] : bins) + { + if (!bin_bytes.empty()) + { + buf.append(bin_idx); + buf.append(bin_bytes); + } + } + } + + /// Encodes a CLOSED lifecycle wire record. + std::unique_ptr encodeLifecycleEvt_(uint16_t cid) + { + assert(lifecycle_change_.isValid() && lifecycle_change_.getValue()); + (void)cid; + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + return encoded; + } + + /// Encodes a FULL snapshot of the current sparse container. + std::unique_ptr encodeFull_(uint16_t cid) + { + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + buf.append(Action::FULL); + appendFullTail_(buf); + return encoded; + } + + /// Encodes a classified sparse delta (SWAP, ADD, etc.) to the wire buffer. + std::unique_ptr encodeDelta_(uint16_t cid, const SparseDeltaClassification& classification) + { + auto encoded = std::make_unique(cid); + auto& buf = encoded->getBuffer(); + const auto action = sparseActionFromKind_(classification.kind); + buf.append(action); + + switch (action) + { + case Action::CONTAINER_SWAP: + assert(classification.bin_index.isValid()); + assert(!classification.payload.empty()); + buf.append(classification.bin_index.getValue()); + buf.append(classification.payload); + break; + case Action::CONTAINER_MULTI_SWAP: + assert(classification.bin_indices.size() >= 2); + assert(classification.bin_indices.size() == classification.payloads.size()); + buf.append(static_cast(classification.bin_indices.size())); + for (size_t i = 0; i < classification.bin_indices.size(); ++i) + { + buf.append(classification.bin_indices[i]); + buf.append(classification.payloads[i]); + } + break; + case Action::SPARSE_REMOVE: + assert(classification.bin_index.isValid()); + buf.append(classification.bin_index.getValue()); + break; + case Action::SPARSE_ADD: + assert(classification.bin_index.isValid()); + assert(!classification.payload.empty()); + buf.append(classification.bin_index.getValue()); + buf.append(classification.payload); + break; + case Action::SPARSE_MULTI_REMOVE: + assert(classification.bin_indices.size() >= 2); + buf.append(static_cast(classification.bin_indices.size())); + for (const auto idx : classification.bin_indices) + { + buf.append(idx); + } + break; + case Action::CARRY: + break; + default: + throw DBException("Unexpected sparse delta action"); + } + + return encoded; + } + + std::map> full_bytes_; + ValidValue lifecycle_change_; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/CheckpointDeltas.hpp b/include/simdb/apps/argos/CheckpointDeltas.hpp new file mode 100644 index 00000000..719ac00b --- /dev/null +++ b/include/simdb/apps/argos/CheckpointDeltas.hpp @@ -0,0 +1,565 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/Exceptions.hpp" +#include "simdb/utils/ValidValue.hpp" + +#include +#include +#include +#include + +namespace simdb::argos { + +//! Result of comparing two consecutive scalar payloads. +enum class ScalarDeltaKind { UNCHANGED, CHANGED }; + +//! Compare previous and current scalar payloads. +//! Returns CHANGED when \p prev is empty (first collection has no baseline). +//! Scalars only support CARRY when UNCHANGED; CHANGED implies a FULL snapshot. +inline ScalarDeltaKind classifyScalarChange(const std::vector& prev, const std::vector& curr) +{ + if (prev.empty()) + { + return ScalarDeltaKind::CHANGED; + } + + return prev == curr ? ScalarDeltaKind::UNCHANGED : ScalarDeltaKind::CHANGED; +} + +inline std::ostream& operator<<(std::ostream& os, ScalarDeltaKind kind) +{ + switch (kind) + { + case ScalarDeltaKind::UNCHANGED: + return os << "UNCHANGED"; + case ScalarDeltaKind::CHANGED: + return os << "CHANGED"; + } + throw DBException("Invalid ScalarDeltaKind"); +} + +//! Contiguous-container delta kinds. +enum class ContigDeltaKind { + CARRY, + SWAP, + MULTI_SWAP, + BOOKENDS, + ARRIVE, + DEPART, + MIMO, + FULL, +}; + +//! Result of classifying a contig container transition. +struct ContigDeltaClassification +{ + ContigDeltaKind kind = ContigDeltaKind::FULL; + simdb::ValidValue swap_index; + std::vector payload; + uint8_t depart_count = 0; + uint8_t arrive_count = 0; + std::vector> arrive_payloads; + std::vector swap_indices; + std::vector> swap_payloads; +}; + +inline uint16_t countContigElements(const std::vector>& contig_bins) +{ + uint64_t count = 0; + for (const auto& bytes : contig_bins) + { + if (!bytes.empty()) + { + ++count; + } else + { + break; + } + } + assert(count <= UINT16_MAX); + return count; +} + +inline bool contigPrefixMatchesAfterDepart(const std::vector>& prev, + const std::vector>& curr, uint16_t prev_size, + uint16_t curr_size, uint16_t depart_count) +{ + const auto overlap = static_cast(prev_size - depart_count); + if (overlap + static_cast(curr_size - overlap) != curr_size) + { + return false; + } + for (uint16_t i = 0; i < overlap; ++i) + { + if (curr[i] != prev[i + depart_count]) + { + return false; + } + } + return true; +} + +//! Classify how a contiguous container changed between consecutive collections. +//! +//! Returns FULL when \p prev is empty (no baseline). Heartbeat forcing is handled +//! by the checkpointer, not this helper. +inline ContigDeltaClassification classifyContigChange(const std::vector>& prev, + const std::vector>& curr) +{ + ContigDeltaClassification result; + + if (prev.empty()) + { + result.kind = ContigDeltaKind::FULL; + return result; + } + + const auto curr_size = countContigElements(curr); + const auto prev_size = countContigElements(prev); + + if (curr_size == prev_size) + { + ValidValue changed_idx; + uint16_t num_changes = 0; + std::vector changed_idxs; + for (uint16_t i = 0; i < curr_size; ++i) + { + if (curr[i] != prev[i]) + { + changed_idxs.push_back(i); + changed_idx = i; + ++num_changes; + if (num_changes > 1) + { + changed_idx.clearValid(); + } + } + } + + if (num_changes == 0) + { + result.kind = ContigDeltaKind::CARRY; + return result; + } + + if (num_changes == 1) + { + result.kind = ContigDeltaKind::SWAP; + result.swap_index = changed_idx.getValue(); + result.payload = curr[changed_idx.getValue()]; + return result; + } + + if (curr_size > 0) + { + bool bookends = true; + for (size_t i = 0; i + 1 < curr_size; ++i) + { + if (curr[i] != prev[i + 1]) + { + bookends = false; + break; + } + } + + if (bookends) + { + result.kind = ContigDeltaKind::BOOKENDS; + result.payload = curr[curr_size - 1]; + return result; + } + } + + result.kind = ContigDeltaKind::MULTI_SWAP; + result.swap_indices = std::move(changed_idxs); + result.swap_payloads.reserve(result.swap_indices.size()); + for (const auto idx : result.swap_indices) + { + result.swap_payloads.push_back(curr[idx]); + } + return result; + } + + if (curr_size == prev_size + 1) + { + bool arrive = true; + for (uint16_t i = 0; i < prev_size; ++i) + { + if (curr[i] != prev[i]) + { + arrive = false; + break; + } + } + + if (arrive) + { + result.kind = ContigDeltaKind::ARRIVE; + result.payload = curr[curr_size - 1]; + return result; + } + } else if (prev_size == curr_size + 1) + { + bool depart = true; + for (uint16_t i = 0; i < curr_size; ++i) + { + if (curr[i] != prev[i + 1]) + { + depart = false; + break; + } + } + + if (depart) + { + result.kind = ContigDeltaKind::DEPART; + return result; + } + } + + for (uint16_t depart_count = 0; depart_count <= prev_size; ++depart_count) + { + const int64_t arrive_count64 = static_cast(curr_size) - static_cast(prev_size) + depart_count; + if (arrive_count64 < 0 || arrive_count64 > UINT8_MAX) + { + continue; + } + + const auto arrive_count = static_cast(arrive_count64); + if (depart_count + arrive_count <= 1) + { + continue; + } + if (depart_count == 1 && arrive_count == 1 && curr_size == prev_size) + { + continue; + } + + if (!contigPrefixMatchesAfterDepart(prev, curr, prev_size, curr_size, depart_count)) + { + continue; + } + + const auto overlap = static_cast(prev_size - depart_count); + result.kind = ContigDeltaKind::MIMO; + result.depart_count = static_cast(depart_count); + result.arrive_count = arrive_count; + result.arrive_payloads.reserve(arrive_count); + for (uint8_t j = 0; j < arrive_count; ++j) + { + result.arrive_payloads.push_back(curr[overlap + j]); + } + return result; + } + + result.kind = ContigDeltaKind::FULL; + return result; +} + +inline std::ostream& operator<<(std::ostream& os, ContigDeltaKind kind) +{ + switch (kind) + { + case ContigDeltaKind::CARRY: + return os << "CARRY"; + case ContigDeltaKind::SWAP: + return os << "SWAP"; + case ContigDeltaKind::MULTI_SWAP: + return os << "MULTI_SWAP"; + case ContigDeltaKind::BOOKENDS: + return os << "BOOKENDS"; + case ContigDeltaKind::ARRIVE: + return os << "ARRIVE"; + case ContigDeltaKind::DEPART: + return os << "DEPART"; + case ContigDeltaKind::MIMO: + return os << "MIMO"; + case ContigDeltaKind::FULL: + return os << "FULL"; + } + throw DBException("Invalid ContigDeltaKind"); +} + +//! Sparse-container delta kinds. +enum class SparseDeltaKind { + CARRY, + SWAP, + MULTI_SWAP, + REMOVE, + ADD, + MULTI_REMOVE, + FULL, +}; + +//! Result of classifying a sparse container transition. +struct SparseDeltaClassification +{ + SparseDeltaKind kind = SparseDeltaKind::FULL; + simdb::ValidValue bin_index; + std::vector payload; + std::vector bin_indices; + std::vector> payloads; +}; + +inline uint16_t countSparseElements(const std::map>& sparse_bins) +{ + uint64_t count = 0; + for (const auto& [_, bytes] : sparse_bins) + { + if (!bytes.empty()) + { + ++count; + } + } + assert(count <= UINT16_MAX); + return static_cast(count); +} + +//! Classify how a sparse container changed between consecutive collections. +//! +//! Returns FULL when \p prev is empty (no baseline). Heartbeat forcing is handled +//! by the checkpointer, not this helper. +inline SparseDeltaClassification classifySparseChange(const std::map>& prev, + const std::map>& curr) +{ + SparseDeltaClassification result; + + if (prev.empty()) + { + result.kind = SparseDeltaKind::FULL; + return result; + } + + const auto curr_size = countSparseElements(curr); + const auto prev_size = countSparseElements(prev); + + if (curr == prev) + { + result.kind = SparseDeltaKind::CARRY; + return result; + } + + if (curr_size + 1 == prev_size) + { + ValidValue removed_idx; + uint16_t removed_count = 0; + bool other_change = false; + for (const auto& [prev_idx, prev_bytes] : prev) + { + if (auto it = curr.find(prev_idx); it == curr.end()) + { + ++removed_count; + removed_idx = prev_idx; + } else if (prev_bytes != it->second) + { + other_change = true; + break; + } + } + + if (!other_change && removed_count == 1) + { + for (const auto& [curr_idx, _] : curr) + { + if (prev.find(curr_idx) == prev.end()) + { + other_change = true; + break; + } + } + } + + if (!other_change && removed_count == 1) + { + result.kind = SparseDeltaKind::REMOVE; + result.bin_index = removed_idx.getValue(); + return result; + } + } + + if (curr_size == prev_size + 1) + { + ValidValue added_idx; + uint16_t added_count = 0; + bool other_change = false; + for (const auto& [curr_idx, curr_bytes] : curr) + { + if (auto it = prev.find(curr_idx); it == prev.end()) + { + ++added_count; + added_idx = curr_idx; + result.payload = curr_bytes; + } else if (it->second != curr_bytes) + { + other_change = true; + break; + } + } + + if (!other_change && added_count == 1) + { + for (const auto& [prev_idx, _] : prev) + { + if (curr.find(prev_idx) == curr.end()) + { + other_change = true; + break; + } + } + } + + if (!other_change && added_count == 1) + { + result.kind = SparseDeltaKind::ADD; + result.bin_index = added_idx.getValue(); + return result; + } + } + + if (curr_size + 1 <= prev_size) + { + std::vector removed_idxs; + bool other_change = false; + for (const auto& [prev_idx, prev_bytes] : prev) + { + if (auto it = curr.find(prev_idx); it == curr.end()) + { + removed_idxs.push_back(prev_idx); + } else if (prev_bytes != it->second) + { + other_change = true; + break; + } + } + + if (!other_change && removed_idxs.size() >= 2 && + removed_idxs.size() == static_cast(prev_size - curr_size)) + { + for (const auto& [curr_idx, _] : curr) + { + if (prev.find(curr_idx) == prev.end()) + { + other_change = true; + break; + } + } + } + + if (!other_change && removed_idxs.size() >= 2 && + removed_idxs.size() == static_cast(prev_size - curr_size)) + { + result.kind = SparseDeltaKind::MULTI_REMOVE; + result.bin_indices = std::move(removed_idxs); + return result; + } + } + + if (curr_size != prev_size) + { + result.kind = SparseDeltaKind::FULL; + return result; + } + + std::vector changed_idxs; + std::vector removed_idxs; + std::vector added_idxs; + + for (const auto& [prev_idx, prev_bytes] : prev) + { + if (auto it = curr.find(prev_idx); it != curr.end()) + { + if (prev_bytes != it->second) + { + changed_idxs.push_back(prev_idx); + } + } else + { + removed_idxs.push_back(prev_idx); + } + } + + for (const auto& [curr_idx, _] : curr) + { + if (prev.find(curr_idx) == prev.end()) + { + added_idxs.push_back(curr_idx); + } + } + + if (!added_idxs.empty() || !removed_idxs.empty()) + { + if (added_idxs.size() == 1 && removed_idxs.empty() && changed_idxs.empty()) + { + result.kind = SparseDeltaKind::ADD; + result.bin_index = added_idxs[0]; + result.payload = curr.at(added_idxs[0]); + return result; + } + + if (removed_idxs.size() == 1 && added_idxs.empty() && changed_idxs.empty()) + { + result.kind = SparseDeltaKind::REMOVE; + result.bin_index = removed_idxs[0]; + return result; + } + + if (removed_idxs.size() >= 2 && added_idxs.empty() && changed_idxs.empty()) + { + result.kind = SparseDeltaKind::MULTI_REMOVE; + result.bin_indices = std::move(removed_idxs); + return result; + } + + result.kind = SparseDeltaKind::FULL; + return result; + } + + if (changed_idxs.size() == 1) + { + result.kind = SparseDeltaKind::SWAP; + result.bin_index = changed_idxs[0]; + result.payload = curr.at(changed_idxs[0]); + return result; + } + + if (changed_idxs.size() >= 2) + { + result.kind = SparseDeltaKind::MULTI_SWAP; + result.bin_indices = std::move(changed_idxs); + result.payloads.reserve(result.bin_indices.size()); + for (const auto idx : result.bin_indices) + { + result.payloads.push_back(curr.at(idx)); + } + return result; + } + + result.kind = SparseDeltaKind::FULL; + return result; +} + +inline std::ostream& operator<<(std::ostream& os, SparseDeltaKind kind) +{ + switch (kind) + { + case SparseDeltaKind::CARRY: + return os << "CARRY"; + case SparseDeltaKind::SWAP: + return os << "SWAP"; + case SparseDeltaKind::MULTI_SWAP: + return os << "MULTI_SWAP"; + case SparseDeltaKind::REMOVE: + return os << "REMOVE"; + case SparseDeltaKind::ADD: + return os << "ADD"; + case SparseDeltaKind::MULTI_REMOVE: + return os << "MULTI_REMOVE"; + case SparseDeltaKind::FULL: + return os << "FULL"; + } + throw DBException("Invalid SparseDeltaKind"); +} + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/CheckpointEncodings.hpp b/include/simdb/apps/argos/CheckpointEncodings.hpp new file mode 100644 index 00000000..9ff54abc --- /dev/null +++ b/include/simdb/apps/argos/CheckpointEncodings.hpp @@ -0,0 +1,109 @@ +// -*- C++ -*- + +// clang-format off + +#pragma once + +/*! + * \file CheckpointEncodings.hpp + * \brief Reference for wire delta encodings emitted by PipelineStager checkpointers. + * + * The PipelineStager stage in ArgosCollector calls each collectable's checkpointer + * (`ScalarCheckpointer`, `ContigContainerCheckpointer`, `SparseContainerCheckpointer`) + * to turn staged checkpoints into wire records. Each record is prefixed with: + * + * [uint16_t cid] collectable ID + * [uint8_t action] encoding (see Action in Checkpoint.hpp) + * + * Delta classification lives in CheckpointDeltas.hpp (`classifyContigChange`, + * `classifySparseChange`). Heartbeat forcing, absent-CID refresh, and CLOSED priming + * are handled in Checkpointer.hpp (`CollectableCheckpointer`). + * + * \par Encoding reference + * + * \verbatim + * Encoding Applies To Used When + * ------------------------ ---------------- ------------------------------------------------------------------------------ + * CLOSED All Collectable transitions to closed (`closeRecord`). The checkpointer may emit a + * priming FULL first when the last snapshot is outside the current heartbeat + * window. + * FULL All First data checkpoint (no prior baseline); heartbeat forces a snapshot; delta + * chain reaches `heartbeat - 1` consecutive non-FULL records; a CLOSED event + * occurred since the prior data checkpoint; scalar bytes changed; container + * change pattern is not covered by any delta; collectable did not stage this + * window but heartbeat refresh requires re-emitting state + * (`shouldRefreshAbsentCid_`). + * CARRY All Payload unchanged since the prior data checkpoint and none of the FULL forcing + * rules apply. + * CONTAINER_SWAP Contig, Sparse Same occupied count; exactly one bin's value changed (same index). + * CONTAINER_MULTI_SWAP Contig, Sparse Same occupied count; two or more bin values changed with no adds or removes. + * For contig, only when the change is not a CONTIG_BOOKENDS shift pattern. + * CONTIG_ARRIVE Contig only Size +1; prefix unchanged; one new element appended at the tail. + * CONTIG_DEPART Contig only Size -1; front pop — `curr[i] == prev[i+1]` for all remaining indices. + * CONTIG_BOOKENDS Contig only Same size; shift-left with new tail — `curr[i] == prev[i+1]` for all but the + * last index (last bin is a new value). + * CONTIG_MIMO Contig only FIFO MIMO: D elements depart from the front and A arrive at the tail; after + * the depart shift, prefixes match; `D + A > 1`; not a simpler single-op pattern + * (e.g. not 1 depart + 1 arrive at same size). + * SPARSE_REMOVE Sparse only Exactly one bin removed; no value changes or adds elsewhere. + * SPARSE_ADD Sparse only Exactly one new bin added; all overlapping bins unchanged. + * SPARSE_MULTI_REMOVE Sparse only Two or more bins removed; no adds and no value changes on survivors. + * \endverbatim + * + * Scalars only ever emit CLOSED, FULL, or CARRY. Contig and sparse share tier-2 encodings + * (CONTAINER_SWAP, CONTAINER_MULTI_SWAP) with identical wire layout; contig adds tier-3 + * FIFO encodings; sparse adds tier-4 index-based encodings. + * + * \par FULL forcing (checkpointer layer) + * + * These conditions override delta classification and force FULL regardless of change size: + * + * - \c forceSnapshot_(sim_time) — either `distance_to_snapshot_ + 1 >= heartbeat_`, or the + * last FULL sim-time is outside the current heartbeat window (`shouldHeartbeatRefresh_`). + * - No prior data checkpoint — first collection for this CID. + * - Closed since prior data checkpoint — a CLOSED lifecycle node sits between the current + * data checkpoint and the previous one. + * - Absent CID refresh — collectable did not participate in this window but heartbeat + * policy still requires a wire record (replays latest snapshot as FULL, or CLOSED plus + * optional priming FULL if already closed). + * + * \par Wire payloads (after the action byte) + * + * | Action | Payload | + * |------------------------|--------------------------------------------------------| + * | CLOSED | | + * | CARRY | | + * | FULL (scalar) | [scalar bytes] | + * | FULL (contig) | [count][bin bytes]..[bin bytes] | + * | FULL (sparse) | [count][bin idx][bin bytes]..[bin idx][bin bytes] | + * | CONTAINER_SWAP | [bin idx][bin bytes] | + * | CONTAINER_MULTI_SWAP | [count][bin idx][bin bytes]..[bin idx][bin bytes] | + * | CONTIG_ARRIVE | [pushed bytes] | + * | CONTIG_DEPART | | + * | CONTIG_BOOKENDS | [pushed bytes] | + * | CONTIG_MIMO | [num popped][num pushed][pushed bytes]..[pushed bytes] | + * | SPARSE_REMOVE | [removed idx] | + * | SPARSE_ADD | [added idx][added bytes] | + * | SPARSE_MULTI_REMOVE | [num removed][removed idx]..[removed idx] | + * + * \par Action enum tiers (Checkpoint.hpp) + * + * - Tier 1 (`0x00`–`0x0F`): lifecycle / common — CLOSED, FULL, CARRY; `0x03`–`0x0F` reserved + * - Tier 2 (`0x10`–`0x1F`): any-container — CONTAINER_SWAP, CONTAINER_MULTI_SWAP; + * `0x12`–`0x1F` reserved + * - Tier 3 (`0x20`–`0x2F`): contig-specific — CONTIG_ARRIVE, CONTIG_DEPART, CONTIG_BOOKENDS, + * CONTIG_MIMO; `0x24`–`0x2F` reserved + * - Tier 4 (`0x30`–`0x3F`): sparse-specific — SPARSE_REMOVE, SPARSE_ADD, SPARSE_MULTI_REMOVE; + * `0x33`–`0x3F` reserved + * + * \par Related implementation files + * + * - Checkpoint.hpp: Action enum, encode/decode per checkpoint type + * - CheckpointDeltas.hpp: classifyContigChange, classifySparseChange + * - Checkpointer.hpp: heartbeat bookkeeping, encodeForPipeline orchestration + * - ArgosCollector.hpp: PipelineStager invokes checkpointer encoding + * - BlobReplayer.hpp: replayer for testing + * - blob_iterator.py: replayer for UI + */ + +// clang-format on diff --git a/include/simdb/apps/argos/Checkpointer.hpp b/include/simdb/apps/argos/Checkpointer.hpp new file mode 100644 index 00000000..42d85a61 --- /dev/null +++ b/include/simdb/apps/argos/Checkpointer.hpp @@ -0,0 +1,692 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/apps/argos/Checkpoint.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/utils/ValidValue.hpp" + +#include + +namespace simdb::argos { + +namespace detail { +/// Counts occupied front-packed bins in a contig snapshot. +inline uint16_t getContainerSize(const std::vector>& contig_bin_bytes) +{ + uint64_t size = 0; + for (const auto& bin_bytes : contig_bin_bytes) + { + if (!bin_bytes.empty()) + { + ++size; + } else + { + break; + } + } + assert(size <= UINT16_MAX); + return size; +} + +/// Counts occupied bins in a sparse snapshot. +inline uint16_t getContainerSize(const std::map>& sparse_bin_bytes) +{ + uint64_t size = 0; + for (const auto& [_, bin_bytes] : sparse_bin_bytes) + { + if (!bin_bytes.empty()) + { + ++size; + } + } + assert(size <= UINT16_MAX); + return size; +} +} // namespace detail + +//! \class CollectableCheckpointer +class CollectableCheckpointer +{ +public: + /// Destroys the checkpointer and its checkpoint chain. + virtual ~CollectableCheckpointer() = default; + + /// Appends a scalar snapshot checkpoint; \throws DBException on unsupported checkpointer types. + virtual void createCheckpoint(uint64_t window_id, const std::vector& scalar_bytes) + { + (void)window_id; + (void)scalar_bytes; + throw DBException("Not implemented"); + } + + /// Appends a contig container snapshot checkpoint; \throws DBException on unsupported checkpointer types. + virtual void createCheckpoint(uint64_t window_id, const std::vector>& contig_bin_bytes) + { + (void)window_id; + (void)contig_bin_bytes; + throw DBException("Not implemented"); + } + + /// Appends a sparse container snapshot checkpoint; \throws DBException on unsupported checkpointer types. + virtual void createCheckpoint(uint64_t window_id, const std::map>& sparse_bin_bytes) + { + (void)window_id; + (void)sparse_bin_bytes; + throw DBException("Not implemented"); + } + + /// Records a closed lifecycle event at \p window_id. + virtual void closeRecord(uint64_t window_id) = 0; + + /// Encodes wire records for \p cid at \p sim_time, using the checkpoint for \p window_id when present. + virtual std::vector> encodeForPipeline(uint64_t window_id, uint64_t sim_time, + uint16_t cid) = 0; + + /// Returns true if this collectable staged data in \p window_id. + virtual bool participatedInWindow(uint64_t window_id) const = 0; + + /// Writes collectable-specific metadata after simulation teardown; default is a no-op. + virtual void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager*) { (void)cid; } + +protected: + /// Stores the heartbeat interval used to force periodic FULL snapshots. + explicit CollectableCheckpointer(size_t heartbeat) : + heartbeat_(heartbeat) + { + } + + /// Finds the latest checkpoint in \p tail at or before \p window_id. + template + static CheckpointT* getAnchorForWindow_(uint64_t window_id, const std::shared_ptr& tail) + { + CheckpointT* anchor = tail.get(); + CheckpointT* best = nullptr; + while (anchor) + { + if (anchor->getWindowID() > window_id) + { + break; + } + best = anchor; + anchor = anchor->next(); + } + return best; + } + + /// Returns true if \p tail contains a checkpoint whose window equals \p window_id. + template + static bool participatedInWindow_(uint64_t window_id, const std::shared_ptr& tail) + { + if (!tail) + { + return false; + } + auto* anchor = getAnchorForWindow_(window_id, tail); + return anchor && anchor->getWindowID() == window_id; + } + + /// Locates the shared_ptr in \p head that owns \p raw. + template + static std::shared_ptr getSharedCheckpoint_(CheckpointT* raw, const std::shared_ptr& head) + { + std::shared_ptr sp = head; + while (sp) + { + if (sp.get() == raw) + { + return sp; + } + auto prev = sp->getPrevShared(); + if (!prev) + { + return nullptr; + } + sp = std::static_pointer_cast(prev); + } + return nullptr; + } + + /// Reads the action byte from an encoded wire record (after the CID prefix). + static Action readEncodedAction_(const CollectedData& encoded) + { + return static_cast(encoded.getData().at(sizeof(uint16_t))); + } + + /// Wraps a single encoded record in a one-element vector. + template static std::vector makeSingleElemVector_(T&& only) + { + std::vector out; + out.emplace_back(std::move(only)); + return out; + } + + /// Returns true when the next wire must be a FULL snapshot (delta limit or heartbeat window). + bool forceSnapshot_(uint64_t sim_time) const + { + return distance_to_snapshot_ + 1 >= heartbeat_ || shouldHeartbeatRefresh_(sim_time); + } + + /// Returns true when the last FULL wire is outside the current heartbeat window. + bool shouldHeartbeatRefresh_(uint64_t sim_time) const + { + if (!last_snapshot_sim_time_.isValid()) + { + return false; + } + if (sim_time <= last_snapshot_sim_time_.getValue()) + { + return false; + } + const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; + return last_snapshot_sim_time_.getValue() < window_lo; + } + + /// Returns true when a CID that skipped this window still needs a heartbeat refresh on the wire. + bool shouldRefreshAbsentCid_(uint64_t sim_time, bool tip_closed) const + { + if (shouldHeartbeatRefresh_(sim_time)) + { + return true; + } + if (!tip_closed || !last_record_closed_sim_time_.isValid() || + sim_time <= last_record_closed_sim_time_.getValue()) + { + return false; + } + const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; + return last_record_closed_sim_time_.getValue() < window_lo; + } + + /// Returns true when a CLOSED wire should be preceded by a priming FULL snapshot. + bool needsClosedPriming_(uint64_t sim_time) const + { + if (!last_snapshot_sim_time_.isValid()) + { + return true; + } + const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; + return last_snapshot_sim_time_.getValue() < window_lo; + } + + /// Updates delta-chain and heartbeat bookkeeping after a wire is emitted. + void recordCidSent_(Action action, uint64_t sim_time) + { + if (action == Action::FULL) + { + distance_to_snapshot_ = 0; + } else + { + ++distance_to_snapshot_; + } + + if (action == Action::FULL) + { + last_snapshot_sim_time_ = sim_time; + } else if (action == Action::CLOSED) + { + last_record_closed_sim_time_ = sim_time; + } + } + + const size_t heartbeat_; + size_t distance_to_snapshot_ = 0; + ValidValue last_snapshot_sim_time_; + ValidValue last_record_closed_sim_time_; +}; + +//! \class ScalarCheckpointer +class ScalarCheckpointer final : public CollectableCheckpointer +{ +public: + /// Constructs a scalar checkpointer with the given heartbeat interval. + ScalarCheckpointer(size_t heartbeat) : + CollectableCheckpointer(heartbeat) + { + } + + /// Appends a scalar data checkpoint for \p window_id. + void createCheckpoint(uint64_t window_id, const std::vector& scalar_bytes) override + { + head_ = std::make_shared(head_, window_id, scalar_bytes); + if (!tail_) + { + tail_ = head_; + } + } + + using CollectableCheckpointer::createCheckpoint; // un-hide the other two + + /// Appends a scalar close lifecycle checkpoint for \p window_id. + void closeRecord(uint64_t window_id) override + { + constexpr bool closed = true; + head_ = std::make_shared(head_, window_id, closed); + if (!tail_) + { + tail_ = head_; + } + } + + /// Returns true if this scalar collectable staged data in \p window_id. + bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } + + /// Encodes scalar wire records for \p cid at \p sim_time. + std::vector> encodeForPipeline(uint64_t window_id, uint64_t sim_time, + uint16_t cid) override + { + auto* anchor = getAnchorForWindow_(window_id, tail_); + if (anchor && anchor->getWindowID() == window_id) + { + return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); + } + + auto* tip = head_.get(); + const bool tip_closed = tip && tip->isClosedEvent(); + if (shouldRefreshAbsentCid_(sim_time, tip_closed)) + { + if (tip_closed) + { + return emitRecordClosed_(cid, sim_time); + } + + auto* latest = latestDataCheckpoint_(); + if (!latest) + { + return {}; + } + + auto encoded = latest->encodeSnapshotForPipeline(cid); + recordCidSent_(Action::FULL, sim_time); + return makeSingleElemVector_(std::move(encoded)); + } + + return {}; + } + +private: + /// Returns the newest scalar data checkpoint, skipping lifecycle-only nodes. + ScalarCheckpoint* latestDataCheckpoint_() const + { + auto* checkpoint = head_.get(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. + std::vector> emitRecordClosed_(uint16_t cid, uint64_t sim_time) + { + std::vector> out; + if (needsClosedPriming_(sim_time)) + { + if (auto* latest = latestDataCheckpoint_()) + { + out.push_back(latest->encodeSnapshotForPipeline(cid)); + recordCidSent_(Action::FULL, sim_time); + } + } + + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + recordCidSent_(Action::CLOSED, sim_time); + out.push_back(std::move(encoded)); + return out; + } + + /// Encodes the scalar checkpoint at \p anchor and trims the chain through that node. + std::vector> encodeForPipeline_(ScalarCheckpoint* anchor, uint16_t cid, + bool force_snapshot, uint64_t sim_time) + { + auto encoded = anchor->encodeForPipeline(cid, force_snapshot); + if (!encoded) + { + return {}; + } + + const auto action = readEncodedAction_(*encoded); + if (action == Action::CLOSED) + { + cleanupThrough_(anchor, action); + return emitRecordClosed_(cid, sim_time); + } + + recordCidSent_(action, sim_time); + cleanupThrough_(anchor, action); + return makeSingleElemVector_(std::move(encoded)); + } + + /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. + void cleanupThrough_(ScalarCheckpoint* anchor, Action action) + { + auto sp = getSharedCheckpoint_(anchor, head_); + if (!sp) + { + return; + } + if (action == Action::FULL) + { + sp->detachPrev(); + } + tail_ = sp; + } + + std::shared_ptr head_; + std::shared_ptr tail_; +}; + +//! \class ContigContainerCheckpointer +class ContigContainerCheckpointer final : public CollectableCheckpointer +{ +public: + /// Constructs a contig checkpointer with heartbeat and fixed capacity. + ContigContainerCheckpointer(size_t heartbeat, size_t capacity) : + CollectableCheckpointer(heartbeat), + capacity_(capacity) + { + } + + /// Appends a contig container data checkpoint for \p window_id. + void createCheckpoint(uint64_t window_id, const std::vector>& contig_bin_bytes) override + { + auto size = detail::getContainerSize(contig_bin_bytes); + assert(size <= capacity_); + (void)capacity_; + max_container_size_ = std::max(max_container_size_, size); + + head_ = std::make_shared(head_, window_id, contig_bin_bytes); + if (!tail_) + { + tail_ = head_; + } + } + + using CollectableCheckpointer::createCheckpoint; // un-hide the other two + + /// Appends a contig close lifecycle checkpoint for \p window_id. + void closeRecord(uint64_t window_id) override + { + constexpr bool closed = true; + head_ = std::make_shared(head_, window_id, closed); + if (!tail_) + { + tail_ = head_; + } + } + + /// Returns true if this contig collectable staged data in \p window_id. + bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } + + /// Encodes contig container wire records for \p cid at \p sim_time. + std::vector> encodeForPipeline(uint64_t window_id, uint64_t sim_time, + uint16_t cid) override + { + auto* anchor = getAnchorForWindow_(window_id, tail_); + if (anchor && anchor->getWindowID() == window_id) + { + return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); + } + + auto* tip = head_.get(); + const bool tip_closed = tip && tip->isClosedEvent(); + if (shouldRefreshAbsentCid_(sim_time, tip_closed)) + { + if (tip_closed) + { + return emitRecordClosed_(cid, sim_time); + } + + auto* latest = latestDataCheckpoint_(); + if (!latest) + { + return {}; + } + + auto encoded = latest->encodeSnapshotForPipeline(cid); + recordCidSent_(Action::FULL, sim_time); + return makeSingleElemVector_(std::move(encoded)); + } + + return {}; + } + + /// Writes observed max queue occupancy to QueueMaxSizes. + void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager* db_mgr) override + { + db_mgr->INSERT(SQL_TABLE("QueueMaxSizes"), SQL_VALUES((int)cid, (int)max_container_size_)); + } + +private: + /// Returns the newest contig data checkpoint, skipping lifecycle-only nodes. + ContigContainerCheckpoint* latestDataCheckpoint_() const + { + auto* checkpoint = head_.get(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. + std::vector> emitRecordClosed_(uint16_t cid, uint64_t sim_time) + { + std::vector> out; + if (needsClosedPriming_(sim_time)) + { + if (auto* latest = latestDataCheckpoint_()) + { + out.push_back(latest->encodeSnapshotForPipeline(cid)); + recordCidSent_(Action::FULL, sim_time); + } + } + + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + recordCidSent_(Action::CLOSED, sim_time); + out.push_back(std::move(encoded)); + return out; + } + + /// Encodes the contig checkpoint at \p anchor and trims the chain through that node. + std::vector> encodeForPipeline_(ContigContainerCheckpoint* anchor, uint16_t cid, + bool force_snapshot, uint64_t sim_time) + { + auto encoded = anchor->encodeForPipeline(cid, force_snapshot); + if (!encoded) + { + return {}; + } + + const auto action = readEncodedAction_(*encoded); + if (action == Action::CLOSED) + { + cleanupThrough_(anchor, action); + return emitRecordClosed_(cid, sim_time); + } + + recordCidSent_(action, sim_time); + cleanupThrough_(anchor, action); + return makeSingleElemVector_(std::move(encoded)); + } + + /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. + void cleanupThrough_(ContigContainerCheckpoint* anchor, Action action) + { + auto sp = getSharedCheckpoint_(anchor, head_); + if (!sp) + { + return; + } + if (action == Action::FULL) + { + sp->detachPrev(); + } + tail_ = sp; + } + + std::shared_ptr head_; + std::shared_ptr tail_; + uint16_t max_container_size_ = 0; + const uint16_t capacity_; +}; + +//! \class SparseContainerCheckpointer +class SparseContainerCheckpointer final : public CollectableCheckpointer +{ +public: + /// Constructs a sparse checkpointer with heartbeat and fixed capacity. + SparseContainerCheckpointer(size_t heartbeat, size_t capacity) : + CollectableCheckpointer(heartbeat), + capacity_(capacity) + { + } + + /// Appends a sparse container data checkpoint for \p window_id. + void createCheckpoint(uint64_t window_id, const std::map>& sparse_bin_bytes) override + { + auto size = detail::getContainerSize(sparse_bin_bytes); + assert(size <= capacity_); + (void)capacity_; + max_container_size_ = std::max(max_container_size_, size); + + head_ = std::make_shared(head_, window_id, sparse_bin_bytes); + if (!tail_) + { + tail_ = head_; + } + } + + using CollectableCheckpointer::createCheckpoint; // un-hide the other two + + /// Appends a sparse close lifecycle checkpoint for \p window_id. + void closeRecord(uint64_t window_id) override + { + constexpr bool closed = true; + head_ = std::make_shared(head_, window_id, closed); + if (!tail_) + { + tail_ = head_; + } + } + + /// Returns true if this sparse collectable staged data in \p window_id. + bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } + + /// Encodes sparse container wire records for \p cid at \p sim_time. + std::vector> encodeForPipeline(uint64_t window_id, uint64_t sim_time, + uint16_t cid) override + { + auto* anchor = getAnchorForWindow_(window_id, tail_); + if (anchor && anchor->getWindowID() == window_id) + { + return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); + } + + auto* tip = head_.get(); + const bool tip_closed = tip && tip->isClosedEvent(); + if (shouldRefreshAbsentCid_(sim_time, tip_closed)) + { + if (tip_closed) + { + return emitRecordClosed_(cid, sim_time); + } + + auto* latest = latestDataCheckpoint_(); + if (!latest) + { + return {}; + } + + auto encoded = latest->encodeSnapshotForPipeline(cid); + recordCidSent_(Action::FULL, sim_time); + return makeSingleElemVector_(std::move(encoded)); + } + + return {}; + } + + /// Writes observed max queue occupancy to QueueMaxSizes. + void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager* db_mgr) override + { + db_mgr->INSERT(SQL_TABLE("QueueMaxSizes"), SQL_VALUES((int)cid, (int)max_container_size_)); + } + +private: + /// Returns the newest sparse data checkpoint, skipping lifecycle-only nodes. + SparseContainerCheckpoint* latestDataCheckpoint_() const + { + auto* checkpoint = head_.get(); + while (checkpoint && !checkpoint->isDataCheckpoint()) + { + checkpoint = checkpoint->prev(); + } + return checkpoint; + } + + /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. + std::vector> emitRecordClosed_(uint16_t cid, uint64_t sim_time) + { + std::vector> out; + if (needsClosedPriming_(sim_time)) + { + if (auto* latest = latestDataCheckpoint_()) + { + out.push_back(latest->encodeSnapshotForPipeline(cid)); + recordCidSent_(Action::FULL, sim_time); + } + } + + auto encoded = std::make_unique(cid); + encoded->getBuffer().append(Action::CLOSED); + recordCidSent_(Action::CLOSED, sim_time); + out.push_back(std::move(encoded)); + return out; + } + + /// Encodes the sparse checkpoint at \p anchor and trims the chain through that node. + std::vector> encodeForPipeline_(SparseContainerCheckpoint* anchor, uint16_t cid, + bool force_snapshot, uint64_t sim_time) + { + auto encoded = anchor->encodeForPipeline(cid, force_snapshot); + if (!encoded) + { + return {}; + } + + const auto action = readEncodedAction_(*encoded); + if (action == Action::CLOSED) + { + cleanupThrough_(anchor, action); + return emitRecordClosed_(cid, sim_time); + } + + recordCidSent_(action, sim_time); + cleanupThrough_(anchor, action); + return makeSingleElemVector_(std::move(encoded)); + } + + /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. + void cleanupThrough_(SparseContainerCheckpoint* anchor, Action action) + { + auto sp = getSharedCheckpoint_(anchor, head_); + if (!sp) + { + return; + } + if (action == Action::FULL) + { + sp->detachPrev(); + } + tail_ = sp; + } + + std::shared_ptr head_; + std::shared_ptr tail_; + uint16_t max_container_size_ = 0; + const uint16_t capacity_; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/CollectedData.hpp b/include/simdb/apps/argos/CollectedData.hpp new file mode 100644 index 00000000..81ec0ddc --- /dev/null +++ b/include/simdb/apps/argos/CollectedData.hpp @@ -0,0 +1,49 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/apps/argos/StreamBuffer.hpp" + +#include + +namespace simdb::argos { + +/// \class CollectedData +/// \brief Wrapper around StreamBuffer which provides collection-specific APIs +class CollectedData +{ +public: + CollectedData(uint16_t cid) : + CollectedData(cid, nullptr) + { + } + + CollectedData(uint16_t cid, TinyStrings<>* tiny_strings) : + cid_(cid), + buffer_(data_, tiny_strings) + { + reset(); + } + + CollectedData(CollectedData&&) = default; + CollectedData(const CollectedData&) = delete; + + uint16_t getCID() const { return cid_; } + + const std::vector& getData() const { return data_; } + + StreamBuffer& getBuffer() { return buffer_; } + + void reset() + { + data_.clear(); + buffer_.append(cid_); + } + +private: + uint16_t cid_ = 0; + std::vector data_; + StreamBuffer buffer_; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/CollectionBuffer.hpp b/include/simdb/apps/argos/CollectionBuffer.hpp deleted file mode 100644 index f6766d44..00000000 --- a/include/simdb/apps/argos/CollectionBuffer.hpp +++ /dev/null @@ -1,90 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/utils/MetaStructs.hpp" - -#include -#include -#include -#include - -namespace simdb { - -/*! - * \class CollectionBuffer - * - * \brief A helper class to allow collections to write their data - * to a single buffer before sending it to the background - * thread for database insertion. We pack everything into - * one buffer to minimize the number of entries we have in - * the database, and to get maximum compression. - */ -class CollectionBuffer -{ -public: - CollectionBuffer(std::vector& buffer) : - buffer_(buffer) - { - buffer_.clear(); - buffer_.reserve(buffer_.capacity()); - } - - /// Note that the elem_id corresponds to a database record's primary key, - /// and thus typically will not be zero. Passing in elem_id=0 means "do not - /// write elem_id to the buffer". - CollectionBuffer(std::vector& buffer, uint16_t elem_id) : - CollectionBuffer(buffer) - { - if (elem_id != 0) - { - append(&elem_id, sizeof(elem_id)); - } - } - - void append(const void* data, size_t num_bytes) - { - const char* bytes = static_cast(data); - buffer_.insert(buffer_.end(), bytes, bytes + num_bytes); - } - -private: - std::vector& buffer_; -}; - -template -inline typename std::enable_if::value && std::is_scalar::value && - !type_traits::is_any_pointer::value, - CollectionBuffer&>::type -operator<<(CollectionBuffer& buffer, const T& val) -{ - if constexpr (std::is_same::value) - { - buffer << static_cast(val); - } else - { - buffer.append(&val, sizeof(T)); - } - - return buffer; -} - -template -inline typename std::enable_if::value, CollectionBuffer&>::type operator<<(CollectionBuffer& buffer, - const T& val) -{ - using dtype = typename std::underlying_type::type; - return buffer << static_cast(val); -} - -template inline CollectionBuffer& operator<<(CollectionBuffer& buffer, const std::vector& bytes) -{ - buffer.append(bytes.data(), bytes.size() * sizeof(T)); - return buffer; -} - -// Go through StringMap to serialize as uint32_t -inline CollectionBuffer& operator<<(CollectionBuffer& buffer, const std::string& val) = delete; -inline CollectionBuffer& operator<<(CollectionBuffer& buffer, const char* val) = delete; - -} // namespace simdb diff --git a/include/simdb/apps/argos/CollectionPoints.hpp b/include/simdb/apps/argos/CollectionPoints.hpp deleted file mode 100644 index 5e9c8320..00000000 --- a/include/simdb/apps/argos/CollectionPoints.hpp +++ /dev/null @@ -1,795 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/pipeline/Serialize.hpp" - -#include -#include -#include -#include -#include - -namespace simdb { - -/// Raw data held in the SimDB collection "black box". Sent to the database -/// for as long as the Status isn't set to DONT_READ. -struct ArgosRecord -{ - enum class Status { READ, READ_ONCE, DONT_READ }; - Status status = Status::DONT_READ; - - const uint16_t elem_id = 0; - bool write_elem_id = true; - std::vector data; - - ArgosRecord(uint16_t elem_id) : - elem_id(elem_id) - { - } - - void reset() - { - status = Status::DONT_READ; - data.clear(); - } -}; - -class TickReader -{ -public: - virtual ~TickReader() = default; - virtual uint64_t getTick() const = 0; -}; - -/// Base class for all collectables. -class CollectionPointBase -{ -public: - CollectionPointBase(uint16_t elem_id, uint16_t clk_id, size_t heartbeat, const std::string& dtype) : - argos_record_(elem_id), - elem_id_(elem_id), - clk_id_(clk_id), - heartbeat_(heartbeat), - dtype_(dtype) - { - } - - virtual ~CollectionPointBase() = default; - - static void enableMinificationLogging() - { - const bool log = true; - logMinification_(&log); - } - - static bool minificationLoggingEnabled() { return logMinification_(); } - - /// Get the unique ID for this collection point. - uint16_t getElemId() const { return elem_id_; } - - /// We typically serialize [elem_id, value] pairs to the database blobs. - /// Some use cases might not need the elem_id however, so this method - /// helps minimize disk space for those use cases. - void doNotWriteElemId() { argos_record_.write_elem_id = false; } - - /// Get the clock database ID for this collection point. - uint16_t getClockId() const { return clk_id_; } - - /// Get the heartbeat for this collection point. This is the - /// maximum number of cycles SimDB will attempt to perform - /// "minification" on the data before it is forced to write - /// the whole un-minified value to the database again. Note - /// that minification is simply an implementation detail - /// for performance. - size_t getHeartbeat() const { return heartbeat_; } - - /// Data type of this collectable, e.g. "uint64_t" or - /// "MemPacket_contig_capacity32" - const std::string& getDataTypeStr() const { return dtype_; } - - void setTickReader(TickReader& reader) { tick_reader_ = &reader; } - - void setAutoCollect(bool is_auto_collected) { is_auto_collected_ = is_auto_collected; } - - bool isAutoCollected() const { return is_auto_collected_; } - - /// Append the collected data from the black box unless the status is - /// DONT_READ. - void sweep(std::vector& swept_data) - { - if (argos_record_.status != ArgosRecord::Status::DONT_READ) - { - swept_data.insert(swept_data.end(), argos_record_.data.begin(), argos_record_.data.end()); - } - - if (argos_record_.status == ArgosRecord::Status::READ_ONCE) - { - argos_record_.reset(); - } - } - - /// Called at the end of simulation / when the pipeline collector is - /// destroyed. Given the DatabaseManager in case the collectable needs to - /// write any final metadata etc. - /// - /// Note that postSim() is called inside a BEGIN/COMMIT TRANSACTION block. - virtual void postSim(DatabaseManager*) {} - -protected: - ArgosRecord argos_record_; - - static bool logMinification_(const bool* log = nullptr) - { - static bool log_minification = false; - if (log) - log_minification = *log; - return log_minification; - } - - uint64_t getTick_() const { return tick_reader_ ? tick_reader_->getTick() : 0; } - -private: - const uint16_t elem_id_; - const uint16_t clk_id_; - const size_t heartbeat_; - const std::string dtype_; - TickReader* tick_reader_ = nullptr; - bool is_auto_collected_ = false; -}; - -#define LOG_MINIFICATION simdb::CollectionPointBase::minificationLoggingEnabled() - -template struct is_std_vector : std::false_type -{ -}; - -template struct is_std_vector> : std::true_type -{ -}; - -template static constexpr bool is_std_vector_v = is_std_vector::value; - -/// Collectable for non-iterable data (not a queue/vector/deque/etc.) -/// - POD -/// - struct -class CollectionPoint : public CollectionPointBase -{ -public: - /// Minification for these collectables can only write the whole value - /// (WRITE) or say that the value hasn't changed (CARRY). - enum class Action : uint8_t { WRITE, CARRY }; - - template - CollectionPoint(Args&&... args) : - CollectionPointBase(std::forward(args)...) - { - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T& val, bool once = false) - { - if (val) - { - activate(*val, once); - } else - { - deactivate(); - } - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T& val, - bool once = false) - { - minify_(val); - argos_record_.status = once ? ArgosRecord::Status::READ_ONCE : ArgosRecord::Status::READ; - } - - /// Remove this collectable from the black box (do not collect anymore until - /// activate() is called again). - void deactivate() { argos_record_.status = ArgosRecord::Status::DONT_READ; } - -private: - /// Write the collectable bytes in the smallest form possible. - template void minify_(const T& val) - { - const uint16_t elem_id = argos_record_.write_elem_id ? getElemId() : 0; - CollectionBuffer buffer(argos_record_.data, elem_id); - if (LOG_MINIFICATION) - std::cout << "\n\n[simdb verbose] tick " << getTick_() << ", cid " << getElemId() << "\n"; - - const auto num_bytes = getNumBytes_(val); - if (!isAutoCollected()) - { - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] Manually collected, " << num_bytes << " bytes\n"; - if constexpr (std::is_trivial::value && std::is_standard_layout::value) - { - buffer << val; - } else - { - StructSerializer::getInstance()->extract(&val, curr_data_); - buffer << curr_data_; - } - return; - } - - if constexpr (std::is_trivial::value && std::is_standard_layout::value) - { - if (num_bytes < 16) - { - // Collectables that are this small are written directly, - // as any attempts at a DB size optimization would actually - // make the database larger. - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] " << num_bytes << "<16, " << demangle(typeid(T).name()) << ": " << val - << "\n"; - buffer << val; - return; - } - } - - StructSerializer::getInstance()->extract(&val, curr_data_); - if (num_carry_overs_ < getHeartbeat() && curr_data_ == prev_data_) - { - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] CARRY\n"; - buffer << CollectionPoint::Action::CARRY; - ++num_carry_overs_; - } else - { - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] WRITE " << curr_data_.size() << " bytes\n"; - buffer << CollectionPoint::Action::WRITE; - buffer << curr_data_; - prev_data_ = curr_data_; - num_carry_overs_ = 0; - } - } - - template size_t getNumBytes_(const T& val) - { - if constexpr (std::is_same_v) - { - return sizeof(int); - } else if constexpr (std::is_same_v) - { - return sizeof(uint8_t); - } else if constexpr (std::is_same_v) - { - return sizeof(int8_t); - } else if constexpr (std::is_same_v) - { - return sizeof(uint16_t); - } else if constexpr (std::is_same_v) - { - return sizeof(int16_t); - } else if constexpr (std::is_same_v) - { - return sizeof(uint32_t); - } else if constexpr (std::is_same_v) - { - return sizeof(int32_t); - } else if constexpr (std::is_same_v) - { - return sizeof(uint64_t); - } else if constexpr (std::is_same_v) - { - return sizeof(int64_t); - } else if constexpr (std::is_same_v) - { - return sizeof(float); - } else if constexpr (std::is_same_v) - { - return sizeof(double); - } else if constexpr (std::is_same_v) - { - return sizeof(uint32_t); - } else if constexpr (std::is_enum::value) - { - return sizeof(typename std::underlying_type::type); - } else - { - return StructSerializer::getInstance()->getStructNumBytes(); - } - } - - std::vector curr_data_; - std::vector prev_data_; - size_t num_carry_overs_ = 0; -}; - -/// Collectable for contiguous (non-sparse) iterable data e.g. -/// queue/vector/deque/etc. -class ContigIterableCollectionPoint : public CollectionPointBase -{ -public: - /// Minification for these collectables can do the following: - /// - ARRIVE: A new element has been added to the end of the container - /// - DEPART: An element has been removed from the front of the - /// container - /// - BOOKENDS: An element was popped off the front and another pushed to - /// the back of the container - /// - CHANGE: Exactly one element in the container has changed - /// - CARRY: Nothing has changed - /// - FULL: Capture the entire container (during heartbeats or when - /// the number - /// of changes do not fall neatly into one of the other - /// categories) - enum class Action : uint8_t { ARRIVE, DEPART, BOOKENDS, CHANGE, CARRY, FULL }; - - ContigIterableCollectionPoint(uint16_t elem_id, uint16_t clk_id, size_t heartbeat, const std::string& dtype, - size_t capacity) : - CollectionPointBase(elem_id, clk_id, heartbeat, dtype), - curr_snapshot_(capacity), - prev_snapshot_(capacity) - { - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T container, - bool once = false) - { - activate(*container, once); - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T& container, - bool once = false) - { - minify_(container); - argos_record_.status = once ? ArgosRecord::Status::READ_ONCE : ArgosRecord::Status::READ; - } - - /// Remove this collectable from the black box (do not collect anymore until - /// activate() is called again). - void deactivate() { argos_record_.status = ArgosRecord::Status::DONT_READ; } - -private: - /// This class is used to compare the current container elements to the - /// previous (last collected cycle). We do this to determine the most - /// efficient way to write the container to the database (minification). - class IterableSnapshot - { - public: - IterableSnapshot(size_t expected_capacity) : - bytes_by_bin_(expected_capacity) - { - } - - uint16_t size() const - { - uint16_t num_valid = 0; - for (const auto& bin : bytes_by_bin_) - { - if (!bin.empty()) - { - ++num_valid; - } - } - return num_valid; - } - - uint16_t capacity() const { return bytes_by_bin_.size(); } - - std::vector& operator[](size_t idx) { return bytes_by_bin_[idx]; } - - const std::vector& operator[](size_t idx) const { return bytes_by_bin_[idx]; } - - void clear() - { - for (auto& bin : bytes_by_bin_) - { - bin.clear(); - } - } - - void compareAndMinify(IterableSnapshot& prev, CollectionBuffer& buffer, uint64_t tick, uint16_t elem_id, - bool is_auto_collected) - { - uint16_t changed_idx = 0; - switch (getMinificationAction_(prev, changed_idx, is_auto_collected)) - { - case Action::CARRY: - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", CARRY\n"; - buffer << ContigIterableCollectionPoint::Action::CARRY; - break; - case Action::ARRIVE: - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", ARRIVE " - << bytes_by_bin_[changed_idx].size() << " bytes\n"; - buffer << ContigIterableCollectionPoint::Action::ARRIVE; - buffer << bytes_by_bin_[changed_idx]; - break; - case Action::DEPART: - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", DEPART\n"; - buffer << ContigIterableCollectionPoint::Action::DEPART; - break; - case Action::CHANGE: - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", CHANGE index " - << changed_idx << "," << bytes_by_bin_[changed_idx].size() << " bytes\n"; - buffer << ContigIterableCollectionPoint::Action::CHANGE; - buffer << changed_idx; - buffer << bytes_by_bin_[changed_idx]; - break; - case Action::BOOKENDS: - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", BOOKENDS, appended " - << bytes_by_bin_[changed_idx].size() << " bytes\n"; - buffer << ContigIterableCollectionPoint::Action::BOOKENDS; - buffer << bytes_by_bin_[changed_idx]; - break; - case Action::FULL: - auto num_elems = size(); - buffer << ContigIterableCollectionPoint::Action::FULL; - buffer << num_elems; - - uint64_t num_bytes_per_bin = 0; - for (uint16_t idx = 0; idx < num_elems; ++idx) - { - if (idx == 0) - { - num_bytes_per_bin = bytes_by_bin_[idx].size(); - } else if (bytes_by_bin_[idx].size() != num_bytes_per_bin) - { - throw DBException("All elements in the container must " - "have the same number of bytes"); - } - buffer << bytes_by_bin_[idx]; - } - - if (LOG_MINIFICATION) - { - if (is_auto_collected) - { - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", FULL with " - << num_elems << " elements\n"; - } else - { - std::cout << "[simdb verbose] cid " << elem_id << ", tick " << tick << ", FULL with " - << num_elems << " elements, manually collected\n"; - } - } - break; - } - } - - private: - Action getMinificationAction_(IterableSnapshot& prev, uint16_t& changed_idx, bool is_auto_collected) - { - if (!is_auto_collected) - { - return Action::FULL; - } - - if (++action_count_ == capacity()) - { - action_count_ = 0; - return Action::FULL; - } - - if (prev.bytes_by_bin_ == bytes_by_bin_) - { - if (bytes_by_bin_.empty()) - { - return Action::FULL; - } - return Action::CARRY; - } - - auto prev_size = prev.size(); - auto curr_size = size(); - if (prev_size == curr_size) - { - // If the (ith+1) of the prev container == the ith of the - // current container, then return BOOKEND - bool bookends = prev_size > 1; - for (size_t idx = 1; idx < prev_size; ++idx) - { - if (prev.bytes_by_bin_[idx] != bytes_by_bin_[idx - 1]) - { - bookends = false; - break; - } - } - if (bookends) - { - changed_idx = bytes_by_bin_.size() - 1; - while (changed_idx) - { - if (!bytes_by_bin_[changed_idx].empty()) - { - break; - } - --changed_idx; - } - return Action::BOOKENDS; - } - - // If exactly one element has changed, then return CHANGE - changed_idx = 0; - for (size_t idx = 0; idx < curr_size; ++idx) - { - if (prev.bytes_by_bin_[idx] != bytes_by_bin_[idx]) - { - if (changed_idx) - { - // If there is more than one change, then return - // FULL - return Action::FULL; - } - changed_idx = idx; - return Action::CHANGE; - } - } - - // Not reachable - assert(false); - } else if (prev_size + 1 == curr_size) - { - bool arrive = true; - for (size_t idx = 0; idx < prev_size; ++idx) - { - if (prev.bytes_by_bin_[idx] != bytes_by_bin_[idx]) - { - arrive = false; - break; - } - } - if (arrive) - { - changed_idx = bytes_by_bin_.size() - 1; - while (changed_idx) - { - if (!bytes_by_bin_[changed_idx].empty()) - { - break; - } - --changed_idx; - } - return Action::ARRIVE; - } - } else if (prev_size - 1 == curr_size) - { - bool depart = true; - for (size_t idx = 0; idx < curr_size; ++idx) - { - if (prev.bytes_by_bin_[idx + 1] != bytes_by_bin_[idx]) - { - depart = false; - break; - } - } - if (depart) - { - return Action::DEPART; - } - } - - return Action::FULL; - } - - std::deque> bytes_by_bin_; - size_t action_count_ = 0; - }; - - /// Write the collectable bytes in the smallest form possible. - template void minify_(const T& container) - { - auto size = container.size(); - if (size > prev_snapshot_.capacity()) - { - size = prev_snapshot_.capacity(); - } - - queue_max_size_ = std::max(queue_max_size_, (uint16_t)size); - curr_snapshot_.clear(); - - auto itr = container.begin(); - auto eitr = container.end(); - uint16_t bin_idx = 0; - - while (itr != eitr && bin_idx < size) - { - if (!writeStruct_(*itr, curr_snapshot_, bin_idx)) - { - break; - } - ++itr; - ++bin_idx; - } - - // Let the current snapshot take into account the previous - // snapshot, and figure out the most compact way to represent - // the current data. - // - // The only thing we must do for all collection points is to - // write the element ID. - CollectionBuffer buffer(argos_record_.data, getElemId()); - curr_snapshot_.compareAndMinify(prev_snapshot_, buffer, getTick_(), getElemId(), isAutoCollected()); - prev_snapshot_ = curr_snapshot_; - } - - template - typename std::enable_if::value, bool>::type - writeStruct_(const T& el, IterableSnapshot& snapshot, uint16_t bin_idx) - { - if (el) - { - return writeStruct_(*el, snapshot, bin_idx); - } - return false; - } - - template - typename std::enable_if::value, bool>::type - writeStruct_(const T& el, IterableSnapshot& snapshot, uint16_t bin_idx) - { - StructSerializer::getInstance()->extract(&el, snapshot[bin_idx]); - return true; - } - - /// Write the maximum size of this queue during simulation. This is used - /// for Argos' SchedulingLines feature. - /// - /// Note that postSim() is called inside a BEGIN/COMMIT TRANSACTION block. - void postSim(DatabaseManager* db_mgr) override; - - IterableSnapshot curr_snapshot_; - IterableSnapshot prev_snapshot_; - uint16_t queue_max_size_ = 0; -}; - -/// Collectable for sparse iterable data e.g. queue/vector/deque/etc. -class SparseIterableCollectionPoint : public CollectionPointBase -{ -public: - SparseIterableCollectionPoint(uint16_t elem_id, uint16_t clk_id, size_t heartbeat, const std::string& dtype, - size_t capacity) : - CollectionPointBase(elem_id, clk_id, heartbeat, dtype), - expected_capacity_(capacity) - { - prev_data_by_bin_.resize(capacity); - num_carry_overs_by_bin_.resize(capacity, 0); - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T container, - bool once = false) - { - activate(*container, once); - } - - /// Put this collectable in the black box for consumption until deactivate() - /// is called. NOTE: There is no reason to call deactivate() on your own if - /// "bool once = true". - template - typename std::enable_if::value, void>::type activate(const T& container, - bool once = false) - { - minify_(container); - argos_record_.status = once ? ArgosRecord::Status::READ_ONCE : ArgosRecord::Status::READ; - } - - /// Remove this collectable from the black box (do not collect anymore until - /// activate() is called again). - void deactivate() { argos_record_.status = ArgosRecord::Status::DONT_READ; } - -private: - /// Write the collectable bytes in the smallest form possible. - /// - /// TODO cnyce - We are not currently performing any minification - /// for sparse iterables types. - template void minify_(const T& container) - { - uint16_t num_valid = 0; - - { - auto itr = container.begin(); - auto eitr = container.end(); - - while (itr != eitr) - { - if constexpr (is_std_vector_v) - { - if (*itr) - { - ++num_valid; - } - } else if (itr.isValid()) - { - ++num_valid; - } - ++itr; - } - } - - queue_max_size_ = std::max(queue_max_size_, num_valid); - - CollectionBuffer buffer(argos_record_.data, getElemId()); - if (LOG_MINIFICATION) - std::cout << "\n\n[simdb verbose] tick " << getTick_() << ", cid " << getElemId() << "\n"; - - buffer << num_valid; - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] num valid: " << num_valid << "\n"; - - uint16_t bin_idx = 0; - auto itr = container.begin(); - auto eitr = container.end(); - while (itr != eitr && bin_idx < expected_capacity_) - { - bool valid; - if constexpr (is_std_vector_v) - { - valid = *itr != nullptr; - } else - { - valid = itr.isValid(); - } - - if (valid) - { - writeStruct_(*itr, buffer, bin_idx); - } - ++itr; - ++bin_idx; - } - } - - template - typename std::enable_if::value, bool>::type - writeStruct_(const T& el, CollectionBuffer& buffer, uint16_t bin_idx) - { - if (el) - { - return writeStruct_(*el, buffer, bin_idx); - } - return false; - } - - template - typename std::enable_if::value, bool>::type - writeStruct_(const T& el, CollectionBuffer& buffer, uint16_t bin_idx) - { - buffer << bin_idx; - StructSerializer::getInstance()->writeStruct(&el, buffer); - - if (LOG_MINIFICATION) - std::cout << "[simdb verbose] bin " << bin_idx << ", " - << StructSerializer::getInstance()->getStructNumBytes() << " bytes\n"; - return true; - } - - /// Write the maximum size of this queue during simulation. This is used - /// for Argos' SchedulingLines feature. - /// - /// Note that postSim() is called inside a BEGIN/COMMIT TRANSACTION block. - void postSim(DatabaseManager* db_mgr) override; - - const size_t expected_capacity_; - std::vector> prev_data_by_bin_; - std::vector num_carry_overs_by_bin_; - uint16_t queue_max_size_ = 0; -}; - -} // namespace simdb diff --git a/include/simdb/apps/argos/EntryPoint.hpp b/include/simdb/apps/argos/EntryPoint.hpp new file mode 100644 index 00000000..e0eb4a49 --- /dev/null +++ b/include/simdb/apps/argos/EntryPoint.hpp @@ -0,0 +1,97 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/apps/argos/EnumInspector.hpp" +#include "simdb/apps/argos/PipelineDataTypes.hpp" +#include "simdb/apps/argos/PipelineStagerInterface.hpp" +#include "simdb/utils/Demangle.hpp" +#include "simdb/utils/TinyStrings.hpp" +#include "simdb/utils/TypeTraits.hpp" + +namespace simdb::argos { + +//! \class EntryPoint +//! \brief Main entry point into Argos collection. +//! +//! TODO cnyce: These need to be template classes when the collection +//! code from Sparta is moved into SimDB. +class EntryPoint +{ +public: + EntryPoint(PipelineStagerInterface* stager_interface, TinyStrings<>* tiny_strings) : + stager_interface_(stager_interface), + tiny_strings_(tiny_strings) + { + } + + /// Get the unique ID for this collection point. + uint16_t getID() const { return cid_; } + + /// Suppress heartbeat re-emission of previously seen bytes. + void closeRecord() + { + if (!closed_) + { + closed_ = true; + stager_interface_->closeRecord(getID()); + } + } + + /// For testing purposes only. DO NOT CALL IN PRODUCTION. + static void resetCIDs() { nextCID_() = 0; } + + TinyStrings<>* getTinyStrings() const { return tiny_strings_; } + + //! NOTE: We only have setScalarValueBytes(), setContigContainerBinBytes() + //! and setSparseContainerBinBytes() all together in one class temporarily + //! until Sparta/SimDB collection is merged. When the entry point class + //! becomes a template, these will collapse to one method ( decides + //! the input data structure). + void setScalarValueBytes(std::vector&& scalar_bytes) + { + closed_ = false; + stager_interface_->stage(getID(), std::move(scalar_bytes)); + } + + //! \see setScalarValueBytes + void setContigContainerBinBytes(std::vector>&& contig_bin_bytes) + { + closed_ = false; + stager_interface_->stage(getID(), std::move(contig_bin_bytes)); + } + + //! \see setScalarValueBytes + void setSparseContainerBinBytes(std::map>&& sparse_bin_bytes) + { + closed_ = false; + stager_interface_->stage(getID(), std::move(sparse_bin_bytes)); + } + +private: + /// Unique ID generator. + static uint16_t& nextCID_() + { + static uint16_t counter = 0; + if (counter == UINT16_MAX) + { + throw DBException("Max number of collectables exceeded (") << UINT16_MAX << ")"; + } + ++counter; + return counter; + } + + /// Unique collectable ID + const uint16_t cid_{nextCID_()}; + + /// Suppress heartbeat re-emission while true + bool closed_ = false; + + /// Most of what EntryPoint does is forward to the stager (ledger) + PipelineStagerInterface* const stager_interface_; + + /// DB-backed string-to-int mapping + TinyStrings<>* const tiny_strings_; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/EnumInspector.hpp b/include/simdb/apps/argos/EnumInspector.hpp new file mode 100644 index 00000000..fef533f6 --- /dev/null +++ b/include/simdb/apps/argos/EnumInspector.hpp @@ -0,0 +1,114 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/utils/Demangle.hpp" +#include "simdb/utils/TypeTraits.hpp" + +#include +#include +#include +#include + +namespace simdb::argos { + +//! This class allows us to track collected enum values (int) and their corresponding +//! stringified values (operator<<) and store them in the database for the Argos UI +//! to use. Note that enums without operator<< are just shown in Argos as their int +//! values. +class EnumInspector +{ +public: + template std::enable_if_t, void> inspect(E val) + { + static EnumMap* map = [this] { + auto& slot = enum_maps_[simdb::demangle_type()]; + if (!slot) + { + slot = std::make_unique>(); + } + return static_cast*>(slot.get()); + }(); + map->inspect(val); + } + + template std::enable_if_t, void> inspect(E) {} + + void serializeEnumMaps(DatabaseManager* db_mgr) + { + for (auto& [_, map] : enum_maps_) + { + map->dumpEnumMap(db_mgr); + } + } + +private: + class EnumMapBase + { + public: + virtual ~EnumMapBase() = default; + virtual void dumpEnumMap(simdb::DatabaseManager* db_mgr) const = 0; + }; + + template class EnumMap : public EnumMapBase + { + public: + void inspect(E) {} + void dumpEnumMap(simdb::DatabaseManager*) const override final {} + }; + + template + class EnumMap>> : public EnumMapBase + { + public: + void inspect(E val) + { + if (last_seen_ && *last_seen_ == val) + { + return; + } + + for (E seen : all_seen_) + { + if (seen == val) + { + last_seen_ = val; + return; + } + } + + all_seen_.push_back(val); + last_seen_ = val; + } + + void dumpEnumMap(DatabaseManager* db_mgr) const override final + { + using underlying_t = std::underlying_type_t; + const auto itype = demangle_type(); + const auto enum_name = demangle_type(); + const auto enum_id = db_mgr->INSERT(SQL_TABLE("CollectedEnums"), SQL_VALUES(enum_name, itype))->getId(); + + auto inserter = db_mgr->prepareINSERT(SQL_TABLE("EnumMembers")); + for (auto e : all_seen_) + { + std::ostringstream oss; + oss << e; + + const auto member_name = oss.str(); + const auto raw_enum_val = static_cast(e); + const auto raw_enum_str = std::to_string(raw_enum_val); + + inserter->createRecordWithColValues(enum_id, member_name, raw_enum_str); + } + } + + private: + std::vector all_seen_; + std::optional last_seen_; + }; + + std::unordered_map> enum_maps_; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/PipelineDataTypes.hpp b/include/simdb/apps/argos/PipelineDataTypes.hpp new file mode 100644 index 00000000..cb7dbdfd --- /dev/null +++ b/include/simdb/apps/argos/PipelineDataTypes.hpp @@ -0,0 +1,216 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/Exceptions.hpp" +#include "simdb/apps/argos/CollectedData.hpp" +#include "simdb/apps/argos/Timestamps.hpp" +#include "simdb/utils/Demangle.hpp" +#include "simdb/utils/ValidValue.hpp" + +#include +#include +#include +#include +#include +#include +#include + +//! This file contains data types that are used to pass information +//! down ArgosCollector's pipeline to the database. + +namespace simdb::argos { + +//! \brief Holds onto all byte buffers containing collected data and open/close state changes. +//! All records of everything that happened at a given simulation time T are processed together +//! and end up in the DB in the same blob/rowid. Clocks are tracked so we can support multi-clock +//! simulations. The clock_ids are a set (not a scalar) since two clocks can collect at the same +//! time (example: one clock might be exactly twice as fast as another). +//! +//! Used as input to the zlib pipeline stage. +struct QueueCollectionData +{ + ValidValue sim_time; + std::vector> entries; + std::unordered_set clock_ids; +}; + +//! \brief Output data structure from the zlib pipeline stage. +struct CompressedQueueCollectionData +{ + ValidValue sim_time; + std::vector compressed_collection_data; + std::unordered_set clock_ids; +}; + +//! \brief Enum used for warning/error/message filtering in Argos. +enum class NotifType { WARNING, ERROR, MESSAGE, __INVALID__ }; + +//! \brief Notification captured during collection. These appear in a dedicated tab in Argos. +struct Notification +{ + ValidValue cid; + std::string notif; + NotifType type = NotifType::__INVALID__; + ValidValue sim_time; + + Notification(uint16_t cid, const std::string& notif, const NotifType type, uint64_t sim_time) : + cid(cid), + notif(notif), + type(type), + sim_time(sim_time) + { + } + + Notification(uint16_t cid, const std::string& notif, const NotifType type) : + cid(cid), + notif(notif), + type(type) + { + } + + // Default ctor needed for ConcurrentQueue::try_pop + Notification() = default; +}; + +//! \brief Single byte buffer with CID for scalar collectables (including scalar structs). +struct ScalarEntry +{ + uint16_t cid = 0; + std::vector scalar_bytes; + + ScalarEntry(uint16_t cid, std::vector&& scalar_bytes) : + cid(cid), + scalar_bytes(std::move(scalar_bytes)) + { + } + + ScalarEntry() = default; + ScalarEntry(ScalarEntry&&) = default; + ScalarEntry& operator=(ScalarEntry&&) = default; +}; + +//! \brief Vector of byte buffers with CID for contig container collectables. +struct ContigEntry +{ + uint16_t cid = 0; + std::vector> contig_bin_bytes; + + ContigEntry(uint16_t cid, std::vector>&& contig_bin_bytes) : + cid(cid), + contig_bin_bytes(std::move(contig_bin_bytes)) + { + } + + ContigEntry() = default; + ContigEntry(ContigEntry&&) = default; + ContigEntry& operator=(ContigEntry&&) = default; +}; + +//! \brief Bin-mapped byte buffers with CID for sparse container collectables. +struct SparseEntry +{ + uint16_t cid = 0; + std::map> sparse_bin_bytes; + + SparseEntry(uint16_t cid, std::map>&& sparse_bin_bytes) : + cid(cid), + sparse_bin_bytes(std::move(sparse_bin_bytes)) + { + } + + SparseEntry() = default; + SparseEntry(SparseEntry&&) = default; + SparseEntry& operator=(SparseEntry&&) = default; +}; + +//! \brief Timestamped notification. +struct NotifEntry +{ + ValidValue sim_time; + std::string notif; + NotifType type = NotifType::__INVALID__; + + NotifEntry(const Timestamp* timestamp, const std::string& notif, NotifType type) : + notif(notif), + type(type) + { + if (timestamp) + { + sim_time = timestamp->getTime(); + } + } + + NotifEntry() = default; + NotifEntry(NotifEntry&&) = default; + NotifEntry& operator=(NotifEntry&&) = default; +}; + +//! \brief This class is used to create all the *Entries above. Tracks everything that happened +//! across all collectables at a specific simulation time point. A ledger is filled at time T +//! until simulation has moved forward, then the ledger is moved to the 1st async stage of +//! the DB pipeline. ArgosCollector will create a new Ledger and add to it until time advances +//! again. +class Ledger +{ +public: + Ledger(uint64_t sim_time, uint64_t window_id, uint64_t reserve_num_scalars = 0, uint64_t reserve_num_contigs = 0, + uint64_t reserve_num_sparses = 0) : + sim_time_(sim_time), + window_id_(window_id) + { + scalar_records_.reserve(reserve_num_scalars); + contig_records_.reserve(reserve_num_contigs); + sparse_records_.reserve(reserve_num_sparses); + } + + Ledger(Ledger&&) = default; + Ledger(const Ledger&) = delete; + + void recordScalar(uint16_t cid, std::vector&& scalar_bytes) + { + scalar_records_.emplace_back(cid, std::move(scalar_bytes)); + } + + void recordContig(uint16_t cid, std::vector>&& contig_bytes) + { + contig_records_.emplace_back(cid, std::move(contig_bytes)); + } + + void recordSparse(uint16_t cid, std::map>&& sparse_bin_bytes) + { + sparse_records_.emplace_back(cid, std::move(sparse_bin_bytes)); + } + + void closeRecord(uint16_t cid) { closed_cids_.insert(cid); } + + uint64_t getSimTime() const { return sim_time_; } + + uint64_t getWindowId() const { return window_id_; } + + bool hasEntries() const + { + return !scalar_records_.empty() || !contig_records_.empty() || !sparse_records_.empty() || + !closed_cids_.empty(); + } + + std::vector releaseScalarEntries() { return std::move(scalar_records_); } + + std::vector releaseContigEntries() { return std::move(contig_records_); } + + std::vector releaseSparseEntries() { return std::move(sparse_records_); } + + const std::unordered_set& getClosedCIDs() const { return closed_cids_; } + +private: + uint64_t sim_time_ = 0; + uint64_t window_id_ = 0; + std::vector scalar_records_; + std::vector contig_records_; + std::vector sparse_records_; + std::unordered_set closed_cids_; +}; + +using LedgerPtr = std::unique_ptr; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/PipelineStagerInterface.hpp b/include/simdb/apps/argos/PipelineStagerInterface.hpp new file mode 100644 index 00000000..5bdd2fcf --- /dev/null +++ b/include/simdb/apps/argos/PipelineStagerInterface.hpp @@ -0,0 +1,37 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/utils/TinyStrings.hpp" + +#include +#include +#include +#include + +namespace simdb::argos { + +//! \brief Interface class to receive collected data, notifications, and open/close state changes. +class PipelineStagerInterface +{ +public: + virtual ~PipelineStagerInterface() = default; + + virtual void stage(uint16_t cid, std::vector&& scalar_bytes) = 0; + + virtual void stage(uint16_t cid, std::vector>&& contig_bin_bytes) = 0; + + virtual void stage(uint16_t cid, std::map>&& sparse_bin_bytes) = 0; + + virtual void closeRecord(uint16_t cid) = 0; + + virtual void postNotif(const std::string& notif, NotifType type) = 0; + + void postWarning(const std::string& warning) { postNotif(warning, NotifType::WARNING); } + + void postError(const std::string& error) { postNotif(error, NotifType::ERROR); } + + void postMessage(const std::string& msg) { postNotif(msg, NotifType::MESSAGE); } +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/Serialize.hpp b/include/simdb/apps/argos/Serialize.hpp deleted file mode 100644 index 825f1fb6..00000000 --- a/include/simdb/apps/argos/Serialize.hpp +++ /dev/null @@ -1,645 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Exceptions.hpp" -#include "simdb/pipeline/CollectionBuffer.hpp" -#include "simdb/utils/Demangle.hpp" -#include "simdb/utils/TinyStrings.hpp" -#include "simdb/utils/TypeTraits.hpp" - -#include -#include -#include -#include -#include -#include - -namespace simdb { - -class DatabaseManager; - -enum class Format { none = 0, hex = 1, boolalpha = 2 }; - -/// Data types supported by the collection system. Note that -/// enum struct fields use the std::underlying_type of that -/// enum, e.g. int32_t -enum class StructFields { - char_t, - int8_t, - uint8_t, - int16_t, - uint16_t, - int32_t, - uint32_t, - int64_t, - uint64_t, - float_t, - double_t, - string_t -}; - -template inline StructFields getFieldDTypeEnum() -{ - if constexpr (std::is_same_v) - { - return StructFields::char_t; - } else if constexpr (std::is_same_v) - { - return StructFields::int8_t; - } else if constexpr (std::is_same_v) - { - return StructFields::uint8_t; - } else if constexpr (std::is_same_v) - { - return StructFields::int16_t; - } else if constexpr (std::is_same_v) - { - return StructFields::uint16_t; - } else if constexpr (std::is_same_v) - { - return StructFields::int32_t; - } else if constexpr (std::is_same_v) - { - return StructFields::uint32_t; - } else if constexpr (std::is_same_v) - { - return StructFields::int64_t; - } else if constexpr (std::is_same_v) - { - return StructFields::uint64_t; - } else if constexpr (std::is_same_v) - { - return StructFields::float_t; - } else if constexpr (std::is_same_v) - { - return StructFields::double_t; - } else if constexpr (std::is_same_v) - { - return StructFields::string_t; - } else if constexpr (std::is_same_v) - { - return StructFields::int32_t; - } else if constexpr (std::is_enum::value) - { - using enum_int_t = typename std::underlying_type::type; - return getFieldDTypeEnum(); - } else - { - throw DBException("Unsupported data type: ") << demangle(typeid(FieldT).name()); - } -} - -/// These dtype strings are stored in the database to inform the downstream -/// python module how to interpret the structs' serialized raw bytes. -inline std::string getFieldDTypeStr(const StructFields dtype) -{ - switch (dtype) - { - case StructFields::char_t: - return "char_t"; - case StructFields::int8_t: - return "int8_t"; - case StructFields::uint8_t: - return "uint8_t"; - case StructFields::int16_t: - return "int16_t"; - case StructFields::uint16_t: - return "uint16_t"; - case StructFields::int32_t: - return "int32_t"; - case StructFields::uint32_t: - return "uint32_t"; - case StructFields::int64_t: - return "int64_t"; - case StructFields::uint64_t: - return "uint64_t"; - case StructFields::float_t: - return "float_t"; - case StructFields::double_t: - return "double_t"; - case StructFields::string_t: - return "string_t"; - } - - throw DBException("Invalid data type"); -} - -inline size_t getDTypeNumBytes(const StructFields dtype) -{ - switch (dtype) - { - case StructFields::char_t: - return sizeof(char); - case StructFields::int8_t: - return sizeof(int8_t); - case StructFields::uint8_t: - return sizeof(uint8_t); - case StructFields::int16_t: - return sizeof(int16_t); - case StructFields::uint16_t: - return sizeof(uint16_t); - case StructFields::int32_t: - return sizeof(int32_t); - case StructFields::uint32_t: - return sizeof(uint32_t); - case StructFields::int64_t: - return sizeof(int64_t); - case StructFields::uint64_t: - return sizeof(uint64_t); - case StructFields::float_t: - return sizeof(float); - case StructFields::double_t: - return sizeof(double); - default: - break; - } - - throw DBException("Invalid data type"); -} - -/// Avoid all SQLite issues (such as not natively supporting uint64_t, or -/// truncating double values to less precision that you wanted), these -/// convertIntToBlob() methods convert scalar PODs to vector (blob). -template std::vector convertIntToBlob(const IntT val) = delete; - -template <> inline std::vector convertIntToBlob(const char val) -{ - return {val}; -} - -template <> inline std::vector convertIntToBlob(const int8_t val) -{ - std::vector blob(sizeof(int8_t)); - memcpy(blob.data(), &val, sizeof(int8_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const uint8_t val) -{ - std::vector blob(sizeof(uint8_t)); - memcpy(blob.data(), &val, sizeof(uint8_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const int16_t val) -{ - std::vector blob(sizeof(int16_t)); - memcpy(blob.data(), &val, sizeof(int16_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const uint16_t val) -{ - std::vector blob(sizeof(uint16_t)); - memcpy(blob.data(), &val, sizeof(uint16_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const int32_t val) -{ - std::vector blob(sizeof(int32_t)); - memcpy(blob.data(), &val, sizeof(int32_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const uint32_t val) -{ - std::vector blob(sizeof(uint32_t)); - memcpy(blob.data(), &val, sizeof(uint32_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const int64_t val) -{ - std::vector blob(sizeof(int64_t)); - memcpy(blob.data(), &val, sizeof(int64_t)); - return blob; -} - -template <> inline std::vector convertIntToBlob(const uint64_t val) -{ - std::vector blob(sizeof(uint64_t)); - memcpy(blob.data(), &val, sizeof(uint64_t)); - return blob; -} - -/// Users specialize this template so we can serialize the int->string mapping -/// for their enums. Ints are stored in the database, and the python modules -/// turn them back into enums later. -template -void defineEnumMap(std::string& enum_name, - std::map::type>& map) = delete; - -/*! - * \class EnumMap - * \brief This class holds and serializes the int->string mapping for the EnumT. - */ -template class EnumMap -{ -public: - using enum_int_t = typename std::underlying_type::type; - using enum_map_t = std::shared_ptr>; - - static EnumMap* instance() - { - static EnumMap map; - return ↦ - } - - const enum_map_t getMap() const { return map_; } - - const std::string& getEnumName() const { return enum_name_; } - - void serializeDefn(DatabaseManager* db_mgr) const; - -private: - EnumMap() - { - map_ = std::make_shared>(); - defineEnumMap(enum_name_, *map_); - } - - enum_map_t map_; - std::string enum_name_; - mutable bool serialized_ = false; -}; - -/// \class FieldBase -/// \brief This class is used to serialize information about a non-enum, -/// non-string field. -class FieldBase -{ -public: - FieldBase(const std::string& name, const StructFields type, const Format format = Format::none) : - name_(name), - dtype_(type), - format_(format) - { - } - - virtual ~FieldBase() = default; - - const std::string& getName() const { return name_; } - - Format getFormat() const { return format_; } - - StructFields getType() const { return dtype_; } - - virtual size_t getNumBytes() const { return getDTypeNumBytes(dtype_); } - - virtual void serializeDefn(DatabaseManager* db_mgr, const std::string& struct_name) const; - - void setIsAutocolorizeKey(bool is_autocolorize_key) - { - if (is_autocolorize_key_ && !is_autocolorize_key) - { - throw DBException("Only one column can be used as the autocolorize key"); - } - is_autocolorize_key_ = is_autocolorize_key; - } - - bool isAutocolorizeKey() const { return is_autocolorize_key_; } - - void setIsDisplayedByDefault(bool is_displayed_by_default) { is_displayed_by_default_ = is_displayed_by_default; } - - bool isDisplayedByDefault() const { return is_displayed_by_default_; } - -private: - std::string name_; - StructFields dtype_; - Format format_; - bool is_autocolorize_key_ = false; - bool is_displayed_by_default_ = true; -}; - -/// \class EnumField -/// \brief This class is used to serialize information about an enum field. -template class EnumField : public FieldBase -{ -public: - EnumField(const char* name) : - FieldBase(name, getFieldDTypeEnum::enum_int_t>()), - map_(EnumMap::instance()->getMap()), - enum_name_(EnumMap::instance()->getEnumName()) - { - } - - virtual void serializeDefn(DatabaseManager* db_mgr, const std::string& struct_name) const override; - -private: - const typename EnumMap::enum_map_t map_; - std::string enum_name_; -}; - -/// \class StringField -/// \brief This class is used to serialize information about a string field. -class StringField : public FieldBase -{ -public: - StringField(const char* name) : - FieldBase(name, StructFields::string_t) - { - } - - size_t getNumBytes() const override { return getDTypeNumBytes(StructFields::uint32_t); } -}; - -template class StructFieldSerializer; - -/// Users specialize this template to write the struct fields one by one into -/// the serializer. -template void writeStructFields(const StructT* s, StructFieldSerializer* serializer) -{ - (void)serializer; -} - -/// This helper class is used by the writeStructFields() specializations -/// supplied by the user. -/// -/// namespace simdb -/// { -/// template <> void -/// defineStructSchema(StructSchema& schema) -/// { -/// schema.addField("int32"); -/// schema.addField("dbl"); -/// schema.addBool("bool"); -/// schema.addString("str"); -/// } -/// -/// template <> void writeStructFields(const DummyPacket* all, -/// StructFieldSerializer* serializer) -/// { -/// serializer->writeField(all->int32); <-- here -/// serializer->writeField(all->dbl); <-- here -/// serializer->writeField(all->b); <-- here -/// serializer->writeField(all->str); <-- here -/// } -/// } // namespace simdb -/// -template class StructFieldSerializer -{ -public: - StructFieldSerializer(const std::vector>& fields, CollectionBuffer& buffer) : - fields_(fields), - buffer_(buffer) - { - } - - /// Write an entire struct. - void writeFields(const StructT* s) { writeStructFields(s, this); } - - /// Write a non-string, non-enum field. - template - typename std::enable_if::value && !std::is_same::value, void>::type - writeField(const FieldT val) - { - auto dtype = getFieldDTypeEnum(); - auto num_bytes = getDTypeNumBytes(dtype); - - if (fields_[current_field_idx_]->getNumBytes() != num_bytes) - { - throw DBException("Data type mismatch in writing struct field"); - } - - buffer_ << val; - ++current_field_idx_; - num_bytes_written_ += num_bytes; - } - - /// Write an enum field. - template - typename std::enable_if::value, void>::type writeField(const FieldT val) - { - using dtype = typename std::underlying_type::type; - writeField(static_cast(val)); - } - - /// Write a string field. - void writeField(const std::string& val) - { - if (dynamic_cast(fields_[current_field_idx_].get())) - { - uint32_t string_id = StringMap::instance()->getStringId(val); - writeField(string_id); - } else - { - throw DBException("Data type mismatch in writing struct field"); - } - } - - /// Write a string field. - void writeField(const char* val) { writeField(std::string(val)); } - - size_t numBytesWritten() const { return num_bytes_written_; } - - /// Get the buffer that the serializer is writing to. - CollectionBuffer& getBuffer() { return buffer_; } - -private: - const std::vector>& fields_; - size_t current_field_idx_ = 0; - CollectionBuffer& buffer_; - size_t num_bytes_written_ = 0; -}; - -template class StructSerializer; - -/// This class is used in order to define a struct-like collectable , which -/// only needs to be defined for non-trivial types. -/// -/// namespace simdb -/// { -/// template <> void -/// defineStructSchema(StructSchema& schema) -/// { -/// schema.addField("int32"); <-- here -/// schema.addField("dbl"); <-- here -/// schema.addBool("bool"); <-- here -/// schema.addString("str"); <-- here -/// } -/// } // namespace simdb -/// -template class StructSchema -{ -public: - const std::string& getStructName() const { return struct_name_; } - - size_t getStructNumBytes() const - { - size_t num_bytes = 0; - for (const auto& field : fields_) - { - num_bytes += field->getNumBytes(); - } - return num_bytes; - } - - template void addField(const char* name) - { - static_assert(!std::is_enum::value && !std::is_same::value && - !std::is_same::value, - "Use addEnum(), addString(), or addBool() instead"); - - fields_.emplace_back(new FieldBase(name, getFieldDTypeEnum())); - if (fields_.size() == 1) - { - setAutoColorizeColumn(name); - } - } - - template void addHex(const char* name) - { - static_assert(std::is_same::value || std::is_same::value, - "Hex modifier only supported for uint32_t and uint64_t"); - fields_.emplace_back(new FieldBase(name, getFieldDTypeEnum(), Format::hex)); - if (fields_.size() == 1) - { - setAutoColorizeColumn(name); - } - } - - void addBool(const char* name) - { - fields_.emplace_back(new FieldBase(name, getFieldDTypeEnum(), Format::boolalpha)); - if (fields_.size() == 1) - { - setAutoColorizeColumn(name); - } - } - - void addString(const char* name) - { - fields_.emplace_back(new StringField(name)); - if (fields_.size() == 1) - { - setAutoColorizeColumn(name); - } - } - - template void addEnum(const char* name) - { - static_assert(std::is_enum::value, "Use addField() for non-enum types"); - fields_.emplace_back(new EnumField(name)); - if (fields_.size() == 1) - { - setAutoColorizeColumn(name); - } - } - - void setAutoColorizeColumn(const char* name) - { - bool found = false; - for (auto& field : fields_) - { - if (field->getName() == name) - { - field->setIsAutocolorizeKey(true); - found = true; - } else - { - field->setIsAutocolorizeKey(false); - } - } - - if (!found) - { - throw DBException("Field not found: ") << name; - } - } - - void makeColumnHiddenByDefault(const char* name) - { - for (auto& field : fields_) - { - if (field->getName() == name) - { - field->setIsDisplayedByDefault(false); - return; - } - } - - throw DBException("Field not found: ") << name; - } - - void serializeDefn(DatabaseManager* db_mgr) const - { - static std::unordered_set serialized_structs; - if (serialized_structs.insert(struct_name_).second) - { - for (auto& field : fields_) - { - field->serializeDefn(db_mgr, struct_name_); - } - } - } - -private: - const std::string struct_name_ = demangle(typeid(StructT).name()); - std::vector> fields_; - - friend class StructSerializer; -}; - -/// This method is used in order to define a struct-like collectable , which -/// only needs to be defined for non-trivial types. -/// -/// namespace simdb -/// { -/// template <> void -/// defineStructSchema(StructSchema& schema) -/// { -/// schema.addField("int32"); <-- here -/// schema.addField("dbl"); <-- here -/// schema.addBool("bool"); <-- here -/// schema.addString("str"); <-- here -/// } -/// } // namespace simdb -/// -template inline void defineStructSchema(StructSchema& schema) -{ - (void)schema; -} - -/// This class is used to read a struct's fields and write them into a -/// CollectionBuffer or directly into a std::vector. It is also -/// responsible for serializing the struct's fields into the database so they -/// can be deserialized in Python (e.g. Argos). -template class StructSerializer -{ -public: - static StructSerializer* getInstance() - { - static_assert(!type_traits::is_any_pointer::value, "StructSerializer does not support pointer types"); - - static StructSerializer serializer; - return &serializer; - } - - const std::string& getStructName() const { return schema_.getStructName(); } - - size_t getStructNumBytes() const { return schema_.getStructNumBytes(); } - - void serializeDefn(DatabaseManager* db_mgr) const { schema_.serializeDefn(db_mgr); } - - size_t writeStruct(const StructT* s, CollectionBuffer& buffer) const - { - StructFieldSerializer field_serializer(schema_.fields_, buffer); - field_serializer.writeFields(s); - return field_serializer.numBytesWritten(); - } - - void extract(const StructT* s, std::vector& bytes) const - { - CollectionBuffer buffer(bytes); - writeStruct(s, buffer); - } - -private: - StructSerializer() { defineStructSchema(schema_); } - - StructSchema schema_; -}; - -} // namespace simdb diff --git a/include/simdb/apps/argos/StreamBuffer.hpp b/include/simdb/apps/argos/StreamBuffer.hpp new file mode 100644 index 00000000..661f2a25 --- /dev/null +++ b/include/simdb/apps/argos/StreamBuffer.hpp @@ -0,0 +1,89 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/utils/TinyStrings.hpp" +#include "simdb/utils/TypeTraits.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace simdb::argos { + +/// \class StreamBuffer +/// \brief Utility class which wraps a char buffer with typed append operators. +class StreamBuffer +{ +public: + StreamBuffer(std::vector& out, TinyStrings<>* tiny_strings = nullptr, bool clear_first = true) : + out_(&out), + tiny_strings_(tiny_strings) + { + if (clear_first) + { + out_->clear(); + } + } + + StreamBuffer(const StreamBuffer&) = delete; + StreamBuffer(StreamBuffer&&) = default; + + void append(const void* data, const size_t num_bytes) + { + auto bytes = static_cast(data); + out_->insert(out_->end(), bytes, bytes + num_bytes); + } + + void append(const bool val) { append(static_cast(val)); } + + void append(const std::string& s) { append(tiny_strings_->getStringID(s)); } + + template + std::enable_if_t && std::is_standard_layout_v && !std::is_enum_v, void> + append(const T& val) + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + append(&val, sizeof(T)); + } + + template void append(const std::vector& val) + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + append(val.data(), val.size() * sizeof(T)); + } + + template void append(const T (&val)[N]) + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + append(val, N * sizeof(T)); + } + + template void append(const std::array& val) + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + append(val.data(), N * sizeof(T)); + } + + template std::enable_if_t, void> append(const T val) + { + using underlying_t = std::underlying_type_t; + append(static_cast(val)); + } + + size_t size() const { return out_->size(); } + + bool operator==(const StreamBuffer& other) const { return *out_ == *other.out_; } + + bool operator==(const std::vector& other) const { return *out_ == other; } + +private: + std::vector* out_ = nullptr; + TinyStrings<>* tiny_strings_ = nullptr; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/Timestamps.hpp b/include/simdb/apps/argos/Timestamps.hpp new file mode 100644 index 00000000..73dd62b5 --- /dev/null +++ b/include/simdb/apps/argos/Timestamps.hpp @@ -0,0 +1,83 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/schema/SchemaDef.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" + +#include + +namespace simdb::argos { + +/// \class Timestamp +/// \brief Timestamp that can get current time values via a backpointer, +/// C-style function, or a std::function +class Timestamp +{ +public: + /// \brief Construct with a backpointer to get the current time value + Timestamp(const uint64_t* backpointer) : + backpointer_(backpointer) + { + assert(backpointer); + } + + /// \brief Construct with a C-style function pointer to get the current time value + Timestamp(uint64_t (*fn)()) : + cfuncpointer_(fn) + { + assert(fn); + } + + /// \brief Construct with a std::function to get the current time value + Timestamp(std::function fn) : + stdfunction_(fn) + { + assert(fn); + } + + /// Read the current simulation time + uint64_t getTime() const + { + if (backpointer_) + { + return *backpointer_; + } + if (cfuncpointer_) + { + return cfuncpointer_(); + } + return stdfunction_(); + } + + /// Create or look up a Timestamps table row for the given simulation time; return rowid. + static int createTimestampInDatabase(DatabaseManager* db_mgr, uint64_t time) + { + // Ensure we don't create multiple entries in the Timestamps table + // that have the same Timestamp column value (it will throw; must + // be unique). + // + // Note that this is called on the DB thread and it is not a big + // performance issue to query alongside the INSERT. + auto query = db_mgr->createQuery("Timestamps"); + query->addConstraintForUInt64("Timestamp", Constraints::EQUAL, time); + + int id; + query->select("Id", id); + + if (query->getResultSet().getNextRecord()) + { + assert(id > 0); + return id; + } + + return db_mgr->INSERT(SQL_TABLE("Timestamps"), SQL_VALUES(time))->getId(); + } + +private: + const uint64_t* backpointer_ = nullptr; + uint64_t (*cfuncpointer_)() = nullptr; + std::function stdfunction_ = nullptr; +}; + +} // namespace simdb::argos diff --git a/include/simdb/apps/argos/TreeNode.hpp b/include/simdb/apps/argos/TreeNode.hpp deleted file mode 100644 index a9b52358..00000000 --- a/include/simdb/apps/argos/TreeNode.hpp +++ /dev/null @@ -1,74 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include -#include -#include -#include -#include - -namespace simdb { - -/// This class is used to build a tree structure from a list of string -/// locations. For example, given the three locations: -/// -/// "top.mid1.bottom1" -/// "top.mid1.bottom2" -/// "top.bottom3" -/// -/// The buildTree() method will create a TreeNode structure that looks like: -/// -/// root -/// | -/// +-- top -/// | -/// +-- mid1 -/// | | -/// | +-- bottom1 -/// | | -/// | +-- bottom2 -/// | -/// +-- bottom3 -struct TreeNode -{ - std::string name; - std::vector> children; - const TreeNode* parent = nullptr; - - int db_id = 0; - int clk_id = 0; - bool is_collectable = false; - - TreeNode(const std::string& name, const TreeNode* parent = nullptr) : - name(name), - parent(parent) - { - } - - std::string getLocation() const - { - std::vector node_names; - auto node = this; - while (node && node->parent) - { - node_names.push_back(node->name); - node = node->parent; - } - - std::reverse(node_names.begin(), node_names.end()); - std::ostringstream oss; - for (size_t idx = 0; idx < node_names.size(); ++idx) - { - oss << node_names[idx]; - if (idx != node_names.size() - 1) - { - oss << "."; - } - } - - return oss.str(); - } -}; - -} // namespace simdb diff --git a/include/simdb/pipeline/PollingThread.hpp b/include/simdb/pipeline/PollingThread.hpp index 0c906af0..63b036eb 100644 --- a/include/simdb/pipeline/PollingThread.hpp +++ b/include/simdb/pipeline/PollingThread.hpp @@ -4,6 +4,7 @@ #include "simdb/Exceptions.hpp" #include "simdb/pipeline/Runnable.hpp" +#include "simdb/utils/StreamFormatters.hpp" #include #include @@ -200,6 +201,7 @@ class PollingThread runnable->print(std::cout, 4); } + [[maybe_unused]] ios_format_saver fmt_saver(std::cout); std::cout << "\n"; std::cout << " Performance report:\n"; std::cout << " Num times run: " << num_times_run_ << "\n"; diff --git a/include/simdb/schema/SchemaDef.hpp b/include/simdb/schema/SchemaDef.hpp index 2188a6b6..d1496792 100644 --- a/include/simdb/schema/SchemaDef.hpp +++ b/include/simdb/schema/SchemaDef.hpp @@ -138,6 +138,10 @@ class Column { throw DBException("Unable to set default value string (data type mismatch)"); } + if (unique_col_) + { + throw DBException("Cannot set default column value on a unique column"); + } setDefaultValueStr_(val); } @@ -150,6 +154,23 @@ class Column /// e.g. "CREATE TABLE ..." const std::string& getDefaultValueAsString() const { return default_val_string_; } + /// Ensure the given column is unique: + /// CREATE TABLE my_table ( + /// Id INTEGER PRIMARY KEY, + /// Timestamp INTEGER UNIQUE <--- + /// ); + void ensureUnique() + { + if (hasDefaultValue()) + { + throw DBException("Column has default value; cannot make unique"); + } + unique_col_ = true; + } + + /// Check if this column should be created with the UNIQUE tag. + bool isUnique() const { return unique_col_; } + private: /// Default values are stringified. For doubles, we need maximum precision. void writeDefaultValue_(std::ostringstream& oss, const double val) const @@ -190,6 +211,9 @@ class Column /// Optional default value (stringified) std::string default_val_string_; + + /// Should this column be defined as e.g. "Timestamp INTEGER UNIQUE"? + bool unique_col_ = false; }; /*! @@ -270,6 +294,23 @@ class Table return *this; } + /// Ensure the given column is unique: + /// CREATE TABLE my_table ( + /// Id INTEGER PRIMARY KEY, + /// Timestamp INTEGER UNIQUE <--- + /// ); + Table& ensureUnique(const std::string& col_name) + { + auto iter = columns_by_name_.find(col_name); + if (iter == columns_by_name_.end()) + { + throw DBException("No column named ") << col_name << " in table " << name_; + } + + iter->second->ensureUnique(); + return *this; + } + /// Index this table's records on the given column. /// CREATE INDEX IndexName ON TableName(ColumnName) Table& createIndexOn(const std::string& col_name) { return createCompoundIndexOn({col_name}); } diff --git a/include/simdb/sqlite/Connection.hpp b/include/simdb/sqlite/Connection.hpp index 0bb864ee..c41c1463 100644 --- a/include/simdb/sqlite/Connection.hpp +++ b/include/simdb/sqlite/Connection.hpp @@ -234,6 +234,10 @@ class Connection : public Transaction { oss << " DEFAULT " << column->getDefaultValueAsString(); } + if (column->isUnique() && column->getName() != pkey) + { + oss << " UNIQUE"; + } if (idx != columns.size() - 1) { oss << ", "; diff --git a/include/simdb/sqlite/DatabaseManager.hpp b/include/simdb/sqlite/DatabaseManager.hpp index a70f9d47..d0a4e3ff 100644 --- a/include/simdb/sqlite/DatabaseManager.hpp +++ b/include/simdb/sqlite/DatabaseManager.hpp @@ -544,6 +544,11 @@ class DatabaseManager /// \see See public INSERT() method for details on how to call this method. std::unique_ptr INSERT_(SqlTable&& table, SqlColumns&& cols, SqlValues&& vals) { + if (cols.getColNames().size() != vals.getNumValues()) + { + throw DBException("Mismatching number of columns and values"); + } + std::unique_ptr record; db_conn_->safeTransaction([&]() { diff --git a/include/simdb/sqlite/Dump.hpp b/include/simdb/sqlite/Dump.hpp new file mode 100644 index 00000000..d0b71f9a --- /dev/null +++ b/include/simdb/sqlite/Dump.hpp @@ -0,0 +1,199 @@ +// -*- C++ -*- + +#pragma once + +#include "simdb/sqlite/DatabaseManager.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace simdb { + +namespace detail { +template inline bool to_string(const std::optional& opt, std::string& out) +{ + if (!opt.has_value()) + { + return false; + } + + if constexpr (std::is_same_v) + { + out = opt.value(); + } else + { + out = std::to_string(opt.value()); + } + return true; +} + +inline void dumpTable(const std::string& table_name, const std::vector& headers, + const std::vector>& row_strings) +{ + std::vector col_sizes; + for (const auto& h : headers) + { + col_sizes.push_back(h.size()); + } + + auto num_dashes = std::accumulate(col_sizes.begin(), col_sizes.end(), 0); + + for (size_t i = 0; i < row_strings.size(); ++i) + { + for (size_t j = 0; j < row_strings[i].size(); ++j) + { + const auto& s = row_strings[i][j]; + auto& size = col_sizes.at(j); + size = std::max(size, s.size()); + } + } + + // Give some extra space + for (auto& size : col_sizes) + { + size += 4; + } + + // Begin printing + std::cout << "=========================================================\n"; + std::cout << "Table: " << table_name << "\n"; + + // Print headers + for (size_t i = 0; i < headers.size(); ++i) + { + std::cout << std::setw(col_sizes[i]) << std::left << headers[i]; + } + std::cout << "\n"; + + // Print dashes + std::cout << std::string(num_dashes, '-') << "\n"; + + // Print rows + if (row_strings.empty()) + { + std::cout << "(table has no records)\n"; + } else + { + for (size_t i = 0; i < row_strings.size(); ++i) + { + for (size_t j = 0; j < row_strings[i].size(); ++j) + { + const auto& s = row_strings[i][j]; + std::cout << std::setw(col_sizes[j]) << std::left << s; + } + std::cout << "\n"; + } + } + + std::cout << std::endl; +} +} // namespace detail + +inline void dumpTable(DatabaseManager* db_mgr, const std::string& table_name) +{ + // TODO cnyce: Can we call sqlite3 directly and simply parse the dumped output on stdout? + struct SelectedValueUnion + { + std::optional i; + std::optional I; + std::optional q; + std::optional Q; + std::optional d; + std::optional s; + + SelectedValueUnion(SqlQuery* query, const Column* col) : + SelectedValueUnion(query, col->getName(), col->getDataType()) + { + } + + SelectedValueUnion(SqlQuery* query, const std::string& col_name, const SqlDataType dtype) + { + using dt = SqlDataType; + switch (dtype) + { + case dt::int32_t: + query->select(col_name.c_str(), i); + break; + case dt::uint32_t: + query->select(col_name.c_str(), I); + break; + case dt::int64_t: + query->select(col_name.c_str(), q); + break; + case dt::uint64_t: + query->select(col_name.c_str(), Q); + break; + case dt::double_t: + query->select(col_name.c_str(), d); + break; + case dt::string_t: + query->select(col_name.c_str(), s); + break; + default: + break; + } + } + + std::optional stringify() const + { + std::string out; + if (!detail::to_string(i, out) && !detail::to_string(I, out) && !detail::to_string(q, out) && + !detail::to_string(Q, out) && !detail::to_string(d, out) && !detail::to_string(s, out)) + { + return std::nullopt; + } + return out; + } + }; + + const auto& schema = db_mgr->getSchema(); + const auto& table = schema.getTable(table_name); + const auto& columns = table.getColumns(); + + std::vector headers; + std::deque selects; + + auto query = db_mgr->createQuery(table_name.c_str()); + if (table.getPrimaryKey() == "Id") + { + headers.push_back("Id"); + selects.emplace_back(query.get(), "Id", SqlDataType::int32_t); + } + + for (const auto& col : columns) + { + headers.push_back(col->getName()); + selects.emplace_back(query.get(), col.get()); + } + + std::vector> row_strings; + + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + std::vector col_strings; + for (const auto& col_result : selects) + { + auto s = col_result.stringify(); + if (s.has_value()) + { + col_strings.push_back(s.value()); + } else + { + col_strings.push_back(""); + } + } + row_strings.emplace_back(std::move(col_strings)); + } + + detail::dumpTable(table_name, headers, row_strings); +} + +} // namespace simdb diff --git a/include/simdb/sqlite/Iterator.hpp b/include/simdb/sqlite/Iterator.hpp index dabf4edc..6d0a1c2b 100644 --- a/include/simdb/sqlite/Iterator.hpp +++ b/include/simdb/sqlite/Iterator.hpp @@ -6,13 +6,60 @@ #include "simdb/utils/utf16.hpp" #include +#include #include #include #include +#include #include namespace simdb { +namespace detail { +template struct unrolled_type +{ + using type = T; +}; + +template struct unrolled_type> +{ + using type = T; +}; + +template using unrolled_type_t = typename unrolled_type::type; + +template +struct unrolled_is_same + : std::conditional_t, unrolled_type_t>, std::true_type, std::false_type> +{ +}; + +template inline constexpr bool unrolled_is_same_v = unrolled_is_same::value; + +template struct is_optional : std::false_type +{ +}; + +template struct is_optional> : std::true_type +{ +}; + +template inline constexpr bool is_optional_v = is_optional::value; + +inline bool columnIsNull(sqlite3_stmt* stmt, const int idx) +{ + return sqlite3_column_type(stmt, idx) == SQLITE_NULL; +} + +inline void requireNonNullColumn(sqlite3_stmt* stmt, const int idx, const std::string& col_name) +{ + if (columnIsNull(stmt, idx)) + { + throw DBException("Column '") << col_name << "' is NULL; use std::optional overload"; + } +} +} // namespace detail + /*! * \class ResultWriterBase * @@ -55,14 +102,16 @@ class ResultWriterBase * \brief Responsible for writing int32 record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterInt32 : public ResultWriterBase +template class ResultWriterInt32 : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterInt32(const char* col_name, int32_t* user_var) : + ResultWriterInt32(const char* col_name, IntT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -72,14 +121,26 @@ class ResultWriterInt32 : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else + { + detail::requireNonNullColumn(stmt, idx, getColName()); + } + *user_var_ = sqlite3_column_int(stmt, idx); } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterInt32(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override { return new ResultWriterInt32(getColName().c_str(), user_var_); } private: - int32_t* user_var_; + IntT* user_var_; }; /*! @@ -88,14 +149,16 @@ class ResultWriterInt32 : public ResultWriterBase * \brief Responsible for writing uint32 record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterUInt32 : public ResultWriterBase +template class ResultWriterUInt32 : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterUInt32(const char* col_name, uint32_t* user_var) : + ResultWriterUInt32(const char* col_name, UIntT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -105,6 +168,18 @@ class ResultWriterUInt32 : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else + { + detail::requireNonNullColumn(stmt, idx, getColName()); + } + sqlite3_int64 tmp = sqlite3_column_int64(stmt, idx); if (tmp < 0 || tmp > UINT32_MAX) @@ -116,10 +191,10 @@ class ResultWriterUInt32 : public ResultWriterBase } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterUInt32(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override { return new ResultWriterUInt32(getColName().c_str(), user_var_); } private: - uint32_t* user_var_; + UIntT* user_var_; }; /*! @@ -128,14 +203,16 @@ class ResultWriterUInt32 : public ResultWriterBase * \brief Responsible for writing int64 record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterInt64 : public ResultWriterBase +template class ResultWriterInt64 : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterInt64(const char* col_name, int64_t* user_var) : + ResultWriterInt64(const char* col_name, IntT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -145,14 +222,26 @@ class ResultWriterInt64 : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else + { + detail::requireNonNullColumn(stmt, idx, getColName()); + } + *user_var_ = sqlite3_column_int64(stmt, idx); } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterInt64(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override { return new ResultWriterInt64(getColName().c_str(), user_var_); } private: - int64_t* user_var_; + IntT* user_var_; }; /*! @@ -161,14 +250,16 @@ class ResultWriterInt64 : public ResultWriterBase * \brief Responsible for writing uint64 record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterUInt64 : public ResultWriterBase +template class ResultWriterUInt64 : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterUInt64(const char* col_name, uint64_t* user_var) : + ResultWriterUInt64(const char* col_name, UIntT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -178,6 +269,18 @@ class ResultWriterUInt64 : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else + { + detail::requireNonNullColumn(stmt, idx, getColName()); + } + const void* blob = sqlite3_column_text16(stmt, idx); if (sqlite3_column_type(stmt, idx) == SQLITE_TEXT && blob != nullptr) { @@ -190,10 +293,10 @@ class ResultWriterUInt64 : public ResultWriterBase } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterUInt64(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override { return new ResultWriterUInt64(getColName().c_str(), user_var_); } private: - uint64_t* user_var_; + UIntT* user_var_; }; /*! @@ -202,14 +305,16 @@ class ResultWriterUInt64 : public ResultWriterBase * \brief Responsible for writing double record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterDouble : public ResultWriterBase +template class ResultWriterDouble : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterDouble(const char* col_name, double* user_var) : + ResultWriterDouble(const char* col_name, DoubleT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -219,14 +324,29 @@ class ResultWriterDouble : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else + { + detail::requireNonNullColumn(stmt, idx, getColName()); + } + *user_var_ = sqlite3_column_double(stmt, idx); } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterDouble(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override + { + return new ResultWriterDouble(getColName().c_str(), user_var_); + } private: - double* user_var_; + DoubleT* user_var_; }; /*! @@ -235,14 +355,16 @@ class ResultWriterDouble : public ResultWriterBase * \brief Responsible for writing text record values to the user's local * variables whenever a query's result set iterator is advanced. */ -class ResultWriterString : public ResultWriterBase +template class ResultWriterString : public ResultWriterBase { + static_assert(detail::unrolled_is_same_v); + public: /// \brief Construction /// \param col_name Name of the selected column /// \param user_var Pointer to the local variable where result values are /// written to - ResultWriterString(const char* col_name, std::string* user_var) : + ResultWriterString(const char* col_name, StringT* user_var) : ResultWriterBase(col_name), user_var_(user_var) { @@ -252,14 +374,31 @@ class ResultWriterString : public ResultWriterBase /// and copy it to the user's local variable. void writeToUserVar(sqlite3_stmt* stmt, const int idx) const override { - *user_var_ = (const char*)sqlite3_column_text(stmt, idx); + if constexpr (detail::is_optional_v) + { + if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = std::nullopt; + return; + } + } else if (detail::columnIsNull(stmt, idx)) + { + *user_var_ = ""; + return; + } + + const char* text = reinterpret_cast(sqlite3_column_text(stmt, idx)); + *user_var_ = text ? text : ""; } /// Return a new copy of this writer. - ResultWriterBase* clone() const override { return new ResultWriterString(getColName().c_str(), user_var_); } + ResultWriterBase* clone() const override + { + return new ResultWriterString(getColName().c_str(), user_var_); + } private: - std::string* user_var_; + StringT* user_var_; }; /*! diff --git a/include/simdb/sqlite/PreparedINSERT.hpp b/include/simdb/sqlite/PreparedINSERT.hpp index 88a9fdff..ee0c1e53 100644 --- a/include/simdb/sqlite/PreparedINSERT.hpp +++ b/include/simdb/sqlite/PreparedINSERT.hpp @@ -210,6 +210,8 @@ class PreparedINSERT return id; } + /// TODO cnyce: createRecordWithColValues using move semantics + private: SQLitePreparedStatement prepared_stmt_; sqlite3_stmt* stmt_ = nullptr; diff --git a/include/simdb/sqlite/Query.hpp b/include/simdb/sqlite/Query.hpp index af44a634..6f935d4b 100644 --- a/include/simdb/sqlite/Query.hpp +++ b/include/simdb/sqlite/Query.hpp @@ -410,6 +410,9 @@ class SqlQuery /// Reset the query constraints. void resetConstraints() { constraint_clauses_.clear(); } + /// Non-optional scalar SELECT overloads throw if a column value is SQL NULL. + /// Use the std::optional overloads when a column may be unset. + /// /// SELECT column values and write to the local variable on each iteration /// (int32_t). /// @@ -420,6 +423,12 @@ class SqlQuery result_writers_.emplace_back(new ResultWriterInt32(col_name, &user_var)); } + /// Use std::optional overload to query record values that may be NULL/unset + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterInt32>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (uint32_t). void select(const char* col_name, uint32_t& user_var) @@ -427,6 +436,12 @@ class SqlQuery result_writers_.emplace_back(new ResultWriterUInt32(col_name, &user_var)); } + /// Use std::optional overload to query record values that may be NULL/unset + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterUInt32>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (int64_t). /// @@ -437,6 +452,12 @@ class SqlQuery result_writers_.emplace_back(new ResultWriterInt64(col_name, &user_var)); } + /// Use std::optional overload to query record values that may be NULL/unset + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterInt64>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (uint64_t). /// @@ -447,6 +468,12 @@ class SqlQuery result_writers_.emplace_back(new ResultWriterUInt64(col_name, &user_var)); } + /// Use std::optional overload to query record values that may be NULL/unset + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterUInt64>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (double). /// @@ -457,21 +484,39 @@ class SqlQuery result_writers_.emplace_back(new ResultWriterDouble(col_name, &user_var)); } + /// Use std::optional overload to query record values that may be NULL/unset + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterDouble>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (string). /// /// std::string val; /// query->select("Col", val); + /// + /// NULL columns are written as the empty string. Use the std::optional overload + /// to distinguish SQL NULL from an explicitly stored empty string. void select(const char* col_name, std::string& user_var) { result_writers_.emplace_back(new ResultWriterString(col_name, &user_var)); } + /// Use std::optional overload to distinguish SQL NULL from "". + void select(const char* col_name, std::optional& user_var) + { + result_writers_.emplace_back(new ResultWriterString>(col_name, &user_var)); + } + /// SELECT column values and write to the local variable on each iteration /// (blob). /// /// std::vector val; /// query->select("Col", val); + /// + /// Note that this does not have a std::optional overload since std::vector::empty() + /// makes NULL obvious. template void select(const char* col_name, std::vector& user_var) { result_writers_.emplace_back(new ResultWriterBlob(col_name, &user_var)); diff --git a/include/simdb/sqlite/Table.hpp b/include/simdb/sqlite/Table.hpp index fe951692..f701157c 100644 --- a/include/simdb/sqlite/Table.hpp +++ b/include/simdb/sqlite/Table.hpp @@ -157,6 +157,8 @@ class SqlValues } } + size_t getNumValues() const { return col_vals_.size(); } + private: std::list> col_vals_; }; diff --git a/include/simdb/utils/CircularBuffer.hpp b/include/simdb/utils/CircularBuffer.hpp deleted file mode 100644 index f69d1bae..00000000 --- a/include/simdb/utils/CircularBuffer.hpp +++ /dev/null @@ -1,88 +0,0 @@ -// -*- C++ -*- - -#pragma once - -#include "simdb/Exceptions.hpp" -#include -#include - -namespace simdb { - -template class CircularBuffer -{ -public: - // Add an element using move semantics - void push(DataT&& item) - { - array_[head_] = std::move(item); - advanceHead_(); - } - - // Emplace construct in-place - template void emplace(Args&&... args) - { - array_[head_] = DataT(std::forward(args)...); - advanceHead_(); - } - - // Pop the oldest element (moved out) - DataT pop() - { - if (empty()) - { - throw simdb::DBException("Buffer is empty"); - } - - DataT item = std::move(array_[tail_]); - full_ = false; - tail_ = (tail_ + 1) % BufferLen; - return item; - } - - // Check if empty - bool empty() const { return !full_ && head_ == tail_; } - - // Check if full - bool full() const { return full_; } - - // Get number of elements in buffer - size_t size() const - { - if (full_) - { - return BufferLen; - } - - if (head_ >= tail_) - { - return head_ - tail_; - } - - return BufferLen + head_ - tail_; - } - - // Reset the buffer (does not deallocate DataT's - based on std::array) - void reset() - { - head_ = tail_; - full_ = false; - } - -private: - void advanceHead_() - { - if (full_) - { - tail_ = (tail_ + 1) % BufferLen; - } - head_ = (head_ + 1) % BufferLen; - full_ = (head_ == tail_); - } - - std::array array_; - size_t head_ = 0; - size_t tail_ = 0; - bool full_ = false; -}; - -} // namespace simdb diff --git a/include/simdb/utils/Compress.hpp b/include/simdb/utils/Compress.hpp index 95b6c64b..8ed3b6ec 100644 --- a/include/simdb/utils/Compress.hpp +++ b/include/simdb/utils/Compress.hpp @@ -26,7 +26,10 @@ inline void compressData(const void* data_ptr, size_t num_bytes, std::vector(zlib_empty), + reinterpret_cast(zlib_empty) + sizeof(zlib_empty)); return; } diff --git a/include/simdb/utils/StreamFormatters.hpp b/include/simdb/utils/StreamFormatters.hpp new file mode 100644 index 00000000..9fe1378c --- /dev/null +++ b/include/simdb/utils/StreamFormatters.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace simdb { + +class ios_format_saver +{ +public: + explicit ios_format_saver(std::ios& s) : + stream_(s) + { + state_.copyfmt(s); + } + + ~ios_format_saver() { stream_.copyfmt(state_); } + +private: + std::ios& stream_; + std::ios state_{nullptr}; +}; + +} // namespace simdb diff --git a/include/simdb/utils/TickTock.hpp b/include/simdb/utils/TickTock.hpp index 379aa34f..3b4b1bf0 100644 --- a/include/simdb/utils/TickTock.hpp +++ b/include/simdb/utils/TickTock.hpp @@ -9,6 +9,8 @@ #include #include +#include "simdb/utils/StreamFormatters.hpp" + namespace simdb::utils { /*! @@ -44,6 +46,7 @@ class SelfProfiler sorted_avg_times.push_back(method_results.second / method_results.first); } + [[maybe_unused]] ios_format_saver fmt_saver(std::cout); std::sort(sorted_avg_times.begin(), sorted_avg_times.end(), std::greater()); for (auto avg_time : sorted_avg_times) { diff --git a/include/simdb/utils/TinyStrings.hpp b/include/simdb/utils/TinyStrings.hpp index 1e7f5957..58d36f79 100644 --- a/include/simdb/utils/TinyStrings.hpp +++ b/include/simdb/utils/TinyStrings.hpp @@ -2,34 +2,26 @@ #pragma once +#include "simdb/Exceptions.hpp" #include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/sqlite/PreparedINSERT.hpp" #include "simdb/utils/DeferredLock.hpp" +#include "simdb/utils/TypeTraits.hpp" #include namespace simdb { /// To keep SimDB collection as fast and small as possible, we serialize strings /// not as actual strings, but as ints. This class is used to map strings to -/// ints, and is periodically serialized to a database table. +/// ints in memory, and may be serialized to a database table at teardown. template class TinyStrings { public: - TinyStrings(DatabaseManager* db_mgr, const std::string& table_name = "TinyStringIDs") : - db_mgr_(db_mgr), - table_name_(table_name) - { - const auto& schema = db_mgr_->getSchema(); - if (!schema.hasTable(table_name_)) - { - Schema append_schema; - using dt = simdb::SqlDataType; - auto& tbl = append_schema.addTable(table_name_); - tbl.addColumn("StringValue", dt::string_t); - tbl.addColumn("StringID", dt::int32_t); - db_mgr->appendSchema(append_schema); - } - } + static inline constexpr uint32_t BAD_STRING_ID = 0; + TinyStrings() = default; + + /// Add or get a string ID for the given string. std::pair insert(const std::string& s) { DeferredLock lock(mutex_); @@ -48,8 +40,14 @@ template class TinyStrings } } + /// Get a string ID for the given string uint32_t getStringID(const std::string& s) { + if (s.empty()) + { + return BAD_STRING_ID; + } + DeferredLock lock(mutex_); if constexpr (MutexProtect) { @@ -59,20 +57,52 @@ template class TinyStrings return getStringID_(s); } - /// Serialize the current string map to the database. - void serialize() + /// Get a string ID for the given string + uint32_t getStringID(const char* s) + { + assert(s != nullptr); + return getStringID(std::string(s)); + } + + /// Get a string ID for the given string (pointer) + template + type_traits::enable_if_t && type_traits::is_any_pointer_v, uint32_t> + getStringID(const S& s) + { + if (s) + { + return getStringID(*s); + } + return BAD_STRING_ID; + } + + /// Serialize newly seen string mappings to the database. + void serialize(DatabaseManager* db_mgr) { - db_mgr_->safeTransaction([&]() { + if (!db_mgr) + { + throw DBException("TinyStrings::serialize requires a DatabaseManager"); + } + + db_mgr->safeTransaction([&]() { DeferredLock lock(mutex_); if constexpr (MutexProtect) { lock.lock(); } + if (allowed_db_ == nullptr) + { + allowed_db_ = db_mgr; + } else if (allowed_db_ != db_mgr) + { + throw DBException("TinyStrings::serialize may only target one DatabaseManager"); + } + + auto inserter = db_mgr->prepareINSERT(SQL_TABLE("TinyStringIDs")); for (const auto& [string_id, string_val] : unserialized_map_) { - db_mgr_->INSERT(SQL_TABLE(table_name_), SQL_COLUMNS("StringValue", "StringID"), - SQL_VALUES(string_val, string_id)); + inserter->createRecordWithColValues(string_val, string_id); } unserialized_map_.clear(); @@ -85,7 +115,12 @@ template class TinyStrings auto iter = map_->find(s); if (iter == map_->end()) { - uint32_t id = map_->size(); + if (map_->size() == UINT32_MAX) + { + throw DBException("Too many TinyStrings created. UINT32_MAX has been reached."); + } + + uint32_t id = map_->size() + 1; map_->insert({s, id}); unserialized_map_.insert({id, s}); return id; @@ -101,9 +136,8 @@ template class TinyStrings string_map_t map_ = std::make_shared>(); unserialized_string_map_t unserialized_map_; - DatabaseManager* db_mgr_; - std::string table_name_; - std::mutex mutex_; + DatabaseManager* allowed_db_ = nullptr; + mutable std::mutex mutex_; }; } // namespace simdb diff --git a/include/simdb/utils/TypeTraits.hpp b/include/simdb/utils/TypeTraits.hpp index 855a1785..dd036037 100644 --- a/include/simdb/utils/TypeTraits.hpp +++ b/include/simdb/utils/TypeTraits.hpp @@ -1,5 +1,10 @@ // -*- C++ -*- +/** + * \file TypeTraits.hpp + * \brief Contains various helpers that are useful for template meta-programming. + */ + #pragma once #include @@ -18,7 +23,197 @@ #include namespace simdb::type_traits { +// If compiler is C++11 compliant, then use explicit aliases. +#if __cplusplus == 201103L + +/** + * \brief This templated struct takes a parameter pack and + * return a nested value to be true, if all the elements in + * the pack are unsigned types. + * + * This is the generic template struct. + */ +template struct all_unsigned; + +/** + * \brief This is the empty template specialization. + */ +template <> struct all_unsigned<> : public std::true_type +{ +}; + +/** + * \brief This is the template specialization with one or more + * elements in the pack. + */ +template struct all_unsigned +{ + static constexpr bool value{std::is_unsigned::type>::value and + all_unsigned::value}; +}; + +/** + * \brief This templated struct takes a parameter pack and + * return a nested value to be true, if all the elements in + * the pack are signed types. + * + * This is the generic template struct. + */ +template struct all_signed; + +/** + * \brief This is the empty template specialization. + */ +template <> struct all_signed<> : public std::true_type +{ +}; + +/** + * \brief This is the template specialization with one or more + * elements in the pack. + */ +template struct all_signed +{ + static constexpr bool value{std::is_signed::type>::value and all_signed::value}; +}; + +/** + * \brief This templated struct takes a parameter pack and + * return a nested value to be true, if all the elements in + * the pack are of the same sign. + */ +template struct all_same_sign +{ + static constexpr bool value{all_signed::value or all_unsigned::value}; +}; + +/** + * \brief This templated struct takes a parameter pack and + * return a nested value to be true, if all the elements in + * the pack are integral types. + * + * This is the generic template struct. + */ +template struct all_are_integral; + +/** + * \brief This is the empty template specialization. + */ +template <> struct all_are_integral<> : public std::true_type +{ +}; + +/** + * \brief This is the template specialization with one or more + * elements in the pack. + */ +template struct all_are_integral +{ + static constexpr bool value{std::is_integral::type>::value and + all_are_integral::value}; +}; + +/** \brief Alias Template for std::enable_if. + */ +template using enable_if_t = typename std::enable_if::type; + +/** \brief Alias Template for std::decay + */ +template using decay_t = typename std::decay::type; + +/** \brief Alias Template for std::underlying_type + */ +template using underlying_type_t = typename std::underlying_type::type; + +// If compiler is C++14/17 compliant, then use standard aliases. +#elif __cplusplus > 201103L +/** \brief Alias Template for std::enable_if. + */ +template using enable_if_t = typename std::enable_if_t; + +/** \brief Alias Template for std::decay + */ +template using decay_t = typename std::decay_t; + +/** \brief Alias Template for std::underlying_type + */ +template using underlying_type_t = typename std::underlying_type_t; +#endif + +/** + * \brief This templated struct takes a target type and another + * parameter pack of types and returns a nested boolean value + * of true, if the target type matches any one of the types in + * the parameter pack. + */ +template struct matches_any; + +/** + * \brief This templated struct takes a target type and another + * parameter pack of types and returns a nested boolean value + * of true, if the target type matches any one of the types in + * the parameter pack. + * + * This is template specialization when there are no types left + * in the parameter pack. + */ +template struct matches_any : public std::true_type +{ +}; + +/** + * \brief This templated struct takes a target type and another + * parameter pack of types and returns a nested boolean value + * of true, if the target type matches any one of the types in + * the parameter pack. + * + * This is template specialization when there are one or more types + * left in the parameter pack. + */ +template struct matches_any +{ + static constexpr bool value{std::is_same::value or matches_any::value}; +}; +/** + * \brief This templated struct takes a type and gives + * back a nested typedef of a pointer to that type. + */ +template struct add_pointer +{ + using type = T*; +}; + +template struct add_pointer +{ + using type = T; +}; + +template struct add_pointer +{ + using type = T; +}; + +template struct add_pointer +{ + using type = T; +}; + +template struct add_pointer +{ + using type = T; +}; + +/** \brief Alias Template for add_pointer. + */ +template using add_pointer_t = typename add_pointer::type; + +/** + * \brief This templated struct lets us know about + * whether the datatype is actually an ordinary object or + * pointer to that object. This is specialized for + * a couple different signatures. + */ template struct is_any_pointer : public std::false_type { }; @@ -87,6 +282,20 @@ template struct is_any_pointer const&> : public st { }; +template inline constexpr bool is_any_pointer_v = is_any_pointer::value; + +/*! + * \brief Template type helper that removes any pointer. + * A modeler may call certain APIs with shared pointers to the + * actual Collectable classes, or templatize Collectables with + * pointers to collectable objects. + * To make our API have a single interface and still work when passed + * pointers, we will remove the pointer and then do all the decision + * making work, by default. + * It is harmless if the modeler passes a non pointer type as + * removing a pointer from something which is not a pointer + * results in itself. + */ template struct remove_any_pointer { using type = T; @@ -172,32 +381,474 @@ template struct remove_any_pointer const&> using type = T; }; +/** \brief Alias Template for remove_pointer. + */ template using remove_any_pointer_t = typename remove_any_pointer::type; -template struct is_contiguous : std::false_type +/** + * \brief This templated struct takes a type and tells + * us whether that type is a STL container. + */ +template struct is_stl_container : std::false_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type +{ +}; + +template struct is_stl_container> : public std::true_type { }; -template struct is_contiguous> : std::true_type +template struct is_stl_container> : public std::true_type { }; -template struct is_contiguous> : std::true_type +template struct is_stl_container> : public std::true_type { }; -// TypeAt gets the N-th type in the parameter pack Ts... -template struct TypeAt; +template struct is_stl +{ + static constexpr bool value = is_stl_container::type>::value; +}; -template struct TypeAt<0, T, Ts...> +template inline constexpr bool is_stl_v = is_stl::value; + +/** + * \brief This Variadic templated struct contains a nested value + * which stores the length of any parameter pack it gets templatized on. + */ +template struct parameter_pack_length +{ + static constexpr std::size_t value = sizeof...(Args); +}; + +template struct peek_last_type; + +template struct peek_last_type : public peek_last_type +{ +}; + +template struct peek_last_type +{ + using type = Tail; +}; + +template using peek_last_type_t = typename peek_last_type::type; +/** + * \brief This Variadic templated struct helps us know about the + * type of the very last or tail item in a parameter pack. + * It works by peeling of one parameter at a time from the pack + * and when it hits the last item, it specializes the struct by + * typedefing the template parameter T in its namespace. + */ +template struct last_index_type +{ +}; + +/** + * \brief Base case when we have just the last item of the + * parameter pack. + */ +template struct last_index_type<0, T> { using type = T; }; -template struct TypeAt +/** + * \brief Recursive case when we recursively peel off items + * from the front of the pack until we hit the last item. + */ +template +struct last_index_type : public last_index_type +{ +}; + +/** \brief Alias Template for last_index_type. + */ +template using last_index_type_t = typename last_index_type::type; + +/** + * \brief This Variadic templated struct basically works much like + * std::integer_sequence. It represents a compile-time sequence of + * integers. This is used as a parameter to the Collection function template + * and helps in type deduction, unpacking and transforming our tuple of + * random parameters back into a variadic template. + * + * Given a tuple, this Indices Sequence Generator takes that tuple and + * transforms it back to a variadic template argument. + */ +template struct sequence_generator +{ +}; + +/** + * \brief This is the generic template. + */ +template struct generate_sequence : generate_sequence +{ +}; + +/** + * \brief This is the specialization which kicks in when the first + * template parameter is 0. + */ +template struct generate_sequence<0, S...> +{ + using type = sequence_generator; +}; + +/** \brief Alias Template for generate_sequence. + */ +template using generate_sequence_t = typename generate_sequence::type; + +/** + * \brief This templated struct lets us know about + * the return type from any random function pointer. This is + * specialized for a couple different signatures. + */ +template struct return_type +{ + using type = T; +}; + +template struct return_type> +{ + using type = R; +}; + +template struct return_type const> +{ + using type = R; +}; + +template struct return_type T::*> +{ + using type = R; +}; + +template struct return_type const T::*> { - static_assert(N < sizeof...(Ts) + 1, "Index out of bounds"); - using type = typename TypeAt::type; + using type = R; }; +template struct return_type T::* const&> +{ + using type = R; +}; + +template struct return_type const T::* const> +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type (T::*)() const> +{ + using type = R; +}; + +template struct return_type& (T::*)() const> +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type (T::* const)() const> +{ + using type = R; +}; + +template struct return_type& (T::* const)() const> +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type +{ + using type = R; +}; + +template struct return_type (T::* const&)() const> +{ + using type = R; +}; + +template struct return_type& (T::* const&)() const> +{ + using type = R; +}; + +/** \brief Alias Template for return_type. + */ +template using return_type_t = typename return_type::type; + +/** \brief Replacement for std::is_pod (deprecated in C++20) + */ +template +struct is_pod : std::integral_constant::value && std::is_standard_layout::value> +{ +}; + +template inline constexpr bool is_pod_v = is_pod::value; + +/** + * \brief Templated struct for detecting Boolean Type. + */ +template struct is_bool : public std::false_type +{ +}; + +template <> struct is_bool : public std::true_type +{ +}; + +template <> struct is_bool : public std::true_type +{ +}; + +template <> struct is_bool : public std::true_type +{ +}; + +template <> struct is_bool : public std::true_type +{ +}; + +template inline constexpr bool is_bool_v = is_bool::value; + +/** + * \brief This templated struct lets us know about + * whether the datatype is actually an std::pair object. + * This is specialized for a couple different signatures. + * The collection procedure for a pair object is broken down + * into first collecting the first member and then, the second. + */ +template struct is_pair : public std::false_type +{ +}; + +template struct is_pair> : public std::true_type +{ +}; + +template struct is_pair&> : public std::true_type +{ +}; + +template struct is_pair const&> : public std::true_type +{ +}; + +template struct is_pair const> : public std::true_type +{ +}; + +template inline constexpr bool is_pair_v = is_pair::value; + +/** + * \brief Templated struct for detecting String Type. + */ +template struct is_string : public std::false_type +{ +}; + +template <> struct is_string : public std::true_type +{ +}; + +template <> struct is_string : public std::true_type +{ +}; + +template <> struct is_string : public std::true_type +{ +}; + +template <> struct is_string : public std::true_type +{ +}; + +template inline constexpr bool is_string_v = is_string::value; + +/** + * \brief Templated struct for detecting char pointer type. + */ +template struct is_char_pointer : public std::false_type +{ +}; + +template <> struct is_char_pointer : public std::true_type +{ +}; + +template <> struct is_char_pointer : public std::true_type +{ +}; + +template <> struct is_char_pointer : public std::true_type +{ +}; + +template <> struct is_char_pointer : public std::true_type +{ +}; + +template inline constexpr bool is_char_pointer_v = is_char_pointer::value; + +/** + * \brief Templated struct for detecting "operator POD() const" + * + * Unlike a std::is_convertible_v cascade, this reports the *exact* return type + * of the class' conversion operator. Conversion to the operator's real return + * type uses an identity second standard conversion (Exact Match), which strictly + * outranks any chained conversion (e.g. operator uint32_t() also makes the class + * convertible to bool, but uint32_t wins). Resolves to void for non-class types, + * classes with no cast operator, and classes with more than one cast operator + * (an ambiguous probe that fails via SFINAE). + */ +template struct pod_convertible +{ +private: + using U = std::decay_t; + + template struct tag + { + using type = P; + }; + + static tag pick(bool); + static tag pick(int8_t); + static tag pick(uint8_t); + static tag pick(int16_t); + static tag pick(uint16_t); + static tag pick(int32_t); + static tag pick(uint32_t); + static tag pick(int64_t); + static tag pick(uint64_t); + static tag pick(double); + static tag pick(float); + static tag pick(...); + + template , int> = 0> + static auto detect(int) -> typename decltype(pick(std::declval()))::type; + + template static auto detect(...) -> void; + +public: + using type = decltype(detect(0)); +}; + +template using pod_convertible_t = typename pod_convertible::type; + +template constexpr bool is_pod_convertible_v = !std::is_same_v, void>; + +// TODO cnyce: reuse Sparta's has_ostream_operator utility once it gets moved to SimDB. +template struct has_ostream_operator : std::false_type +{ +}; + +template +struct has_ostream_operator() << std::declval())>> + : std::true_type +{ +}; + +template inline constexpr bool has_ostream_operator_v = has_ostream_operator::value; + +template struct has_sparta_pair_definition_type : std::false_type +{ +}; + +template +struct has_sparta_pair_definition_type> : std::true_type +{ +}; + +template +inline constexpr bool has_sparta_pair_definition_type_v = has_sparta_pair_definition_type::value; + } // namespace simdb::type_traits diff --git a/include/simdb/utils/ValidValue.hpp b/include/simdb/utils/ValidValue.hpp index eed97cf6..9ab61410 100644 --- a/include/simdb/utils/ValidValue.hpp +++ b/include/simdb/utils/ValidValue.hpp @@ -1,23 +1,47 @@ #pragma once #include "simdb/Exceptions.hpp" +#include "simdb/utils/TypeTraits.hpp" +#include +#include namespace simdb { template class ValidValue { private: - T value_ = 0; + T value_{}; bool valid_ = false; public: - ValidValue& operator=(T val) + ValidValue() = default; + + explicit ValidValue(const T& val) : + value_(val), + valid_(true) + { + } + + explicit ValidValue(T&& val) : + value_(std::move(val)), + valid_(true) + { + } + + ValidValue& operator=(const T& val) { value_ = val; valid_ = true; return *this; } + ValidValue& operator=(T&& val) + { + value_ = std::move(val); + valid_ = true; + return *this; + } + const T& getValue() const { if (!valid_) @@ -46,3 +70,13 @@ template class ValidValue }; } // namespace simdb + +template inline std::ostream& operator<<(std::ostream& os, const simdb::ValidValue& vv) +{ + static_assert(simdb::type_traits::has_ostream_operator_v); + if (!vv.isValid()) + { + return os << ""; + } + return os << vv.getValue(); +} diff --git a/python/argos/argos.py b/python/argos/argos.py index 0b4f9ca0..5ae46401 100644 --- a/python/argos/argos.py +++ b/python/argos/argos.py @@ -9,9 +9,8 @@ def OnInit(self): parser = argparse.ArgumentParser() parser.add_argument("--database", required=True, help="Path to the database file") parser.add_argument("--view-file", help="Path to the view file (*.avf) to load") - parser.add_argument("--dev-debug", help="Enable verbose output for development and debugging", action="store_true") args = parser.parse_args() app = MyApp() - workspace = Workspace(args.database, args.view_file, args.dev_debug) + workspace = Workspace(args.database, args.view_file) app.MainLoop() diff --git a/python/argos/viewer/README.md b/python/argos/viewer/README.md index ea93153d..6eaa61ca 100644 --- a/python/argos/viewer/README.md +++ b/python/argos/viewer/README.md @@ -1,3 +1,91 @@ -# Argos +# Argos Viewer -TBD \ No newline at end of file +The Argos viewer is a desktop application for exploring and **replaying** the +data captured by the Argos collector. It opens a SimDB-produced database and +lets you step through simulated time, reconstructing the state of collected +objects at any point and visualizing them with a set of composable widgets. + +The collector answers *"how do I get simulation data out fast and safely?"*; the +viewer answers *"now that it's captured, how do I look at it?"* + +--- + +## Launching + +```bash +python argos.py --database path/to/collected.db +python argos.py --database path/to/collected.db --view-file layout.avf +``` + +- `--database` (required): the Argos-collected database to open. +- `--view-file` (optional): an Argos View File (`*.avf`) describing a saved + layout. If omitted, the viewer restores your last-known view when one exists. + +--- + +## What the viewer shows + +The viewer presents the simulation as a navigable **hierarchy** of the objects +that were collected, alongside the **clocks** that drove the run and the range +of simulated time that data exists for. You pick objects of interest, drop them +into visualization widgets, and scrub through time to watch them change. + +Because collection is checkpoint-based (periodic full snapshots plus small +deltas in between), the viewer reconstructs an object's exact value at any +requested point in time by starting from the most recent snapshot and replaying +the deltas up to that point. This all happens behind the scenes; you just move +the playback control and the widgets update. + +Databases may contain a single clock or many. When there are multiple clocks, +each object only updates on the edges of the clock it was collected against. + +--- + +## The UI + +The main window has three parts: + +1. **Menu bar** -- create, open, and save views (`File > New / Open / Save View`, + with Ctrl+N/O/S) and exit. +2. **Data inspector** -- a set of tabs, each holding a **canvas** you can split + left/right and top/bottom to arrange multiple widgets side by side. A `+` tab + adds new tabs. A **Logs** tab appears when the collection recorded + notifications worth surfacing. +3. **Playback bar** -- a clock selector, the current cycle/tick readout, step + buttons (`-30 / -10 / -3 / -1` and `+1 / +3 / +10 / +30`), and a scrubber + slider spanning the start and end of the run. + +Each widget has its own small toolbar: open its settings, clear it, split its +cell horizontally or vertically, or maximize it to fill the canvas. + +### Widgets + +- **Queue tables** -- a tabular view of container objects (e.g. queues and + buffers), with configurable visible columns and per-column + auto-colorization to make values easy to scan. +- **Queue Utilization** -- occupancy of container objects over time. +- **Scheduling Lines** -- a timeline-style view of scheduled activity. +- **Summary Views** -- aggregate/summary displays across the selection. + +--- + +## View files (`*.avf`) + +A **view file** captures your layout so you can reopen it later or share it: the +tabs, how each canvas is split, which widgets are placed where, the columns and +colorization chosen for each object, and the selected clock. Save with +`File > Save View`; the window title marks unsaved changes. The viewer also +remembers your last layout and reopens it automatically the next time you launch +without a `--view-file`. + +The playback position (the tick you're currently viewing) is treated as a local, +per-session preference and is not stored in the shared view file. + +--- + +## Relationship to the collector + +The viewer is the read side of Argos; the write side is the Argos collector, a +SimDB app that streams collected data into a single database. For the collector +and an end-to-end walkthrough, see the SimDB book's Argos case study in +[`docs/book/parts/part7-argos-case-study.adoc`](../../../docs/book/parts/part7-argos-case-study.adoc). diff --git a/python/argos/viewer/gui/canvas_grid.py b/python/argos/viewer/gui/canvas_grid.py index b549eb70..0bc3c96f 100644 --- a/python/argos/viewer/gui/canvas_grid.py +++ b/python/argos/viewer/gui/canvas_grid.py @@ -1,6 +1,13 @@ -import wx +import wx, wx.adv, wx.aui from viewer.gui.widgets.splitter_window import DirtySplitterWindow from viewer.gui.view_settings import DirtyReasons +from viewer.gui.widgets.queue_utiliz import QueueUtilizWidget +from viewer.gui.widgets.scheduling_lines import SchedulingLinesWidget +from viewer.gui.widgets.summary_views import SummaryViews +from viewer.gui.widgets.iterable_struct import IterableStruct +from viewer.gui.dialogs.widget_data_selections import WidgetDataSelectionsDlg +from viewer.gui.dialogs.widget_data_selections import SchedulingLinesEditDlg +from functools import partial class CanvasGrid(wx.Panel): def __init__(self, parent, rows=1, cols=1): @@ -8,7 +15,6 @@ def __init__(self, parent, rows=1, cols=1): if rows == 1 and cols == 1: self.container = WidgetContainer(self) - self.Bind(wx.EVT_CONTEXT_MENU, self.__OnContextMenu) else: self.container = DirtySplitterWindow(self.frame, self, style=wx.SP_LIVE_UPDATE) self.__BuildGrid(self.container, rows, cols) @@ -29,7 +35,7 @@ def frame(self): frame = frame.GetParent() return frame - + def DestroyAllWidgets(self): self.__DestroyAllWidgets(self.container) @@ -58,15 +64,23 @@ def GetCurrentViewSettings(self): def ApplyViewSettings(self, settings): self.__RecursivelyApplyViewSettings(settings, self) + def AddQuickLinks(self, links, splitters_only=False): + links.append(("Split left/right", self.__OnSplitVertically)) + links.append(("Split top/bottom", self.__OnSplitHorizontally)) + if not splitters_only: + links.append(("Maximize", self.__Explode)) + def __RecursivelyApplyViewSettings(self, settings, window): if settings['window_type'] == 'widget_container': widget_creation_str = settings['widget_creation_str'] - if widget_creation_str: + if widget_creation_str and widget_creation_str != 'NO_WIDGET': widget = self.frame.widget_creator.CreateWidget(widget_creation_str, window.container) if 'widget_settings' in settings: widget.ApplyViewSettings(settings['widget_settings']) window.container.SetWidget(widget) + else: + window.container.SetWidget(None) elif settings['window_type'] == 'splitter': if settings['split_mode'] == 'horizontal': window.__OnSplitHorizontally(None) @@ -141,75 +155,62 @@ def __BuildGrid(self, splitter, rows, cols): elif cols == 1: splitter.SplitHorizontally(CanvasGrid(splitter, rows // 2, cols), CanvasGrid(splitter, rows - rows // 2, cols)) - def __OnContextMenu(self, event): - # Get the position where the user right-clicked - pos = event.GetPosition() - pos = self.ScreenToClient(pos) + def __GetWidgetState(self): + if not isinstance(self.container, WidgetContainer): + return None, None - menu = wx.Menu() + widget = self.container.GetWidget() + if widget is None: + return None, None - split_vertically = menu.Append(-1, "Split left/right") - split_horizontally = menu.Append(-1, "Split top/bottom") + return widget.GetWidgetCreationString(), widget.GetCurrentViewSettings() - if isinstance(self.GetParent(), wx.SplitterWindow): - menu.AppendSeparator() - explode = menu.Append(-1, "Explode") - self.Bind(wx.EVT_MENU, self.__Explode, explode) - elif isinstance(self.container, WidgetContainer): - widget = self.container.GetWidget() - if widget: - menu.AppendSeparator() - clear = menu.Append(-1, "Clear widget") - self.Bind(wx.EVT_MENU, lambda event: self.__DestroyAllWidgets(self.container), clear) + @staticmethod + def __PlaceWidgetInContainer(frame, widget_container, widget_creation_str, widget_settings): + if not widget_creation_str: + return - self.Bind(wx.EVT_MENU, self.__OnSplitVertically, split_vertically) - self.Bind(wx.EVT_MENU, self.__OnSplitHorizontally, split_horizontally) + widget = frame.widget_creator.CreateWidget(widget_creation_str, widget_container) + if widget_settings is not None: + widget.ApplyViewSettings(widget_settings) - self.PopupMenu(menu, pos) + widget_container.SetWidget(widget) - def __OnSplitVertically(self, event): - widget_creation_str = None - if self.container: - widget = self.container.GetWidget() - widget_creation_str = widget.GetWidgetCreationString() if widget else None + def __OnSplitVertically(self, event=None): + widget_creation_str, widget_settings = self.__GetWidgetState() if self.container: self.container.DestroyAllWidgets() + self.container.Destroy() + self.container = None self.GetSizer().Clear() self.container = CanvasGrid(self, rows=1, cols=2) self.GetSizer().Add(self.container, 1, wx.EXPAND) self.Layout() - if widget_creation_str: - win1 = self.container.container.GetWindow1() - widget_container = win1.container - widget = self.container.frame.widget_creator.CreateWidget(widget_creation_str, widget_container) - widget_container.SetWidget(widget) + win1 = self.container.container.GetWindow1() + self.__PlaceWidgetInContainer(self.frame, win1.container, widget_creation_str, widget_settings) splitter = self.container.container splitter.SetSashPosition(splitter.GetSize().GetWidth() // 2) self.frame.view_settings.SetDirty(reason=DirtyReasons.WidgetSplit) - def __OnSplitHorizontally(self, event): - widget_creation_str = None - if self.container: - widget = self.container.GetWidget() - widget_creation_str = widget.GetWidgetCreationString() if widget else None + def __OnSplitHorizontally(self, event=None): + widget_creation_str, widget_settings = self.__GetWidgetState() if self.container: self.container.DestroyAllWidgets() + self.container.Destroy() + self.container = None self.GetSizer().Clear() self.container = CanvasGrid(self, rows=2, cols=1) self.GetSizer().Add(self.container, 1, wx.EXPAND) self.Layout() - if widget_creation_str: - win1 = self.container.container.GetWindow1() - widget_container = win1.container - widget = self.container.frame.widget_creator.CreateWidget(widget_creation_str, widget_container) - widget_container.SetWidget(widget) + win1 = self.container.container.GetWindow1() + self.__PlaceWidgetInContainer(self.frame, win1.container, widget_creation_str, widget_settings) splitter = self.container.container splitter.SetSashPosition(splitter.GetSize().GetHeight() // 2) @@ -225,17 +226,17 @@ def __FindFirstWidgetContainer(self, container): return None def __Explode(self, event): - widget = self.container.GetWidget() - widget_creation_str = widget.GetWidgetCreationString() if widget else None + widget_creation_str, widget_settings = self.__GetWidgetState() - frame = self.container.frame + frame = self.frame inspector = frame.inspector inspector.ResetCurrentTab() if widget_creation_str: containers = inspector.GetCurrentTabWidgetContainers() - widget = frame.widget_creator.CreateWidget(widget_creation_str, containers[0]) - containers[0].SetWidget(widget) + CanvasGrid.__PlaceWidgetInContainer( + frame, containers[0], widget_creation_str, widget_settings, + ) frame.view_settings.SetDirty(reason=DirtyReasons.CanvasExploded) @@ -258,14 +259,10 @@ class WidgetContainer(wx.Panel): def __init__(self, parent): super(WidgetContainer, self).__init__(parent) self._widget = None - - frame = parent - while frame and not isinstance(frame, wx.Frame): - frame = frame.GetParent() - - self.SetDropTarget(WidgetContainerDropTarget(self, self.frame.widget_creator)) + self._quick_links = WidgetQuickLinks(self) sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(self._quick_links, 1, wx.EXPAND | wx.LEFT | wx.TOP, 5) self.SetSizer(sizer) @property @@ -275,19 +272,31 @@ def frame(self): frame = frame.GetParent() return frame - + def SetWidget(self, widget): + sizer = self.GetSizer() + if widget: + dirty_reason = DirtyReasons.WidgetDropped + elif self._widget: + dirty_reason = DirtyReasons.WidgetRemoved + else: + dirty_reason = None + if self._widget: - self.GetSizer().Detach(self._widget) + sizer.Detach(self._widget) self._widget.Destroy() + self._widget = None - self._widget = widget if widget: - sizer = self.GetSizer() + self.__HideQuickLinks() + self._widget = widget sizer.Add(widget, 1, wx.EXPAND) + else: + self.__ShowQuickLinks() self.Layout() - self.frame.view_settings.SetDirty(reason=DirtyReasons.WidgetDropped) + if dirty_reason is not None: + self.frame.view_settings.SetDirty(reason=dirty_reason) wx.CallAfter(self.__RefreshWidget) def SetWidgetFocus(self): @@ -295,10 +304,7 @@ def SetWidgetFocus(self): self._widget.SetFocus() def DestroyAllWidgets(self): - if self._widget: - self.GetSizer().Detach(self._widget) - self._widget.Destroy() - self._widget = None + self.SetWidget(None) def UpdateWidgets(self): if self._widget: @@ -306,54 +312,139 @@ def UpdateWidgets(self): def GetWidget(self): return self._widget - - def SplitCanvas(self, split_mode, sash_position): - assert not self.GetSizer().GetChildren(), 'Cannot split a non-empty widget container' - if split_mode == 'horizontal': - self.SplitHorizontally(WidgetContainer(self), WidgetContainer(self)) - elif split_mode == 'vertical': - self.SplitVertically(WidgetContainer(self), WidgetContainer(self)) + def LaunchQueueViewer(self): + selected = [self._widget.elem_path] if self._widget else [] + dlg = WidgetDataSelectionsDlg(self, self.frame, selected, queues_only=True, single_selection=True, title="Select queue") + result = dlg.ShowModal() + + if result == wx.ID_OK: + selected = dlg.GetSelectedElemPaths() + if len(selected) == 0: + wx.MessageBox('No data selected', 'Error', wx.OK | wx.ICON_ERROR) + selected = None + else: + selected = selected[0] + else: + selected = None + dlg.Destroy() + + if not selected: + return + + widget = IterableStruct(self, self.frame, selected) + self.SetWidget(widget) + + def LaunchQueueUtilizViewer(self): + widget = QueueUtilizWidget(self, self.frame) + self.SetWidget(widget) + + def LaunchSchedulingLinesViewer(self): + if isinstance(self._widget, SchedulingLinesWidget): + elem_paths = self._widget.caption_mgr.GetAllMatchingElemPaths() + num_ticks_before = self._widget.num_ticks_before + num_ticks_after = self._widget.num_ticks_after + show_details = self._widget.show_detailed_queue_packets + hide_empty_rows = self._widget.hide_empty_rows + show_full_paths = self._widget.show_full_paths + enable_tooltips = self._widget.enable_tooltips + show_did = self._widget.show_did + else: + elem_paths = [] + num_ticks_before = SchedulingLinesWidget.DEFAULT_TICKS_BEFORE + num_ticks_after = SchedulingLinesWidget.DEFAULT_TICKS_AFTER + show_details = SchedulingLinesWidget.DEFAULT_SHOW_DETAILS + hide_empty_rows = SchedulingLinesWidget.DEFAULT_HIDE_EMPTY_ROWS + show_full_paths = SchedulingLinesWidget.DEFAULT_SHOW_FULL_PATHS + enable_tooltips = SchedulingLinesWidget.DEFAULT_ENABLE_TOOLTIPS + show_did = SchedulingLinesWidget.DEFAULT_SHOW_DID + + dlg = SchedulingLinesEditDlg(self, self.frame, elem_paths, num_ticks_before, num_ticks_after, show_details, hide_empty_rows, show_full_paths, enable_tooltips, show_did) + result = dlg.ShowModal() + if result == wx.ID_OK: + elem_paths = dlg.GetSelectedElemPaths() + num_ticks_before = dlg.num_ticks_before + num_ticks_after = dlg.num_ticks_after + show_details = dlg.show_details + hide_empty_rows = dlg.hide_empty_rows + show_full_paths = dlg.show_full_paths + enable_tooltips = dlg.enable_tooltips + show_did = dlg.show_did + if len(elem_paths) > 0: + widget = SchedulingLinesWidget(self, self.frame, elem_paths, num_ticks_before, num_ticks_after, show_details, hide_empty_rows, show_full_paths, enable_tooltips, show_did) + self.SetWidget(widget) + else: + wx.MessageBox("No data selected", "Error", wx.OK | wx.ICON_ERROR) + + dlg.Destroy() + + def LaunchWatchlistBuilder(self): + widget = SummaryViews(self, self.frame) + widget.Hide() + if widget.EditWidget(None): + widget.Show() + self.SetWidget(widget) + else: + widget.Destroy() + + def AddQuickLinks(self, links): + self.GetParent().AddQuickLinks(links, True) + + def __HideQuickLinks(self): + if not self._quick_links.IsShown(): + return + + sizer = self.GetSizer() + for item in sizer.GetChildren(): + if item.GetWindow() == self._quick_links: + sizer.Detach(self._quick_links) + break + self._quick_links.Hide() + + def __ShowQuickLinks(self): + sizer = self.GetSizer() + self._quick_links.Show() + for item in sizer.GetChildren(): + if item.GetWindow() == self._quick_links: + return + sizer.Add(self._quick_links, 1, wx.EXPAND | wx.LEFT | wx.TOP, 5) - self.SetSashPosition(sash_position) - def __RefreshWidget(self): self.UpdateWidgets() self.SetWidgetFocus() -class WidgetContainerDropTarget(wx.TextDropTarget): - def __init__(self, widget_container, widget_creator): - super(WidgetContainerDropTarget, self).__init__() - self.widget_container = widget_container - self.widget_creator = widget_creator +class WidgetQuickLinks(wx.Panel): + def __init__(self, widget_container): + wx.Panel.__init__(self, widget_container) + + def AddLinks(label, links): + label = wx.StaticText(self, label=label) + vsizer.Add(label) + for label, callback in links: + row = wx.BoxSizer(wx.HORIZONTAL) + bullet = wx.StaticText(self, label="\u2022") + link = wx.adv.HyperlinkCtrl( + self, label=label, style=wx.adv.HL_DEFAULT_STYLE & ~wx.adv.HL_CONTEXTMENU) + link.Bind(wx.adv.EVT_HYPERLINK, lambda evt, cb=callback: cb()) + row.Add(bullet, 0, wx.RIGHT | wx.ALIGN_CENTER_VERTICAL, 8) + row.Add(link) + vsizer.Add(row, 0, wx.LEFT, 5) + + vsizer = wx.BoxSizer(wx.VERTICAL) + self.SetSizer(vsizer) + + # Links to create widgets + links = [ + ("Queue Inspector", widget_container.LaunchQueueViewer), + ("Queue Utilizations", widget_container.LaunchQueueUtilizViewer), + ("Scheduling Lines", widget_container.LaunchSchedulingLinesViewer), + ("Watchlist", widget_container.LaunchWatchlistBuilder) + ] + AddLinks("Widgets:", links) + + # Links to split canvas + links = [] + widget_container.AddQuickLinks(links) + AddLinks("Canvas:", links) - def OnDropText(self, x, y, text): - widget = self.widget_container.GetWidget() - if widget: - current_widget_is_tool = widget.GetWidgetCreationString().find('$') == -1 - incoming_widget_is_tool = text.find('$') == -1 - - if current_widget_is_tool and incoming_widget_is_tool: - return self.__DropWidget(text) - elif current_widget_is_tool and not incoming_widget_is_tool: - elem_path = text.split('$')[1] - err_msg = widget.GetErrorIfDroppedNodeIncompatible(elem_path) - if err_msg: - msg, title = err_msg - wx.MessageBox(msg, title, wx.OK | wx.ICON_ERROR) - return False - - widget.AddElement(elem_path) - return True - else: - return self.__DropWidget(text) - else: - return self.__DropWidget(text) - - def __DropWidget(self, text): - widget = self.widget_creator.CreateWidget(text, self.widget_container) - if not widget: - return False - - self.widget_container.SetWidget(widget) - return True + self.Layout() diff --git a/python/argos/viewer/gui/navtree.py b/python/argos/viewer/gui/collectable_tree.py similarity index 69% rename from python/argos/viewer/gui/navtree.py rename to python/argos/viewer/gui/collectable_tree.py index b615c1a1..c0339f44 100644 --- a/python/argos/viewer/gui/navtree.py +++ b/python/argos/viewer/gui/collectable_tree.py @@ -1,19 +1,20 @@ -import wx, os +import wx from functools import partial -class NavTree(wx.TreeCtrl): - def __init__(self, parent, frame): - super(NavTree, self).__init__(parent, style=wx.TR_DEFAULT_STYLE | wx.TR_HIDE_ROOT | wx.TR_LINES_AT_ROOT) +class CollectableTree(wx.TreeCtrl): + def __init__(self, parent, frame, leaf_elem_paths): + super(CollectableTree, self).__init__(parent, style=wx.TR_DEFAULT_STYLE | wx.TR_HIDE_ROOT | wx.TR_LINES_AT_ROOT) self.frame = frame self.simhier = frame.simhier + self._visible_elem_paths = self.__BuildVisibleElemPaths(leaf_elem_paths) self._root = self.AddRoot("root") - self._tree_items_by_db_id = {self.simhier.GetRootID(): self._root } + self._tree_items_by_id = {0: self._root } self._tree_items_by_elem_path = {} - self.__RecurseBuildTree(self.simhier.GetRootID()) + self.__RecurseBuildTree(self.simhier.GetTree().GetRoot()) self._leaf_elem_paths_by_tree_item = {} - for db_id, tree_item in self._tree_items_by_db_id.items(): + for db_id, tree_item in self._tree_items_by_id.items(): if not self.GetChildrenCount(tree_item): self._leaf_elem_paths_by_tree_item[tree_item] = self.simhier.GetElemPath(db_id).replace('root.','') @@ -22,9 +23,6 @@ def __init__(self, parent, frame): self.Bind(wx.EVT_RIGHT_DOWN, self.__OnRightClick) self.Bind(wx.EVT_TREE_ITEM_EXPANDED, self.__OnItemExpanded) - self._utiliz_image_list = frame.widget_renderer.utiliz_handler.CreateUtilizImageList() - self.SetImageList(self._utiliz_image_list) - self._tooltips_by_item = {} self.Bind(wx.EVT_TREE_ITEM_GETTOOLTIP, self.__ProcessTooltip) @@ -36,32 +34,11 @@ def __init__(self, parent, frame): assert elem_path.find('root.') == -1 def UpdateUtilizBitmaps(self): - for elem_path in self.simhier.GetElemPaths(): - elem_id = self.simhier.GetElemID(elem_path) - if self.simhier.GetWidgetType(elem_id) == 'QueueTable': - utiliz_pct = self.frame.widget_renderer.utiliz_handler.GetUtilizPct(elem_path) - image_idx = int(utiliz_pct * 100) - item = self._leaf_tree_items_by_elem_path[elem_path] - self.SetItemImage(item, image_idx) - - capacity = self.simhier.GetCapacityByElemPath(elem_path) - size = int(capacity * utiliz_pct) - tooltip = '{}\nUtilization: {}% ({}/{} bins filled)'.format(elem_path, round(utiliz_pct*100), size, capacity) - elif self.simhier.GetWidgetType(elem_id) == 'Timeseries': - image_idx = self._utiliz_image_list.GetImageCount() - 1 - item = self._leaf_tree_items_by_elem_path[elem_path] - self.SetItemImage(item, image_idx) - tooltip = '{}\nNo utilization data available for timeseries stats'.format(elem_path) - else: - item = None - tooltip = None - - if item and tooltip: - self._tooltips_by_item[item] = tooltip + pass def ExpandAll(self): self.Unbind(wx.EVT_TREE_ITEM_EXPANDED) - super(NavTree, self).ExpandAll() + super(CollectableTree, self).ExpandAll() self.UpdateUtilizBitmaps() self.Bind(wx.EVT_TREE_ITEM_EXPANDED, self.__OnItemExpanded) @@ -130,13 +107,36 @@ def __GetExpandedElemPaths(self, start_item): return expanded_items - def __RecurseBuildTree(self, parent_id): - for child_id in self.simhier.GetChildIDs(parent_id): - child_name = self.simhier.GetName(child_id) - child = self.AppendItem(self._tree_items_by_db_id[parent_id], child_name) - self._tree_items_by_db_id[child_id] = child + @staticmethod + def __BuildVisibleElemPaths(leaf_elem_paths): + visible_paths = set() + for leaf_path in leaf_elem_paths: + parts = leaf_path.split('.') + for i in range(1, len(parts) + 1): + visible_paths.add('.'.join(parts[:i])) + return visible_paths + + def __RecurseBuildTree(self, node): + if node is self.simhier.GetTree().GetRoot(): + for child in node.GetChildren(): + self.__RecurseBuildTree(child) + else: + if node.GetPath() not in self._visible_elem_paths: + return + + if node.GetParent(): + parent_id = node.GetParent().GetID() + else: + parent_id = 0 + + child_name = node.GetName() + child_id = node.GetID() + child = self.AppendItem(self._tree_items_by_id[parent_id], child_name) + self._tree_items_by_id[child_id] = child self._tree_items_by_elem_path[self.simhier.GetElemPath(child_id)] = child - self.__RecurseBuildTree(child_id) + + for child in node.GetChildren(): + self.__RecurseBuildTree(child) def __OnRightClick(self, event): item = self.HitTest(event.GetPosition()) @@ -152,42 +152,18 @@ def __OnRightClick(self, event): menu = wx.Menu() def ExpandAll(event, **kwargs): - kwargs['navtree'].ExpandAll() + kwargs['tree'].ExpandAll() event.Skip() def CollapseAll(event, **kwargs): - kwargs['navtree'].CollapseAll() + kwargs['tree'].CollapseAll() event.Skip() expand_all = menu.Append(-1, "Expand All") - self.Bind(wx.EVT_MENU, partial(ExpandAll, navtree=self), expand_all) + self.Bind(wx.EVT_MENU, partial(ExpandAll, tree=self), expand_all) collapse_all = menu.Append(-1, "Collapse All") - self.Bind(wx.EVT_MENU, partial(CollapseAll, navtree=self), collapse_all) - - def AddToWatchlist(*args, **kwargs): - navtree = kwargs['navtree'] - elem_path = kwargs['elem_path'] - navtree.frame.explorer.watchlist.AddToWatchlist(elem_path) - - def ShowWatchlistHelp(*args, **kwargs): - wx.MessageBox("Right-click nodes in the NavTree to add them to the Watchlist.\n" \ - "You will only see 'Add to Watchlist' on the right-click menu if the node \n" \ - "has a widget associated with it and is not already in the Watchlist.", "Watchlist Help") - - menu.AppendSeparator() - - if item in self._leaf_elem_paths_by_tree_item: - elem_path = self._leaf_elem_paths_by_tree_item[item] - if elem_path not in self.frame.explorer.watchlist.GetWatchedSimElems(): - add_to_watchlist = menu.Append(-1, "Add to Watchlist") - self.Bind(wx.EVT_MENU, partial(AddToWatchlist, navtree=self, elem_path=elem_path), add_to_watchlist) - else: - show_watchlist_help = menu.Append(-1, "Watchlist Help") - self.Bind(wx.EVT_MENU, ShowWatchlistHelp, show_watchlist_help) - else: - show_watchlist_help = menu.Append(-1, "Watchlist Help") - self.Bind(wx.EVT_MENU, ShowWatchlistHelp, show_watchlist_help) + self.Bind(wx.EVT_MENU, partial(CollapseAll, tree=self), collapse_all) self.PopupMenu(menu) menu.Destroy() @@ -198,3 +174,41 @@ def __OnItemExpanded(self, event): def __ProcessTooltip(self, event): item = event.GetItem() event.SetToolTip(self._tooltips_by_item.get(item, "")) + +class QueuesTree(CollectableTree): + def __init__(self, parent, frame): + super().__init__(parent, frame, frame.simhier.GetContainerElemPaths()) + self._utiliz_image_list = frame.widget_renderer.utiliz_handler.CreateUtilizImageList() + self.SetImageList(self._utiliz_image_list) + + def UpdateUtilizBitmaps(self): + for elem_path in self.simhier.GetContainerElemPaths(): + collectable_id = self.simhier.GetCollectionID(elem_path) + widget_type = self.simhier.GetWidgetType(collectable_id) + if widget_type == 'QueueTable': + utiliz_pct = self.frame.widget_renderer.utiliz_handler.GetUtilizPct(elem_path) + image_idx = int(utiliz_pct * 100) + item = self._leaf_tree_items_by_elem_path[elem_path] + self.SetItemImage(item, image_idx) + + capacity = self.simhier.GetCapacityByElemPath(elem_path) + size = int(capacity * utiliz_pct) + tooltip = '{}\nUtilization: {}% ({}/{} bins filled)'.format(elem_path, round(utiliz_pct*100), size, capacity) + elif widget_type == 'Timeseries': + image_idx = self._utiliz_image_list.GetImageCount() - 1 + item = self._leaf_tree_items_by_elem_path[elem_path] + self.SetItemImage(item, image_idx) + tooltip = '{}\nNo utilization data available for timeseries stats'.format(elem_path) + else: + item = None + tooltip = None + + if item and tooltip: + self._tooltips_by_item[item] = tooltip + +class ScalarsTree(CollectableTree): + def __init__(self, parent, frame): + simhier = frame.simhier + leaf_paths = simhier.GetScalarStatsElemPaths() + simhier.GetScalarStructsElemPaths() + leaf_paths.sort() + super().__init__(parent, frame, leaf_paths) diff --git a/python/argos/viewer/gui/dialogs/save_view_file.py b/python/argos/viewer/gui/dialogs/save_view_file.py new file mode 100644 index 00000000..da4aa18d --- /dev/null +++ b/python/argos/viewer/gui/dialogs/save_view_file.py @@ -0,0 +1,57 @@ +import wx +from viewer.model.dirty_reasons import DIRTY_REASONS + +class SaveViewFileDlg(wx.Dialog): + def __init__(self, prompt='Save to view file?', reasons=None, include_skip_btn=False): + super().__init__(None, title='Save View') + + self._reasons = set(reasons) if reasons else None + sizer = wx.BoxSizer(wx.VERTICAL) + instruction = wx.StaticText(self, label=prompt) + instruction.Wrap(400) + sizer.Add(instruction, 0, wx.ALL | wx.ALIGN_CENTER, 10) + + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + + # Save button + save_btn = wx.Button(self, label="Save") + save_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_YES)) + btn_sizer.Add(save_btn, 0, wx.ALL, 5) + + # Skip button + if include_skip_btn: + skip_btn = wx.Button(self, label="Skip") + skip_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_NO)) + btn_sizer.Add(skip_btn, 0, wx.ALL, 5) + + # Cancel button + cancel_btn = wx.Button(self, label="Cancel") + cancel_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_CANCEL)) + btn_sizer.Add(cancel_btn, 0, wx.ALL, 5) + + # "What changed?" button + if self._reasons: + what_changed_btn = wx.Button(self, wx.ID_ANY, label="What changed?") + what_changed_btn.Bind(wx.EVT_BUTTON, self.__ShowWhatChanged) + btn_sizer.Add(what_changed_btn, 0, wx.ALL, 5) + + sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 10) + self.SetSizer(sizer) + self.Fit() + self.Layout() + + def __ShowWhatChanged(self, event): + if self._reasons is None: + return + + if len(self._reasons) > 1: + msg = 'The following changes were made to the view:\n\n' + for i, reason in enumerate(self._reasons): + msg += str(i) + '. ' + DIRTY_REASONS[reason] + '\n' + else: + reason = next(iter(self._reasons)) + msg = DIRTY_REASONS[reason] + + dlg = wx.MessageDialog(self, msg, 'Changes', wx.OK | wx.ICON_INFORMATION) + dlg.ShowModal() + dlg.Destroy() diff --git a/python/argos/viewer/gui/dialogs/string_list_selection.py b/python/argos/viewer/gui/dialogs/string_list_selection.py deleted file mode 100644 index 77bd9f7f..00000000 --- a/python/argos/viewer/gui/dialogs/string_list_selection.py +++ /dev/null @@ -1,59 +0,0 @@ -import wx - -class StringListSelectionDlg(wx.Dialog): - def __init__(self, widget, all_strings, checked_strings, prompt='Make selections:'): - super().__init__(widget, title='Customize Widget') - - self.checkboxes = [] - panel = wx.Panel(self) - sizer = wx.BoxSizer(wx.VERTICAL) - - tbox = wx.StaticText(panel, label=prompt) - sizer.Add(tbox, 0, wx.ALL | wx.EXPAND, 5) - - # Create a checkbox for each item - for s in all_strings: - checkbox = wx.CheckBox(panel, label=s) - checkbox.Bind(wx.EVT_CHECKBOX, self.__OnCheckbox) - sizer.Add(checkbox, 1, wx.ALL | wx.EXPAND, 5) - - if s in checked_strings: - checkbox.SetValue(True) - else: - checkbox.SetValue(False) - - self.checkboxes.append(checkbox) - - # OK and Cancel buttons - self._ok_btn = wx.Button(panel, wx.ID_OK) - btn_sizer = wx.StdDialogButtonSizer() - btn_sizer.AddButton(self._ok_btn) - btn_sizer.AddButton(wx.Button(panel, wx.ID_CANCEL)) - btn_sizer.Realize() - - sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 10) - panel.SetSizerAndFit(sizer) - - # Find the longest string length of all our checkbox labels - dc = wx.ClientDC(panel) - dc.SetFont(wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL)) - longest_length = 0 - - for s in all_strings: - text_width, _ = dc.GetTextExtent(s) - longest_length = max(longest_length, text_width) - - w,h = sizer.GetMinSize() - h += 100 - w = max(w, longest_length + 100) - self.SetSize((w,h)) - self.Layout() - self.Refresh() - - def GetSelectedStrings(self): - # Return a list of selected items - return [checkbox.GetLabel() for checkbox in self.checkboxes if checkbox.IsChecked()] - - def __OnCheckbox(self, event): - any_selected = any(checkbox.IsChecked() for checkbox in self.checkboxes) - self._ok_btn.Enable(any_selected) diff --git a/python/argos/viewer/gui/dialogs/widget_data_selections.py b/python/argos/viewer/gui/dialogs/widget_data_selections.py new file mode 100644 index 00000000..5dfaff1b --- /dev/null +++ b/python/argos/viewer/gui/dialogs/widget_data_selections.py @@ -0,0 +1,643 @@ +import wx +from functools import partial + +class WidgetDataSelectionsDlg(wx.Dialog): + def __init__( + self, parent, frame, elem_paths, queues_only=False, single_selection=False, + settings_chkboxes=None, title="Edit Data Selections", + ): + _, screen_h = wx.GetDisplaySize() + super().__init__(parent, title=title, size=(600, int(screen_h * 0.75))) + + self.frame = frame + self.simhier = frame.simhier + self._settings_chkboxes = settings_chkboxes or [] + self._settings_chkboxes_by_label = {} + if queues_only: + self._all_leaf_paths = sorted(self.simhier.GetContainerElemPaths()) + else: + self._all_leaf_paths = sorted(self.simhier.GetElemPaths(True)) + + initial_paths_set = set(elem_paths) + self._selected_paths = [p for p in elem_paths if p in self._all_leaf_paths] + for p in self._all_leaf_paths: + if p in initial_paths_set and p not in self._selected_paths: + self._selected_paths.append(p) + + self._initial_paths = list(self._selected_paths) + self._single_selection = single_selection + self._single_selected_path = None + + self._tree_items_by_id = {} + self._leaf_paths_by_tree_item = {} + self._list_indices_by_path = {} + self._syncing_list_checkboxes = False + + if single_selection and queues_only: + instruction_text = 'Select a leaf queue from the tree' + elif single_selection: + instruction_text = 'Select a leaf node from the tree' + else: + instruction_text = 'Right-click nodes to add/remove from widget' + + instruction_label = wx.StaticText(self, label=instruction_text) + tree_style = wx.TR_DEFAULT_STYLE | wx.TR_HIDE_ROOT | wx.TR_LINES_AT_ROOT + if not single_selection: + tree_style = tree_style | wx.TR_MULTIPLE + self.hier_tree = wx.TreeCtrl(self, style=tree_style) + + if not single_selection: + self.selections_list = wx.ListCtrl(self, style=wx.LC_REPORT | wx.LC_SINGLE_SEL) + self.selections_list.EnableCheckBoxes(True) + self.selections_list.InsertColumn(0, 'Current Selections', width=550) + + self.move_up_btn = wx.BitmapButton(self, bitmap=wx.ArtProvider.GetBitmap(wx.ART_GO_UP, wx.ART_BUTTON)) + self.move_up_btn.Bind(wx.EVT_BUTTON, self.__MoveSelectedElemUp) + + self.move_down_btn = wx.BitmapButton(self, bitmap=wx.ArtProvider.GetBitmap(wx.ART_GO_DOWN, wx.ART_BUTTON)) + self.move_down_btn.Bind(wx.EVT_BUTTON, self.__MoveSelectedElemDown) + + btn_sizer = wx.StdDialogButtonSizer() + self.ok_btn = wx.Button(self, wx.ID_OK) + btn_sizer.AddButton(self.ok_btn) + btn_sizer.AddButton(wx.Button(self, wx.ID_CANCEL)) + btn_sizer.Realize() + + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(instruction_label, 0, wx.ALL, 5) + sizer.Add(self.hier_tree, 1, wx.LEFT | wx.RIGHT | wx.BOTTOM | wx.EXPAND, 5) + if not single_selection: + list_sizer = wx.BoxSizer(wx.HORIZONTAL) + list_sizer.Add(self.selections_list, 1, wx.LEFT | wx.RIGHT | wx.BOTTOM | wx.EXPAND, 5) + + arrow_btns_sizer = wx.BoxSizer(wx.VERTICAL) + arrow_btns_sizer.Add(self.move_up_btn) + arrow_btns_sizer.Add(self.move_down_btn) + list_sizer.Add(arrow_btns_sizer) + sizer.Add(list_sizer) + + self._BuildSettingsArea(sizer) + + sizer.Add(btn_sizer, 0, wx.ALL | wx.ALIGN_RIGHT, 10) + self.SetSizer(sizer) + + if not single_selection: + self.hier_tree.Bind(wx.EVT_RIGHT_DOWN, partial(self.__OnTreeRightClick, tree=self.hier_tree)) + self.selections_list.Bind(wx.EVT_LIST_ITEM_CHECKED, self.__OnListItemChecked) + self.selections_list.Bind(wx.EVT_LIST_ITEM_UNCHECKED, self.__OnListItemUnchecked) + self.selections_list.Bind(wx.EVT_LIST_ITEM_SELECTED, self.__UpdateButtonStates) + else: + self.hier_tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.__OnTreeSelectionChanged) + self.hier_tree.Bind(wx.EVT_RIGHT_DOWN, partial(self.__OnTreeRightClick, tree=self.hier_tree)) + + self.__BuildTree() + self.__BuildSelectionsList() + self.__UpdateButtonStates() + + def _OnWidgetCheckbox(self, label, checked): + raise RuntimeError("Not implemented") + + def GetSettingCheckbox(self, label): + return self._settings_chkboxes_by_label[label].IsChecked() + + def _BuildSettingsArea(self, sizer): + if not self._settings_chkboxes: + return + + for i, (label, checked) in enumerate(self._settings_chkboxes): + chkbox = wx.CheckBox(self, label=label) + chkbox.SetValue(checked) + if i == 0: + sizer.Add(chkbox) + else: + sizer.Add(chkbox, 0, wx.TOP, 5) + chkbox.Bind(wx.EVT_CHECKBOX, partial(self.__OnSettingsCheckbox, label=label)) + self._settings_chkboxes_by_label[label] = chkbox + + def __OnSettingsCheckbox(self, evt, label): + if self._settings_chkboxes: + self._OnWidgetCheckbox(label, evt.IsChecked()) + evt.Skip() + + def GetSelectedElemPaths(self): + if self._single_selection: + return [self._single_selected_path] if self._single_selected_path else [] + + selected_paths = [] + for idx in range(self.selections_list.GetItemCount()): + if self.selections_list.IsItemChecked(idx): + selected_paths.append(self.selections_list.GetItemText(idx)) + + return selected_paths + + def __RebuildSelectedPaths(self, selected_paths_set): + kept = [p for p in self._selected_paths if p in selected_paths_set] + for p in self._all_leaf_paths: + if p in selected_paths_set and p not in kept: + kept.append(p) + self._selected_paths = kept + + def __OnListItemChecked(self, evt): + if self._syncing_list_checkboxes: + evt.Skip() + return + + path = self.selections_list.GetItemText(evt.GetIndex()) + if path not in self._selected_paths: + selected_paths = set(self._selected_paths) + selected_paths.add(path) + self.__RebuildSelectedPaths(selected_paths) + self.__UpdateButtonStates() + evt.Skip() + + def __OnListItemUnchecked(self, evt): + if self._syncing_list_checkboxes: + evt.Skip() + return + + path = self.selections_list.GetItemText(evt.GetIndex()) + if path in self._selected_paths: + selected_paths = set(self._selected_paths) + selected_paths.remove(path) + self.__RebuildSelectedPaths(selected_paths) + self.__UpdateButtonStates() + evt.Skip() + + def __OnTreeSelectionChanged(self, evt): + item = self.hier_tree.GetSelection() + if item.IsOk() and item in self._leaf_paths_by_tree_item: + self._single_selected_path = self._leaf_paths_by_tree_item[item] + else: + self._single_selected_path = None + + self.__UpdateButtonStates() + evt.Skip() + + def __OnTreeRightClick(self, evt, tree): + item = tree.HitTest(evt.GetPosition()) + if not item: + return + + item = item[0] + if not item.IsOk(): + return + + if self._single_selection: + tree.SelectItem(item) + else: + selections = tree.GetSelections() + if item not in selections: + tree.SelectItem(item) + self.__PopupTreeContextMenu(tree, item) + + def __MoveSelectedElemUp(self, evt): + selected_rows = self.__GetListCtrlSelectedRows() + assert len(selected_rows) == 1 + src_row = selected_rows[0] + assert src_row > 0 + dst_row = src_row - 1 + self.__SwapListCtrlItems(src_row, dst_row) + + def __MoveSelectedElemDown(self, evt): + selected_rows = self.__GetListCtrlSelectedRows() + assert len(selected_rows) == 1 + src_row = selected_rows[0] + dst_row = src_row + 1 + assert dst_row < self.selections_list.GetItemCount() + self.__SwapListCtrlItems(src_row, dst_row) + + def __SwapListCtrlItems(self, src_row, dst_row): + src_text = self.selections_list.GetItemText(src_row) + dst_text = self.selections_list.GetItemText(dst_row) + self.selections_list.SetItemText(dst_row, src_text) + self.selections_list.SetItemText(src_row, dst_text) + + src_checked = self.selections_list.IsItemChecked(src_row) + dst_checked = self.selections_list.IsItemChecked(dst_row) + self.selections_list.CheckItem(src_row, dst_checked) + self.selections_list.CheckItem(dst_row, src_checked) + + src_selected = self.selections_list.IsSelected(src_row) + dst_selected = self.selections_list.IsSelected(dst_row) + self.selections_list.Select(src_row, dst_selected) + self.selections_list.Select(dst_row, src_selected) + + self.selections_list.EnsureVisible(dst_row) + + self._selected_paths = [ + self.selections_list.GetItemText(i) + for i in range(self.selections_list.GetItemCount()) + if self.selections_list.IsItemChecked(i) + ] + + def __GetSelectedLeafPaths(self, tree): + paths = [] + for selected_item in tree.GetSelections(): + if selected_item.IsOk() and selected_item in self._leaf_paths_by_tree_item: + paths.append(self._leaf_paths_by_tree_item[selected_item]) + return paths + + def __GetTargetLeafPathsForMenu(self, tree, item): + selected_leaf_paths = self.__GetSelectedLeafPaths(tree) + if item in tree.GetSelections() and selected_leaf_paths: + return selected_leaf_paths + return [self._leaf_paths_by_tree_item[item]] + + def __PopupTreeContextMenu(self, tree, item): + menu = wx.Menu() + if not self._single_selection: + if item in self._leaf_paths_by_tree_item: + target_paths = self.__GetTargetLeafPathsForMenu(tree, item) + in_widget = [path for path in target_paths if path in self._selected_paths] + not_in_widget = [path for path in target_paths if path not in self._selected_paths] + if not_in_widget: + add_item = menu.Append(-1, 'Add to Widget') + self.Bind( + wx.EVT_MENU, + partial(self.__OnAddLeavesFromBranch, paths=not_in_widget), + add_item, + ) + if in_widget: + remove_item = menu.Append(-1, 'Remove from Widget') + self.Bind( + wx.EVT_MENU, + partial(self.__OnRemoveLeavesFromBranch, paths=in_widget), + remove_item, + ) + else: + leaves = self.__CollectLeavesFromItem(tree, item) + selected_leaves = [path for path in leaves if path in self._selected_paths] + if len(selected_leaves) < len(leaves): + add_leaves = menu.Append(-1, 'Add leaves to widget') + self.Bind( + wx.EVT_MENU, + partial(self.__OnAddLeavesFromBranch, paths=leaves), + add_leaves, + ) + if selected_leaves: + remove_leaves = menu.Append(-1, 'Remove leaves from widget') + self.Bind( + wx.EVT_MENU, + partial(self.__OnRemoveLeavesFromBranch, paths=selected_leaves), + remove_leaves, + ) + menu.AppendSeparator() + self.__AppendExpandCollapseSubmenu(menu, tree) + tree.PopupMenu(menu) + menu.Destroy() + + def __AppendExpandCollapseSubmenu(self, menu, tree): + expand_submenu = wx.Menu() + all_expanded, all_collapsed = self.__GetTreeExpandCollapseState(tree) + def ExpandAll(evt, **kwargs): + kwargs['tree'].ExpandAll() + evt.Skip() + def CollapseAll(evt, **kwargs): + kwargs['tree'].CollapseAll() + evt.Skip() + if not all_expanded: + expand_all = expand_submenu.Append(-1, 'Expand All') + self.Bind(wx.EVT_MENU, partial(ExpandAll, tree=tree), expand_all) + if not all_collapsed: + collapse_all = expand_submenu.Append(-1, 'Collapse All') + self.Bind(wx.EVT_MENU, partial(CollapseAll, tree=tree), collapse_all) + menu.AppendSubMenu(expand_submenu, 'Expand / Collapse') + + def __OnAddLeavesFromBranch(self, evt, paths): + self.__SetPathsSelected(paths, True) + evt.Skip() + + def __OnRemoveLeavesFromBranch(self, evt, paths): + self.__SetPathsSelected(paths, False) + evt.Skip() + + def __SetPathsSelected(self, paths, selected): + selected_paths = set(self._selected_paths) + changed = False + for path in paths: + if selected: + if path not in selected_paths: + selected_paths.add(path) + changed = True + elif path in selected_paths: + selected_paths.remove(path) + changed = True + + if changed: + self.__RebuildSelectedPaths(selected_paths) + self.__ApplySelectionToList() + self.__UpdateButtonStates() + + def __UpdateButtonStates(self, *args): + if self._single_selection: + return + + list_ctrl_count = self.selections_list.GetItemCount() + selected_rows = self.__GetListCtrlSelectedRows() + if list_ctrl_count <= 1 or len(selected_rows) > 1 or len(selected_rows) == 0: + self.move_up_btn.Disable() + self.move_down_btn.Disable() + else: + selected_row = selected_rows[0] + if selected_row == 0: + self.move_up_btn.Disable() + self.move_down_btn.Enable() + elif selected_row == list_ctrl_count - 1: + self.move_up_btn.Enable() + self.move_down_btn.Disable() + else: + self.move_up_btn.Enable() + self.move_down_btn.Enable() + + def __GetListCtrlSelectedRows(self): + rows = [] + for idx in range(self.selections_list.GetItemCount()): + if self.selections_list.IsSelected(idx): + rows.append(idx) + + return rows + + def __BuildTree(self): + self._tree_items_by_id = {} + self._leaf_paths_by_tree_item = {} + + self.hier_tree.DeleteAllItems() + root = self.hier_tree.AddRoot('root') + self._tree_items_by_id[0] = root + + visible_paths = self.__BuildVisibleElemPaths(self._all_leaf_paths) + self.__RecurseBuildTree( + self.hier_tree, self.simhier.GetTree().GetRoot(), visible_paths, + ) + + def __RecurseBuildTree(self, tree_ctrl, node, visible_paths): + if node is self.simhier.GetTree().GetRoot(): + for child in node.GetChildren(): + self.__RecurseBuildTree(tree_ctrl, child, visible_paths) + return + + elem_path = node.GetPath() + if elem_path not in visible_paths: + return + + if node.GetParent(): + parent_id = node.GetParent().GetID() + else: + parent_id = 0 + + tree_item = tree_ctrl.AppendItem(self._tree_items_by_id[parent_id], node.GetName()) + node_id = node.GetID() + self._tree_items_by_id[node_id] = tree_item + + if not node.children: + self._leaf_paths_by_tree_item[tree_item] = elem_path + + for child in node.GetChildren(): + self.__RecurseBuildTree(tree_ctrl, child, visible_paths) + + def __ApplySelectionToList(self): + self.__SyncSelectionCheckboxes() + + def __BuildSelectionsList(self): + if self._single_selection: + return + + self.selections_list.Freeze() + try: + self.selections_list.DeleteAllItems() + self._list_indices_by_path = {} + + selected_paths = set(self._selected_paths) + for path in self.__GetDisplayedListPaths(): + idx = self.selections_list.InsertItem(self.selections_list.GetItemCount(), path) + self.selections_list.CheckItem(idx, path in selected_paths) + self._list_indices_by_path[path] = idx + finally: + self.selections_list.Thaw() + + def __SyncSelectionCheckboxes(self): + if not self._list_indices_by_path: + self.__BuildSelectionsList() + return + + selected_paths = set(self._selected_paths) + self._syncing_list_checkboxes = True + try: + for path, idx in self._list_indices_by_path.items(): + checked = path in selected_paths + if self.selections_list.IsItemChecked(idx) != checked: + self.selections_list.CheckItem(idx, checked) + finally: + self._syncing_list_checkboxes = False + + def __GetDisplayedListPaths(self): + selected = list(self._selected_paths) + unselected = [p for p in self._all_leaf_paths if p not in selected] + return selected + unselected + + def __CollectLeavesFromItem(self, tree, item): + if item in self._leaf_paths_by_tree_item: + return [self._leaf_paths_by_tree_item[item]] + + paths = [] + child, cookie = tree.GetFirstChild(item) + while child.IsOk(): + paths.extend(self.__CollectLeavesFromItem(tree, child)) + child = tree.GetNextSibling(child) + + return paths + + def __GetTreeExpandCollapseState(self, tree): + all_expanded = True + all_collapsed = True + has_expandable = False + + child, cookie = tree.GetFirstChild(tree.GetRootItem()) + while child.IsOk(): + branch_expanded, branch_collapsed, branch_has_expandable = ( + self.__BranchExpandCollapseState(tree, child) + ) + if branch_has_expandable: + has_expandable = True + if not branch_expanded: + all_expanded = False + if not branch_collapsed: + all_collapsed = False + child = tree.GetNextSibling(child) + + if not has_expandable: + return False, False + + return all_expanded, all_collapsed + + def __BranchExpandCollapseState(self, tree, item): + has_expandable = tree.ItemHasChildren(item) + all_expanded = True + all_collapsed = True + + if has_expandable: + if tree.IsExpanded(item): + all_collapsed = False + else: + all_expanded = False + + child, cookie = tree.GetFirstChild(item) + while child.IsOk(): + child_expanded, child_collapsed, child_has_expandable = ( + self.__BranchExpandCollapseState(tree, child) + ) + if child_has_expandable: + has_expandable = True + if not child_expanded: + all_expanded = False + if not child_collapsed: + all_collapsed = False + child = tree.GetNextSibling(child) + + return all_expanded, all_collapsed, has_expandable + + @staticmethod + def __BuildVisibleElemPaths(leaf_elem_paths): + visible_paths = set() + for leaf_path in leaf_elem_paths: + parts = leaf_path.split('.') + for i in range(1, len(parts) + 1): + visible_paths.add('.'.join(parts[:i])) + return visible_paths + +class QueueUtilizEditDlg(WidgetDataSelectionsDlg): + SHOW_FULL_PATHS_LABEL = 'Show full paths' + + def __init__(self, parent, frame, elem_paths, show_full_paths, title="Edit Data Selections"): + chkboxes = [(self.SHOW_FULL_PATHS_LABEL, show_full_paths)] + WidgetDataSelectionsDlg.__init__( + self, parent, frame, elem_paths, queues_only=True, + settings_chkboxes=chkboxes, title=title, + ) + + def _OnWidgetCheckbox(self, label, checked): + pass + + @property + def show_full_paths(self): + return self.GetSettingCheckbox(self.SHOW_FULL_PATHS_LABEL) + +class SummaryViewsEditDlg(WidgetDataSelectionsDlg): + SHOW_FULL_PATHS_LABEL = 'Show full paths' + SHOW_DID_LABEL = 'Show DID' + + def __init__(self, parent, frame, elem_paths, show_full_paths, show_did, title="Edit Data Selections"): + chkboxes = [ + (self.SHOW_FULL_PATHS_LABEL, show_full_paths), + (self.SHOW_DID_LABEL, show_did), + ] + WidgetDataSelectionsDlg.__init__( + self, parent, frame, elem_paths, + settings_chkboxes=chkboxes, title=title, + ) + + def _OnWidgetCheckbox(self, label, checked): + pass + + @property + def show_full_paths(self): + return self.GetSettingCheckbox(self.SHOW_FULL_PATHS_LABEL) + + @property + def show_did(self): + return self.GetSettingCheckbox(self.SHOW_DID_LABEL) + +class SchedulingLinesEditDlg(WidgetDataSelectionsDlg): + SHOW_DETAILS_LABEL = 'Show detailed queue packets' + HIDE_EMPTY_ROWS_LABEL = 'Hide empty rows' + SHOW_FULL_PATHS_LABEL = 'Show full paths' + ENABLE_TOOLTIPS_LABEL = 'Enable tooltips' + SHOW_DID_LABEL = 'Show DID' + + def __init__( + self, parent, frame, elem_paths, num_ticks_before, num_ticks_after, show_details, hide_empty_rows, show_full_paths, enable_tooltips, show_did, title="Edit Data Selections", + ): + self._num_ticks_before = num_ticks_before + self._num_ticks_after = num_ticks_after + chkboxes = [ + (self.SHOW_DETAILS_LABEL, show_details), + (self.HIDE_EMPTY_ROWS_LABEL, hide_empty_rows), + (self.SHOW_FULL_PATHS_LABEL, show_full_paths), + (self.ENABLE_TOOLTIPS_LABEL, enable_tooltips), + (self.SHOW_DID_LABEL, show_did), + ] + WidgetDataSelectionsDlg.__init__( + self, parent, frame, elem_paths, queues_only=True, settings_chkboxes=chkboxes, + ) + + def _BuildSettingsArea(self, sizer): + assert self._num_ticks_before >= 1 and self._num_ticks_before <= 25 + info_ticks_before = wx.StaticText(self, label='Num ticks before current tick:') + self._label_ticks_before = wx.StaticText(self, label=f'({self._num_ticks_before})') + self._slider_ticks_before = wx.Slider( + self, value=self._num_ticks_before, minValue=1, maxValue=25) + self._slider_ticks_before.Bind(wx.EVT_SLIDER, self.__SyncWithSliderTicks) + + assert self._num_ticks_after >= 1 and self._num_ticks_after <= 25 + info_ticks_after = wx.StaticText(self, label='Num ticks after current tick:') + self._label_ticks_after = wx.StaticText(self, label=f'({self._num_ticks_after})') + self._slider_ticks_after = wx.Slider( + self, value=self._num_ticks_after, minValue=1, maxValue=25) + self._slider_ticks_after.Bind(wx.EVT_SLIDER, self.__SyncWithSliderTicks) + + gb_sizer = wx.GridBagSizer(vgap=10, hgap=12) + gb_sizer.Add(info_ticks_before, pos=(0, 0)) + gb_sizer.Add(self._slider_ticks_before, pos=(0, 1), flag=wx.EXPAND) + gb_sizer.Add(self._label_ticks_before, pos=(0, 2)) + + gb_sizer.Add(info_ticks_after, pos=(1, 0)) + gb_sizer.Add(self._slider_ticks_after, pos=(1, 1), flag=wx.EXPAND) + gb_sizer.Add(self._label_ticks_after, pos=(1, 2)) + + gb_sizer.AddGrowableCol(1) + sizer.Add(gb_sizer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5) + + WidgetDataSelectionsDlg._BuildSettingsArea(self, sizer) + + def _OnWidgetCheckbox(self, label, checked): + pass + + @property + def num_ticks_before(self): + return self._slider_ticks_before.GetValue() + + @property + def num_ticks_after(self): + return self._slider_ticks_after.GetValue() + + @property + def show_details(self): + return self.GetSettingCheckbox(self.SHOW_DETAILS_LABEL) + + @property + def hide_empty_rows(self): + return self.GetSettingCheckbox(self.HIDE_EMPTY_ROWS_LABEL) + + @property + def show_full_paths(self): + return self.GetSettingCheckbox(self.SHOW_FULL_PATHS_LABEL) + + @property + def enable_tooltips(self): + return self.GetSettingCheckbox(self.ENABLE_TOOLTIPS_LABEL) + + @property + def show_did(self): + return self.GetSettingCheckbox(self.SHOW_DID_LABEL) + + def __UpdateButtonStates(self, *args): + WidgetDataSelectionsDlg.__UpdateButtonStates(self, *args) + + def __SyncWithSliderTicks(self, evt): + value = self._slider_ticks_before.GetValue() + self._label_ticks_before.SetLabel(f'({value})') + + value = self._slider_ticks_after.GetValue() + self._label_ticks_after.SetLabel(f'({value})') + + evt.Skip() diff --git a/python/argos/viewer/gui/explorer.py b/python/argos/viewer/gui/explorer.py deleted file mode 100644 index c4c569f0..00000000 --- a/python/argos/viewer/gui/explorer.py +++ /dev/null @@ -1,18 +0,0 @@ -import wx -from viewer.gui.navtree import NavTree -from viewer.gui.watchlist import Watchlist -from viewer.gui.tools import SystemwideTools - -class DataExplorer(wx.Notebook): - def __init__(self, parent, frame): - super(DataExplorer, self).__init__(parent, style=wx.NB_LEFT) - self.frame = frame - self.navtree = NavTree(self, frame) - self.watchlist = Watchlist(self, frame) - self.tools = SystemwideTools(self, frame) - - self.AddPage(self.navtree, "NavTree") - self.AddPage(self.watchlist, "Watchlist") - self.AddPage(self.tools, "Tools") - - self.SetMinSize((200, 200)) diff --git a/python/argos/viewer/gui/inspector.py b/python/argos/viewer/gui/inspector.py index 52f87613..68c9dd44 100644 --- a/python/argos/viewer/gui/inspector.py +++ b/python/argos/viewer/gui/inspector.py @@ -1,61 +1,92 @@ +import re import wx +import wx.aui from viewer.gui.canvas_grid import CanvasGrid +from viewer.gui.logs import CollectionLogs from viewer.gui.view_settings import DirtyReasons +from contextlib import contextmanager from functools import partial -class DataInspector(wx.Notebook): +class DataInspector(wx.aui.AuiNotebook): def __init__(self, parent, frame): - super(DataInspector, self).__init__(parent, style=wx.NB_TOP) + super(DataInspector, self).__init__(parent, style=wx.aui.AUI_NB_TOP | wx.aui.AUI_NB_SCROLL_BUTTONS) self.frame = frame self.tabs = [] + + # AuiNotebook (unlike wx.Notebook) emits EVT_AUINOTEBOOK_PAGE_CHANGED + # during programmatic page mutations (e.g. DeletePage/InsertPage). This + # guard ensures the "Add Tab" dialog only opens on a genuine user click + # of the plus tab, not while we are rebuilding tabs ourselves. + self.__suppress_add_tab_dialog = False + + # The "Logs" tab is a fixed, non-editable page pinned to index 0 when + # present. It is not part of self.tabs (the user-managed CanvasGrid + # tabs), so page index math below offsets past it when it exists. + self.__has_logs_tab = False self.__AddPlusTab() + if self.__HasNotifications(): + self.__AddLogsTab() self.__AddInspectorTab("Tab 1") - self.SetSelection(0) + self.__SelectDefaultTab() self.SetMinSize((200, 200)) - self.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.__OnPageChanged) - self.Bind(wx.EVT_CONTEXT_MENU, self.__OnContextMenu) + self.Bind(wx.aui.EVT_AUINOTEBOOK_PAGE_CHANGED, self.__OnPageChanged) + self.Bind(wx.aui.EVT_AUINOTEBOOK_TAB_RIGHT_DOWN, self.__OnContextMenu) + + @contextmanager + def __SuppressAddTabDialog(self): + # Save/restore (rather than a plain True/False) so nested mutations, + # e.g. ApplyViewSettings calling __AddInspectorTab, behave correctly. + prev = self.__suppress_add_tab_dialog + self.__suppress_add_tab_dialog = True + try: + yield + finally: + self.__suppress_add_tab_dialog = prev def GetCurrentTabWidgetContainers(self): - selected_tab = self.GetSelection() - if selected_tab == self.GetPageCount() - 1: + tab_idx = self.__SelectedTabIndex() + if tab_idx is None: return None - - return self.tabs[selected_tab].GetWidgetContainers() + + return self.tabs[tab_idx].GetWidgetContainers() def ResetCurrentTab(self): - selected_tab = self.GetSelection() - if selected_tab == self.GetPageCount() - 1: + tab_idx = self.__SelectedTabIndex() + if tab_idx is None: return - - self.tabs[selected_tab].ResetLayout() + + self.tabs[tab_idx].ResetLayout() def GetCurrentViewSettings(self): settings = {} - settings['tab_names'] = [self.GetPageText(i) for i in range(self.GetPageCount() - 1)] + # Skip the fixed "Logs" tab (when present) and the trailing "Add Tab" page. + settings['tab_names'] = [self.GetPageText(i) for i in range(self.__FirstUserTabIndex(), self.GetPageCount() - 1)] settings['tab_settings'] = [tab.GetCurrentViewSettings() for tab in self.tabs] return settings def ApplyViewSettings(self, settings): - # Reset all tabs - for tab in self.tabs: - tab.ResetLayout() + with self.__SuppressAddTabDialog(): + # Reset all tabs + for tab in self.tabs: + tab.ResetLayout() - self.tabs = [] + self.tabs = [] - # Delete all tabs except the last one - while self.GetPageCount() > 1: - self.DeletePage(0) + # Delete all user tabs, preserving the fixed "Logs" tab (when present) + # and the trailing "Add Tab" page. + while self.GetPageCount() > self.__FirstUserTabIndex() + 1: + self.DeletePage(self.__FirstUserTabIndex()) - # Add new tabs - for tab_name in settings['tab_names']: - self.__AddInspectorTab(tab_name) + # Add new tabs + for tab_name in settings['tab_names']: + self.__AddInspectorTab(tab_name) - # Apply settings to each tab - if 'tab_settings' in settings: - for i, tab_settings in enumerate(settings['tab_settings']): - self.tabs[i].ApplyViewSettings(tab_settings) + # Apply settings to each tab + if 'tab_settings' in settings: + for i, tab_settings in enumerate(settings['tab_settings']): + self.tabs[i].ApplyViewSettings(tab_settings) def GetCurrentUserSettings(self): settings = {} @@ -63,24 +94,35 @@ def GetCurrentUserSettings(self): return settings def ApplyUserSettings(self, settings): - # Select the tab that was selected before saving the settings + selected_tab = settings.get('selected_tab') + if not selected_tab: + self.__SelectDefaultTab() + return + + # "Tab 1" was the historical default before the Logs tab existed. + if self.__has_logs_tab and selected_tab == 'Tab 1': + self.__SelectDefaultTab() + return + for i in range(self.GetPageCount() - 1): - if self.GetPageText(i) == settings['selected_tab']: + if self.GetPageText(i) == selected_tab: self.SetSelection(i) - break + return + + self.__SelectDefaultTab() def ResetToDefaultViewSettings(self, update_widgets=True): self.ApplyViewSettings({'tab_names': ['Tab 1']}) - self.ApplyUserSettings({'selected_tab': 'Tab 1'}) + self.__SelectDefaultTab() def RefreshWidgetsOnCurrentTab(self): - selected_tab = self.GetSelection() - if selected_tab == self.GetPageCount() - 1: + tab_idx = self.__SelectedTabIndex() + if tab_idx is None: return - self.tabs[selected_tab].UpdateWidgets() - self.tabs[selected_tab].Layout() - self.tabs[selected_tab].Refresh() + self.tabs[tab_idx].UpdateWidgets() + self.tabs[tab_idx].Layout() + self.tabs[tab_idx].Refresh() def RefreshWidgetsOnAllTabs(self): for tab in self.tabs: @@ -88,24 +130,68 @@ def RefreshWidgetsOnAllTabs(self): tab.Layout() tab.Refresh() + def __GetDefaultTabName(self): + # Suggest a "Tab N" name that does not collide with any existing tab. + # N is one past the highest existing "Tab N" number, but at least + # (num_tabs + 1) so the suggestion keeps growing as tabs are added. + highest = 0 + for i in range(self.GetPageCount() - 1): + match = re.fullmatch(r"Tab (\d+)", self.GetPageText(i)) + if match: + highest = max(highest, int(match.group(1))) + + return "Tab %d" % max(len(self.tabs) + 1, highest + 1) + + def __HasNotifications(self): + cursor = self.frame.db.cursor() + cursor.execute("SELECT COUNT(*) FROM Notifications") + return cursor.fetchone()[0] > 0 + + def __FirstUserTabIndex(self): + return 1 if self.__has_logs_tab else 0 + + def __SelectDefaultTab(self): + # Index 0 is the Logs tab when present, otherwise the first user tab. + self.SetSelection(0) + + def __SelectedTabIndex(self): + # Map the currently selected page to an index into self.tabs, or None + # when the selection is the fixed "Logs" tab or the "Add Tab" page. + selected_page = self.GetSelection() + if selected_page == self.GetPageCount() - 1: + return None + if self.__has_logs_tab and selected_page == 0: + return None + + return selected_page - self.__FirstUserTabIndex() + def __AddPlusTab(self): super(DataInspector, self).AddPage(wx.Panel(self), "Add Tab") + def __AddLogsTab(self): + with self.__SuppressAddTabDialog(): + self.logs = CollectionLogs(self, self.frame) + super(DataInspector, self).InsertPage(0, self.logs, "Logs") + self.__has_logs_tab = True + def __AddInspectorTab(self, name): - canvas_grid = CanvasGrid(self) - super(DataInspector, self).InsertPage(self.GetPageCount() - 1, canvas_grid, name) - self.tabs.append(canvas_grid) - self.SetSelection(self.GetPageCount() - 2) + with self.__SuppressAddTabDialog(): + canvas_grid = CanvasGrid(self) + super(DataInspector, self).InsertPage(self.GetPageCount() - 1, canvas_grid, name) + self.tabs.append(canvas_grid) + self.SetSelection(self.GetPageCount() - 2) def __OnPageChanged(self, event): new_page_index = event.GetSelection() - if new_page_index == self.GetPageCount() - 1: + if not self.__suppress_add_tab_dialog and new_page_index == self.GetPageCount() - 1: self.__ShowAddTabDialog() + else: + self.RefreshWidgetsOnCurrentTab() event.Skip() def __ShowAddTabDialog(self): - dlg = wx.TextEntryDialog(self, "Enter name for the new tab:", "New Tab", value="Tab %d" % (len(self.tabs) + 1)) + dlg = wx.TextEntryDialog(self, "Enter name for the new tab:", "New Tab", value=self.__GetDefaultTabName()) if dlg.ShowModal() == wx.ID_OK: new_tab_name = dlg.GetValue().strip() @@ -113,6 +199,7 @@ def __ShowAddTabDialog(self): for i in range(self.GetPageCount() - 1): if self.GetPageText(i) == new_tab_name: wx.MessageBox("A tab with that name already exists.", "Error", wx.OK | wx.ICON_ERROR) + self.SetSelection(self.GetPageCount() - 2) return self.__AddInspectorTab(new_tab_name) @@ -123,38 +210,34 @@ def __ShowAddTabDialog(self): dlg.Destroy() def __OnContextMenu(self, event): - # Get the position where the user right-clicked - pos = event.GetPosition() - pos = self.ScreenToClient(pos) # Convert to client coordinates - - # Check if the right-click was within the tab area - hit = self.HitTest(pos) - if hit == wx.NOT_FOUND or hit[0] == self.GetPageCount() - 1 or hit[0] == -1: + # The tab that was right-clicked is reported directly by the event. + tab_idx = event.GetSelection() + # Ignore the fixed "Logs" tab (when present) and the "Add Tab" page. + if tab_idx < self.__FirstUserTabIndex() or tab_idx == self.GetPageCount() - 1: return - + # Show the context menu menu = wx.Menu() rename_item = menu.Append(wx.ID_ANY, "Rename tab") - self.Bind(wx.EVT_MENU, partial(self.__OnRenameTab, tab_idx=hit[0]), rename_item) + self.Bind(wx.EVT_MENU, partial(self.__OnRenameTab, tab_idx=tab_idx), rename_item) if len(self.tabs) > 1: delete_item = menu.Append(wx.ID_ANY, "Delete tab") - self.Bind(wx.EVT_MENU, partial(self.__OnDeleteTab, tab_idx=hit[0]), delete_item) + self.Bind(wx.EVT_MENU, partial(self.__OnDeleteTab, tab_idx=tab_idx), delete_item) - # Popup the menu - pos = wx.Point(pos.x, pos.y - 40) - self.PopupMenu(menu, self.ClientToScreen(pos)) + # Popup at the current cursor position (wxDefaultPosition). + self.PopupMenu(menu) menu.Destroy() def __OnRenameTab(self, event, tab_idx): # Show a dialog to enter the new name - dlg = wx.TextEntryDialog(self, "Enter new name:", "Rename Tab", - "Tab %d" % (len(self.tabs) + 1)) + dlg = wx.TextEntryDialog(self, "Enter new name:", "Rename Tab", + self.__GetDefaultTabName()) if dlg.ShowModal() == wx.ID_OK: new_name = dlg.GetValue().strip() if new_name: - if new_name == 'Add Tab': + if new_name in ('Add Tab', 'Logs'): wx.MessageBox("You cannot rename this tab.", "Error", wx.OK | wx.ICON_ERROR) return @@ -171,8 +254,15 @@ def __OnDeleteTab(self, event, tab_idx): if dlg.ShowModal() == wx.ID_YES: # Delete the selected tab - self.DeletePage(tab_idx) - self.tabs.pop(tab_idx) + with self.__SuppressAddTabDialog(): + self.DeletePage(tab_idx) + self.tabs.pop(tab_idx - self.__FirstUserTabIndex()) + + # Deleting the rightmost real tab can leave the plus tab + # selected; move selection back to a real tab. + if self.GetSelection() == self.GetPageCount() - 1: + self.SetSelection(self.GetPageCount() - 2) + self.frame.view_settings.SetDirty(reason=DirtyReasons.TabDeleted) dlg.Destroy() diff --git a/python/argos/viewer/gui/logs.py b/python/argos/viewer/gui/logs.py new file mode 100644 index 00000000..68475d47 --- /dev/null +++ b/python/argos/viewer/gui/logs.py @@ -0,0 +1,55 @@ +import wx +import wx.lib.scrolledpanel as scrolled + +class CollectionLogs(scrolled.ScrolledPanel): + NOTIF_TYPE_LABELS = {0: 'WARNING', 1: 'ERROR', 2: 'MESSAGE'} + + def __init__(self, parent, frame): + super().__init__(parent) + self.frame = frame + self.simhier = frame.simhier + + cursor = frame.db.cursor() + cursor.execute("SELECT Timestamp, NotifStr, NotifType " + "FROM Notifications ORDER BY Timestamp ASC") + rows = cursor.fetchall() + + cursor.execute("SELECT CID, FullPath FROM CollectableTreeNodes") + paths_by_cid = {cid: fp.replace('root.', '') for cid, fp in cursor.fetchall()} + + mono10 = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) + + sizer = wx.BoxSizer(wx.VERTICAL) + + grid = wx.GridBagSizer(vgap=2, hgap=10) + + row = 0 + for timestamp, notif_str, notif_type in rows: + tick = str(timestamp) if timestamp is not None else '0' + type_label = self.NOTIF_TYPE_LABELS.get(notif_type, str(notif_type)) + + tick_text = wx.StaticText(self, label='Tick:{}'.format(int(tick))) + tick_text.SetFont(mono10) + grid.Add(tick_text, pos=(row, 0)) + + type_text = wx.StaticText(self, label='Type:{}'.format(type_label)) + type_text.SetFont(mono10) + grid.Add(type_text, pos=(row, 1)) + + body = wx.StaticText(self, label=notif_str) + body.SetFont(mono10) + grid.Add(body, pos=(row + 1, 0), span=(1, 3), + flag=wx.EXPAND | wx.BOTTOM, border=6) + + row += 2 + + if row > 0: + grid.AddGrowableCol(2, 1) + + sizer.Add(grid, 1, wx.EXPAND | wx.ALL, 5) + self.SetSizer(sizer) + self.SetupScrolling() + self.Layout() + + def UpdateWidgets(self): + pass diff --git a/python/argos/viewer/gui/tools.py b/python/argos/viewer/gui/tools.py deleted file mode 100644 index bce6adc1..00000000 --- a/python/argos/viewer/gui/tools.py +++ /dev/null @@ -1,19 +0,0 @@ -import wx - -class SystemwideTools(wx.TreeCtrl): - def __init__(self, parent, frame): - super(SystemwideTools, self).__init__(parent, style=wx.TR_DEFAULT_STYLE | wx.TR_LINES_AT_ROOT) - self.frame = frame - - self._root = self.AddRoot("Systemwide Tools") - self.AppendItem(self._root, "Queue Utilization") - self.AppendItem(self._root, "Scheduling Lines") - - #cursor = frame.db.cursor() - #cmd = 'SELECT COUNT(Id) FROM TimeseriesData WHERE ElementPath="IPC"' - #cursor.execute(cmd) - - #if cursor.fetchone()[0] > 0: - # self.AppendItem(self._root, "IPC") - - self.ExpandAll() diff --git a/python/argos/viewer/gui/view_settings.py b/python/argos/viewer/gui/view_settings.py index 30aa11ea..2dc50b9f 100644 --- a/python/argos/viewer/gui/view_settings.py +++ b/python/argos/viewer/gui/view_settings.py @@ -1,43 +1,6 @@ -import wx, yaml, os, enum, shutil, tempfile - -class DirtyReasons(enum.Enum): - WidgetDropped = 1 - WidgetSplit = 2 - CanvasExploded = 3 - TabAdded = 4 - TabRenamed = 5 - TabDeleted = 6 - WatchlistAdded = 7 - WatchlistRemoved = 8 - WatchlistOrgChanged = 9 - QueueUtilizDispQueueChanged = 10 - TimeseriesPlotSettingsChanged = 11 - QueueTableDispColsChanged = 12 - QueueTableAutoColorizeChanged = 13 - SchedulingLinesWidgetChanged = 14 - SashPositionChanged = 15 - WidgetRemoved = 16, - TrackedPacketChanged = 17 - -DIRTY_REASONS = { - DirtyReasons.WidgetDropped: 'A widget was dropped onto the widget canvas', - DirtyReasons.WidgetSplit: 'A widget was split horizontally or vertically', - DirtyReasons.CanvasExploded: 'Widget canvas was exploded', - DirtyReasons.TabAdded: 'A new tab was added', - DirtyReasons.TabRenamed: 'A tab was renamed', - DirtyReasons.TabDeleted: 'A tab was deleted', - DirtyReasons.WatchlistAdded: 'Something was added to the Watchlist', - DirtyReasons.WatchlistRemoved: 'Something was removed from the Watchlist', - DirtyReasons.WatchlistOrgChanged: 'The Watchlist organization (flat/hier) was changed', - DirtyReasons.QueueUtilizDispQueueChanged: 'The displayed queues in a Queue Utilization widget were changed', - DirtyReasons.TimeseriesPlotSettingsChanged: 'Settings were changed for a timeseries plot', - DirtyReasons.QueueTableDispColsChanged: 'Displayed columns were changed for a Queue Table widget', - DirtyReasons.QueueTableAutoColorizeChanged: 'Auto-colorize column was changed for a Queue Table widget', - DirtyReasons.SchedulingLinesWidgetChanged: 'Displayed queues were changed for a Scheduling Lines widget', - DirtyReasons.SashPositionChanged: 'Widget canvas splitter window sash position changed', - DirtyReasons.WidgetRemoved: 'A widget was removed from the inspector canvas', - DirtyReasons.TrackedPacketChanged: 'Changes were made to tracked packet(s)' -} +import wx, yaml, os, shutil, tempfile +from viewer.model.dirty_reasons import DirtyReasons +from viewer.gui.dialogs.save_view_file import SaveViewFileDlg class ViewSettings: def __init__(self): @@ -46,6 +9,7 @@ def __init__(self): self._frame = None self._dirty = False self._dirty_reasons = set() + self._last_known_view = os.path.expanduser('~/.argos/last_known_view.avf') @property def view_file(self): @@ -76,37 +40,43 @@ def SetDirty(self, dirty=True, reason=None): def PostLoad(self, frame, view_file): self._frame = frame if view_file: - self.Load(view_file) + self.Load(view_file) # Not dirty + elif os.path.exists(self._last_known_view): + self.Load(self._last_known_view, set_as_current=False) + self.SetDirty(True) # Implicitly loading "last known view" does not clear dirty flag + else: + self.__ResetDefaultViewSettings() - self.__ResetDefaultViewSettings() self.__ApplyUserSettings() - self.SetDirty(False) - def Load(self, view_file): + def Load(self, view_file, set_as_current=True): if not os.path.isfile(view_file): msg = f"View file '{view_file}' does not exist" dlg = wx.MessageDialog(None, msg, 'Error', wx.OK | wx.ICON_ERROR) dlg.ShowModal() dlg.Destroy() return - - with open(view_file, 'r') as fin: - settings = yaml.load(fin, Loader=yaml.FullLoader) - self._frame.explorer.navtree.ApplyViewSettings(settings['NavTree']) - self._frame.explorer.watchlist.ApplyViewSettings(settings['Watchlist']) - self._frame.playback_bar.ApplyViewSettings(settings['PlaybackBar']) - self._frame.data_retriever.ApplyViewSettings(settings['DataRetriever']) - self._frame.inspector.ApplyViewSettings(settings['Inspector']) - self._frame.widget_renderer.ApplyViewSettings(settings['WidgetRenderer']) + + try: + with open(view_file, 'r') as fin: + settings = yaml.load(fin, Loader=yaml.FullLoader) + self._frame.playback_bar.ApplyViewSettings(settings['PlaybackBar']) + self._frame.data_retriever.ApplyViewSettings(settings['DataRetriever']) + self._frame.inspector.ApplyViewSettings(settings['Inspector']) + self._frame.widget_renderer.ApplyViewSettings(settings['WidgetRenderer']) + except Exception as ex: + print (f"Error loading view file '{view_file}': '{ex}'") + self.__ResetDefaultViewSettings() self._frame.inspector.RefreshWidgetsOnAllTabs() - self.view_file = view_file + if set_as_current: + self.view_file = view_file self.SetDirty(False) def CreateNewView(self): if self._dirty: if self.view_file: - result = self.__AskToSaveChangesToCurrentView("Save changes to '{}'?".format(self.view_file)) + result = self.__AskToSaveChangesToCurrentView("Save changes to '{}'?".format(os.path.basename(self.view_file)), True) if result == wx.ID_CANCEL: return @@ -115,7 +85,7 @@ def CreateNewView(self): self.__ResetDefaultViewSettings() else: - result = self.__AskToSaveChangesToCurrentView("Save current view to new file before creating a new view?") + result = self.__AskToSaveChangesToCurrentView("Save current view to new file before creating a new view?", True) if result == wx.ID_CANCEL: return @@ -185,8 +155,7 @@ def DoLoad(view_settings, view_file): wx.BeginBusyCursor() wx.CallAfter(DoLoad, self, view_file) - # Note that this method returns True if Argos can be closed after calling this method. - def SaveView(self, prompt_if_dirty=True, on_frame_closing=False): + def SaveView(self, prompt_if_dirty=True): self.__SaveUserSettings() if not self._dirty: @@ -196,7 +165,7 @@ def SaveView(self, prompt_if_dirty=True, on_frame_closing=False): # Ask the user if they want to save the view to a new file dlg = SaveViewFileDlg(prompt="Save current Argos view to a new file?", reasons=self._dirty_reasons) elif self.view_file is not None and prompt_if_dirty: - dlg = SaveViewFileDlg(prompt="Save changes to '{}'?".format(self.view_file), reasons=self._dirty_reasons) + dlg = SaveViewFileDlg(prompt="Save changes to '{}'?".format(os.path.basename(self.view_file)), reasons=self._dirty_reasons) else: dlg = None result = wx.ID_YES @@ -224,26 +193,52 @@ def SaveView(self, prompt_if_dirty=True, on_frame_closing=False): # Do not close Argos if the user cancels the save dialog if not view_file: return False - - if result == wx.ID_NO: + + self.__WriteViewSettings(view_file) + self.view_file = view_file + self.SetDirty(False) + return True + + def OnFrameClosing(self): + # Returns True if Argos can be closed after calling this method. + self.__SaveUserSettings() + + if self.view_file is None: + # Common case: no named AVF in use. Snapshot the current view so the + # next launch (without --view-file) reloads where the user left off. + self.__WriteViewSettings(self._last_known_view) return True + # A named AVF is in use. Offer to save changes back to it, then drop any + # last_known_view so it does not shadow the named view on the next launch. + if self._dirty: + result = self.__AskToSaveChangesToCurrentView("Save changes to '{}'?".format(os.path.basename(self.view_file)), True) + if result == wx.ID_CANCEL: + return False + + if result == wx.ID_YES: + self.SaveView(prompt_if_dirty=False) + + if os.path.exists(self._last_known_view): + os.remove(self._last_known_view) + + return True + + def __WriteViewSettings(self, view_file): settings = { - 'NavTree': self._frame.explorer.navtree.GetCurrentViewSettings(), - 'Watchlist': self._frame.explorer.watchlist.GetCurrentViewSettings(), 'PlaybackBar': self._frame.playback_bar.GetCurrentViewSettings(), 'DataRetriever': self._frame.data_retriever.GetCurrentViewSettings(), 'Inspector': self._frame.inspector.GetCurrentViewSettings(), 'WidgetRenderer': self._frame.widget_renderer.GetCurrentViewSettings() } + settings_dir = os.path.dirname(view_file) + if settings_dir and not os.path.exists(settings_dir): + os.makedirs(settings_dir) + with open(view_file, 'w') as fout: yaml.dump(settings, fout) - self.view_file = view_file - self.SetDirty(False) - return True - def __UpdateTitle(self): if self._frame is None: return @@ -268,8 +263,6 @@ def __SaveUserSettings(self): settings_file = os.path.join(settings_dir, 'user_settings.yaml') settings = { - 'NavTree': self._frame.explorer.navtree.GetCurrentUserSettings(), - 'Watchlist': self._frame.explorer.watchlist.GetCurrentUserSettings(), 'PlaybackBar': self._frame.playback_bar.GetCurrentUserSettings(), 'DataRetriever': self._frame.data_retriever.GetCurrentUserSettings(), 'Inspector': self._frame.inspector.GetCurrentUserSettings(), @@ -295,25 +288,22 @@ def __ApplyUserSettings(self): try: with open(settings_file, 'r') as fin: settings = yaml.load(fin, Loader=yaml.FullLoader) - self._frame.explorer.navtree.ApplyUserSettings(settings['NavTree']) - self._frame.explorer.watchlist.ApplyUserSettings(settings['Watchlist']) self._frame.playback_bar.ApplyUserSettings(settings['PlaybackBar']) self._frame.data_retriever.ApplyUserSettings(settings['DataRetriever']) self._frame.inspector.ApplyUserSettings(settings['Inspector']) self._frame.widget_renderer.ApplyUserSettings(settings['WidgetRenderer']) - except: - print ("Error loading user settings. Deleting settings file.") + except Exception as ex: + print (f"Error loading user settings. Deleting settings file. Error: '{ex}'") os.remove(settings_file) + self.__ResetDefaultViewSettings() - def __AskToSaveChangesToCurrentView(self, prompt): - dlg = SaveViewFileDlg(prompt=prompt, reasons=self._dirty_reasons) + def __AskToSaveChangesToCurrentView(self, prompt, include_skip_btn=False): + dlg = SaveViewFileDlg(prompt=prompt, reasons=self._dirty_reasons, include_skip_btn=include_skip_btn) result = dlg.ShowModal() dlg.Destroy() return result def __ResetDefaultViewSettings(self): - self._frame.explorer.navtree.ResetToDefaultViewSettings(False) - self._frame.explorer.watchlist.ResetToDefaultViewSettings(False) self._frame.playback_bar.ResetToDefaultViewSettings(False) self._frame.data_retriever.ResetToDefaultViewSettings(False) self._frame.inspector.ResetToDefaultViewSettings(False) @@ -341,54 +331,3 @@ def __GetViewFileForSave(self): path += '.avf' return path - -class SaveViewFileDlg(wx.Dialog): - def __init__(self, prompt='Save to view file?', reasons=None): - super().__init__(None, title='Save View', size=(550, 200)) - - self._reasons = reasons - panel = wx.Panel(self) - sizer = wx.BoxSizer(wx.VERTICAL) - - instruction = wx.StaticText(panel, label=prompt) - sizer.Add(instruction, 0, wx.ALL | wx.CENTER, 10) - - btn_sizer = wx.BoxSizer(wx.HORIZONTAL) - - # Save button - save_btn = wx.Button(panel, label="Save") - save_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_YES)) - btn_sizer.Add(save_btn, 0, wx.ALL | wx.CENTER, 5) - - # Do not save button (closes Argos) - do_not_save_btn = wx.Button(panel, label="Do not save") - do_not_save_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_NO)) - btn_sizer.Add(do_not_save_btn, 0, wx.ALL | wx.CENTER, 5) - - # Cancel button - cancel_btn = wx.Button(panel, label="Cancel") - cancel_btn.Bind(wx.EVT_BUTTON, lambda event: self.EndModal(wx.ID_CANCEL)) - btn_sizer.Add(cancel_btn, 0, wx.ALL | wx.CENTER, 5) - - # "What changed?" button - what_changed_btn = wx.Button(panel, wx.ID_ANY, label="What changed?") - what_changed_btn.Bind(wx.EVT_BUTTON, self.__ShowWhatChanged) - btn_sizer.Add(what_changed_btn, 0, wx.ALL | wx.CENTER, 5) - - sizer.Add(btn_sizer, 0, wx.ALL | wx.RIGHT, 10) - panel.SetSizer(sizer) - - def __ShowWhatChanged(self, event): - if self._reasons is None: - return - - if len(self._reasons) > 1: - msg = 'The following changes were made to the view:\n\n' - for i,reason in enumerate(self._reasons): - msg += str(i) + '. ' + DIRTY_REASONS[reason] + '\n' - else: - msg = DIRTY_REASONS[self._reasons.pop()] - - dlg = wx.MessageDialog(self, msg, 'Changes', wx.OK | wx.ICON_INFORMATION) - dlg.ShowModal() - dlg.Destroy() diff --git a/python/argos/viewer/gui/watchlist.py b/python/argos/viewer/gui/watchlist.py deleted file mode 100644 index f676e517..00000000 --- a/python/argos/viewer/gui/watchlist.py +++ /dev/null @@ -1,330 +0,0 @@ -import wx -from functools import partial -from viewer.gui.view_settings import DirtyReasons - -class Watchlist(wx.TreeCtrl): - def __init__(self, parent, frame): - super(Watchlist, self).__init__(parent, style=wx.TR_DEFAULT_STYLE | wx.TR_HIDE_ROOT | wx.TR_LINES_AT_ROOT) - self.frame = frame - self._watched_sim_elems = [] - self._mode = 'flat' - - sizer = wx.BoxSizer(wx.VERTICAL) - self.SetSizer(sizer) - self.__RenderWatchlist() - - self.Bind(wx.EVT_TREE_ITEM_GETTOOLTIP, self.__ProcessTooltip) - self.Bind(wx.EVT_RIGHT_DOWN, self.__OnRightClick) - self.Bind(wx.EVT_TREE_ITEM_EXPANDED, self.__OnItemExpanded) - - self._utiliz_image_list = frame.widget_renderer.utiliz_handler.CreateUtilizImageList() - self.SetImageList(self._utiliz_image_list) - - self._tooltips_by_item = {} - - def GetWatchedSimElems(self): - return self._watched_sim_elems - - def GetItemElemPath(self, item): - if not item.IsOk(): - return None - - path_parts = [] - while item and item != self.GetRootItem(): - item_text = self.GetItemText(item) - path_parts.append(item_text) - item = self.GetItemParent(item) - - if item == self.GetRootItem(): - path_parts.append(self.GetItemText(self.GetRootItem())) - - path_parts.reverse() - return '.'.join(path_parts).replace('Root.Watchlist.', '') - - def AddToWatchlist(self, elem_path): - if elem_path in self._watched_sim_elems: - return - - self._watched_sim_elems.append(elem_path) - self.__RenderWatchlist() - self.GetParent().ChangeSelection(1) - self.frame.view_settings.SetDirty(reason=DirtyReasons.WatchlistAdded) - - def UpdateUtilizBitmaps(self): - self._tooltips_by_item = {} - self.__UpdateUtilizBitmaps(self.GetRootItem()) - - def ExpandAll(self): - self.Unbind(wx.EVT_TREE_ITEM_EXPANDED) - super(Watchlist, self).ExpandAll() - self.UpdateUtilizBitmaps() - self.Bind(wx.EVT_TREE_ITEM_EXPANDED, self.__OnItemExpanded) - - def GetCurrentViewSettings(self): - settings = {} - settings['grouping_mode'] = self._mode - settings['watched_sim_elem_paths'] = self._watched_sim_elems - return settings - - def ApplyViewSettings(self, settings): - grouping_mode = settings['grouping_mode'] - watched_sim_elem_paths = settings['watched_sim_elem_paths'] - - self._watched_sim_elems = watched_sim_elem_paths - self._mode = grouping_mode - self.__RenderWatchlist() - - def GetCurrentUserSettings(self): - # All our settings are in the user settings and do not affect the view file - return {} - - def ApplyUserSettings(self, settings): - # All our settings are in the user settings and do not affect the view file - pass - - def ResetToDefaultViewSettings(self, update_widgets=True): - self.ApplyViewSettings({'grouping_mode': 'flat', 'watched_sim_elem_paths': []}) - - def GetItemElemPath(self, item): - if not item or not item.IsOk(): - return None - - node_names = [] - while item and item != self.GetRootItem(): - node_name = self.GetItemText(item) - node_names.append(node_name) - item = self.GetItemParent(item) - - node_names.reverse() - return '.'.join(node_names) - - def __GetExpandedElemPaths(self, start_item): - expanded_items = [] - if start_item.IsOk() and self.IsExpanded(start_item): - if start_item != self.GetRootItem(): - elem_path = self.GetItemElemPath(start_item) - expanded_items.append(elem_path) - - child = self.GetFirstChild(start_item)[0] - while child.IsOk(): - expanded_items.extend(self.__GetExpandedElemPaths(child)) - child = self.GetNextSibling(child) - - return expanded_items - - def __UpdateUtilizBitmaps(self, item): - elem_path = self.GetItemElemPath(item) - if elem_path is None: - return - - if elem_path in self._watched_sim_elems: - simhier = self.frame.explorer.navtree.simhier - elem_id = simhier.GetElemID(elem_path) - if simhier.GetWidgetType(elem_id) == 'QueueTable': - utiliz_pct = self.frame.widget_renderer.utiliz_handler.GetUtilizPct(elem_path) - image_idx = int(utiliz_pct * 100) - self.SetItemImage(item, image_idx) - - capacity = simhier.GetCapacityByElemPath(elem_path) - size = int(capacity * utiliz_pct) - tooltip = '{}\nUtilization: {}% ({}/{} bins filled)'.format(elem_path, round(utiliz_pct*100), size, capacity) - self._tooltips_by_item[item] = tooltip - elif simhier.GetWidgetType(elem_id) == 'Timeseries': - image_idx = self._utiliz_image_list.GetImageCount() - 1 - self.SetItemImage(item, image_idx) - tooltip = '{}\nNo utilization data available for timeseries stats'.format(elem_path) - self._tooltips_by_item[item] = tooltip - - child, cookie = self.GetFirstChild(item) - while child.IsOk(): - self.__UpdateUtilizBitmaps(child) - child, cookie = self.GetNextChild(item, cookie) - - def __RenderWatchlist(self): - if self._mode == 'flat': - self.__RenderFlatView() - else: - self.__RenderHierView() - - def __RenderFlatView(self, *args, **kwargs): - dirty = self._mode == 'hier' - self._mode = 'flat' - self._undeletable_items = [] - - self.DeleteAllItems() - if len(self._watched_sim_elems) > 0: - self.AddRoot("Root") - - sizer = self.GetSizer() - sizer.Clear(True) - - if len(self._watched_sim_elems) == 0: - howto = wx.StaticText(self, label="Right-click nodes in the \nNavTree to add them to \nthe Watchlist.", - style=wx.ST_ELLIPSIZE_END) - - # Make the font size of howto smaller - font = howto.GetFont() - font.SetPointSize(font.GetPointSize() - 2) - howto.SetFont(font) - - sizer.Add(howto, 1, wx.EXPAND | wx.ALL, 10) - else: - watchlist_root = self.AppendItem(self.GetRootItem(), "Watchlist") - for elem in self._watched_sim_elems: - self.AppendItem(watchlist_root, elem) - - self.ExpandAll() - - self.UpdateUtilizBitmaps() - - if dirty: - self.frame.view_settings.SetDirty(reason=DirtyReasons.WatchlistOrgChanged) - - def __RenderHierView(self, *args, **kwargs): - dirty = self._mode = 'flat' - self._mode = 'hier' - self._undeletable_items = [] - - self.DeleteAllItems() - if len(self._watched_sim_elems) > 0: - self.AddRoot("Root") - - sizer = self.GetSizer() - sizer.Clear(True) - - if len(self._watched_sim_elems) == 0: - howto = wx.StaticText(self, label="Right-click nodes in the \nNavTree to add them to \nthe Watchlist.", - style=wx.ST_ELLIPSIZE_END) - - # Make the font size of howto smaller - font = howto.GetFont() - font.SetPointSize(font.GetPointSize() - 2) - howto.SetFont(font) - - sizer.Add(howto, 1, wx.EXPAND | wx.ALL, 10) - else: - items_by_path = {} - items_by_path["Root"] = self.GetRootItem() - - watchlist_root = self.AppendItem(self.GetRootItem(), "Watchlist") - items_by_path["Root.Watchlist"] = watchlist_root - - for _, item in items_by_path.items(): - self._undeletable_items.append(item) - - # Honor the same hierarchy as the NavTree - navtree_leaf_paths = self.frame.explorer.navtree.simhier.GetItemElemPaths() - - for path in navtree_leaf_paths: - if path not in self._watched_sim_elems: - continue - - parts = path.split('.') - current_item = watchlist_root - - for part in parts: - item_key = ".".join(parts[:parts.index(part)+1]) - - # Check if the item already exists - if item_key not in items_by_path: - current_item = self.AppendItem(current_item, part) - items_by_path[item_key] = current_item - else: - current_item = items_by_path[item_key] - - self.ExpandAll() - - self.UpdateUtilizBitmaps() - - if dirty: - self.frame.view_settings.SetDirty(reason=DirtyReasons.WatchlistOrgChanged) - - def __ProcessTooltip(self, event): - item = event.GetItem() - tooltip = self._tooltips_by_item.get(item, '') - event.SetToolTip(tooltip) - - def __OnRightClick(self, event): - item = self.HitTest(event.GetPosition()) - if not item: - return - - item = item[0] - if not item.IsOk(): - return - - self.SelectItem(item) - - menu = wx.Menu() - - def ExpandAll(event, **kwargs): - kwargs['watchlist'].ExpandAll() - event.Skip() - - def CollapseAll(event, **kwargs): - kwargs['watchlist'].CollapseAll() - event.Skip() - - expand_all = menu.Append(-1, "Expand All") - self.Bind(wx.EVT_MENU, partial(ExpandAll, watchlist=self), expand_all) - - collapse_all = menu.Append(-1, "Collapse All") - self.Bind(wx.EVT_MENU, partial(CollapseAll, watchlist=self), collapse_all) - - menu.AppendSeparator() - - if self._mode == 'hier': - flat_view = menu.Append(-1, "Flat View") - self.Bind(wx.EVT_MENU, self.__RenderFlatView, flat_view) - - if self._mode == 'flat': - hier_view = menu.Append(-1, "Hierarchical View") - self.Bind(wx.EVT_MENU, self.__RenderHierView, hier_view) - - elem_path = self.GetItemElemPath(item) - if elem_path in self._watched_sim_elems: - menu.AppendSeparator() - remove_from_watchlist = menu.Append(-1, "Remove from Watchlist") - self.Bind(wx.EVT_MENU, partial(self.__RemoveFromWatchlist, elem_path=elem_path), remove_from_watchlist) - elif item not in self._undeletable_items: - menu.AppendSeparator() - remove_from_watchlist = menu.Append(-1, "Remove from Watchlist") - self.Bind(wx.EVT_MENU, partial(self.__RemoveFromWatchlist, item_to_remove=item), remove_from_watchlist) - - self.PopupMenu(menu) - menu.Destroy() - - event.Skip() - - def __RemoveFromWatchlist(self, *args, **kwargs): - dirty = False - if 'elem_path' in kwargs: - elem_path = kwargs['elem_path'] - dirty = elem_path in self._watched_sim_elems - self._watched_sim_elems.remove(elem_path) - else: - item_to_remove = kwargs['item_to_remove'] - watched_sim_elems = [] - self.__RecurseGetWatchedSimElems(item_to_remove, watched_sim_elems) - - for sim_elem in watched_sim_elems: - dirty |= sim_elem in self._watched_sim_elems - self._watched_sim_elems.remove(sim_elem) - - self.__RenderWatchlist() - - if dirty: - self.frame.view_settings.SetDirty(reason=DirtyReasons.WatchlistRemoved) - - def __RecurseGetWatchedSimElems(self, item, watched_sim_elems): - item_path = self.GetItemElemPath(item) - if item_path in self._watched_sim_elems: - watched_sim_elems.append(item_path) - - child, cookie = self.GetFirstChild(item) - while child.IsOk(): - self.__RecurseGetWatchedSimElems(child, watched_sim_elems) - child, cookie = self.GetNextChild(item, cookie) - - def __OnItemExpanded(self, event): - self.UpdateUtilizBitmaps() diff --git a/python/argos/viewer/gui/widgets/grid.py b/python/argos/viewer/gui/widgets/grid.py index 2d5087d7..111f3606 100644 --- a/python/argos/viewer/gui/widgets/grid.py +++ b/python/argos/viewer/gui/widgets/grid.py @@ -32,6 +32,9 @@ def SetCellValue(self, row, col, value, immediate_refresh=False): def GetCellValue(self, row, col): return self.renderer.GetCellValue(row, col) + def SetCellAlignment(self, row, col, alignment): + self.renderer.SetCellAlignment(row, col, alignment) + def SetCellBackgroundColour(self, row, col, color, immediate_refresh=False): self.renderer.SetCellBackgroundColour(row, col, color) if immediate_refresh: @@ -102,6 +105,9 @@ def SetCellValue(self, row, col, value): def GetCellValue(self, row, col): return self.cells[row][col].GetText() + def SetCellAlignment(self, row, col, alignment): + self.cells[row][col].AlignLabel(alignment) + def SetCellBackgroundColour(self, row, col, color): self.cells[row][col].SetBackgroundColour(color) @@ -144,6 +150,7 @@ def GetBestSize(self, grid, attr, dc, row, col): class GridCell: def __init__(self, font=None): self.text = '' + self.text_alignment = wx.ALIGN_CENTER self.font = font if font else wx.Font(12, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) self.background_colour = (255, 255, 255) self.border_width = 0 @@ -156,6 +163,9 @@ def SetText(self, text): def GetText(self): return self.text + def AlignLabel(self, alignment): + self.text_alignment = alignment + def SetFont(self, font): self.font = font @@ -200,7 +210,7 @@ def Draw(self, grid, dc, rect): if self.text: dc.SetFont(self.font) - dc.DrawLabel(self.text, rect, wx.ALIGN_CENTER) + dc.DrawLabel(self.text, rect, self.text_alignment) if self.border_width: dc.SetPen(wx.Pen(wx.BLACK, self.border_width)) diff --git a/python/argos/viewer/gui/widgets/ipc.py b/python/argos/viewer/gui/widgets/ipc.py deleted file mode 100644 index ac8d6025..00000000 --- a/python/argos/viewer/gui/widgets/ipc.py +++ /dev/null @@ -1,84 +0,0 @@ -import zlib, struct -from viewer.gui.widgets.scalar_statistic import ScalarStatistic - -class IPCWidget(ScalarStatistic): - def __init__(self, parent, frame): - super().__init__(parent, frame, 'IPC', configurable=False, size=parent.GetSize()) - - self.ax.set_title('') - self.ax.set_xlabel('') - self.ax.set_ylabel('IPC') - self.ax.grid(False) - self.ax.tick_params(axis='x', which='both', bottom=False, top=False) - self.ax.set_xlim(left=self.time_vals[0], right=self.time_vals[-1]) - - r,g,b = (240/255 for _ in range(3)) - self.figure.patch.set_facecolor((r,g,b)) - self._avxline = None - - self.UpdateWidgetData() - - # Launch frame inspector - import wx.lib.inspection - wx.lib.inspection.InspectionTool().Show() - - def GetWidgetCreationString(self): - return 'IPC' - - def UpdateWidgetData(self): - self.__UpdatePlotSize() - self.__UpdateCurrentTickVertLine() - self.canvas.draw() - self.Layout() - self.Update() - self.Refresh() - - def GetCurrentViewSettings(self): - return {} - - def GetCurrentUserSettings(self): - return {} - - def ApplyViewSettings(self, settings): - pass - - def GetStatsValues(self, frame, elem_path): - cursor = frame.db.cursor() - - cmd = 'SELECT DataValsBlob, IsCompressed FROM TimeseriesData WHERE ElementPath="{}"'.format(elem_path) - cursor.execute(cmd) - - data_vals_blob, is_compressed = cursor.fetchone() - if is_compressed: - data_vals_blob = zlib.decompress(data_vals_blob) - - data_vals = [] - while len(data_vals_blob) > 0: - val_blob = data_vals_blob[:8] - data_vals.append(struct.unpack('d', val_blob)[0]) - data_vals_blob = data_vals_blob[8:] - - return data_vals - - def __UpdatePlotSize(self): - # Desired size in pixels - desired_width_pixels = self.GetParent().GetSize().GetWidth() - 10 - desired_height_pixels = 50 - - # Get the current DPI - dpi = self.figure.get_dpi() - - # Calculate new size in inches - new_width_inches = desired_width_pixels / dpi - new_height_inches = desired_height_pixels / dpi - - # Set the new size in inches - self.figure.set_size_inches(new_width_inches, new_height_inches) - - def __UpdateCurrentTickVertLine(self): - if self._avxline: - self._avxline.remove() - - tick = self.frame.widget_renderer.tick - if tick in self.time_vals: - self._avxline = self.ax.axvline(x=tick, color='r', linestyle='--') diff --git a/python/argos/viewer/gui/widgets/iterable_struct.py b/python/argos/viewer/gui/widgets/iterable_struct.py index bd6452ff..0b525a71 100644 --- a/python/argos/viewer/gui/widgets/iterable_struct.py +++ b/python/argos/viewer/gui/widgets/iterable_struct.py @@ -1,20 +1,32 @@ import wx, wx.grid from collections.abc import Iterable +from viewer.gui.dialogs.widget_data_selections import WidgetDataSelectionsDlg class IterableStruct(wx.Panel): def __init__(self, parent, frame, elem_path): super(IterableStruct, self).__init__(parent) + self.SetBackgroundColour('white') self.frame = frame self.elem_path = elem_path self.deserializer = frame.data_retriever.GetDeserializer(elem_path) all_field_names = self.deserializer.GetAllFieldNames() - self.capacity = frame.simhier.GetCapacityByElemPath(elem_path) + num_rows = self.capacity + num_cols = len(all_field_names) + if 'DID' in all_field_names: + num_cols -= 1 + assert len(all_field_names) > 0 + self.grid = wx.grid.Grid(self) - self.grid.CreateGrid(self.capacity, len(all_field_names), wx.grid.Grid.GridSelectNone) + self.grid.CreateGrid(num_rows, num_cols, wx.grid.Grid.GridSelectNone) self.grid.EnableEditing(False) + self.grid.SetLabelBackgroundColour('white') self.__SyncGridViewSettings() + mono10 = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) + self.grid.SetDefaultCellFont(mono10) + self.grid.SetLabelFont(mono10) + for i in range(self.capacity): self.grid.SetRowLabelValue(i, str(i)) @@ -24,23 +36,23 @@ def __init__(self, parent, frame, elem_path): font = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) location_elem.SetFont(font) - # Add a gear button (size 16x16) to the left of the time series plot. - # Clicking the button will open a dialog to change the plot settings. - # Note that we do not add the button to the sizer since we want to - # force it to be in the top-left corner of the widget canvas. We do - # this with the 'pos' argument to the wx.BitmapButton constructor. - gear_btn = wx.BitmapButton(self, bitmap=frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) - gear_btn.SetToolTip('Edit widget settings') + gear_btn, clear_btn, split_lr, split_tb, maximize_btn = frame.CreateWidgetStandardButtons( + self, self.__EditWidget, 'Edit widget settings') row1 = wx.BoxSizer(wx.HORIZONTAL) - row1.AddSpacer(30) - row1.Add(self.utiliz_elem, 0, wx.ALL, 5) - row1.Add(location_elem, 1, wx.EXPAND | wx.ALL, 5) + row1.Add(gear_btn, 0, wx.TOP | wx.RIGHT, 5) + row1.Add(clear_btn, 0, wx.TOP | wx.RIGHT, 5) + row1.Add(split_lr, 0, wx.TOP | wx.RIGHT, 5) + row1.Add(split_tb, 0, wx.TOP | wx.RIGHT, 5) + row1.Add(maximize_btn, 0, wx.TOP | wx.RIGHT, 5) + row1.AddSpacer(5) + row1.Add(self.utiliz_elem, 0, wx.TOP, 7) + row1.AddSpacer(5) + row1.Add(location_elem, 0, wx.EXPAND | wx.TOP, 7) sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(row1, 0, wx.EXPAND | wx.ALL, 10) - sizer.Add(self.grid, 1, wx.EXPAND | wx.ALL, 10) + sizer.Add(row1, 0, wx.ALL, 5) + sizer.Add(self.grid, 0, wx.EXPAND | wx.LEFT | wx.TOP, 5) self.SetSizer(sizer) self.Layout() @@ -58,11 +70,9 @@ def UpdateWidgetData(self): widget_renderer = self.frame.widget_renderer tick = widget_renderer.tick - queue_data = self.frame.data_retriever.Unpack(self.elem_path, (tick,tick)) - - auto_colorize_col = self.frame.data_retriever.GetAutoColorizeColumn(self.elem_path) - auto_colorize_col_idx = self.visible_field_names.index(auto_colorize_col) if auto_colorize_col in self.visible_field_names else None + queue_data = self.frame.data_retriever.Unpack(self.elem_path, tick) + auto_colorize_col_name = 'DID' if 'DID' in self.visible_field_names else self.visible_field_names[0] self.__ClearGrid() if len(queue_data['DataVals']) and isinstance(queue_data['DataVals'][0], Iterable): @@ -70,11 +80,23 @@ def UpdateWidgetData(self): if row_data is None: continue - for j, field_name in enumerate(self.visible_field_names): - self.grid.SetCellValue(idx, j, str(row_data[field_name])) - if auto_colorize_col_idx is not None: - color = widget_renderer.GetAutoColor(row_data[self.visible_field_names[auto_colorize_col_idx]]) - self.grid.SetCellBackgroundColour(idx, j, color) + write_col = 0 + for read_col, field_name in enumerate(self.visible_field_names): + if field_name == 'DID': + continue + + self.grid.SetCellValue(idx, write_col, str(row_data[read_col][1])) + + color_keyval = None + for key, keyval in row_data: + if key == auto_colorize_col_name: + color_keyval = keyval + break + + assert color_keyval is not None + color = widget_renderer.GetAutoColor(color_keyval) + self.grid.SetCellBackgroundColour(idx, write_col, color) + write_col += 1 num_rows_shown = len(queue_data['DataVals'][0]) else: @@ -109,16 +131,11 @@ def __SyncGridViewSettings(self): self.grid.SetColLabelValue(i, '') # Only show the visible field names - for i, field_name in enumerate(self.visible_field_names): - self.grid.SetColLabelValue(i, field_name) - - # Make sure we are showing the visible columns - for i in range(len(self.visible_field_names)): - self.grid.ShowCol(i) - - # Hide any columns that are not visible - for i in range(len(self.visible_field_names), self.grid.GetNumberCols()): - self.grid.HideCol(i) + i = 0 + for field_name in self.visible_field_names: + if field_name != 'DID': + self.grid.SetColLabelValue(i, field_name) + i += 1 self.grid.AutoSizeColumns() self.Layout() @@ -130,20 +147,27 @@ def __ClearGrid(self): self.grid.SetCellBackgroundColour(i, j, wx.WHITE) def __EditWidget(self, event): - all_field_names = self.deserializer.GetAllFieldNames() - auto_colorize_col = self.frame.data_retriever.GetAutoColorizeColumn(self.elem_path) - dlg = QueueTableCustomizationDlg(self, - all_field_names, - self.visible_field_names, - auto_colorize_col) - - if dlg.ShowModal() == wx.ID_OK: - # Note that the data retriever will update all widgets when these method is called - self.frame.data_retriever.SetVisibleFieldNames(self.elem_path, dlg.GetDisplayedColumns()) - self.frame.data_retriever.SetAutoColorizeColumn(self.elem_path, dlg.GetAutoColorizeColumn()) - + assert self.elem_path not in (None, '') + selected = [self.elem_path] + dlg = WidgetDataSelectionsDlg(self, self.frame, selected, queues_only=True, single_selection=True) + result = dlg.ShowModal() + + if result == wx.ID_OK: + selected = dlg.GetSelectedElemPaths() + if len(selected) == 0: + wx.MessageBox('No data selected', 'Error', wx.OK | wx.ICON_ERROR) + selected = None + else: + selected = selected[0] + else: + selected = None dlg.Destroy() + if selected: + widget_container = self.GetParent() + widget = IterableStruct(widget_container, self.frame, selected) + widget_container.SetWidget(widget) + class UtilizElement(wx.StaticText): def __init__(self, parent, frame, capacity): super().__init__(parent) @@ -160,112 +184,3 @@ def UpdateUtilizPct(self, utiliz_pct): tooltip = 'Utilization: {}% ({}/{} bins filled)'.format(round(utiliz_pct * 100), int(utiliz_pct * self.capacity), self.capacity) self.SetToolTip(tooltip) - -class QueueTableCustomizationDlg(wx.Dialog): - def __init__(self, widget, all_columns, displayed_columns, auto_colorize_col): - super().__init__(widget, title='Customize Widget') - - self.checkboxes = [] - self.radio_buttons = [] - self.panel = wx.Panel(self) - sizer = wx.BoxSizer(wx.VERTICAL) - - tbox = wx.StaticText(self.panel, label='Select columns to display:') - sizer.Add(tbox, 0, wx.ALL | wx.EXPAND, 5) - - # Create a checkbox for each item - for s in all_columns: - hbox = wx.BoxSizer(wx.HORIZONTAL) - self.radio_buttons.append(wx.RadioButton(self.panel, label='Auto-colorize')) - self.radio_buttons[-1].SetToolTip('Auto-colorize based on this column') - hbox.Add(self.radio_buttons[-1], flag=wx.ALIGN_CENTER_VERTICAL) - - if s == auto_colorize_col: - self.radio_buttons[-1].SetValue(True) - else: - self.radio_buttons[-1].SetValue(False) - - checkbox = wx.CheckBox(self.panel, label=s) - hbox.Add(checkbox, 1, wx.ALL | wx.EXPAND, 5) - - if s in displayed_columns: - checkbox.SetValue(True) - else: - checkbox.SetValue(False) - - self.checkboxes.append(checkbox) - sizer.Add(hbox, 1, wx.ALL | wx.EXPAND, 5) - - # OK and Cancel buttons - self._ok_btn = wx.Button(self.panel, wx.ID_OK) - btn_sizer = wx.StdDialogButtonSizer() - btn_sizer.AddButton(self._ok_btn) - btn_sizer.AddButton(wx.Button(self.panel, wx.ID_CANCEL)) - btn_sizer.Realize() - - sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 10) - self.panel.SetSizerAndFit(sizer) - - # Find the longest string length of all our checkbox labels - dc = wx.ClientDC(self.panel) - dc.SetFont(wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL)) - longest_length = 0 - - for s in all_columns: - text_width, _ = dc.GetTextExtent(s) - longest_length = max(longest_length, text_width) - - w,h = sizer.GetMinSize() - h += 100 - w = max(w, longest_length + 100) - self.SetSize((w,h)) - self.Layout() - self.Refresh() - - for checkbox in self.checkboxes: - checkbox.Bind(wx.EVT_CHECKBOX, self.__OnCheckbox) - - for radio_button in self.radio_buttons: - radio_button.Bind(wx.EVT_RADIOBUTTON, self.__OnRadioButtonGroup) - - def GetDisplayedColumns(self): - # Return a list of selected items - return [checkbox.GetLabel() for checkbox in self.checkboxes if checkbox.IsChecked()] - - def GetAutoColorizeColumn(self): - for i,radio_btn in enumerate(self.radio_buttons): - if radio_btn.GetValue(): - return self.checkboxes[i].GetLabel() - - return None - - def __OnCheckbox(self, event): - any_selected = any(checkbox.IsChecked() for checkbox in self.checkboxes) - self._ok_btn.Enable(any_selected) - - if not any_selected: - self._ok_btn.SetToolTip('Select at least one column to display') - return - else: - self._ok_btn.UnsetToolTip() - - for i,checkbox in enumerate(self.checkboxes): - radio_btn = self.radio_buttons[i] - if not checkbox.IsChecked() and radio_btn.GetValue(): - self._ok_btn.Enable(False) - self._ok_btn.SetToolTip('Auto-colorize column must be displayed') - return - - def __OnRadioButtonGroup(self, event): - for radio_button in self.radio_buttons: - radio_button.SetValue(False) - - event.GetEventObject().SetValue(True) - - for i,radio_btn in enumerate(self.radio_buttons): - if radio_btn == event.GetEventObject(): - self.checkboxes[i].SetValue(True) - break - - self._ok_btn.Enable(True) - self._ok_btn.UnsetToolTip() diff --git a/python/argos/viewer/gui/widgets/playback_bar.py b/python/argos/viewer/gui/widgets/playback_bar.py index 9380c467..f9c0f978 100644 --- a/python/argos/viewer/gui/widgets/playback_bar.py +++ b/python/argos/viewer/gui/widgets/playback_bar.py @@ -3,7 +3,7 @@ class PlaybackBar(wx.Panel): def __init__(self, frame): - super(PlaybackBar, self).__init__(frame, size=(frame.GetSize().width, 100)) + super(PlaybackBar, self).__init__(frame, size=(frame.GetSize().width, -1)) self.SetBackgroundColour('light gray') widget_renderer = self.frame.widget_renderer diff --git a/python/argos/viewer/gui/widgets/queue_utiliz.py b/python/argos/viewer/gui/widgets/queue_utiliz.py index 8f261fb6..83527d6a 100644 --- a/python/argos/viewer/gui/widgets/queue_utiliz.py +++ b/python/argos/viewer/gui/widgets/queue_utiliz.py @@ -1,24 +1,22 @@ import wx, copy -from viewer.gui.dialogs.string_list_selection import StringListSelectionDlg +from viewer.gui.dialogs.widget_data_selections import QueueUtilizEditDlg from viewer.gui.view_settings import DirtyReasons +from viewer.gui.widgets.scheduling_lines import CaptionManager class QueueUtilizWidget(wx.Panel): - def __init__(self, parent, frame): - super().__init__(parent, size=(800, 500)) + DEFAULT_SHOW_FULL_PATHS = False + + def __init__(self, parent, frame, elem_paths=None, show_full_paths=DEFAULT_SHOW_FULL_PATHS): + super().__init__(parent) + self.SetBackgroundColour('white') self.frame = frame + self.show_full_paths = show_full_paths # Get all container sim paths from the simhier - self.container_elem_paths = self.frame.simhier.GetContainerElemPaths() - self.container_elem_paths.sort() - - # Add a gear button (size 16x16) to the left of the time series plot. - # Clicking the button will open a dialog to change the plot settings. - # Note that we do not add the button to the sizer since we want to - # force it to be in the top-left corner of the widget canvas. We do - # this with the 'pos' argument to the wx.BitmapButton constructor. - gear_btn = wx.BitmapButton(self, bitmap=frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) - gear_btn.SetToolTip('Edit widget settings') + self.container_elem_paths = elem_paths if elem_paths else self.frame.simhier.GetContainerElemPaths() + + gear_btn, clear_btn, split_lr, split_tb, maximize_btn = frame.CreateWidgetStandardButtons( + self, self.__EditWidget, 'Edit data selections') # The layout of this widget is like a barchart: # @@ -30,15 +28,25 @@ def __init__(self, parent, frame): # # Where the X's above are shown as a colored heatmap based on the # utilization percentage of each queue. - self.panel = wx.Panel(self) + self.panel = wx.ScrolledWindow(self, style=wx.VSCROLL | wx.HSCROLL) + self.panel.SetScrollRate(10, 10) self._elem_path_text_boxes = [] self._utiliz_bars = [] self.__LayoutComponents() sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(self.panel, 1, 20, wx.TOP | wx.LEFT) + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + btn_sizer.Add(gear_btn, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(clear_btn, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_lr, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_tb, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(maximize_btn, 0, wx.TOP, 5) + sizer.Add(btn_sizer, 0, wx.BOTTOM | wx.LEFT | wx.TOP, 5) + sizer.Add(self.panel, 1, wx.EXPAND | wx.LEFT, 5) self.SetSizer(sizer) + self.panel.FitInside() + self.UpdateWidgetData() self.Layout() def GetWidgetCreationString(self): @@ -58,46 +66,43 @@ def UpdateWidgetData(self): def GetCurrentViewSettings(self): settings = {} settings['displayed_elem_paths'] = self.container_elem_paths + settings['show_full_paths'] = self.show_full_paths return settings def GetCurrentUserSettings(self): return {} def ApplyViewSettings(self, settings): - paths1 = set(self.container_elem_paths) - paths2 = set(settings['displayed_elem_paths']) - if paths1 == paths2: + show_full_paths = settings.get('show_full_paths', self.DEFAULT_SHOW_FULL_PATHS) + if self.container_elem_paths == settings['displayed_elem_paths'] and \ + self.show_full_paths == show_full_paths: return self.container_elem_paths = copy.deepcopy(settings['displayed_elem_paths']) - self.container_elem_paths.sort() + self.show_full_paths = show_full_paths self.__LayoutComponents() self.UpdateWidgetData() self.frame.view_settings.SetDirty(reason=DirtyReasons.QueueUtilizDispQueueChanged) - def __OnSimElemInitDrag(self, event): - text_elem = event.GetEventObject() - - if text_elem in self._elem_path_text_boxes: - if text_elem.HasCapture(): - text_elem.ReleaseMouse() - else: - text_elem.CaptureMouse() - data = wx.TextDataObject('IterableStruct$' + text_elem.GetLabel()) - drag_source = wx.DropSource(text_elem) - drag_source.SetData(data) - drag_source.DoDragDrop(wx.Drag_DefaultMove) - text_elem.ReleaseMouse() - - event.Skip() + wx.CallAfter(self.UpdateWidgetData) def __EditWidget(self, event): - dlg = StringListSelectionDlg(self, self.frame.simhier.GetContainerElemPaths(), self.container_elem_paths, 'Displayed queues:') + dlg = QueueUtilizEditDlg( + self, self.frame, self.container_elem_paths, self.show_full_paths, + ) if dlg.ShowModal() == wx.ID_OK: - self.ApplyViewSettings({'displayed_elem_paths': dlg.GetSelectedStrings()}) + self.ApplyViewSettings({ + 'displayed_elem_paths': dlg.GetSelectedElemPaths(), + 'show_full_paths': dlg.show_full_paths, + }) dlg.Destroy() + def __FormatElemPathLabel(self, elem_path): + if self.show_full_paths: + return elem_path + return CaptionManager.GetMinimumUniqueSuffix(elem_path, self.container_elem_paths) + def __LayoutComponents(self): had_sizer = self.panel.GetSizer() is not None if had_sizer: @@ -113,19 +118,22 @@ def __LayoutComponents(self): self._utiliz_bars = [] else: sizer = wx.FlexGridSizer(2, 0, 10) - sizer.AddGrowableCol(1) + #sizer.AddGrowableCol(1) assert len(self._elem_path_text_boxes) == 0 assert len(self._utiliz_bars) == 0 - self._elem_path_text_boxes = [wx.StaticText(self.panel, label=elem_path) for elem_path in self.container_elem_paths] + self._elem_path_text_boxes = [] + for elem_path in self.container_elem_paths: + label_text = self.__FormatElemPathLabel(elem_path) + label_ctrl = wx.StaticText(self.panel, label=label_text) + CaptionManager.ApplyPartialPathTooltip( + label_ctrl, elem_path, label_text, self.show_full_paths) + self._elem_path_text_boxes.append(label_ctrl) self._utiliz_bars = [UtilizBar(self.panel, self.frame) for _ in range(len(self.container_elem_paths))] for elem_path, utiliz_bar in zip(self._elem_path_text_boxes, self._utiliz_bars): - sizer.Add(elem_path, 1, wx.EXPAND) - sizer.Add(utiliz_bar, 1, wx.EXPAND) - - for text_elem in self._elem_path_text_boxes: - text_elem.Bind(wx.EVT_LEFT_DOWN, self.__OnSimElemInitDrag) + sizer.Add(elem_path) + sizer.Add(utiliz_bar) font = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) for elem in self._elem_path_text_boxes: @@ -139,7 +147,7 @@ def __LayoutComponents(self): class UtilizBar(wx.Panel): def __init__(self, parent, frame): - super().__init__(parent, size=(300, 10)) + super().__init__(parent, size=(200, 20)) self.frame = frame self.static_text = wx.StaticText(self, label='0%') font = wx.Font(8, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) @@ -153,12 +161,13 @@ def UpdateUtilizPct(self, utiliz_pct): self.static_text.SetLabel('{}%'.format(round(utiliz_pct * 100))) color = self.frame.widget_renderer.utiliz_handler.ConvertUtilizPctToColor(utiliz_pct) self.SetBackgroundColour(color) - self.static_text.SetBackgroundColour(color) - - height = self.GetSize().GetHeight() - width = round(utiliz_pct * 500) + + height = 20 + width = round(utiliz_pct * 200) if width == 0: assert color == (255, 255, 255) - width = 500 + width = 200 self.SetSize((width, height)) + self.SetMinSize((width, height)) + self.SetMaxSize((width, height)) diff --git a/python/argos/viewer/gui/widgets/scalar_statistic.py b/python/argos/viewer/gui/widgets/scalar_statistic.py deleted file mode 100644 index 3923b1ea..00000000 --- a/python/argos/viewer/gui/widgets/scalar_statistic.py +++ /dev/null @@ -1,199 +0,0 @@ -import wx -import pylab as pl -import matplotlib -import numpy as np -import zlib, struct -from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas -from viewer.gui.view_settings import DirtyReasons - -class ScalarStatistic(wx.Panel): - def __init__(self, parent, frame, elem_path, configurable=True, **kwargs): - super(ScalarStatistic, self).__init__(parent, **kwargs) - self.frame = frame - self.elem_path = elem_path - - sizer = wx.BoxSizer(wx.VERTICAL) - - data_vals = self.GetStatsValues(frame, elem_path) - - cursor = frame.db.cursor() - cmd = 'SELECT TimeValsBlob, IsCompressed FROM AllTimeVals' - cursor.execute(cmd) - - time_vals_blob, is_compressed = cursor.fetchone() - if is_compressed: - time_vals_blob = zlib.decompress(time_vals_blob) - - time_vals = [] - while len(time_vals_blob) > 0: - time_vals.append(struct.unpack('Q', time_vals_blob[:8])[0]) - time_vals_blob = time_vals_blob[8:] - - stat_values = {'TimeVals': time_vals, 'DataVals': data_vals} - - # Create a timeseries plot - if len(stat_values['TimeVals']) > 0: - self.time_vals = stat_values['TimeVals'] - self.data_vals = stat_values['DataVals'] - self.figure = matplotlib.figure.Figure() - self.canvas = FigureCanvas(self, -1, self.figure) - self.canvas.SetPosition((25,-10)) - - self.ax = self.figure.add_subplot(111) - self.ax.plot(self.time_vals, self.data_vals, 'b-') - self.ax.set_title(self.elem_path) - self.ax.set_xlabel('Ticks') - self.ax.set_ylabel('Values') - self.ax.grid() - self.ax.autoscale() - - sizer.Add(self.canvas, 0, wx.EXPAND) - - # Add a gear button (size 16x16) to the left of the time series plot. - # Clicking the button will open a dialog to change the plot settings. - # Note that we do not add the button to the sizer since we want to - # force it to be in the top-left corner of the widget canvas. We do - # this with the 'pos' argument to the wx.BitmapButton constructor. - if configurable: - gear_btn = wx.BitmapButton(self, bitmap=frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) - gear_btn.SetToolTip('Edit widget settings') - else: - sizer.Add(wx.StaticText(self, label='No data for stat at location:\n%s' % elem_path), 0, wx.EXPAND) - - self.SetSizer(sizer) - self.Layout() - - def GetStatsValues(self, frame, elem_path): - cursor = frame.db.cursor() - - collection_id = frame.simhier.GetCollectionID(elem_path) - cmd = 'SELECT DataType FROM Collections WHERE Id={}'.format(collection_id) - cursor.execute(cmd) - data_type = cursor.fetchone()[0] - - cmd = 'SELECT DataValsBlob, IsCompressed FROM TimeseriesData WHERE ElementPath="{}"'.format(elem_path) - cursor.execute(cmd) - - data_vals_blob, is_compressed = cursor.fetchone() - if is_compressed: - data_vals_blob = zlib.decompress(data_vals_blob) - - if data_type == 'int8_t': - format = 'b' - elif data_type == 'int16_t': - format = 'h' - elif data_type == 'int32_t': - format = 'i' - elif data_type == 'int64_t': - format = 'q' - elif data_type == 'uint8_t': - format = 'B' - elif data_type == 'uint16_t': - format = 'H' - elif data_type == 'uint32_t': - format = 'I' - elif data_type == 'uint64_t': - format = 'Q' - elif data_type == 'float': - format = 'f' - elif data_type == 'double': - format = 'd' - else: - raise ValueError('Invalid enum integer type: ' + data_type) - - data_vals = [] - while len(data_vals_blob) > 0: - data_vals.append(struct.unpack(format, data_vals_blob[:struct.calcsize(format)])[0]) - data_vals_blob = data_vals_blob[struct.calcsize(format):] - - return data_vals - - def GetWidgetCreationString(self): - return 'ScalarStatistic$' + self.elem_path - - def UpdateWidgetData(self): - # Nothing to do since we plot all data - pass - - def GetCurrentViewSettings(self): - settings = {} - settings['title'] = self.ax.get_title() - settings['xlabel'] = self.ax.xaxis.get_label().get_text() - settings['ylabel'] = self.ax.yaxis.get_label().get_text() - settings['show_xlabel'] = self.ax.xaxis.get_label().get_visible() - settings['show_ylabel'] = self.ax.yaxis.get_label().get_visible() - return settings - - def GetCurrentUserSettings(self): - return {} - - def ApplyViewSettings(self, settings): - if settings == self.GetCurrentViewSettings(): - return - - self.ax.set_title(settings['title']) - self.ax.set_xlabel(settings['xlabel']) - self.ax.set_ylabel(settings['ylabel']) - self.ax.xaxis.get_label().set_visible(settings['show_xlabel']) - self.ax.yaxis.get_label().set_visible(settings['show_ylabel']) - self.canvas.draw() - self.Update() - self.Refresh() - self.frame.view_settings.SetDirty(reason=DirtyReasons.TimeseriesPlotSettingsChanged) - - def __EditWidget(self, event): - dlg = PlotCustomizationDialog(self, **self.GetCurrentViewSettings()) - if dlg.ShowModal() == wx.ID_OK: - self.ApplyViewSettings(dlg.GetSettings()) - - dlg.Destroy() - -class PlotCustomizationDialog(wx.Dialog): - def __init__(self, parent, title='Timeseries', xlabel='Time', ylabel='Values', show_xlabel=True, show_ylabel=True): - super().__init__(parent, title="Customize Plot", size=(500, 600)) - - # Create the main sizer - sizer = wx.BoxSizer(wx.VERTICAL) - - # Title - self.title_text = wx.TextCtrl(self, value=title) - sizer.Add(wx.StaticText(self, label="Title:"), 0, wx.ALL | wx.EXPAND, 5) - sizer.Add(self.title_text, 0, wx.ALL | wx.EXPAND, 5) - - # X-axis Label - self.x_label_text = wx.TextCtrl(self, value=xlabel) - sizer.Add(wx.StaticText(self, label="X-axis Label:"), 0, wx.ALL | wx.EXPAND, 5) - sizer.Add(self.x_label_text, 0, wx.ALL | wx.EXPAND, 5) - - # Y-axis Label - self.y_label_text = wx.TextCtrl(self, value=ylabel) - sizer.Add(wx.StaticText(self, label="Y-axis Label:"), 0, wx.ALL | wx.EXPAND, 5) - sizer.Add(self.y_label_text, 0, wx.ALL | wx.EXPAND, 5) - - # Checkboxes for showing labels - self.show_x_label_checkbox = wx.CheckBox(self, label="Show X-axis Label") - self.show_x_label_checkbox.SetValue(show_xlabel) - sizer.Add(self.show_x_label_checkbox, 0, wx.ALL | wx.EXPAND, 5) - - self.show_y_label_checkbox = wx.CheckBox(self, label="Show Y-axis Label") - self.show_y_label_checkbox.SetValue(show_ylabel) - sizer.Add(self.show_y_label_checkbox, 0, wx.ALL | wx.EXPAND, 5) - - # OK and Cancel buttons - btn_sizer = wx.StdDialogButtonSizer() - btn_sizer.AddButton(wx.Button(self, wx.ID_OK)) - btn_sizer.AddButton(wx.Button(self, wx.ID_CANCEL)) - btn_sizer.Realize() - - sizer.Add(btn_sizer, 0, wx.ALIGN_CENTER | wx.ALL, 10) - self.SetSizer(sizer) - - def GetSettings(self): - return { - 'title': self.title_text.GetValue(), - 'xlabel': self.x_label_text.GetValue(), - 'ylabel': self.y_label_text.GetValue(), - 'show_xlabel': self.show_x_label_checkbox.GetValue(), - 'show_ylabel': self.show_y_label_checkbox.GetValue(), - } diff --git a/python/argos/viewer/gui/widgets/scalar_struct.py b/python/argos/viewer/gui/widgets/scalar_struct.py deleted file mode 100644 index ee2f57d6..00000000 --- a/python/argos/viewer/gui/widgets/scalar_struct.py +++ /dev/null @@ -1,43 +0,0 @@ -import wx - -class ScalarStruct(wx.Panel): - def __init__(self, parent, frame, elem_path): - super(ScalarStruct, self).__init__(parent) - self.frame = frame - self.elem_path = elem_path - self.struct_text_elem = wx.StaticText(self, label='Scalar Struct:\n%s' % elem_path) - self._field_names = frame.data_retriever.GetDeserializer(elem_path).GetAllFieldNames() - - font = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) - self.struct_text_elem.SetFont(font) - - self.sizer = wx.BoxSizer(wx.VERTICAL) - self.sizer.Add(self.struct_text_elem, 0, wx.EXPAND) - self.SetSizer(self.sizer) - self.Layout() - - def GetWidgetCreationString(self): - return 'ScalarStruct$' + self.elem_path - - def UpdateWidgetData(self): - widget_renderer = self.frame.widget_renderer - tick = widget_renderer.tick - queue_data = self.frame.data_retriever.Unpack(self.elem_path, (tick,tick)) - - field_max_len = max([len(field_name) for field_name in self._field_names]) - struct_str = [] - for field_name in self._field_names: - field_val = queue_data['DataVals'][0][field_name] - field_name = field_name.ljust(field_max_len) - struct_str.append('%s: %s' % (field_name, field_val)) - - self.struct_text_elem.SetLabel('\n'.join(struct_str)) - - def GetCurrentViewSettings(self): - return {} - - def GetCurrentUserSettings(self): - return {} - - def ApplyViewSettings(self, settings): - pass diff --git a/python/argos/viewer/gui/widgets/scheduling_lines.py b/python/argos/viewer/gui/widgets/scheduling_lines.py index 96f34006..360c39c6 100644 --- a/python/argos/viewer/gui/widgets/scheduling_lines.py +++ b/python/argos/viewer/gui/widgets/scheduling_lines.py @@ -5,38 +5,67 @@ from functools import partial class SchedulingLinesWidget(wx.Panel): - def __init__(self, parent, frame): + DEFAULT_TICKS_BEFORE = 10 + DEFAULT_TICKS_AFTER = 10 + DEFAULT_SHOW_DETAILS = True + DEFAULT_HIDE_EMPTY_ROWS = True + DEFAULT_SHOW_FULL_PATHS = False + DEFAULT_ENABLE_TOOLTIPS = False + DEFAULT_SHOW_DID = False + + def __init__(self, parent, frame, elem_paths=None, num_ticks_before=DEFAULT_TICKS_BEFORE, num_ticks_after=DEFAULT_TICKS_AFTER, show_details=DEFAULT_SHOW_DETAILS, hide_empty_rows=DEFAULT_HIDE_EMPTY_ROWS, show_full_paths=DEFAULT_SHOW_FULL_PATHS, enable_tooltips=DEFAULT_ENABLE_TOOLTIPS, show_did=DEFAULT_SHOW_DID): super().__init__(parent) self.frame = frame - self.num_ticks_before = 5 - self.num_ticks_after = 25 - self.show_detailed_queue_packets = True + self.num_ticks_before = num_ticks_before + self.num_ticks_after = num_ticks_after + self.show_detailed_queue_packets = show_details + self.hide_empty_rows = hide_empty_rows + self.show_full_paths = show_full_paths + self.enable_tooltips = enable_tooltips + self.show_did = show_did self.caption_mgr = CaptionManager(frame.simhier) self.tracked_annos = {} - self.grid = None - self.info = None - self.gear_btn = None self.rasterizers = {} - self.grid = None - - self.info = wx.StaticText(self, label='Drag queues from the NavTree to create scheduling lines.', size=(600,18)) - self.info.SetFont(wx.Font(14, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL)) - - vsizer = wx.BoxSizer(wx.VERTICAL) - vsizer.AddStretchSpacer() - vsizer.Add(self.info, 1, wx.ALL | wx.CENTER | wx.EXPAND, 5) - vsizer.AddStretchSpacer() cursor = frame.db.cursor() - cmd = 'SELECT CollectableTreeNodeID,MaxSize FROM QueueMaxSizes' + cmd = 'SELECT CID,MaxSize FROM QueueMaxSizes' cursor.execute(cmd) self.queue_max_sizes_by_collection_id = {} for collection_id,max_size in cursor.fetchall(): self.queue_max_sizes_by_collection_id[collection_id] = max_size - self.__ShowUsageInfo() + if elem_paths: + self.SetElements(elem_paths) + + @staticmethod + def GetRootDataTypeName(frame, elem_path): + collection_id = frame.simhier.GetCollectionID(elem_path) + dtype = frame.dtype_inspector.GetDataTypeForCollectionID(collection_id) + if not dtype: + return None + + idx = dtype.find('_sparse_capacity') + if idx != -1: + return dtype[:idx] + + idx = dtype.find('_contig_capacity') + if idx != -1: + return dtype[:idx] + + return dtype + + @classmethod + def ElemPathHasDidField(cls, frame, elem_path): + dtype = cls.GetRootDataTypeName(frame, elem_path) + if not dtype: + return False + return frame.dtype_inspector.GetEffectiveColorKey(dtype) == 'DID' + + @classmethod + def AnyElemPathHasDidField(cls, frame, elem_paths): + return any(cls.ElemPathHasDidField(frame, elem_path) for elem_path in elem_paths) def GetWidgetCreationString(self): return 'Scheduling Lines' @@ -59,7 +88,7 @@ def GetErrorIfDroppedNodeIncompatible(self, elem_path): for row in range(self.grid.GetNumberRows()): existing_captions.add(self.grid.GetCellValue(row, 0).rstrip()) - todo_captions = self.__GetCaptionsForElement(elem_path) + todo_captions = self.__GetCaptionsForElement(elem_path, self.__GetCaptionElemPathsIncluding(elem_path)) for caption in todo_captions: if caption in existing_captions: msg = 'Adding this to the Scheduling Lines widget would result in a duplicate caption(s). ' @@ -71,19 +100,38 @@ def GetErrorIfDroppedNodeIncompatible(self, elem_path): def AddElement(self, elem_path): self.__AddElement(elem_path) self.__Refresh() + self.frame.view_settings.SetDirty(reason=DirtyReasons.SchedulingLinesWidgetChanged) + + def SetElements(self, elem_paths): + # Filter out all the collectables that did not collect any data + warning = [] + elem_paths_with_data = [] + for elem_path in elem_paths: + elem_cid = self.frame.simhier.GetCollectionID(elem_path) + has_data = self.queue_max_sizes_by_collection_id.get(elem_cid, 0) > 0 + if has_data: + elem_paths_with_data.append(elem_path) + else: + if not warning: + warning.append('No data collected and will not be displayed:') + warning.append(' - ' + elem_path) - if not self.gear_btn: - self.gear_btn = wx.BitmapButton(self, bitmap=self.frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - self.gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) - self.gear_btn.SetToolTip('Edit widget settings') + if warning: + warning = '\n'.join(warning) + wx.MessageBox(warning, 'Warning', wx.OK | wx.ICON_WARNING) + self.caption_mgr.ClearSelections() + for elem_path in elem_paths_with_data: + self.__AddElement(elem_path) + + self.__Refresh() self.frame.view_settings.SetDirty(reason=DirtyReasons.SchedulingLinesWidgetChanged) - def UpdateWidgetData(self): - if not self.grid and not self.info and not self.gear_btn: + def UpdateWidgetData(self, regenerate_grid=False): + if not self.grid: return - self.__Refresh() + self.__Refresh(regenerate_grid) def GetCurrentViewSettings(self): settings = {} @@ -91,6 +139,10 @@ def GetCurrentViewSettings(self): settings['num_ticks_before'] = self.num_ticks_before settings['num_ticks_after'] = self.num_ticks_after settings['show_detailed_queue_packets'] = self.show_detailed_queue_packets + settings['hide_empty_rows'] = self.hide_empty_rows + settings['show_full_paths'] = self.show_full_paths + settings['enable_tooltips'] = self.enable_tooltips + settings['show_did'] = self.show_did settings['tracked_annos'] = copy.deepcopy(self.tracked_annos) return settings @@ -102,6 +154,10 @@ def ApplyViewSettings(self, settings): self.num_ticks_before != settings['num_ticks_before'] or \ self.num_ticks_after != settings['num_ticks_after'] or \ self.show_detailed_queue_packets != settings['show_detailed_queue_packets'] or \ + self.hide_empty_rows != settings['hide_empty_rows'] or \ + self.show_full_paths != settings['show_full_paths'] or \ + self.enable_tooltips != settings['enable_tooltips'] or \ + self.show_did != settings['show_did'] or \ self.tracked_annos != settings['tracked_annos'] if not dirty: @@ -111,6 +167,10 @@ def ApplyViewSettings(self, settings): self.num_ticks_before = settings['num_ticks_before'] self.num_ticks_after = settings['num_ticks_after'] self.show_detailed_queue_packets = settings['show_detailed_queue_packets'] + self.hide_empty_rows = settings['hide_empty_rows'] + self.show_full_paths = settings['show_full_paths'] + self.enable_tooltips = settings['enable_tooltips'] + self.show_did = settings['show_did'] self.tracked_annos = settings['tracked_annos'] self.__Refresh() @@ -154,92 +214,67 @@ def __AddElement(self, elem_path): # NumInstsRetired1[0] # # The user can adjust these settings in the widget settings dialog. - regex_replacement = GetHeadsUpCamelCaseQueueName(elem_path) - self.caption_mgr.SetElemPathRegexReplacement(elem_path, regex_replacement) + self.caption_mgr.SetElemPathRegexReplacement(elem_path, elem_path) - def __Refresh(self): + def __Refresh(self, new_grid=True): if len(self.caption_mgr.GetAllMatchingElemPaths()) > 0: - if self.info: - self.info.Hide() + # Preserve the scrollbar position across the grid regeneration below. + # The old grid is destroyed and a brand-new one is created, which + # would otherwise reset the scroll position back to the top. + saved_view_start = self.grid.GetViewStart() if self.grid else None + + start_time = self.frame.widget_renderer.tick - self.num_ticks_before + end_time = self.frame.widget_renderer.tick + self.num_ticks_after + elem_paths = self.caption_mgr.GetAllMatchingElemPaths() + self._caption_elem_paths = elem_paths + self._ranges = self.frame.data_retriever.UnpackRange(start_time, end_time, elem_paths) + self._bins_with_data_by_elem_path = self.__GetBinsWithDataByElemPath(self._ranges, elem_paths) + self._layouts_by_elem_path = {} + for elem_path in elem_paths: + bins_with_data = self._bins_with_data_by_elem_path[elem_path] + self._layouts_by_elem_path[elem_path] = self.__BuildRowLayout(elem_path, bins_with_data) self.SetBackgroundColour('white') - self.__RegenerateSchedulingLinesGrid() + self.__RegenerateSchedulingLinesGrid(new_grid) self.__RasterizeAllCells() - if not self.gear_btn: - self.gear_btn = wx.BitmapButton(self, bitmap=self.frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - self.gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) - self.gear_btn.SetToolTip('Edit widget settings') - else: - self.__ShowUsageInfo() - - def __ShowUsageInfo(self): - if self.gear_btn: - self.gear_btn.Destroy() - self.gear_btn = None - - if self.info: - self.info.Destroy() - self.info = None - - if self.grid: - self.grid.Destroy() - self.grid = None - - if self.GetSizer(): - self.GetSizer().Clear() - - self.SetBackgroundColour('light gray') - - self.info = wx.StaticText(self, label='Drag queues from the NavTree to create scheduling lines.', size=(600,18)) - self.info.SetFont(wx.Font(14, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL)) - - vsizer = wx.BoxSizer(wx.VERTICAL) - vsizer.AddStretchSpacer() - vsizer.Add(self.info, 1, wx.ALL | wx.CENTER | wx.EXPAND, 5) - vsizer.AddStretchSpacer() - - hsizer = wx.BoxSizer(wx.HORIZONTAL) - hsizer.AddStretchSpacer() - hsizer.Add(vsizer, 0, wx.CENTER | wx.EXPAND) - hsizer.AddStretchSpacer() - - self.SetSizer(hsizer) - self.Layout() + # Restore the scroll position after the new grid has been laid out + # and auto-sized (which establishes its scroll range). + if saved_view_start is not None: + #wx.CallAfter(self.grid.Scroll, saved_view_start[0], saved_view_start[1]) + self.grid.Scroll(saved_view_start[0], saved_view_start[1]) - def __RegenerateSchedulingLinesGrid(self): + def __RegenerateSchedulingLinesGrid(self, new_grid): if self.grid: self.grid.Destroy() self.grid = None - if self.info: - self.info.Destroy() - self.info = None - - if self.gear_btn: - self.gear_btn.Destroy() - self.gear_btn = None - sizer = self.GetSizer() if sizer: sizer.Clear() sizer = wx.BoxSizer(wx.VERTICAL) - # The number of rows can be calculated as: - # 1. Go through each element path we are tracking (each is a queue) - # a. For each element path, get the number of bins in each queue (A) - # b. For each element path, note the maximum number of elements seen in the simulation (B) - # c. If A>B, then the number of rows is B+1. Otherwise, the number of rows is A. (C) - # >>> The required number of rows is the sum of all (C) values in the (1) loop. - + self._struct_dtypes_by_row = {} num_rows = 0 for elem_path in self.caption_mgr.GetAllMatchingElemPaths(): collection_id = self.frame.simhier.GetCollectionID(elem_path) - num_bins = self.frame.simhier.GetCapacityByCollectionID(collection_id) # (A) - max_size = self.queue_max_sizes_by_collection_id[collection_id] # (B) - elem_num_rows = max_size + 1 if max_size < num_bins else num_bins # (C) - num_rows += elem_num_rows + layout = self._layouts_by_elem_path[elem_path] + + dtype = self.frame.dtype_inspector.GetDataTypeForCollectionID(collection_id) + idx = dtype.find('_sparse_capacity') + if idx != -1: + dtype = dtype[:idx] + else: + idx = dtype.find('_contig_capacity') + if idx != -1: + dtype = dtype[:idx] + + for i, segment in enumerate(layout): + row = num_rows + i + self._struct_dtypes_by_row[row] = dtype + + num_rows += len(layout) # The number of columns can be calculated as: # 1. Start with the sum of self.num_ticks_before and self.num_ticks_after (A) @@ -251,7 +286,7 @@ def __RegenerateSchedulingLinesGrid(self): num_cols = self.num_ticks_before + self.num_ticks_after + 1 if self.show_detailed_queue_packets: - num_cols += 3 + num_cols += 2 # Create 8-point monospace font for the grid cells font8 = wx.Font(8, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) @@ -259,38 +294,55 @@ def __RegenerateSchedulingLinesGrid(self): # Create 10-point font for the grid column labels font10 = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) - self.grid = Grid(self, self.frame, num_rows, num_cols, cell_font=font8, label_font=font10, cell_selection_allowed=False) + if new_grid or self.grid is None: + self.grid = Grid(self, self.frame, num_rows, num_cols, cell_font=font8, label_font=font10, cell_selection_allowed=False) self.grid.GetGridWindow().Bind(wx.EVT_MOTION, self.__OnGridMouseMotion) - self.grid.GetGridWindow().Bind(wx.EVT_RIGHT_DOWN, self.__OnGridRightClick) self.grid.EnableGridLines(False) self.grid.SetLabelBackgroundColour('white') - self.gear_btn = wx.BitmapButton(self, bitmap=self.frame.CreateResourceBitmap('gear.png'), pos=(5,5)) - self.gear_btn.Bind(wx.EVT_BUTTON, self.__EditWidget) + gear_btn, clear_btn, split_lr, split_tb, maximize_btn = self.frame.CreateWidgetStandardButtons( + self, self.__EditWidget, 'Edit widget settings') current_tick = self.frame.widget_renderer.tick col_labels = [] time_vals = self.frame.data_retriever.GetAllTimeVals() - time_vals = {float(val) for val in time_vals} + time_vals = {int(val) for val in time_vals} for col in range(1, self.num_ticks_before + self.num_ticks_after + 1): tick = current_tick - self.num_ticks_before + col - 1 - if float(tick) in time_vals: + if int(tick) in time_vals: self.grid.SetColLabelValue(col, str(tick)) col_labels.append(str(tick)) else: self.grid.SetColLabelValue(col, '') + if self.show_detailed_queue_packets: + detailed_pkt_col = self.num_ticks_before + self.num_ticks_after + 2 + self.grid.SetColLabelValue(detailed_pkt_col - 1, '') + if int(current_tick) in time_vals: + self.grid.SetColLabelValue(detailed_pkt_col, str(current_tick)) + col_labels.append(str(current_tick)) + else: + self.grid.SetColLabelValue(detailed_pkt_col, '') + # Use a DC to get the length of the longest col label dc = wx.ScreenDC() dc.SetFont(self.grid.GetLabelFont()) - max_col_label_len = max([dc.GetTextExtent(col_label)[0] for col_label in col_labels]) + max_col_label_len = max([dc.GetTextExtent(col_label)[0] for col_label in col_labels]) if col_labels else 0 self.grid.SetColLabelSize(max_col_label_len + 4) self.grid.SetColLabelValue(0, '') self.grid.SetColLabelTextOrientation(wx.VERTICAL) self.grid.HideRowLabels() - sizer.Add(self.grid, 0, wx.EXPAND, 5) + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + btn_sizer.Add(gear_btn, 0, wx.TOP | wx.RIGHT | wx.LEFT, 5) + btn_sizer.Add(clear_btn, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_lr, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_tb, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(maximize_btn, 0, wx.TOP, 5) + sizer.Add(btn_sizer, 0, wx.BOTTOM, 5) + + sizer.Add(self.grid, 0, wx.EXPAND) self.SetSizer(sizer) self.grid.ClearGrid() @@ -301,19 +353,9 @@ def __RegenerateSchedulingLinesGrid(self): self.grid.SetCellBorder(row, self.num_ticks_before + 1, 1, wx.LEFT) self.__SetElementCaptions(0) - if self.show_detailed_queue_packets: - self.__SetElementCaptions(self.num_ticks_before + self.num_ticks_after + 2) - - # Clear the column labels for the detailed queue packets section - for i in range(self.num_ticks_before + self.num_ticks_after + 1, self.grid.GetNumberCols()): - self.grid.SetColLabelValue(i, '') def __RasterizeAllCells(self): - for elem_path in self.caption_mgr.GetAllMatchingElemPaths(): - start_time = self.frame.widget_renderer.tick - self.num_ticks_before - end_time = self.frame.widget_renderer.tick + self.num_ticks_after - - vals = self.frame.data_retriever.Unpack(elem_path, (start_time, end_time)) + for elem_path, vals in self._ranges.items(): time_vals = vals['TimeVals'] data_vals = vals['DataVals'] @@ -327,12 +369,56 @@ def __RasterizeAllCells(self): # Left-justify the detailed packet column if self.show_detailed_queue_packets: - col = self.num_ticks_before + self.num_ticks_after + 3 - labels = [self.grid.GetCellValue(row,col) for row in range(self.grid.GetNumberRows())] - max_num_chars = max([len(label) for label in labels]) - for row in range(self.grid.GetNumberRows()): - label = self.grid.GetCellValue(row, col).strip() - label = label.strip() + ' '*(max_num_chars - len(label)) + col = self.num_ticks_before + self.num_ticks_after + 2 + + def GetMaxFieldVarLengths(strings): + result = {} + + for row in strings: + for field, value in re.findall(r'(\w+)\((.*?)\)', row): + result[field] = max( + result.get(field, 0), + len(value) + ) + + return result + + def AlignLabel(label, max_varlens_by_field): + parts = [] + + for field, value in re.findall(r'(\w+)\((.*?)\)', label): + target = max_varlens_by_field[field] + + part = f'{field}({value})' + + # Pad based on value width difference + pad = target - len(value) + parts.append(part + (' ' * pad)) + + return ' '.join(parts) + + labels = [self.grid.GetCellValue(row,col).strip() for row in range(self.grid.GetNumberRows())] + labels_by_dtype = {} + for row, dtype in self._struct_dtypes_by_row.items(): + if dtype not in labels_by_dtype: + labels_by_dtype[dtype] = [] + labels_by_dtype[dtype].append(labels[row]) + + for row, label in enumerate(labels): + if not self.show_did and 'DID' in label: + parts = label.split() + new_label_parts = [] + for p in parts: + if p.find('DID(') != 0: + new_label_parts.append(p) + label = ' '.join(new_label_parts) + + if row in self._struct_dtypes_by_row: + row_dtype = self._struct_dtypes_by_row[row] + row_align_labels = labels_by_dtype[row_dtype] + max_varlens_by_field = GetMaxFieldVarLengths(row_align_labels) + label = AlignLabel(label, max_varlens_by_field) + self.grid.SetCellValue(row, col, label) self.grid.AutoSize() @@ -358,15 +444,15 @@ def __SetElementCaptions(self, col): row = 0 captions = [] for elem_path in self.caption_mgr.GetAllMatchingElemPaths(): - elem_captions = self.__GetCaptionsForElement(elem_path) - for caption in elem_captions: - match = re.match(r'(.+)\[(\d+)\]', caption) - assert match - bin_idx = match.group(2) - tooltip = elem_path + '[' + bin_idx + ']' + for segment in self._layouts_by_elem_path[elem_path]: + caption = self.__FormatSegmentCaption(elem_path, segment) + tooltip = self.__GetCaptionColumnTooltip(elem_path, segment, caption) captions.append(caption) - self.grid.SetCellToolTip(row, col, tooltip) + if tooltip: + self.grid.SetCellToolTip(row, col, tooltip) + else: + self.grid.UnsetCellToolTip(row, col) row += 1 max_num_chars = max([len(caption) for caption in captions]) @@ -386,663 +472,177 @@ def __SetElementCaptions(self, col): row_offset += self.__SetCaptionsForElement(elem_path, row_offset, col, max_num_chars, detailed_pkt_col) def __SetCaptionsForElement(self, elem_path, row_offset, col, max_num_chars, detailed_pkt_col): - collection_id = self.frame.simhier.GetCollectionID(elem_path) - num_bins = self.frame.simhier.GetCapacityByCollectionID(collection_id) - max_size = self.queue_max_sizes_by_collection_id[collection_id] + layout = self._layouts_by_elem_path[elem_path] - if max_size < num_bins: - caption_prefix = self.caption_mgr.GetCaptionPrefix(elem_path) - caption = '{}[{}-{}]'.format(caption_prefix, max_size-1, num_bins-1) - caption += ' '*(max_num_chars - len(caption)) - self.grid.SetCellValue(row_offset, col, caption) + for i, segment in enumerate(layout): + caption = self.__FormatSegmentCaption(elem_path, segment) + caption += ' ' * (max_num_chars - len(caption)) + self.grid.SetCellValue(row_offset + i, col, caption) - for i in range(1, max_size): - bin_idx = max_size - i - 1 - caption = self.caption_mgr.GetCaption(elem_path, bin_idx) - caption += ' '*(max_num_chars - len(caption)) - self.grid.SetCellValue(row_offset + i, col, caption) - self.rasterizers[(elem_path, bin_idx)] = Rasterizer(self.frame, self.grid, self, elem_path, bin_idx, row_offset + i, detailed_pkt_col) + if segment['kind'] == 'bin': + bin_idx = segment['bin'] + self.rasterizers[(elem_path, bin_idx)] = Rasterizer( + self.frame, self.grid, self, elem_path, bin_idx, row_offset + i, detailed_pkt_col) - return max_size + 1 - else: - for i in range(num_bins): - bin_idx = num_bins - i - 1 - caption = self.caption_mgr.GetCaption(elem_path, bin_idx) - caption += ' '*(max_num_chars - len(caption)) - self.grid.SetCellValue(row_offset + i, col, caption) - self.rasterizers[(elem_path, bin_idx)] = Rasterizer(self.frame, self.grid, self, elem_path, bin_idx, row_offset + i, detailed_pkt_col) + return len(layout) + + def __GetCaptionElemPathsIncluding(self, elem_path): + elem_paths = list(self.caption_mgr.GetAllMatchingElemPaths()) + if elem_path not in elem_paths: + elem_paths.append(elem_path) + return elem_paths - return num_bins + def __GetCaptionsForElement(self, elem_path, elem_paths=None): + segments = self.__BuildStaticRowLayout(elem_path) + return [self.__FormatSegmentCaption(elem_path, segment, elem_paths) for segment in segments] - def __GetCaptionsForElement(self, elem_path): + def __GetBinsWithDataByElemPath(self, ranges, elem_paths): + bins_with_data_by_elem_path = {} + for elem_path in elem_paths: + bins_with_data = set() + vals = ranges.get(elem_path, {'DataVals': []}) + for data_dicts in vals['DataVals']: + if data_dicts is None: + continue + for bin_idx, annos in enumerate(data_dicts): + if annos is not None: + bins_with_data.add(bin_idx) + bins_with_data_by_elem_path[elem_path] = bins_with_data + return bins_with_data_by_elem_path + + def __BuildRowLayout(self, elem_path, bins_with_data): + if self.hide_empty_rows: + return self.__BuildDynamicRowLayout(elem_path, bins_with_data) + return self.__BuildStaticRowLayout(elem_path) + + def __BuildStaticRowLayout(self, elem_path): collection_id = self.frame.simhier.GetCollectionID(elem_path) num_bins = self.frame.simhier.GetCapacityByCollectionID(collection_id) max_size = self.queue_max_sizes_by_collection_id[collection_id] - captions = [] - if max_size < num_bins: - caption_prefix = self.caption_mgr.GetCaptionPrefix(elem_path) - captions.append('{}[{}-{}]'.format(caption_prefix, max_size-1, num_bins-1)) + if max_size == 0: + return [{'kind': 'no_data'}] + segments = [] + if max_size < num_bins: + segments.append({'kind': 'range', 'lo': max_size - 1, 'hi': num_bins - 1}) for i in range(1, max_size): bin_idx = max_size - i - 1 - caption = self.caption_mgr.GetCaption(elem_path, bin_idx) - captions.append(caption) + segments.append({'kind': 'bin', 'bin': bin_idx}) else: for i in range(num_bins): bin_idx = num_bins - i - 1 - caption = self.caption_mgr.GetCaption(elem_path, bin_idx) - captions.append(caption) + segments.append({'kind': 'bin', 'bin': bin_idx}) + return segments + + def __BuildDynamicRowLayout(self, elem_path, bins_with_data): + collection_id = self.frame.simhier.GetCollectionID(elem_path) + num_bins = self.frame.simhier.GetCapacityByCollectionID(collection_id) + + segments = [] + run_hi = None + for bin_idx in range(num_bins - 1, -1, -1): + if bin_idx in bins_with_data: + if run_hi is not None: + segments.append({'kind': 'range', 'lo': bin_idx + 1, 'hi': run_hi}) + run_hi = None + segments.append({'kind': 'bin', 'bin': bin_idx}) + elif run_hi is None: + run_hi = bin_idx + + if run_hi is not None: + segments.append({'kind': 'range', 'lo': 0, 'hi': run_hi}) + return segments + + def __FormatSegmentCaption(self, elem_path, segment, elem_paths=None): + if elem_paths is None: + elem_paths = self._caption_elem_paths + + if segment['kind'] == 'no_data': + return '{}(no data)'.format( + self.caption_mgr.GetCaptionPrefix(elem_path, elem_paths, self.show_full_paths)) + if segment['kind'] == 'range': + caption_prefix = self.caption_mgr.GetCaptionPrefix(elem_path, elem_paths, self.show_full_paths) + lo = segment['lo'] + hi = segment['hi'] + if lo == hi: + return '{}[{}]'.format(caption_prefix, lo) + return '{}[{}-{}]'.format(caption_prefix, lo, hi) + return self.caption_mgr.GetCaption(elem_path, segment['bin'], elem_paths, self.show_full_paths) + + def __GetCaptionColumnTooltip(self, elem_path, segment, caption): + full_tooltip = self.__SegmentElemPathTooltip(elem_path, segment) + if caption.rstrip() == full_tooltip: + return None + + if not self.show_full_paths: + return full_tooltip + + if self.enable_tooltips: + return full_tooltip + return None - return captions + def __SegmentElemPathTooltip(self, elem_path, segment): + if segment['kind'] == 'no_data': + return elem_path + if segment['kind'] == 'range': + lo = segment['lo'] + hi = segment['hi'] + if lo == hi: + return '{}[{}]'.format(elem_path, lo) + return '{}[{}-{}]'.format(elem_path, lo, hi) + return '{}[{}]'.format(elem_path, segment['bin']) def __OnGridMouseMotion(self, evt): x, y = self.grid.CalcUnscrolledPosition(evt.GetX(), evt.GetY()) row, col = self.grid.XYToCell(x, y) - tooltip = self.grid.GetCellToolTip(row, col) + + if col == 0 or self.enable_tooltips: + tooltip = self.grid.GetCellToolTip(row, col) + else: + tooltip = None if tooltip: self.grid.SetToolTip(tooltip) else: self.grid.UnsetToolTip() - def __OnGridRightClick(self, evt): - x, y = self.grid.CalcUnscrolledPosition(evt.GetX(), evt.GetY()) - row, col = self.grid.XYToCell(x, y) - - if col == 0: - return - - if self.show_detailed_queue_packets and col > self.num_ticks_before + self.num_ticks_after: - return - - # Extract the element path from the row label's tooltip e.g. "top.cpu.core0.rob.stats.num_insts_retired" - elem_path = self.grid.GetCellToolTip(row, 0) - - # Extract the caption for this row e.g. "NumInstsRetired[3]" - caption = self.grid.GetCellValue(row, 0) - - # Extract the bin index from the caption - match = re.match(r'(.+)\[(\d+)\]', caption) - assert match - bin_idx = match.group(2) - - # Remove the [bin_idx] suffix from elem_path - elem_path = elem_path.replace('[{}]'.format(bin_idx), '') - - # Get the tick for the cell we right-clicked - cell_tick = float(self.grid.GetColLabelValue(col)) - - auto_colorize_column = self.frame.data_retriever.GetAutoColorizeColumn(elem_path) - unpacked = self.frame.data_retriever.Unpack(elem_path, cell_tick) - data_vals = unpacked['DataVals'][0] - if data_vals is None: - return - - bin_data = data_vals[int(bin_idx)] - if auto_colorize_column not in bin_data: - return - - auto_colorize_value = bin_data[auto_colorize_column] - menu_anno = '{}({})'.format(auto_colorize_column, auto_colorize_value) - - menu = wx.Menu() - - if menu_anno not in {'{}({})'.format(k,v) for k,v in self.tracked_annos.items()}: - opt = menu.Append(wx.ID_ANY, 'Highlight cells with annotation "{}"'.format(menu_anno)) - self.grid.Bind(wx.EVT_MENU, partial(self.__HighlightCellsWithTag, key=auto_colorize_column, value=auto_colorize_value, highlight=True), opt) - else: - opt = menu.Append(wx.ID_ANY, 'Unhighlight cells with annotation "{}"'.format(menu_anno)) - self.grid.Bind(wx.EVT_MENU, partial(self.__HighlightCellsWithTag, key=auto_colorize_column, value=auto_colorize_value, highlight=False), opt) - - #opt = menu.Append(wx.ID_ANY, 'Go to next cycle where different') - #self.grid.Bind(wx.EVT_MENU, partial(self.__GoToNextCycleWhereDifferent, elem_path=elem_path, bin_idx=bin_idx), opt) - - #opt = menu.Append(wx.ID_ANY, 'Go to previous cycle where different') - #self.grid.Bind(wx.EVT_MENU, partial(self.__GoToPrevCycleWhereDifferent, elem_path=elem_path, bin_idx=bin_idx), opt) - - self.grid.PopupMenu(menu) - - def __HighlightCellsWithTag(self, evt, key, value, highlight): - if highlight: - self.tracked_annos[key] = value - dirty = True - else: - if key in self.tracked_annos: - del self.tracked_annos[key] - dirty = True - else: - dirty = False - - if dirty: - self.frame.view_settings.SetDirty(reason=DirtyReasons.TrackedPacketChanged) - - self.UpdateWidgetData() - - def __GoToNextCycleWhereDifferent(self, evt, elem_path, bin_idx): - print ('TODO: Go to next cycle where different') - - def __GoToPrevCycleWhereDifferent(self, evt, elem_path, bin_idx): - print ('TODO: Go to previous cycle where different') - def __EditWidget(self, evt): - dlg = SchedulingLinesCustomizationDialog(self, self.caption_mgr, self.num_ticks_before, self.num_ticks_after, self.show_detailed_queue_packets) - result = dlg.ShowModal() - dlg.Destroy() - - if result == wx.ID_OK: - self.ApplyViewSettings({'regexes': dlg.GetElementPathCaptionRegexes(as_list=True), - 'num_ticks_before': dlg.GetNumTicksBefore(), - 'num_ticks_after': dlg.GetNumTicksAfter(), - 'show_detailed_queue_packets': dlg.ShowDetailedQueuePackets(), - 'tracked_annos': copy.deepcopy(self.tracked_annos)}) - -class SchedulingLinesCustomizationDialog(wx.Dialog): - def __init__(self, parent, caption_mgr, num_ticks_before, num_ticks_after, show_detailed_queue_packets): - super().__init__(parent, title="Customize Scheduling Lines") - - self.caption_mgr = copy.deepcopy(caption_mgr) - self.show_detailed_queue_packets = show_detailed_queue_packets - self.ok_btn = None - - self.move_up_btn = wx.BitmapButton(self, bitmap=wx.ArtProvider.GetBitmap(wx.ART_GO_UP, wx.ART_BUTTON)) - self.move_up_btn.Bind(wx.EVT_BUTTON, self.__MoveSelectedElemUp) - - self.move_down_btn = wx.BitmapButton(self, bitmap=wx.ArtProvider.GetBitmap(wx.ART_GO_DOWN, wx.ART_BUTTON)) - self.move_down_btn.Bind(wx.EVT_BUTTON, self.__MoveSelectedElemDown) - - self.remove_btn = wx.BitmapButton(self, bitmap=wx.ArtProvider.GetBitmap(wx.ART_DELETE, wx.ART_BUTTON)) - self.remove_btn.Bind(wx.EVT_BUTTON, self.__RemoveSelectedElems) - self.remove_btn.Disable() - - edit_btns_sizer = wx.BoxSizer(wx.VERTICAL) - edit_btns_sizer.Add(self.move_up_btn, 0, wx.ALL, 5) - edit_btns_sizer.Add(self.move_down_btn, 0, wx.ALL, 5) - edit_btns_sizer.Add(self.remove_btn, 0, wx.ALL, 5) - - self.element_path_regexes_list_ctrl = wx.ListCtrl(self, style=wx.LC_REPORT) - self.element_path_regexes_list_ctrl.InsertColumn(0, "Path Regex") - self.element_path_regexes_list_ctrl.InsertColumn(1, "Caption") - self.element_path_regexes_list_ctrl.Bind(wx.EVT_LIST_ITEM_SELECTED, self.__OnListCtrlItemSelected) - self.element_path_regexes_list_ctrl.Bind(wx.EVT_LIST_ITEM_DESELECTED, self.__OnListCtrlItemSelected) - self.element_path_regexes_list_ctrl.Bind(wx.EVT_LEFT_DCLICK, self.__OnListCtrlItemDClicked) - self.element_path_regexes_list_ctrl.Bind(wx.EVT_LIST_END_LABEL_EDIT, self.__ValidateRegexSettings) - - element_path_caption_regexes = self.caption_mgr.GetElemPathRegexReplacements() - idx = 0 - for path_regex, caption_replacements in element_path_caption_regexes.items(): - self.element_path_regexes_list_ctrl.InsertItem(idx, path_regex) - self.element_path_regexes_list_ctrl.SetItem(idx, 1, caption_replacements) - - if len(element_path_caption_regexes) == 1: - self.move_up_btn.Disable() - self.move_down_btn.Disable() - else: - idx += 1 - - hsizer = wx.BoxSizer(wx.HORIZONTAL) - hsizer.Add(self.element_path_regexes_list_ctrl, 1, wx.ALL | wx.EXPAND, 5) - hsizer.Add(edit_btns_sizer) - - self.ok_btn = wx.Button(self, wx.ID_OK) - self.cancel_btn = wx.Button(self, wx.ID_CANCEL) - - exit_btn_sizer = wx.BoxSizer(wx.HORIZONTAL) - exit_btn_sizer.Add(self.ok_btn, 0, wx.ALL | wx.EXPAND, 5) - exit_btn_sizer.Add(self.cancel_btn, 0, wx.ALL | wx.EXPAND, 5) - - num_ticks_before_min_val = 1 - num_ticks_before_max_val = 25 - num_ticks_after_min_val = 5 - num_ticks_after_max_val = 100 - - num_ticks_before_info_label = wx.StaticText(self, label='Cycles to show before current tick:') - num_ticks_after_info_label = wx.StaticText(self, label='Cycles to show after current tick:') - - self.num_ticks_before_value_label = wx.StaticText(self, label=str(num_ticks_before)) - self.num_ticks_after_value_label = wx.StaticText(self, label=str(num_ticks_after)) - - num_ticks_before_min_val_text_label = wx.StaticText(self, label=str(num_ticks_before_min_val)) - num_ticks_before_max_val_text_label = wx.StaticText(self, label=str(num_ticks_before_max_val)) - num_ticks_after_min_val_text_label = wx.StaticText(self, label=str(num_ticks_after_min_val)) - num_ticks_after_max_val_text_label = wx.StaticText(self, label=str(num_ticks_after_max_val)) - - self.num_ticks_before_slider = wx.Slider(self, minValue=num_ticks_before_min_val, maxValue=num_ticks_before_max_val, value=num_ticks_before) - self.num_ticks_after_slider = wx.Slider(self, minValue=num_ticks_after_min_val, maxValue=num_ticks_after_max_val, value=num_ticks_after) - - self.num_ticks_before_slider.Bind(wx.EVT_SCROLL, self.__OnNumTicksBeforeSliderChanged) - self.num_ticks_after_slider.Bind(wx.EVT_SCROLL, self.__OnNumTicksAfterSliderChanged) - - w,h = self.num_ticks_before_slider.GetSize() - self.num_ticks_before_slider.SetMinSize((300, h)) - - w,h = self.num_ticks_after_slider.GetSize() - self.num_ticks_after_slider.SetMinSize((300, h)) - - num_ticks_sizer = wx.FlexGridSizer(rows=2, cols=6, hgap=0, vgap=0) - num_ticks_sizer.Add(num_ticks_before_info_label) - num_ticks_sizer.Add(self.num_ticks_before_value_label, 0, wx.LEFT, 20) - num_ticks_sizer.Add(wx.StaticLine(self, style=wx.LI_VERTICAL), 0, wx.LEFT | wx.EXPAND, 20) - num_ticks_sizer.Add(num_ticks_before_min_val_text_label, 0, wx.LEFT, 20) - num_ticks_sizer.Add(self.num_ticks_before_slider, 0, wx.TOP | wx.EXPAND, -7) - num_ticks_sizer.Add(num_ticks_before_max_val_text_label) - - num_ticks_sizer.Add(num_ticks_after_info_label) - num_ticks_sizer.Add(self.num_ticks_after_value_label, 0, wx.LEFT, 20) - num_ticks_sizer.Add(wx.StaticLine(self, style=wx.LI_VERTICAL), 0, wx.LEFT | wx.EXPAND, 20) - num_ticks_sizer.Add(num_ticks_after_min_val_text_label, 0, wx.LEFT, 20) - num_ticks_sizer.Add(self.num_ticks_after_slider, 0, wx.TOP | wx.EXPAND, -7) - num_ticks_sizer.Add(num_ticks_after_max_val_text_label) - - show_detailed_queue_packets_checkbox = wx.CheckBox(self, label='Show detailed queue packets') - show_detailed_queue_packets_checkbox.SetValue(show_detailed_queue_packets) - show_detailed_queue_packets_checkbox.Bind(wx.EVT_CHECKBOX, self.__OnShowDetailedQueuePacketsChanged) - - regex_example_label = wx.StaticText(self, label='Regex examples:') - regex_example_text = wx.StaticText(self, label='top.cpu.core([0-9]+).rob.stats.num_insts_retired') - regex_example_text2 = wx.StaticText(self, label='top.cpu.core0.rob.stats.ipc') - - caption_example_label = wx.StaticText(self, label='Caption examples:') - caption_example_text = wx.StaticText(self, label='NumInstsRetired\\1') - caption_example_text2 = wx.StaticText(self, label='IPC') - - font = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) - regex_example_label.SetFont(font.Bold()) - regex_example_text.SetFont(font) - regex_example_text2.SetFont(font) - caption_example_label.SetFont(font.Bold()) - caption_example_text.SetFont(font) - caption_example_text2.SetFont(font) - self.element_path_regexes_list_ctrl.SetFont(font) - - dc = wx.ScreenDC() - dc.SetFont(self.element_path_regexes_list_ctrl.GetFont()) - col0_width = max([dc.GetTextExtent(s)[0] for s in element_path_caption_regexes.keys()]) + 50 - col1_width = max([dc.GetTextExtent(s)[0] for s in element_path_caption_regexes.values()]) + 50 - - col0_width = max(col0_width, dc.GetTextExtent(regex_example_text.GetLabel())[0]) - col0_width += dc.GetTextExtent('([0-9]+)')[0] - - hgap = col0_width - dc.GetTextExtent(regex_example_text.GetLabel())[0] - example_sizer = wx.FlexGridSizer(rows=3, cols=2, hgap=hgap, vgap=0) - example_sizer.Add(regex_example_label) - example_sizer.Add(caption_example_label) - example_sizer.Add(regex_example_text) - example_sizer.Add(caption_example_text) - example_sizer.Add(regex_example_text2) - example_sizer.Add(caption_example_text2) - - vsizer = wx.BoxSizer(wx.VERTICAL) - vsizer.Add(hsizer, 1, wx.ALL | wx.EXPAND, 5) - vsizer.Add(example_sizer, 0, wx.ALL | wx.EXPAND, 5) - vsizer.AddSpacer(10) - vsizer.Add(num_ticks_sizer, 0, wx.ALL | wx.EXPAND, 5) - vsizer.Add(show_detailed_queue_packets_checkbox, 0, wx.ALL | wx.EXPAND, 5) - vsizer.Add(exit_btn_sizer, 0, wx.ALL | wx.EXPAND, 5) - - self.errors_label = wx.StaticText(self, label='No issues found', size=(col0_width,-1)) - hsizer2 = wx.BoxSizer(wx.HORIZONTAL) - hsizer2.Add(vsizer) - hsizer2.AddSpacer(10) - hsizer2.Add(wx.StaticLine(self, style=wx.LI_VERTICAL), 0, wx.EXPAND) - hsizer2.AddSpacer(10) - hsizer2.Add(self.errors_label) - - sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(hsizer2, 1, wx.ALL | wx.EXPAND, 5) - self.SetSizer(sizer) - self.Layout() - - self.element_path_regexes_list_ctrl.SetColumnWidth(0, col0_width) - self.element_path_regexes_list_ctrl.SetColumnWidth(1, col1_width) - - dlg_width = col0_width + col1_width - dlg_min_width = dlg_width + hsizer.CalcMin()[0] + 50 - dlg_min_height = sizer.CalcMin()[1] + 75 - self.SetSize((dlg_min_width, dlg_min_height)) - self.Layout() - - @property - def frame(self): - return self.GetParent().frame - - def GetNumTicksBefore(self): - return self.num_ticks_before_slider.GetValue() - - def GetNumTicksAfter(self): - return self.num_ticks_after_slider.GetValue() - - def ShowDetailedQueuePackets(self): - return self.show_detailed_queue_packets - - def GetElementPathCaptionRegexes(self, as_list=False): - regexes = OrderedDict() - for row in range(self.element_path_regexes_list_ctrl.GetItemCount()): - path_regex = self.element_path_regexes_list_ctrl.GetItemText(row, 0) - caption_replacements = self.element_path_regexes_list_ctrl.GetItemText(row, 1) - regexes[path_regex] = caption_replacements - - if as_list: - return list(regexes.items()) - - return regexes - - def __OnListCtrlItemSelected(self, evt): - selected_elem_idxs = self.__GetListCtrlSelectedItemIdxs() - if len(selected_elem_idxs) == 0: - self.move_up_btn.Disable() - self.move_down_btn.Disable() - self.remove_btn.Disable() - return - - if len(selected_elem_idxs) != 1: - self.move_up_btn.Disable() - self.move_down_btn.Disable() - - if len(selected_elem_idxs) > 0: - self.remove_btn.Enable() - else: - self.remove_btn.Disable() - - return - - selected_elem_idx = selected_elem_idxs[0] - if selected_elem_idx == wx.NOT_FOUND: - self.move_up_btn.Disable() - self.move_down_btn.Disable() - self.remove_btn.Disable() - return - else: - self.remove_btn.Enable() - - if selected_elem_idx == 0: - self.move_up_btn.Disable() - else: - self.move_up_btn.Enable() - - if selected_elem_idx == self.element_path_regexes_list_ctrl.GetItemCount() - 1: - self.move_down_btn.Disable() - else: - self.move_down_btn.Enable() - - def __MoveSelectedElemUp(self, evt): - selected_elem_idxs = self.__GetListCtrlSelectedItemIdxs() - assert len(selected_elem_idxs) == 1 - selected_elem_idx = selected_elem_idxs[0] - assert selected_elem_idx > 0 - - orig_top_idx = selected_elem_idx-1 - orig_bottom_idx = selected_elem_idx - - orig_top_item_col0_text = self.element_path_regexes_list_ctrl.GetItemText(orig_top_idx, 0) - orig_top_item_col1_text = self.element_path_regexes_list_ctrl.GetItemText(orig_top_idx, 1) - - orig_bottom_item_col0_text = self.element_path_regexes_list_ctrl.GetItemText(orig_bottom_idx, 0) - orig_bottom_item_col1_text = self.element_path_regexes_list_ctrl.GetItemText(orig_bottom_idx, 1) - - self.element_path_regexes_list_ctrl.SetItem(orig_top_idx, 0, orig_bottom_item_col0_text) - self.element_path_regexes_list_ctrl.SetItem(orig_top_idx, 1, orig_bottom_item_col1_text) - - self.element_path_regexes_list_ctrl.SetItem(orig_bottom_idx, 0, orig_top_item_col0_text) - self.element_path_regexes_list_ctrl.SetItem(orig_bottom_idx, 1, orig_top_item_col1_text) - - self.element_path_regexes_list_ctrl.Select(orig_bottom_idx, False) - self.element_path_regexes_list_ctrl.Select(orig_top_idx, True) - - element_path_caption_regexes = self.GetElementPathCaptionRegexes(as_list=True) - tmp = element_path_caption_regexes[orig_top_idx] - element_path_caption_regexes[orig_top_idx] = element_path_caption_regexes[orig_bottom_idx] - element_path_caption_regexes[orig_bottom_idx] = tmp - self.caption_mgr.SetElemPathRegexReplacements(element_path_caption_regexes) - - def __MoveSelectedElemDown(self, evt): - selected_elem_idxs = self.__GetListCtrlSelectedItemIdxs() - assert len(selected_elem_idxs) == 1 - selected_elem_idx = selected_elem_idxs[0] - assert selected_elem_idx < self.element_path_regexes_list_ctrl.GetItemCount() - 1 - - orig_top_idx = selected_elem_idx - orig_bottom_idx = selected_elem_idx+1 - - orig_top_item_col0_text = self.element_path_regexes_list_ctrl.GetItemText(orig_top_idx, 0) - orig_top_item_col1_text = self.element_path_regexes_list_ctrl.GetItemText(orig_top_idx, 1) - - orig_bottom_item_col0_text = self.element_path_regexes_list_ctrl.GetItemText(orig_bottom_idx, 0) - orig_bottom_item_col1_text = self.element_path_regexes_list_ctrl.GetItemText(orig_bottom_idx, 1) - - self.element_path_regexes_list_ctrl.SetItem(orig_top_idx, 0, orig_bottom_item_col0_text) - self.element_path_regexes_list_ctrl.SetItem(orig_top_idx, 1, orig_bottom_item_col1_text) - - self.element_path_regexes_list_ctrl.SetItem(orig_bottom_idx, 0, orig_top_item_col0_text) - self.element_path_regexes_list_ctrl.SetItem(orig_bottom_idx, 1, orig_top_item_col1_text) - - self.element_path_regexes_list_ctrl.Select(orig_top_idx, False) - self.element_path_regexes_list_ctrl.Select(orig_bottom_idx, True) - - element_path_caption_regexes = self.GetElementPathCaptionRegexes(as_list=True) - tmp = element_path_caption_regexes[orig_top_idx] - element_path_caption_regexes[orig_top_idx] = element_path_caption_regexes[orig_bottom_idx] - element_path_caption_regexes[orig_bottom_idx] = tmp - self.caption_mgr.SetElemPathRegexReplacements(element_path_caption_regexes) - - def __RemoveSelectedElems(self, evt): - selected_elem_idxs = self.__GetListCtrlSelectedItemIdxs() - selected_elem_idxs.reverse() - - for i in selected_elem_idxs: - self.element_path_regexes_list_ctrl.DeleteItem(i) - - for i in range(self.element_path_regexes_list_ctrl.GetItemCount()): - self.element_path_regexes_list_ctrl.Select(i, False) - - element_path_caption_regexes = self.GetElementPathCaptionRegexes() - self.caption_mgr.SetElemPathRegexReplacements(element_path_caption_regexes) - self.remove_btn.Disable() - - def __GetListCtrlSelectedItemIdxs(self): - idxs = [] - item_count = self.element_path_regexes_list_ctrl.GetItemCount() - - for i in range(item_count): - if self.element_path_regexes_list_ctrl.GetItemState(i, wx.LIST_STATE_SELECTED): - idxs.append(i) - - return idxs - - def __OnShowDetailedQueuePacketsChanged(self, evt): - self.show_detailed_queue_packets = evt.IsChecked() - - def __OnListCtrlItemDClicked(self, evt): - # Get the mouse position in the list control - mouse_x, mouse_y = evt.GetPosition() - - hit = self.element_path_regexes_list_ctrl.HitTest((mouse_x, mouse_y)) - if not hit or hit[0] == -1: - return - - row = hit[0] - if mouse_x < self.element_path_regexes_list_ctrl.GetColumnWidth(0): - col = 0 - elif mouse_x > self.element_path_regexes_list_ctrl.GetColumnWidth(0): - col = 1 - else: - return - - row_rect = self.element_path_regexes_list_ctrl.GetItemRect(row) - if col == 0: - cell_rect = wx.Rect(row_rect.GetLeft(), - row_rect.GetTop(), - self.element_path_regexes_list_ctrl.GetColumnWidth(0), - row_rect.GetHeight()) - else: - cell_rect = wx.Rect(row_rect.GetLeft() + self.element_path_regexes_list_ctrl.GetColumnWidth(0), - row_rect.GetTop(), - self.element_path_regexes_list_ctrl.GetColumnWidth(1), - row_rect.GetHeight()) - - pos = cell_rect.GetTopLeft() - size = cell_rect.GetSize() - self.__EditListCtrlCell(row, col, pos, size) - - def __EditListCtrlCell(self, row, col, text_ctrl_pos, text_ctrl_size): - # Get the current value of the cell - current_text = self.element_path_regexes_list_ctrl.GetItemText(row, col) - - # Create a text control to edit the cell - self.text_ctrl = wx.TextCtrl(self, value=current_text, style=wx.TE_PROCESS_ENTER, pos=text_ctrl_pos, size=text_ctrl_size) - - # Callback when editing the cell - self.text_ctrl.Bind(wx.EVT_TEXT, lambda evt: self.__OnListCtrlEdit(evt, row, col, current_text)) - - # Callbacks when finished editing the cell - self.text_ctrl.Bind(wx.EVT_TEXT_ENTER, lambda evt: self.__OnListCtrlEditComplete(evt, row, col, current_text)) - self.text_ctrl.Bind(wx.EVT_KILL_FOCUS, lambda evt: self.__OnListCtrlEditComplete(evt, row, col, current_text)) - - # Focus the text control right away - self.text_ctrl.SetFocus() - - def __OnListCtrlEdit(self, evt, row, col, current_text): - if col == 0: - regex = self.text_ctrl.GetValue() - try: - re.compile(regex) - self.text_ctrl.SetBackgroundColour(wx.WHITE) - evt.Skip() - except: - self.text_ctrl.SetBackgroundColour((255, 192, 203)) # Pink - else: - regex = self.element_path_regexes_list_ctrl.GetItemText(row, 0) - caption = self.text_ctrl.GetValue() - valid = False - - for elem_path in self.frame.simhier.GetContainerElemPaths(): - try: - re.compile(regex) - re.sub(regex, caption, elem_path) - self.text_ctrl.SetBackgroundColour(wx.WHITE) - evt.Skip() - valid = True - break - except: - pass - - if not valid: - self.text_ctrl.SetBackgroundColour((255, 192, 203)) # Pink - - def __OnListCtrlEditComplete(self, evt, row, col, orig_text): - if not self.text_ctrl: - return - - self.text_ctrl.Unbind(wx.EVT_TEXT_ENTER) - self.text_ctrl.Unbind(wx.EVT_KILL_FOCUS) - evt.Skip() - - new_value = self.text_ctrl.GetValue() - - def DestroyTextCtrl(text_ctrl): - text_ctrl.Destroy() - text_ctrl = None - - wx.CallAfter(DestroyTextCtrl, self.text_ctrl) - - def SetListCtrlItem(list_ctrl, row, col, text): - list_ctrl.SetItem(row, col, text) - - if self.text_ctrl.GetBackgroundColour() == (255, 192, 203): - # The regex or caption is invalid. Revert to the original text. - wx.CallAfter(SetListCtrlItem, self.element_path_regexes_list_ctrl, row, col, orig_text) - else: - # The regex or caption is valid. Update the list control. - wx.CallAfter(SetListCtrlItem, self.element_path_regexes_list_ctrl, row, col, new_value) - - wx.CallAfter(self.__ValidateRegexSettings, None) - - def __OnNumTicksBeforeSliderChanged(self, evt): - self.num_ticks_before_value_label.SetLabel(str(self.num_ticks_before_slider.GetValue())) - - def __OnNumTicksAfterSliderChanged(self, evt): - self.num_ticks_after_value_label.SetLabel(str(self.num_ticks_after_slider.GetValue())) - - def __ValidateRegexSettings(self, evt): - if not self.ok_btn: - return - - orig_regex_replacements = self.caption_mgr.GetElemPathRegexReplacements() - validate_regex_replacements = OrderedDict() - for row in range(self.element_path_regexes_list_ctrl.GetItemCount()): - path_regex = self.element_path_regexes_list_ctrl.GetItemText(row, 0) - caption_replacements = self.element_path_regexes_list_ctrl.GetItemText(row, 1) - validate_regex_replacements[path_regex] = caption_replacements - - self.caption_mgr.SetElemPathRegexReplacements(validate_regex_replacements) - - error_msgs = [] - for row in range(self.element_path_regexes_list_ctrl.GetItemCount()): - path_regex = self.element_path_regexes_list_ctrl.GetItemText(row, 0) - caption_replacements = self.element_path_regexes_list_ctrl.GetItemText(row, 1) - - try: - re.compile(path_regex) - except: - error_msgs.append('Invalid regex in row {}:\n {}'.format(row, path_regex)) - continue - - replacements_valid = False - for elem_path in self.frame.simhier.GetContainerElemPaths(): - try: - re.sub(path_regex, caption_replacements, elem_path) - replacements_valid = True - break - except: - continue - - if not replacements_valid: - error_msgs.append('Invalid replacements in row {}:\n {}'.format(row, caption_replacements)) - - caption_prefixes = set() - for elem_path in self.frame.simhier.GetContainerElemPaths(): - caption_prefix = self.caption_mgr.GetCaptionPrefix(elem_path) - if caption_prefix is None: - continue - - if caption_prefix in caption_prefixes: - error_msgs.append('Duplicate caption prefix found: {}'.format(caption_prefix)) - else: - caption_prefixes.add(caption_prefix) - - if len(error_msgs) > 0: - self.ok_btn.Disable() - self.caption_mgr.SetElemPathRegexReplacements(orig_regex_replacements) - self.errors_label.SetLabel('\n'.join(error_msgs)) - self.errors_label.SetForegroundColour(wx.RED) - - current_font = self.errors_label.GetFont() - - # Create a new font based on the current one but bold - bold_font = wx.Font(current_font.GetPointSize(), - current_font.GetFamily(), - current_font.GetStyle(), - wx.FONTWEIGHT_BOLD) - - self.errors_label.SetFont(bold_font) - else: - self.ok_btn.Enable() - self.errors_label.SetLabel('No issues found') - self.errors_label.SetForegroundColour(wx.BLACK) - - current_font = self.errors_label.GetFont() - - # Create a new font based on the current one but without bold - normal_font = wx.Font(current_font.GetPointSize(), - current_font.GetFamily(), - current_font.GetStyle(), - wx.FONTWEIGHT_NORMAL) - - self.errors_label.SetFont(normal_font) + widget_container = self.GetParent() + widget_container.LaunchSchedulingLinesViewer() class CaptionManager: + MINIMUM_CAPTION_PATH_PARTS = 2 + def __init__(self, simhier): self.simhier = simhier + self.ClearSelections() + + @classmethod + def GetMinimumUniqueSuffix(cls, elem_path, elem_paths, min_parts=MINIMUM_CAPTION_PATH_PARTS): + parts = elem_path.split('.') + all_parts = [path.split('.') for path in elem_paths] + + for suffix_len in range(1, len(parts) + 1): + my_suffix = parts[-suffix_len:] + matches = sum( + 1 for other_parts in all_parts + if len(other_parts) >= suffix_len and other_parts[-suffix_len:] == my_suffix + ) + if matches == 1: + display_len = max(min_parts, suffix_len) + return '.'.join(parts[-display_len:]) + + return elem_path + + @classmethod + def ApplyPartialPathTooltip(cls, control, full_path, label_text, show_full_paths): + if not show_full_paths and label_text.rstrip() != full_path: + control.SetToolTip(full_path) + else: + control.UnsetToolTip() + + def ClearSelections(self): self.regex_replacements_by_elem_path_regex = OrderedDict() def SetElemPathRegexReplacement(self, elem_path_regex, regex_replacement): @@ -1063,7 +663,10 @@ def GetElemPathRegexReplacements(self, as_list=False): return d - def GetCaption(self, elem_path, bin_idx): + def GetCaption(self, elem_path, bin_idx, elem_paths=None, show_full_paths=False): + if elem_paths is None: + elem_paths = self.GetAllMatchingElemPaths() + for regex, replacements in self.regex_replacements_by_elem_path_regex.items(): if regex == elem_path: # No regex was supplied in the settings dialog. The full path was given e.g. @@ -1074,7 +677,7 @@ def GetCaption(self, elem_path, bin_idx): # # We will just return the last part of the path as the caption using # heads-up camel case e.g. "NumInstsRetired[3]" - return self.GetCaptionPrefix(elem_path) + '[{}]'.format(bin_idx) + return self.GetCaptionPrefix(elem_path, elem_paths, show_full_paths) + '[{}]'.format(bin_idx) if re.compile(regex).match(elem_path): # This matched an elem path e.g. @@ -1090,11 +693,19 @@ def GetCaption(self, elem_path, bin_idx): # core index return re.sub(regex, replacements, elem_path) + '[{}]'.format(bin_idx) - return GetHeadsUpCamelCaseQueueName(elem_path) + '[{}]'.format(bin_idx) + prefix = elem_path if show_full_paths else self.GetMinimumUniqueSuffix(elem_path, elem_paths) + return f'{prefix}[{bin_idx}]' - def GetCaptionPrefix(self, elem_path): + def GetCaptionPrefix(self, elem_path, elem_paths=None, show_full_paths=False): + if elem_paths is None: + elem_paths = self.GetAllMatchingElemPaths() + for regex, replacements in self.regex_replacements_by_elem_path_regex.items(): if regex == elem_path: + if replacements == elem_path: + if show_full_paths: + return elem_path + return self.GetMinimumUniqueSuffix(elem_path, elem_paths) return replacements if re.compile(regex).match(elem_path): @@ -1103,16 +714,31 @@ def GetCaptionPrefix(self, elem_path): return None def GetAllMatchingElemPaths(self): + # The display order of the queues/rows follows the order of the regex + # OrderedDict (i.e. the order the user set in the customization dialog), + # NOT the static order of self.simhier.GetContainerElemPaths(). The regex + # dict is therefore the outer loop here. + container_elem_paths = self.simhier.GetContainerElemPaths() + elem_paths = [] - for elem_path in self.simhier.GetContainerElemPaths(): - for regex, _ in self.regex_replacements_by_elem_path_regex.items(): - if elem_path == regex: - elem_paths.append(elem_path) - break + seen = set() + for regex, _ in self.regex_replacements_by_elem_path_regex.items(): + # Exact path match first (the regex is a full element path). + if regex in container_elem_paths and regex not in seen: + elem_paths.append(regex) + seen.add(regex) + continue + + # Otherwise treat it as a regex and append all matching container + # paths (in stable simhier order within this single regex group). + compiled = re.compile(regex) + for elem_path in container_elem_paths: + if elem_path in seen: + continue - if re.compile(regex).match(elem_path): + if compiled.match(elem_path): elem_paths.append(elem_path) - break + seen.add(elem_path) return elem_paths @@ -1131,21 +757,6 @@ def GetMatchingElemPaths(self, regex): return elem_paths -def GetHeadsUpCamelCaseQueueName(elem_path): - parts = elem_path.split('.') - queue_name = parts[-1] - parts = queue_name.split('_') - - for i,part in enumerate(parts): - if len(part) == 1: - part = part.upper() - else: - part = part[0].upper() + part[1:] - - parts[i] = part - - return ''.join(parts) - class Rasterizer: def __init__(self, frame, grid, widget, elem_path, bin_idx, row, detailed_pkt_col): self.frame = frame @@ -1157,20 +768,31 @@ def __init__(self, frame, grid, widget, elem_path, bin_idx, row, detailed_pkt_co self.detailed_pkt_col = detailed_pkt_col def Draw(self, elem_path, bin_idx, time_val, annos): + if not annos: + return + assert elem_path == self.elem_path assert bin_idx == self.bin_idx auto_colorize_column = self.frame.data_retriever.GetAutoColorizeColumn(elem_path) - auto_colorize_key = annos[auto_colorize_column] + auto_colorize_key = None + for key, keyval in annos: + if key == auto_colorize_column: + auto_colorize_key = keyval + break + + assert auto_colorize_key is not None auto_color = self.frame.widget_renderer.GetAutoColor(auto_colorize_key) auto_label = self.frame.widget_renderer.GetAutoTag(auto_colorize_key) anno = [] - for k,v in annos.items(): - anno.append('{}({})'.format(k,v)) + for k, v in annos: + if k == 'DID' and not self.widget.show_did: + continue + anno.append('{}({})'.format(k, v)) - stringized_anno = ' '.join(anno) stringized_tooltip = '\n'.join(anno) + stringized_anno = ' '.join(anno) tracked_annos = self.widget.tracked_annos show_border = auto_colorize_column in tracked_annos and tracked_annos[auto_colorize_column] == auto_colorize_key @@ -1181,14 +803,15 @@ def Draw(self, elem_path, bin_idx, time_val, annos): col_label = self.grid.GetColLabelValue(col) try: - col_label = float(col_label) + col_label = int(col_label) except: continue - if col_label == float(time_val): + if col_label == int(time_val): self.grid.SetCellValue(self.row, col, auto_label) self.grid.SetCellBackgroundColour(self.row, col, auto_color) - self.grid.SetCellToolTip(self.row, col, stringized_tooltip) + if self.widget.enable_tooltips: + self.grid.SetCellToolTip(self.row, col, stringized_tooltip) border_width = 1 if show_border else self.grid.GetCellBorderWidth(self.row, col) border_side = wx.ALL if show_border else self.grid.GetCellBorderSide(self.row, col) @@ -1196,8 +819,17 @@ def Draw(self, elem_path, bin_idx, time_val, annos): break if self.detailed_pkt_col != -1 and time_val == self.frame.widget_renderer.tick: + def Strip(stringized_anno, string, replace): + while string in stringized_anno: + stringized_anno = stringized_anno.replace(string, replace) + return stringized_anno + + stringized_anno = Strip(stringized_anno, '((', '(') + stringized_anno = Strip(stringized_anno, '))', ')') self.grid.SetCellValue(self.row, self.detailed_pkt_col, stringized_anno) + self.grid.SetCellAlignment(self.row, self.detailed_pkt_col, wx.ALIGN_CENTER_VERTICAL) self.grid.SetCellBackgroundColour(self.row, self.detailed_pkt_col, auto_color) - self.grid.SetCellToolTip(self.row, self.detailed_pkt_col, stringized_tooltip) + if self.widget.enable_tooltips: + self.grid.SetCellToolTip(self.row, self.detailed_pkt_col, stringized_tooltip) if show_border: self.grid.SetCellBorder(self.row, self.detailed_pkt_col, 1, wx.ALL) diff --git a/python/argos/viewer/gui/widgets/summary_views.py b/python/argos/viewer/gui/widgets/summary_views.py new file mode 100644 index 00000000..9830775a --- /dev/null +++ b/python/argos/viewer/gui/widgets/summary_views.py @@ -0,0 +1,334 @@ +import wx, re +from collections import OrderedDict +from viewer.model.data_deserializers import StructDeserializer +from viewer.gui.view_settings import DirtyReasons +from viewer.gui.dialogs.widget_data_selections import SummaryViewsEditDlg +from viewer.gui.widgets.scheduling_lines import CaptionManager + +class SummaryViews(wx.Panel): + DEFAULT_SHOW_FULL_PATHS = True + DEFAULT_SHOW_DID = False + + def __init__(self, parent, frame, show_full_paths=DEFAULT_SHOW_FULL_PATHS, show_did=DEFAULT_SHOW_DID): + super().__init__(parent) + self.frame = frame + self.show_full_paths = show_full_paths + self.show_did = show_did + self.summary_scroller = None + self.summary = None + self._summary_grid_dirty = True + self._elem_paths_by_cid = {} + + @property + def elem_paths(self): + return list(self._elem_paths_by_cid.values()) + + @property + def cids(self): + return list(self._elem_paths_by_cid.keys()) + + def GetWidgetCreationString(self): + return 'Summary Views' + + def GetErrorIfDroppedNodeIncompatible(self, elem_path): + if elem_path in self.elem_paths: + return 'This collectable is already being displayed.', 'Duplicate Collectable' + + return None + + def UpdateWidgetData(self): + self.__Refresh() + + def GetCurrentViewSettings(self): + settings = {} + settings['elem_paths'] = self.elem_paths + settings['show_full_paths'] = self.show_full_paths + settings['show_did'] = self.show_did + return settings + + def GetCurrentUserSettings(self): + return {} + + def ApplyViewSettings(self, settings): + show_full_paths = settings.get('show_full_paths', self.DEFAULT_SHOW_FULL_PATHS) + show_did = settings.get('show_did', self.DEFAULT_SHOW_DID) + paths_changed = self.elem_paths != settings['elem_paths'] + dirty = ( + paths_changed + or self.show_full_paths != show_full_paths + or self.show_did != show_did + or self._summary_grid_dirty + ) + if not dirty: + return + + if paths_changed or self.show_full_paths != show_full_paths: + self._summary_grid_dirty = True + + self._elem_paths_by_cid = { + self.frame.simhier.GetCollectionID(path):path + for path in settings['elem_paths'] + } + self.show_full_paths = show_full_paths + self.show_did = show_did + + self.frame.view_settings.SetDirty(reason=DirtyReasons.SummaryViewsWidgetChanged) + self.__Refresh() + + def AddElement(self, elem_path): + if elem_path in self.elem_paths: + return + + cid = self.frame.simhier.GetCollectionID(elem_path) + self._elem_paths_by_cid[cid] = elem_path + self._summary_grid_dirty = True + self.__Refresh() + + def EditWidget(self, evt, title="Edit Data Selections"): + dlg = SummaryViewsEditDlg( + self, self.frame, self.elem_paths, self.show_full_paths, self.show_did, title=title, + ) + result = dlg.ShowModal() + if result == wx.ID_OK: + elem_paths = dlg.GetSelectedElemPaths() + show_full_paths = dlg.show_full_paths + show_did = dlg.show_did + dlg.Destroy() + if result == wx.ID_OK: + self.ApplyViewSettings({ + 'elem_paths': elem_paths, + 'show_full_paths': show_full_paths, + 'show_did': show_did, + }) + return True + else: + return False + + def __Refresh(self): + if len(self.elem_paths): + self.SetBackgroundColour('white') + self.__RegenerateSummary() + else: + if self.summary_scroller: + self.summary_scroller.Destroy() + self.summary_scroller = None + self.summary = None + elif self.summary: + self.summary.Destroy() + self.summary = None + + self._summary_grid_dirty = True + + def __RegenerateSummary(self): + assert len(self.elem_paths) + if self._summary_grid_dirty: + if self.summary_scroller: + self.summary_scroller.Destroy() + self.summary_scroller = None + self.summary = None + + self.summary_scroller = wx.ScrolledWindow(self, style=wx.VSCROLL | wx.HSCROLL) + self.summary_scroller.SetScrollRate(10, 10) + self.summary = SummaryGrid(self.summary_scroller, self.frame, self.elem_paths, self) + + scroller_sizer = wx.BoxSizer(wx.VERTICAL) + scroller_sizer.Add(self.summary, 0) + self.summary_scroller.SetSizer(scroller_sizer) + + if not self.GetSizer(): + self.SetSizer(wx.BoxSizer(wx.HORIZONTAL)) + hsizer = self.GetSizer() + hsizer.Clear() + hsizer.AddSpacer(5) + hsizer.Add(self.summary_scroller, 1, wx.EXPAND | wx.ALL, 5) + + self.summary.Layout() + self.summary_scroller.Layout() + self.summary_scroller.FitInside() + + self._summary_grid_dirty = False + + self.summary.UpdateWidgetData() + self.Layout() + +class SummaryGrid(wx.Panel): + def __init__(self, parent, frame, elem_paths, summary_views): + wx.Panel.__init__(self, parent) + self.frame = frame + self.summary_views = summary_views + self.value_handlers = {} + + gear_btn, clear_btn, split_lr, split_tb, maximize_btn = frame.CreateWidgetStandardButtons( + self, summary_views.EditWidget, 'Edit data selections') + + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + btn_sizer.Add(gear_btn, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(clear_btn, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_lr, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(split_tb, 0, wx.TOP | wx.RIGHT, 5) + btn_sizer.Add(maximize_btn, 0, wx.TOP, 5) + + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(btn_sizer, 0, wx.BOTTOM, 5) + + # Group all element leaf paths by their parents + collectable_grps = OrderedDict() + for p in elem_paths: + idx = p.rfind('.') + assert idx != -1 + assert idx + 1 < len(p) + parent = p[:idx] + if parent not in collectable_grps: + collectable_grps[parent] = [] + collectable_grps[parent].append(p[idx+1:]) + + # Now we have a dict that looks like this: + # + # top.cpu.core0.decode + # foo + # bar + # top.cpu.core0.retire + # baz + grid_sizer = wx.GridBagSizer(vgap=5, hgap=5) + mono10 = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) + mono10_bold = wx.Font(10, wx.FONTFAMILY_MODERN, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD) + + max_label_len = 0 + parent_paths = list(collectable_grps.keys()) + for parent, leaves in collectable_grps.items(): + if summary_views.show_full_paths: + parent_label_text = parent + else: + parent_label_text = CaptionManager.GetMinimumUniqueSuffix(parent, parent_paths) + max_label_len = max(max_label_len, len(parent_label_text)) + for leaf in leaves: + max_label_len = max(max_label_len, len(leaf)) + + row = 0 + for parent, leaves in collectable_grps.items(): + if summary_views.show_full_paths: + parent_label_text = parent + else: + parent_label_text = CaptionManager.GetMinimumUniqueSuffix(parent, parent_paths) + parent_label = wx.StaticText(self, label=parent_label_text) + parent_label.SetFont(mono10_bold) + CaptionManager.ApplyPartialPathTooltip( + parent_label, parent, parent_label_text, summary_views.show_full_paths) + grid_sizer.Add(parent_label, pos=(row,0)) + row += 1 + + for leaf in leaves: + full_path = parent + '.' + leaf + num_dashes = max_label_len - len(leaf) + 1 + if num_dashes > 0: + label = '-'*num_dashes + ' ' + else: + label = '' + label += leaf + leaf_label = wx.StaticText(self, label=label) + leaf_label.SetFont(mono10) + CaptionManager.ApplyPartialPathTooltip( + leaf_label, full_path, label, summary_views.show_full_paths) + grid_sizer.Add(leaf_label, pos=(row,0)) + + leaf_cid = frame.simhier.GetCollectionID(full_path) + if leaf_cid in frame.simhier.GetContainerIDs(): + capacity = frame.simhier.GetCapacityByCollectionID(leaf_cid) + summary_handler = SummaryGrid.ContainerSummary(self, frame, capacity) + else: + dtype_name = frame.dtype_inspector.GetDataTypeForCollectionID(leaf_cid) + if dtype_name in ('char', 'unsigned char', 'short', 'unsigned short', 'int', 'unsigned int', 'long', 'unsigned long'): + summary_handler = SummaryGrid.IntegerSummary(self, frame) + elif isinstance(frame.dtype_inspector.GetDeserializer(dtype_name), StructDeserializer): + summary_handler = SummaryGrid.StructSummary(self, frame, summary_views) + else: + summary_handler = SummaryGrid.SimpleSummary(self, frame) + + summary_handler.SetFont(mono10) + grid_sizer.Add(summary_handler, pos=(row,4)) + grid_sizer.AddGrowableRow(row) + self.value_handlers[full_path] = summary_handler + row += 1 + + sizer.Add(grid_sizer) + self.SetSizer(sizer) + self.Layout() + + def UpdateWidgetData(self): + if not self.value_handlers: + return + + current_tick = self.frame.widget_renderer.tick + elem_paths = list(self.value_handlers.keys()) + all_data = self.frame.data_retriever.UnpackRange(current_tick, current_tick, elem_paths) + + for elem_path, elem_data in all_data.items(): + handler = self.value_handlers[elem_path] + if len(elem_data['DataVals']) == 1: + value = elem_data['DataVals'][0] + else: + value = None + + handler.UpdateValue(value) + + class SimpleSummary(wx.StaticText): + def __init__(self, parent, frame): + wx.StaticText.__init__(self, parent, label='TODO') + self.frame = frame + + def UpdateValue(self, value): + self.SetLabel(str(value)) + + class IntegerSummary(wx.StaticText): + def __init__(self, parent, frame): + wx.StaticText.__init__(self, parent, label='TODO') + self.frame = frame + self.hex = False + + def UpdateValue(self, value): + value = str(value) + if self.hex: + value = hex(value) + self.SetLabel(value) + + class StructSummary(wx.StaticText): + def __init__(self, parent, frame, summary_views): + wx.StaticText.__init__(self, parent, label='TODO') + self.frame = frame + self.summary_views = summary_views + self.auto_color = True + + def UpdateValue(self, value): + label = [] + if value is not None: + for field_name, field_value in value: + if field_name == 'DID' and not self.summary_views.show_did: + continue + field_value = str(field_value) + field_value = re.sub(r'\s+', ' ', field_value) + label.append(f'{field_name}({field_value})') + label = ' '.join(label) + self.SetLabel(label) + self.SetToolTip(label) + else: + self.SetLabel('(no data)') + self.UnsetToolTip() + + def __HandleContextMenu(self, evt, auto_color): + self.auto_color = auto_color + + class ContainerSummary(wx.StaticText): + def __init__(self, parent, frame, capacity): + wx.StaticText.__init__(self, parent, label='TODO') + self.frame = frame + self.capacity = capacity + assert self.capacity > 0 + + def UpdateValue(self, value): + if value: + size = len(value) + pct = f"{100.0 * size / self.capacity:.1f}%" + label = f'{pct} full ({size}/{self.capacity})' + self.SetLabel(label) + else: + self.SetLabel('(no data)') diff --git a/python/argos/viewer/gui/widgets/widget_creator.py b/python/argos/viewer/gui/widgets/widget_creator.py index 23685b19..45b0d636 100644 --- a/python/argos/viewer/gui/widgets/widget_creator.py +++ b/python/argos/viewer/gui/widgets/widget_creator.py @@ -1,70 +1,24 @@ -import wx from viewer.gui.widgets.queue_utiliz import QueueUtilizWidget from viewer.gui.widgets.scheduling_lines import SchedulingLinesWidget -#from viewer.gui.widgets.scalar_statistic import ScalarStatistic -from viewer.gui.widgets.scalar_struct import ScalarStruct +from viewer.gui.widgets.summary_views import SummaryViews from viewer.gui.widgets.iterable_struct import IterableStruct -#from viewer.gui.widgets.ipc import IPCWidget class WidgetCreator: def __init__(self, frame): self.frame = frame - def BindToWidgetSource(self, widget_source): - if isinstance(widget_source, wx.TreeCtrl): - widget_source.Unbind(wx.EVT_TREE_BEGIN_DRAG) - widget_source.Bind(wx.EVT_TREE_BEGIN_DRAG, self.__BeginDragFromTree) - def CreateWidget(self, widget_creation_key, widget_container): if widget_creation_key == 'Queue Utilization': return QueueUtilizWidget(widget_container, self.frame) elif widget_creation_key == 'Scheduling Lines': return SchedulingLinesWidget(widget_container, self.frame) - elif widget_creation_key == 'IPC': - #return IPCWidget(widget_container, self.frame) - wx.MessageBox('IPC widget is not implemented', 'Error', wx.OK | wx.ICON_ERROR) + elif widget_creation_key == 'Summary Views': + return SummaryViews(widget_container, self.frame) elif widget_creation_key.find('$') != -1: widget_name, elem_path = widget_creation_key.split('$') - if widget_name == 'ScalarStatistic': - #return ScalarStatistic(widget_container, self.frame, elem_path) - wx.MessageBox('ScalarStatistic widget is not implemented', 'Error', wx.OK | wx.ICON_ERROR) - elif widget_name == 'ScalarStruct': - return ScalarStruct(widget_container, self.frame, elem_path) - elif widget_name == 'IterableStruct': + if widget_name == 'IterableStruct': return IterableStruct(widget_container, self.frame, elem_path) elif widget_creation_key == 'NO_WIDGET': return None - else: - raise ValueError(f"Unknown widget creation key: {widget_creation_key}") - - def __BeginDragFromTree(self, event): - tree = event.GetEventObject() - item = event.GetItem() - item_parent = tree.GetItemParent(item) - if not item_parent.IsOk(): - event.Skip() - return - - widget_creation_str = None - if tree.GetItemText(item_parent) == 'Systemwide Tools': - widget_creation_str = tree.GetItemText(item) - else: - elem_path = tree.GetItemElemPath(item) - simhier = self.frame.explorer.navtree.simhier - - if elem_path in simhier.GetScalarStatsElemPaths(): - widget_creation_str = f'ScalarStatistic${elem_path}' - elif elem_path in simhier.GetScalarStructsElemPaths(): - widget_creation_str = f'ScalarStruct${elem_path}' - elif elem_path in simhier.GetContainerElemPaths(): - widget_creation_str = f'IterableStruct${elem_path}' - - if widget_creation_str is None: - event.Skip() - return - data = wx.TextDataObject() - data.SetText(widget_creation_str) - source = wx.DropSource(tree) - source.SetData(data) - source.DoDragDrop(True) + raise ValueError(f"Unknown widget creation key: {widget_creation_key}") diff --git a/python/argos/viewer/gui/widgets/widget_renderer.py b/python/argos/viewer/gui/widgets/widget_renderer.py index da7a7e7f..adfd3bf3 100644 --- a/python/argos/viewer/gui/widgets/widget_renderer.py +++ b/python/argos/viewer/gui/widgets/widget_renderer.py @@ -5,8 +5,14 @@ class WidgetRenderer: def __init__(self, frame): self.frame = frame cursor = frame.db.cursor() - cursor.execute('SELECT MIN(Tick), MAX(Tick) FROM CollectionRecords') + cursor.execute('SELECT MIN(Timestamp), MAX(Timestamp) FROM Timestamps') self._start_tick, self._end_tick = cursor.fetchone() + + # Recall that uint64_t is stored as a string + if isinstance(self._start_tick, str): + self._start_tick = int(self._start_tick) + self._end_tick = int(self._end_tick) + self._current_tick = self._start_tick self._utiliz_handler = IterableUtiliz(self, frame.simhier) self._auto_colors_by_key = {} @@ -103,9 +109,6 @@ def GetAutoTag(self, value): return tag def __UpdateWidgetsOnCurrentTab(self): - self.frame.explorer.navtree.UpdateUtilizBitmaps() - self.frame.explorer.watchlist.UpdateUtilizBitmaps() - notebook = self.frame.inspector page_idx = notebook.GetSelection() page = notebook.GetPage(page_idx) diff --git a/python/argos/viewer/model/blob_handlers.py b/python/argos/viewer/model/blob_handlers.py new file mode 100644 index 00000000..46689da1 --- /dev/null +++ b/python/argos/viewer/model/blob_handlers.py @@ -0,0 +1,245 @@ +import copy +from abc import abstractmethod +from collections import OrderedDict + +# Base class for all blob handlers (for blob iteration) +class BlobHandler: + @abstractmethod + def HandleScalarClosed(self, context): pass + + @abstractmethod + def HandleScalarCarried(self, context): pass + + @abstractmethod + def HandleScalarFullDump(self, context, deserialized): pass + + @abstractmethod + def HandleContigContainerClosed(self, context): pass + + @abstractmethod + def HandleContigContainerCarried(self, context): pass + + @abstractmethod + def HandleContigContainerFullDump(self, context, deserialized): pass + + @abstractmethod + def HandleContigContainerSwap(self, context, bin_idx, deserialized): pass + + @abstractmethod + def HandleContigContainerArrival(self, context, deserialized): pass + + @abstractmethod + def HandleContigContainerDeparture(self, context): pass + + @abstractmethod + def HandleContigContainerBookends(self, context, deserialized): pass + + @abstractmethod + def HandleSparseContainerClosed(self, context): pass + + @abstractmethod + def HandleSparseContainerCarried(self, context): pass + + @abstractmethod + def HandleSparseContainerFullDump(self, context, deserialized): pass + + @abstractmethod + def HandleSparseContainerExchangedBin(self, context, bin_idx, bin_deserialized): pass + + @abstractmethod + def HandleSparseContainerRemovedBin(self, context, bin_idx): pass + + @abstractmethod + def HandleSparseContainerAddedBin(self, context, bin_idx, bin_deserialized): pass + + # Called by the blob iterator after each blob (tick) has been fully + # processed. Handlers that track per-tick state can override this. + def SnapshotTick(self, context): pass + +# Blob handler for sanity checking (ensure that we can simply iterate over any blob without parsing bugs) +class SmokeTestHandler(BlobHandler): + def HandleScalarClosed(self, context): + print(f'At tick {context.current_tick}, scalar cid {context.current_cid} was closed') + + def HandleScalarCarried(self, context): + print(f'At tick {context.current_tick}, scalar cid {context.current_cid} was carried') + + def HandleScalarFullDump(self, context, deserialized): + print(f'At tick {context.current_tick}, scalar cid {context.current_cid} was fully dumped') + + def HandleContigContainerClosed(self, context): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} was closed') + + def HandleContigContainerCarried(self, context): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} was carried') + + def HandleContigContainerFullDump(self, context, deserialized): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} was fully dumped') + + def HandleContigContainerSwap(self, context, bin_idx, deserialized): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} swapped bin {bin_idx}') + + def HandleContigContainerArrival(self, context, deserialized): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} had an arrival') + + def HandleContigContainerDeparture(self, context): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} had a departure') + + def HandleContigContainerBookends(self, context, deserialized): + print(f'At tick {context.current_tick}, contig container cid {context.current_cid} updated its bookends') + + def HandleSparseContainerClosed(self, context): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} was closed') + + def HandleSparseContainerCarried(self, context): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} was carried') + + def HandleSparseContainerFullDump(self, context, deserialized): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} was fully dumped') + + def HandleSparseContainerExchangedBin(self, context, bin_idx, bin_deserialized): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} exchanged bin {bin_idx}') + + def HandleSparseContainerRemovedBin(self, context, bin_idx): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} removed bin {bin_idx}') + + def HandleSparseContainerAddedBin(self, context, bin_idx, bin_deserialized): + print(f'At tick {context.current_tick}, sparse container cid {context.current_cid} added bin {bin_idx}') + +# Blob handler for extracting collected values/structs/containers +class DataExtractionHandler(BlobHandler): + def __init__(self, simhier, snapshot_cids=None): + self._simhier = simhier + self._values_by_cid = {} + + # When snapshot_cids is None, per-tick tracking is disabled and + # SnapshotTick() is a no-op (zero overhead for callers that only + # need the final reconstructed value). When a set of CIDs is given, + # we record a deep copy of each of those CIDs' values at every tick. + self._snapshot_cids = snapshot_cids + self._values_by_cid_by_tick = {} + + def HandleScalarClosed(self, context): + if context.current_cid in self._values_by_cid: + self._values_by_cid[context.current_cid] = None + + def HandleScalarCarried(self, context): + pass + + def HandleScalarFullDump(self, context, deserialized): + self._values_by_cid[context.current_cid] = deserialized + + def HandleContigContainerClosed(self, context): + self._values_by_cid[context.current_cid] = [] + + def HandleContigContainerCarried(self, context): + pass + + def HandleContigContainerFullDump(self, context, deserialized): + assert isinstance(deserialized, list) + assert len(deserialized) <= self._simhier.GetCapacityByCollectionID(context.current_cid) + self._values_by_cid[context.current_cid] = deserialized + + def HandleContigContainerSwap(self, context, bin_idx, deserialized): + if context.current_cid in self._values_by_cid: + container_values = self._values_by_cid[context.current_cid] + assert bin_idx < self._simhier.GetCapacityByCollectionID(context.current_cid) + assert bin_idx < len(container_values) + container_values[bin_idx] = deserialized + + def HandleContigContainerArrival(self, context, deserialized): + if context.current_cid in self._values_by_cid: + container_values = self._values_by_cid[context.current_cid] + container_values.append(deserialized) + assert len(container_values) <= self._simhier.GetCapacityByCollectionID(context.current_cid) + + def HandleContigContainerDeparture(self, context): + if context.current_cid in self._values_by_cid: + container_values = self._values_by_cid[context.current_cid] + assert len(container_values) > 0 + container_values.pop(0) + + def HandleContigContainerBookends(self, context, deserialized): + self.HandleContigContainerDeparture(context) + self.HandleContigContainerArrival(context, deserialized) + + def HandleSparseContainerClosed(self, context): + if context.current_cid in self._values_by_cid: + self._values_by_cid[context.current_cid] = None + + def HandleSparseContainerCarried(self, context): + pass + + def HandleSparseContainerFullDump(self, context, deserialized): + self._ResetSparseContainer(context, deserialized) + + def HandleSparseContainerExchangedBin(self, context, bin_idx, bin_deserialized): + if context.current_cid in self._values_by_cid: + assert bin_idx < len(self._values_by_cid[context.current_cid]) + self._values_by_cid[context.current_cid][bin_idx] = bin_deserialized + + def HandleSparseContainerRemovedBin(self, context, bin_idx): + self.HandleSparseContainerExchangedBin(context, bin_idx, None) + + def HandleSparseContainerAddedBin(self, context, bin_idx, bin_deserialized): + if context.current_cid in self._values_by_cid: + assert bin_idx < len(self._values_by_cid[context.current_cid]) + self._values_by_cid[context.current_cid][bin_idx] = bin_deserialized + + def SnapshotTick(self, context): + if self._snapshot_cids is None: + return + + for cid in self._snapshot_cids: + if cid not in self._values_by_cid: + continue + + # Deep copy is required because contig/sparse container values are + # lists mutated in place Without copying, every tick's snapshot + # would alias the same mutated list. + value = copy.deepcopy(self._values_by_cid[cid]) + if cid not in self._values_by_cid_by_tick: + self._values_by_cid_by_tick[cid] = OrderedDict() + self._values_by_cid_by_tick[cid][context.current_tick] = value + + def HasDataFor(self, ident): + cid = self._ResolveCID(ident) + return cid in self._values_by_cid + + def GetValuesByTick(self, ident): + cid = self._ResolveCID(ident) + return self._values_by_cid_by_tick.get(cid, OrderedDict()) + + def GetFinalValue(self, ident): + cid = self._ResolveCID(ident) + return self._values_by_cid.get(cid) + + def GetAllFinalValues(self, use_path_keys=True): + elem_paths = self._simhier.GetItemElemPaths() + if use_path_keys: + identifiers = elem_paths + else: + identifiers = [self._simhier.GetCollectionID(path) for path in elem_paths] + + final_values = { + ident:self.GetFinalValue(ident) + for ident in identifiers + } + + return final_values + + def _ResolveCID(self, ident): + if isinstance(ident, str): + try: + ident = int(ident) + except: + ident = self._simhier.GetCollectionID(ident) + + assert isinstance(ident, int) + return ident + + def _ResetSparseContainer(self, context, deserialized): + capacity = self._simhier.GetCapacityByCollectionID(context.current_cid) + self._values_by_cid[context.current_cid] = [None]*capacity + for bin_idx, bin_deserialized in deserialized.items(): + self._values_by_cid[context.current_cid][bin_idx] = bin_deserialized diff --git a/python/argos/viewer/model/blob_iterator.py b/python/argos/viewer/model/blob_iterator.py new file mode 100644 index 00000000..b0ada2d4 --- /dev/null +++ b/python/argos/viewer/model/blob_iterator.py @@ -0,0 +1,404 @@ +import os, sys, zlib +from typing import Any +from functools import partial + +_PACKAGE_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if _PACKAGE_ROOT not in sys.path: + sys.path.insert(0, _PACKAGE_ROOT) + +from viewer.model.data_deserializers import ByteBuffer +from viewer.model.blob_handlers import * + +# Tier 1: 0x00–0x0F — lifecycle / common (scalars + all collectables) +_CLOSED = 0x00 +_FULL = 0x01 +_CARRY = 0x02 +# 0x03–0x0F reserved + +# Tier 2: 0x10–0x1F — any-container (identical wire for contig & sparse) +_CONTAINER_SWAP = 0x10 +_CONTAINER_MULTI_SWAP = 0x11 +# 0x12–0x1F reserved + +# Tier 3: 0x20–0x2F — contig-specific (FIFO queue semantics) +_CONTIG_ARRIVE = 0x20 +_CONTIG_DEPART = 0x21 +_CONTIG_BOOKENDS = 0x22 +_CONTIG_MIMO = 0x23 +# 0x24–0x2F reserved + +# Tier 4: 0x30–0x3F — sparse-specific (explicit bin indices) +_SPARSE_REMOVE = 0x30 +_SPARSE_ADD = 0x31 +_SPARSE_MULTI_REMOVE = 0x32 +# 0x33–0x3F reserved + +_VALID_COMMON_ACTIONS = { + _CLOSED, + _FULL, + _CARRY +} + +_VALID_SCALAR_ACTIONS = _VALID_COMMON_ACTIONS + +_VALID_ANY_CONTAINER_ACTIONS = { + _CONTAINER_SWAP, + _CONTAINER_MULTI_SWAP, +} + +_VALID_CONTIG_CONTAINER_ACTIONS = _VALID_COMMON_ACTIONS | _VALID_ANY_CONTAINER_ACTIONS | \ + { + _CONTIG_ARRIVE, + _CONTIG_DEPART, + _CONTIG_BOOKENDS, + _CONTIG_MIMO, + } + +_VALID_SPARSE_CONTAINER_ACTIONS = _VALID_COMMON_ACTIONS | _VALID_ANY_CONTAINER_ACTIONS | \ + { + _SPARSE_REMOVE, + _SPARSE_ADD, + _SPARSE_MULTI_REMOVE, + } + +class Resources: + def __init__(self, dtype_inspector, simhier, handler): + self.dtype_inspector = dtype_inspector + self.simhier = simhier + self.handler = handler + self.buf = None + + def GetDeserializer(self, cid): + type_name = self.dtype_inspector.GetDataTypeForCollectionID(cid) + deserializer = self.dtype_inspector.GetDeserializer(type_name) + if cid in self.simhier.GetContainerIDs(): + deserializer = deserializer._bin_deserializer + + return deserializer + +class Context: + def __init__(self): + self.current_cid = None + self.current_tick = None + +def HandleCID(resources, context): + if resources.buf.Done(): + return + + cid = resources.buf.Read('H') + context.current_cid = cid + return HandleAction + +def HandleAction(resources, context): + cid = context.current_cid + action = int(resources.buf.Read('B')) + + if cid not in resources.simhier.GetContainerIDs(): + return partial(HandleScalarAction, action=action) + elif resources.simhier.GetSparseFlagByCollectionID(cid): + return partial(HandleSparseContainerAction, action=action) + else: + return partial(HandleContigContainerAction, action=action) + +def HandleScalarAction(resources, context, action): + assert action in _VALID_SCALAR_ACTIONS + + if action == _CLOSED: + resources.handler.HandleScalarClosed(context) + elif action == _CARRY: + resources.handler.HandleScalarCarried(context) + else: + type_deserializer = resources.GetDeserializer(context.current_cid) + deserialized = type_deserializer.Deserialize(resources.buf) + + if action == _FULL: + resources.handler.HandleScalarFullDump(context, deserialized) + + return HandleCID + +def HandleContigContainerAction(resources, context, action): + assert action in _VALID_CONTIG_CONTAINER_ACTIONS + + if action == _CLOSED: + resources.handler.HandleContigContainerClosed(context) + elif action == _CARRY: + resources.handler.HandleContigContainerCarried(context) + else: + type_deserializer = resources.GetDeserializer(context.current_cid) + + if action == _FULL: + size = resources.buf.Read('H') + deserialized = [] + for _ in range(size): + deserialized.append(type_deserializer.Deserialize(resources.buf)) + + resources.handler.HandleContigContainerFullDump(context, deserialized) + + elif action == _CONTAINER_SWAP: + bin_idx = resources.buf.Read('H') + deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleContigContainerSwap(context, bin_idx, deserialized) + + elif action == _CONTAINER_MULTI_SWAP: + count = resources.buf.Read('B') + for _ in range(count): + bin_idx = resources.buf.Read('H') + deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleContigContainerSwap(context, bin_idx, deserialized) + + elif action == _CONTIG_ARRIVE: + deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleContigContainerArrival(context, deserialized) + + elif action == _CONTIG_DEPART: + resources.handler.HandleContigContainerDeparture(context) + + elif action == _CONTIG_BOOKENDS: + deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleContigContainerBookends(context, deserialized) + + elif action == _CONTIG_MIMO: + depart_count = resources.buf.Read('B') + arrive_count = resources.buf.Read('B') + for _ in range(depart_count): + resources.handler.HandleContigContainerDeparture(context) + for _ in range(arrive_count): + deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleContigContainerArrival(context, deserialized) + + return HandleCID + +def HandleSparseContainerAction(resources, context, action): + assert action in _VALID_SPARSE_CONTAINER_ACTIONS + + if action == _CLOSED: + resources.handler.HandleSparseContainerClosed(context) + elif action == _CARRY: + resources.handler.HandleSparseContainerCarried(context) + else: + type_deserializer = resources.GetDeserializer(context.current_cid) + + if action == _FULL: + size = resources.buf.Read('H') + deserialized = {} + for _ in range(size): + bin_idx = resources.buf.Read('H') + bin_deserialized = type_deserializer.Deserialize(resources.buf) + deserialized[bin_idx] = bin_deserialized + + resources.handler.HandleSparseContainerFullDump(context, deserialized) + + elif action == _CONTAINER_SWAP: + bin_idx = resources.buf.Read('H') + bin_deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleSparseContainerExchangedBin(context, bin_idx, bin_deserialized) + + elif action == _CONTAINER_MULTI_SWAP: + count = resources.buf.Read('B') + for _ in range(count): + bin_idx = resources.buf.Read('H') + bin_deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleSparseContainerExchangedBin(context, bin_idx, bin_deserialized) + + elif action == _SPARSE_REMOVE: + bin_idx = resources.buf.Read('H') + resources.handler.HandleSparseContainerRemovedBin(context, bin_idx) + + elif action == _SPARSE_ADD: + bin_idx = resources.buf.Read('H') + bin_deserialized = type_deserializer.Deserialize(resources.buf) + resources.handler.HandleSparseContainerAddedBin(context, bin_idx, bin_deserialized) + + elif action == _SPARSE_MULTI_REMOVE: + count = resources.buf.Read('B') + for _ in range(count): + bin_idx = resources.buf.Read('H') + resources.handler.HandleSparseContainerRemovedBin(context, bin_idx) + + return HandleCID + +class BlobIterator: + def __init__(self, dtype_inspector, simhier): + self._dtype_inspector = dtype_inspector + self._simhier = simhier + self._final_tick = None + + @property + def connection(self): + return self._dtype_inspector.connection + + def GetFinalTick(self): + assert self._final_tick is not None, 'Iterate() never called' + return self._final_tick + + def Iterate(self, handler: BlobHandler, time_range: Any = None, lookback: bool = False, + clock_ids: Any = None): + cursor = self.connection.cursor() + if time_range is None: + rows = self._fetch_all_rows(cursor, clock_ids) + else: + if isinstance(time_range, int): + time_range = [time_range, time_range] + elif not isinstance(time_range, list): + time_range = list(time_range) + if len(time_range) == 1: + time_range *= 2 + + lo, hi = int(time_range[0]), int(time_range[1]) + if lookback: + cursor.execute('SELECT Heartbeat FROM CollectionGlobals') + heartbeat = int(cursor.fetchone()[0]) + rows = self._fetch_lookback_rows(cursor, lo, hi, heartbeat, clock_ids) + else: + rows = self._fetch_time_range_rows(cursor, lo, hi, clock_ids) + + resources = Resources(self._dtype_inspector, self._simhier, handler) + context = Context() + for tsid, ts, compressed_blob in rows: + resources.buf = ByteBuffer(zlib.decompress(compressed_blob)) + context.current_tick = int(ts) + + handler_func = HandleCID + while handler_func: + handler_func = handler_func(resources, context) + handler.SnapshotTick(context) + + self._final_tick = context.current_tick + + def _has_timestamp_clocks(self, cursor): + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='TimestampClocks'" + ) + return cursor.fetchone() is not None + + def _normalize_clock_ids(self, clock_ids): + if clock_ids is None: + return None + if isinstance(clock_ids, int): + return [clock_ids] + return list(clock_ids) + + def _use_clock_filter(self, cursor, clock_ids): + clock_ids = self._normalize_clock_ids(clock_ids) + return bool(clock_ids) and self._has_timestamp_clocks(cursor) + + def _fetch_all_rows(self, cursor, clock_ids=None): + clock_ids = self._normalize_clock_ids(clock_ids) + if self._use_clock_filter(cursor, clock_ids): + placeholders = ','.join('?' for _ in clock_ids) + cursor.execute( + 'SELECT DISTINCT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'INNER JOIN TimestampClocks tc ON tc.TimestampID = t.Id ' + f'WHERE tc.ClockID IN ({placeholders}) ' + 'ORDER BY t.Id ASC', + tuple(clock_ids), + ) + else: + cursor.execute( + 'SELECT DISTINCT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'ORDER BY t.Id ASC' + ) + return cursor.fetchall() + + def _fetch_time_range_rows(self, cursor, lo, hi, clock_ids=None): + clock_ids = self._normalize_clock_ids(clock_ids) + if self._use_clock_filter(cursor, clock_ids): + placeholders = ','.join('?' for _ in clock_ids) + cursor.execute( + 'SELECT DISTINCT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'INNER JOIN TimestampClocks tc ON tc.TimestampID = t.Id ' + f'WHERE tc.ClockID IN ({placeholders}) ' + 'AND CAST(t.Timestamp AS INTEGER) >= ? AND CAST(t.Timestamp AS INTEGER) <= ? ' + 'ORDER BY t.Id ASC', + tuple(clock_ids) + (lo, hi), + ) + else: + cursor.execute( + 'SELECT DISTINCT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'WHERE CAST(t.Timestamp AS INTEGER) >= ? AND CAST(t.Timestamp AS INTEGER) <= ? ' + 'ORDER BY t.Id ASC', + (lo, hi), + ) + return cursor.fetchall() + + def _fetch_lookback_rows(self, cursor, start_tick, end_tick, heartbeat, clock_ids=None): + clock_ids = self._normalize_clock_ids(clock_ids) + if self._use_clock_filter(cursor, clock_ids): + placeholders = ','.join('?' for _ in clock_ids) + params = tuple(clock_ids) + (start_tick, heartbeat) + tuple(clock_ids) + (end_tick,) + cursor.execute( + 'WITH eligible AS (' + ' SELECT DISTINCT t.Id ' + ' FROM Timestamps t ' + ' INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + ' INNER JOIN TimestampClocks tc ON tc.TimestampID = t.Id ' + f' WHERE tc.ClockID IN ({placeholders}) ' + ' AND CAST(t.Timestamp AS INTEGER) <= ?' + '), warmup AS (' + ' SELECT Id FROM eligible ORDER BY Id DESC LIMIT ?' + ') ' + 'SELECT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'INNER JOIN TimestampClocks tc ON tc.TimestampID = t.Id ' + f'WHERE tc.ClockID IN ({placeholders}) ' + ' AND t.Id >= (SELECT MIN(Id) FROM warmup) ' + ' AND CAST(t.Timestamp AS INTEGER) <= ? ' + 'ORDER BY t.Id ASC', + params, + ) + else: + cursor.execute( + 'WITH warmup AS (' + ' SELECT t.Id ' + ' FROM Timestamps t ' + ' INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + ' WHERE CAST(t.Timestamp AS INTEGER) <= ? ' + ' ORDER BY t.Id DESC ' + ' LIMIT ?' + ') ' + 'SELECT t.Id, t.Timestamp, cr.Records ' + 'FROM Timestamps t ' + 'INNER JOIN CollectionRecords cr ON cr.TimestampID = t.Id ' + 'WHERE t.Id >= (SELECT MIN(Id) FROM warmup) ' + ' AND CAST(t.Timestamp AS INTEGER) <= ? ' + 'ORDER BY t.Id ASC', + (start_tick, heartbeat, end_tick), + ) + return cursor.fetchall() + +def main(): + from viewer.model.dtype_inspector import DataTypeInspector + from viewer.model.simhier import SimHierarchy + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--tick", type=int, help="Extract the data values at this tick") + parser.add_argument("database", help="Path to the database file") + args = parser.parse_args() + + dtype_inspector = DataTypeInspector(args.database) + simhier = SimHierarchy(dtype_inspector.connection, dtype_inspector) + iterator = BlobIterator(dtype_inspector, simhier) + + # No tick provided? Use smoke test handler. + if args.tick is None: + handler = SmokeTestHandler() + iterator.Iterate(handler) + + # Tick provided? Use the last collection records at or before tick. + else: + handler = DataExtractionHandler(simhier) + iterator.Iterate(handler, args.tick, lookback=True) + print(handler.GetAllFinalValues()) + +if __name__ == '__main__': + main() diff --git a/python/argos/viewer/model/data_deserializers.py b/python/argos/viewer/model/data_deserializers.py new file mode 100644 index 00000000..549bfcc0 --- /dev/null +++ b/python/argos/viewer/model/data_deserializers.py @@ -0,0 +1,396 @@ +# Argos collection-record blob decoding. +from collections import OrderedDict +import copy + +# POD types whose ``CONVERTERS`` yield integers suitable for ``hex()`` display. +_INTEGRAL_DISPLAY_DTYPES = frozenset( + { + "signed char", + "unsigned char", + "short", + "unsigned short", + "int", + "unsigned int", + "long", + "unsigned long", + "long long", + "unsigned long long", + } +) + + +def _make_simple_output_formatter(dtype_name, special_formatter): + # type: (str, str) -> object + fmt = (special_formatter or "").strip().lower() + if fmt not in ("hex", "oct"): + return lambda x: x + if dtype_name not in _INTEGRAL_DISPLAY_DTYPES: + return lambda x: x + + if fmt == "hex": + + def _fmt_hex(val): + if isinstance(val, int) and not isinstance(val, bool): + return hex(val) + return val + + return _fmt_hex + + def _fmt_oct(val): + if isinstance(val, int) and not isinstance(val, bool): + return oct(val) + return val + + return _fmt_oct + + +UNPACK_FORMATS = { + 'char': 'b', + 'signed char': 'b', + 'unsigned char': 'B', + 'short': 'h', + 'unsigned short': 'H', + 'int': 'i', + 'unsigned int': 'I', + 'long': 'q', + 'unsigned long': 'Q', + 'double': 'd', + 'float': 'f', + 'bool': 'B' +} + +# Helper class used to minimize the number of byte array copies +# we make during deserialization. +import struct +class ByteBuffer: + def __init__(self, data_bytes): + assert isinstance(data_bytes, bytes) + self._read_idx = 0 + self._end_idx = len(data_bytes) + self._data_bytes = data_bytes + + def Read(self, fmt): + if fmt in UNPACK_FORMATS: + fmt = UNPACK_FORMATS[fmt] + nbytes = struct.calcsize(fmt) + assert self._read_idx + nbytes <= self._end_idx + + val_bytes = self._data_bytes[self._read_idx : self._read_idx + nbytes] + val = struct.unpack(fmt, val_bytes) + if len(fmt) == 1: + val = val[0] + + self._read_idx += nbytes + return val + + def Extract(self, num_bytes): + assert self._read_idx + num_bytes <= self._end_idx + extracted_bytes = self._data_bytes[self._read_idx : self._read_idx + num_bytes] + self._read_idx += num_bytes + return extracted_bytes + + def Jump(self, num_bytes): + assert self._read_idx + num_bytes <= self._end_idx + self._read_idx += num_bytes + + def Done(self): + assert self._read_idx <= self._end_idx + return self._read_idx == self._end_idx + + @classmethod + def CreateFrom(cls, obj): + if isinstance(obj, cls): + return obj + else: + return cls(obj) + +def CreateDeserializer(inspector, dtype_name, tiny_strings=None): + # Extract sparse/contig and capacity from a data type name + def GetContainerMeta(dtype_name): + def GetMeta(which): + key = f'_{which}_capacity' + idx = dtype_name.find(key) + if idx != -1: + base_dtype_name = dtype_name[:idx] + capacity = dtype_name.replace(base_dtype_name + key, '') + capacity = int(capacity) + return (base_dtype_name, capacity) + + return None + + meta = GetMeta('sparse') + if meta: + return (meta[0], meta[1], 'sparse') + if not meta: + meta = GetMeta('contig') + if meta: + return (meta[0], meta[1], 'contig') + + return None + + # Simple types (non-enum) + if dtype_name in UNPACK_FORMATS: + special = ( + inspector.GetSimpleTypeSpecialFormatter(dtype_name) if inspector is not None else "" + ) + return SimpleDeserializer(dtype_name, special) + + # String types (collected as uint32_t) + if dtype_name == 'string': + return StringDeserializer(tiny_strings) + + # Container types + container_meta = GetContainerMeta(dtype_name) + if container_meta: + dtype_name, capacity, sparse_mode = container_meta + bin_deserializer = inspector.GetDeserializer(dtype_name) + if sparse_mode == 'sparse': + return SparseContainerDeserializer(bin_deserializer, capacity) + else: + return ContigContainerDeserializer(bin_deserializer, capacity) + + # Struct types + struct_defn = inspector.GetStructDefn(dtype_name) + if struct_defn is not None: + return StructDeserializer(dtype_name, struct_defn, inspector, tiny_strings) + + # Enum types + enum_map = inspector.GetEnumMap(dtype_name) + if enum_map: + backing_kind = inspector.GetEnumBackingKind(dtype_name) + if backing_kind in SimpleDeserializer.CONVERTERS: + val_deserializer = SimpleDeserializer(backing_kind, "") + return EnumDeserializer(val_deserializer, enum_map) + + return None + +# This class deserializes non-enum POD types. +class SimpleDeserializer: + CONVERTERS = { + 'char': lambda x: str(x), + 'signed char': lambda x: int(x), + 'unsigned char': lambda x: int(x), + 'short': lambda x: int(x), + 'unsigned short': lambda x: int(x), + 'int': lambda x: int(x), + 'unsigned int': lambda x: int(x), + 'long': lambda x: int(x), + 'unsigned long': lambda x: int(x), + 'double': lambda x: float(x), + 'float': lambda x: float(x), + 'bool': lambda x: bool(x) + } + + NUM_BYTES = { + 'char': 1, + 'signed char': 1, + 'unsigned char': 1, + 'short': 2, + 'unsigned short': 2, + 'int': 4, + 'unsigned int': 4, + 'long': 8, + 'unsigned long': 8, + 'double': 8, + 'float': 4, + 'bool': 1 + } + + def __init__(self, dtype_name, special_formatter=""): + # type: (str, str) -> None + self._fmt = UNPACK_FORMATS[dtype_name] + self._converter = self.CONVERTERS[dtype_name] + self._num_bytes = self.NUM_BYTES[dtype_name] + self._formatter = _make_simple_output_formatter(dtype_name, special_formatter or "") + + def GetNumBytes(self): + return self._num_bytes + + def Deserialize(self, data_bytes): + buf = ByteBuffer.CreateFrom(data_bytes) + val = buf.Read(self._fmt) + val = self._converter(val) + return self._formatter(val) + +# This class deserializes string types. +class StringDeserializer: + def __init__(self, tiny_strings): + assert tiny_strings is not None + self._tiny_strings = tiny_strings + + def GetNumBytes(self): + # Always stored as uint32_t + return 4 + + def Deserialize(self, data_bytes): + buf = ByteBuffer.CreateFrom(data_bytes) + string_id = buf.Read('I') + return self._tiny_strings.GetString(string_id, must_exist=True) + +# This class deserializes enum types. +class EnumDeserializer: + def __init__(self, val_deserializer, enum_map): + self._val_deserializer = val_deserializer + + self._enum_map = {} + for e_name, e_int in enum_map.items(): + assert isinstance(e_name, str) + assert isinstance(e_int, int) + self._enum_map[e_int] = e_name + + self._enum_map = {k:v for v,k in enum_map.items()} + + def GetNumBytes(self): + # Defer to the int deserializer + return self._val_deserializer.GetNumBytes() + + def Deserialize(self, data_bytes): + enum_val = self._val_deserializer.Deserialize(data_bytes) + enum_val = int(enum_val) + return self._enum_map[enum_val] + +# This class deserializes contiguous container types. +class ContigContainerDeserializer: + def __init__(self, bin_deserializer, capacity): + self._bin_deserializer = bin_deserializer + self._capacity = capacity + + def GetCapacity(self): + return self._capacity + + def GetAllFieldNames(self): + return self._bin_deserializer.GetAllFieldNames() + + def GetVisibleFieldNames(self): + # TODO cnyce + return self.GetAllFieldNames() + + def GetBinNumBytes(self): + return self._bin_deserializer.GetNumBytes() + + def Deserialize(self, data_bytes): + buf = ByteBuffer.CreateFrom(data_bytes) + + # First 2 bytes always give the container size + size = buf.Read('H') + assert size <= self._capacity + + # Create the container + container = [] + while len(container) < size: + bin_val = self._bin_deserializer.Deserialize(buf) + container.append(bin_val) + + return container + +# This class deserializes sparse container types. +class SparseContainerDeserializer: + def __init__(self, bin_deserializer, capacity): + self._bin_deserializer = bin_deserializer + self._capacity = capacity + + def GetCapacity(self): + return self._capacity + + def GetAllFieldNames(self): + return self._bin_deserializer.GetAllFieldNames() + + def GetVisibleFieldNames(self): + # TODO cnyce + return self.GetAllFieldNames() + + def GetBinNumBytes(self): + # Account for uint16_t bin index for each element + return self._bin_deserializer.GetNumBytes() + 2 + + def Deserialize(self, data_bytes): + buf = ByteBuffer.CreateFrom(data_bytes) + + # First 2 bytes always give the container size + size = buf.Read('H') + assert size <= self._capacity + + # Create the container; recall that sparse containers + # store None wherever there is no data + container = [None] * self._capacity + + # Read each element (bin), noting that each one is + # preceeded by a uint16_t which gives the bin idx. + while size > 0: + bin_idx = buf.Read('H') + bin_val = self._bin_deserializer.Deserialize(buf) + container[bin_idx] = bin_val + size -= 1 + + return container + +# This class deserializes struct types. +class StructDeserializer: + def __init__(self, struct_name, struct_defn, inspector, tiny_strings): + self.struct_name = struct_name + self._field_deserializers = [] + self._flattened_field_names = [] + StructDeserializer.__RecurseFindCollectableFields( + struct_defn, inspector, tiny_strings, self._field_deserializers, self._flattened_field_names + ) + assert self._field_deserializers + assert self._flattened_field_names + self._visible_field_names = self._flattened_field_names + + def GetAllFieldNames(self): + return copy.deepcopy(self._flattened_field_names) + + def GetVisibleFieldNames(self): + return copy.deepcopy(self._visible_field_names) + + def SetVisibleFieldNames(self, field_names): + # type: (list) -> None + if not field_names: + return + allowed = set(self._flattened_field_names) + visible = [] + for field_name in field_names: + if field_name in allowed and field_name not in visible: + visible.append(field_name) + if visible: + self._visible_field_names = visible + + def GetNumBytes(self): + num_bytes = 0 + for field_name, deserializer in self._field_deserializers: + num_bytes += deserializer.GetNumBytes() + return num_bytes + + def Deserialize(self, data_bytes): + buf = ByteBuffer.CreateFrom(data_bytes) + deserialized = [] + + for field_name, deserializer in self._field_deserializers: + deserialized.append((field_name, deserializer.Deserialize(buf))) + + return deserialized + + @staticmethod + def __RecurseFindCollectableFields( + struct_defn, inspector, tiny_strings, field_deserializers, flattened_field_names + ): + for field in struct_defn.children: + if field.name and field.kind != 'struct': + flattened_field_names.append(field.name) + if field.kind == 'pod' and field.type_name != 'string': + deserializer = SimpleDeserializer( + field.type_name, field.special_formatter or "" + ) + field_deserializers.append((field.name, deserializer)) + elif field.type_name == 'string': + deserializer = StringDeserializer(tiny_strings) + field_deserializers.append((field.name, deserializer)) + elif field.kind == 'enum': + deserializer = CreateDeserializer(inspector, field.type_name) + field_deserializers.append((field.name, deserializer)) + elif field.kind == 'struct': + StructDeserializer.__RecurseFindCollectableFields( + field, inspector, tiny_strings, field_deserializers, flattened_field_names + ) + else: + raise ValueError(f'Unknown field data type: {field.kind}') diff --git a/python/argos/viewer/model/data_retriever.py b/python/argos/viewer/model/data_retriever.py index 7eb8ef10..9dc7ca67 100644 --- a/python/argos/viewer/model/data_retriever.py +++ b/python/argos/viewer/model/data_retriever.py @@ -1,159 +1,87 @@ -import zlib, struct, copy, re -from enum import IntEnum -from viewer.gui.view_settings import DirtyReasons +import zlib, copy, sqlite3 +from viewer.model.dirty_reasons import DirtyReasons +from viewer.model.data_deserializers import ContigContainerDeserializer +from viewer.model.data_deserializers import SparseContainerDeserializer +from viewer.model.data_deserializers import StructDeserializer +from viewer.model.blob_iterator import BlobIterator +from viewer.model.blob_handlers import DataExtractionHandler class DataRetriever: - def __init__(self, frame, db, simhier): + def __init__(self, frame, db_path, simhier, dtype_inspector): self.frame = frame - self._db = db + self._db = sqlite3.connect(db_path) self.simhier = simhier - self.cursor = db.cursor() + self.dtype_inspector = dtype_inspector + self.cursor = self._db.cursor() cursor = self.cursor cursor.execute('SELECT Heartbeat FROM CollectionGlobals') self._heartbeat = cursor.fetchone()[0] - cursor.execute('SELECT DISTINCT(Tick) FROM CollectionRecords ORDER BY Tick ASC') + cursor.execute('SELECT DISTINCT(Timestamp) FROM Timestamps ORDER BY Timestamp ASC') self._time_vals = [row[0] for row in cursor.fetchall()] - self._displayed_columns_by_struct_name = {} - self._auto_colorize_column_by_struct_name = {} + for i,t in enumerate(self._time_vals): + if isinstance(t, str): + self._time_vals[i] = int(t) + + self._displayed_columns_by_elem_path = {} + self._auto_colorize_column_by_elem_path = {} self._cached_utiliz_sizes = {} self._cached_utiliz_time_val = None - self._collection_ids_by_elem_path = {} - for elem_path in simhier.GetElemPaths(): - collection_id = simhier.GetCollectionID(elem_path) - if collection_id: - self._collection_ids_by_elem_path[elem_path] = collection_id - - cursor.execute('SELECT IntVal,String FROM StringMap') - strings_by_int = {} - for intval,stringval in cursor.fetchall(): - strings_by_int[intval] = stringval - - cursor.execute('SELECT EnumName,EnumValStr,EnumValBlob,IntType FROM EnumDefns') - enums_by_name = {} - for enum_name,enum_str,enum_blob,int_type in cursor.fetchall(): - if enum_name not in enums_by_name: - enums_by_name[enum_name] = EnumDef(enum_name, int_type) - - enums_by_name[enum_name].AddEntry(enum_str, enum_blob) - - cursor.execute('SELECT StructName,FieldType FROM StructFields') - struct_num_bytes_by_struct_name = {} - for struct_name, field_type in cursor.fetchall(): - if struct_name not in struct_num_bytes_by_struct_name: - struct_num_bytes_by_struct_name[struct_name] = 0 - - if field_type in ('char', 'int8_t', 'uint8_t'): - struct_num_bytes_by_struct_name[struct_name] += 1 - elif field_type in ('int16_t', 'uint16_t'): - struct_num_bytes_by_struct_name[struct_name] += 2 - elif field_type in ('int32_t', 'uint32_t', 'float_t'): - struct_num_bytes_by_struct_name[struct_name] += 4 - elif field_type in ('int64_t', 'uint64_t', 'double_t'): - struct_num_bytes_by_struct_name[struct_name] += 8 - elif field_type == 'bool': - struct_num_bytes_by_struct_name[struct_name] += 4 # bools are stored as int32_t - elif field_type == 'string_t': - struct_num_bytes_by_struct_name[struct_name] += 4 # strings are stored as uint32_t - elif field_type in enums_by_name: - struct_num_bytes_by_struct_name[struct_name] += enums_by_name[field_type].GetNumBytes() - else: - raise ValueError('Invalid field type: ' + field_type) - - self._deserializers_by_dtype = {} - pod_dtypes = ('char', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'float_t', 'double_t', 'bool') - for dtype in pod_dtypes: - self._deserializers_by_dtype[dtype] = PODDeserializer(dtype) - - for dtype in enums_by_name.keys(): - self._deserializers_by_dtype[dtype] = EnumDeserializer(enums_by_name[dtype]) - - self._deserializers_by_dtype['string_t'] = StringDeserializer(strings_by_int) - - for struct_name in struct_num_bytes_by_struct_name.keys(): - deserializer = StructDeserializer(self, struct_name, strings_by_int, enums_by_name) - cmd = 'SELECT FieldName,FieldType,FormatCode FROM StructFields WHERE StructName="{}"'.format(struct_name) - cursor.execute(cmd) - - for field_name, field_type, format_code in cursor.fetchall(): - field_deserializer = self._deserializers_by_dtype[field_type] - deserializer.AddField(field_name, field_deserializer, format_code) - - self._deserializers_by_dtype[struct_name] = deserializer - - cursor.execute('SELECT ElementTreeNodeID,DataType,AutoCollected FROM CollectableTreeNodes') - self._replayers_by_elem_path = {} + self._cids_by_elem_path = {} + self._elem_paths_by_cid = {} + def HandleLeaf(leaf): + cid = leaf.GetMeta('CID') + elem_path = leaf.GetPath() + self._cids_by_elem_path[elem_path] = cid + self._elem_paths_by_cid[cid] = elem_path + + simhier.GetTree().VisitLeaves(HandleLeaf) + + cursor.execute('SELECT FullPath,TypeName FROM CollectableTreeNodes') self._dtypes_by_elem_path = {} - self._auto_collected_cids = set() - for id, dtype, auto_collected in cursor.fetchall(): - elem_path = simhier.GetElemPath(id) + for elem_path, dtype in cursor.fetchall(): self._dtypes_by_elem_path[elem_path] = dtype - if auto_collected: - self._auto_collected_cids.add(id) - - if dtype in ('char', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'float_t', 'double_t', 'bool'): - self._replayers_by_elem_path[elem_path] = PODReplayer(dtype) - elif dtype == 'string_t': - self._replayers_by_elem_path[elem_path] = StringReplayer(strings_by_int) - elif dtype in enums_by_name: - self._replayers_by_elem_path[elem_path] = EnumReplayer(dtype, enums_by_name) - elif '_contig_capacity' in dtype: - pattern = re.compile(r'(.*)_contig_capacity(\d+)') - match = pattern.match(dtype) - if match is None: - raise ValueError('Invalid data type: ' + dtype) - - struct_name = match.group(1) - capacity = int(match.group(2)) - struct_num_bytes = struct_num_bytes_by_struct_name[struct_name] - self._replayers_by_elem_path[elem_path] = ContigIterableReplayer(struct_num_bytes, capacity) - elif '_sparse_capacity' in dtype: - pattern = re.compile(r'(.*)_sparse_capacity(\d+)') - match = pattern.match(dtype) - if match is None: - raise ValueError('Invalid data type: ' + dtype) - - struct_name = match.group(1) - capacity = int(match.group(2)) - struct_num_bytes = struct_num_bytes_by_struct_name[struct_name] - self._replayers_by_elem_path[elem_path] = SparseIterableReplayer(struct_num_bytes, capacity) - else: - struct_name = dtype - struct_num_bytes = struct_num_bytes_by_struct_name[struct_name] - self._replayers_by_elem_path[elem_path] = ScalarStructReplayer(struct_name, struct_num_bytes) - - def IsDevDebug(self): - return self.frame.dev_debug + self.ResetToDefaultViewSettings(update_widgets=False) def GetCurrentViewSettings(self): settings = {} - assert set(self._displayed_columns_by_struct_name.keys()) == set(self._auto_colorize_column_by_struct_name.keys()) + assert set(self._displayed_columns_by_elem_path.keys()) == set(self._auto_colorize_column_by_elem_path.keys()) - for struct_name,displayed_columns in self._displayed_columns_by_struct_name.items(): + for elem_path, displayed_columns in self._displayed_columns_by_elem_path.items(): assert len(displayed_columns) > 0 - settings[struct_name] = {'auto_colorize_column':None} - settings[struct_name]['displayed_columns'] = copy.deepcopy(displayed_columns) - - for struct_name,auto_colorize_column in self._auto_colorize_column_by_struct_name.items(): - assert struct_name in settings - settings[struct_name]['auto_colorize_column'] = None if not auto_colorize_column else auto_colorize_column + settings[elem_path] = {'auto_colorize_column': None} + settings[elem_path]['displayed_columns'] = copy.deepcopy(displayed_columns) + + for elem_path, auto_colorize_column in self._auto_colorize_column_by_elem_path.items(): + assert elem_path in settings + settings[elem_path]['auto_colorize_column'] = None if not auto_colorize_column else auto_colorize_column return settings - + def ApplyViewSettings(self, settings): - self._displayed_columns_by_struct_name = {} - self._auto_colorize_column_by_struct_name = {} + self._displayed_columns_by_elem_path = {} + self._auto_colorize_column_by_elem_path = {} - for struct_name,struct_settings in settings.items(): + for elem_path, struct_settings in settings.items(): displayed_columns = struct_settings['displayed_columns'] auto_colorize_column = struct_settings['auto_colorize_column'] + self._displayed_columns_by_elem_path[elem_path] = copy.deepcopy(displayed_columns) + self._auto_colorize_column_by_elem_path[elem_path] = auto_colorize_column - self._displayed_columns_by_struct_name[struct_name] = copy.deepcopy(displayed_columns) - self._auto_colorize_column_by_struct_name[struct_name] = auto_colorize_column + for elem_path in self.simhier.GetContainerElemPaths(): + struct_name, struct_deserializer = self.__GetStructViewMeta(elem_path) + if struct_name is None or struct_deserializer is None: + continue + visible = self._displayed_columns_by_elem_path.get(elem_path) + if visible: + struct_deserializer.SetVisibleFieldNames(visible) + ac = self._auto_colorize_column_by_elem_path.get(elem_path) + if ac not in struct_deserializer.GetVisibleFieldNames(): + self._auto_colorize_column_by_elem_path[elem_path] = None self.frame.inspector.RefreshWidgetsOnAllTabs() @@ -166,688 +94,190 @@ def ApplyUserSettings(self, settings): pass def ResetToDefaultViewSettings(self, update_widgets=True): - self.cursor.execute('SELECT DISTINCT(StructName) FROM StructFields') - struct_names = [] - for struct_name in self.cursor.fetchall(): - struct_names.append(struct_name[0]) - - self._auto_colorize_column_by_struct_name = {} - self._displayed_columns_by_struct_name = {} - - for struct_name in struct_names: - cmd = 'SELECT FieldName FROM StructFields WHERE StructName="{}" AND IsAutoColorizeKey=1'.format(struct_name) - self.cursor.execute(cmd) - auto_colorize_column = [row[0] for row in self.cursor.fetchall()] - assert len(auto_colorize_column) <= 1 - if len(auto_colorize_column) == 1: - self._auto_colorize_column_by_struct_name[struct_name] = auto_colorize_column[0] - else: - self._auto_colorize_column_by_struct_name[struct_name] = None - - cmd = 'SELECT FieldName FROM StructFields WHERE StructName="{}" AND IsDisplayedByDefault=1'.format(struct_name) - self.cursor.execute(cmd) - displayed_columns = [row[0] for row in self.cursor.fetchall()] - assert len(displayed_columns) > 0 - self._displayed_columns_by_struct_name[struct_name] = displayed_columns + self._auto_colorize_column_by_elem_path = {} + self._displayed_columns_by_elem_path = {} + for elem_path in self.simhier.GetContainerElemPaths(): + struct_name, struct_deserializer = self.__GetStructViewMeta(elem_path) + if struct_name is None or struct_deserializer is None: + continue + + all_columns = struct_deserializer.GetAllFieldNames() + hidden_csv = self.simhier.GetMetaAtPath(elem_path, 'ArgosDefaultHiddenColumns') or '' + hidden_set = {x.strip() for x in hidden_csv.split(',') if x.strip()} + visible = [c for c in all_columns if c not in hidden_set] + if not visible: + visible = all_columns + struct_deserializer.SetVisibleFieldNames(visible) + self._displayed_columns_by_elem_path[elem_path] = list(visible) + + default_color_col = self.dtype_inspector.GetEffectiveColorKey(struct_name) + if default_color_col not in visible: + default_color_col = None + self._auto_colorize_column_by_elem_path[elem_path] = default_color_col if update_widgets: self.frame.inspector.RefreshWidgetsOnAllTabs() def SetVisibleFieldNames(self, elem_path, field_names): - deserializer = self.GetDeserializer(elem_path) - struct_name = deserializer.struct_name - assert struct_name in self._displayed_columns_by_struct_name + struct_name, struct_deserializer = self.__GetStructViewMeta(elem_path) + if struct_name is None or struct_deserializer is None: + return + assert elem_path in self._displayed_columns_by_elem_path - if self._displayed_columns_by_struct_name[struct_name] == field_names: + if self._displayed_columns_by_elem_path[elem_path] == field_names: return auto_colorize_col = self.GetAutoColorizeColumn(elem_path) if auto_colorize_col not in field_names: self.SetAutoColorizeColumn(elem_path, None) - self._displayed_columns_by_struct_name[struct_name] = copy.deepcopy(field_names) + struct_deserializer.SetVisibleFieldNames(field_names) + self._displayed_columns_by_elem_path[elem_path] = struct_deserializer.GetVisibleFieldNames() self.frame.inspector.RefreshWidgetsOnAllTabs() self.frame.view_settings.SetDirty(reason=DirtyReasons.QueueTableDispColsChanged) def SetAutoColorizeColumn(self, elem_path, field_name): - deserializer = self.GetDeserializer(elem_path) - struct_name = deserializer.struct_name - assert struct_name in self._auto_colorize_column_by_struct_name + struct_name, struct_deserializer = self.__GetStructViewMeta(elem_path) + if struct_name is None or struct_deserializer is None: + return + assert elem_path in self._auto_colorize_column_by_elem_path - if self._auto_colorize_column_by_struct_name[struct_name] == field_name: + if field_name is not None and field_name not in struct_deserializer.GetVisibleFieldNames(): return - self._auto_colorize_column_by_struct_name[struct_name] = field_name + if self._auto_colorize_column_by_elem_path[elem_path] == field_name: + return + + self._auto_colorize_column_by_elem_path[elem_path] = field_name self.frame.inspector.RefreshWidgetsOnAllTabs() self.frame.view_settings.SetDirty(reason=DirtyReasons.QueueTableAutoColorizeChanged) def GetAutoColorizeColumn(self, elem_path): + struct_name, deserializer = self.__GetStructViewMeta(elem_path) + if struct_name is None: + return None + + col = self._auto_colorize_column_by_elem_path.get(elem_path) + if col is None: + col = deserializer.GetVisibleFieldNames()[0] + + return col + + def __GetStructViewMeta(self, elem_path): deserializer = self.GetDeserializer(elem_path) - struct_name = deserializer.struct_name - return self._auto_colorize_column_by_struct_name.get(struct_name) + if isinstance(deserializer, (ContigContainerDeserializer, SparseContainerDeserializer)): + deserializer = deserializer._bin_deserializer + if isinstance(deserializer, StructDeserializer): + return deserializer.struct_name, deserializer + return None, None def GetDeserializer(self, elem_path): dtype = self._dtypes_by_elem_path[elem_path] - if dtype not in self._deserializers_by_dtype: - if '_contig_capacity' in dtype: - dtype = dtype.split('_contig_capacity')[0] - elif '_sparse_capacity' in dtype: - dtype = dtype.split('_sparse_capacity')[0] - else: - raise ValueError('Invalid data type: ' + dtype) - - return self._deserializers_by_dtype[dtype] + return self.dtype_inspector.GetDeserializer(dtype) + + def _has_timestamp_clocks(self): + self.cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='TimestampClocks'" + ) + return self.cursor.fetchone() is not None + + def _clock_collected_at_tick(self, clock_id, tick): + if clock_id is None or not self._has_timestamp_clocks(): + return True + + self.cursor.execute( + 'SELECT 1 FROM Timestamps t ' + 'INNER JOIN TimestampClocks tc ON tc.TimestampID = t.Id ' + 'WHERE tc.ClockID = ? AND CAST(t.Timestamp AS INTEGER) = ? ' + 'LIMIT 1', + (clock_id, int(tick)), + ) + return self.cursor.fetchone() is not None + + def _database_has_multiple_clocks(self): + self.cursor.execute('SELECT COUNT(*) FROM Clocks') + return int(self.cursor.fetchone()[0]) > 1 + + def _clock_ids_for_elem_paths(self, elem_paths): + if not self._database_has_multiple_clocks(): + return None + + clock_ids = set() + for elem_path in elem_paths: + clk_id = self.simhier.GetMetaAtPath(elem_path, 'ClkID') + if clk_id is not None: + clock_ids.add(int(clk_id)) + return sorted(clock_ids) if clock_ids else None + + def _clock_id_for_elem_path(self, elem_path): + if not self._database_has_multiple_clocks(): + return None + + clk_id = self.simhier.GetMetaAtPath(elem_path, 'ClkID') + return int(clk_id) if clk_id is not None else None def GetIterableSizesByCollectionID(self, time_val): if self._cached_utiliz_time_val is not None and time_val == self._cached_utiliz_time_val: return self._cached_utiliz_sizes - return {id:0 for id in self.simhier.GetContainerIDs()} - - def Unpack(self, elem_path, time_range=None): - cmd = 'SELECT Tick,Data,IsCompressed FROM CollectionRecords ' - if time_range is not None: - cmd += 'WHERE ' - if type(time_range) in (int, float): - time_range = (time_range, time_range) - - assert type(time_range) in (list,tuple) and len(time_range) == 2 - where_clauses = [] - if time_range[0] >= 0: - # Find the first time value that is greater than or equal to - # the lower bound. - start_time = None - for i,time_val in enumerate(self._time_vals): - if time_val >= time_range[0]: - # Go back number of ticks to get to the - # start of the data. The entire [t1-heartbeat:t2] - # range will be "replayed" to find the current data - # values. We have to go back number of - # ticks to ensure that we encounter a "full" data - # dump -- followed by incremental dumps such as - # "carry-over" values (the data didn't change from - # the previous tick) etc. - start_idx = i - self._heartbeat - start_idx = max(0, start_idx) - start_time = self._time_vals[start_idx] - break - - where_clauses.append(' Tick>={} '.format(start_time)) - - if time_range[1] >= 0: - where_clauses.append(' Tick<={} '.format(time_range[1])) - - cmd += ' AND '.join(where_clauses) - - cmd += ' ORDER BY Tick ASC' - self.cursor.execute(cmd) - - # We have to reset all the replayers even though all but one - # of them are NOT for . This is because the replayers - # are stateful and we have to reset them all to ensure that the - # one we are interested in is in a clean state. - for replayer in self._replayers_by_elem_path.values(): - replayer.Reset() - - requested_elem_path = elem_path - for tick, data_blob, is_compressed in self.cursor.fetchall(): - if is_compressed: - data_blob = zlib.decompress(data_blob) - - while True: - # The first 2 bytes of any blob is a collectable ID, - # followed by the raw bytes of that collectable, then - # another collectable ID, and so on. - cid = struct.unpack('H', data_blob[:2])[0] - if self.IsDevDebug(): - print ('[simdb verbose] tick {}, cid {}'.format(tick, cid)) - - data_blob = data_blob[2:] - - replayer = self._replayers_by_elem_path[self.simhier.GetElemPath(cid)] - is_auto_collected = cid in self._auto_collected_cids - is_dev_debug = self.IsDevDebug() - num_bytes_read = replayer.Replay(tick, data_blob, is_auto_collected, is_dev_debug) - if num_bytes_read == 0: - break - - data_blob = data_blob[num_bytes_read:] - if len(data_blob) == 0: - break - - time_vals = [] - data_vals = [] - - ticks = list(self._replayers_by_elem_path[requested_elem_path].values_by_tick.keys()) - ticks.sort() - - deserializer = self.GetDeserializer(requested_elem_path) - for tick in ticks: - if time_range is None or (tick >= time_range[0] and tick <= time_range[1]): - time_vals.append(tick) - - data_vals.append([]) - data_blobs = self._replayers_by_elem_path[requested_elem_path].values_by_tick[tick] - for data_blob in data_blobs: - data_vals[-1].append(deserializer.Deserialize(data_blob)) - - return {'TimeVals': time_vals, 'DataVals': data_vals} + tick = int(time_val) + iterator = BlobIterator(self.dtype_inspector, self.simhier) + handler = DataExtractionHandler(self.simhier) + clock_ids = self._clock_ids_for_elem_paths(self.simhier.GetContainerElemPaths()) + iterator.Iterate(handler, tick, lookback=True, clock_ids=clock_ids) + + sizes_by_cid = {} + for cid in self.simhier.GetContainerIDs(): + elems = handler.GetFinalValue(cid) + sizes_by_cid[cid] = len([e for e in elems if e is not None]) if elems else 0 + + self._cached_utiliz_time_val = time_val + self._cached_utiliz_sizes = sizes_by_cid + return sizes_by_cid + + def Unpack(self, elem_path, tick): + iterator = BlobIterator(self.dtype_inspector, self.simhier) + handler = DataExtractionHandler(self.simhier) + clock_id = self._clock_id_for_elem_path(elem_path) + iterator.Iterate(handler, tick, lookback=True, clock_ids=clock_id) + + if handler.HasDataFor(elem_path) and self._clock_collected_at_tick(clock_id, tick): + unpacked = { + 'TimeVals': [iterator.GetFinalTick()], + 'DataVals': [handler.GetFinalValue(elem_path)] + } + else: + unpacked = { + 'TimeVals': [], + 'DataVals': [] + } + + return unpacked + + def UnpackRange(self, start_tick, end_tick, elem_paths=None): + if elem_paths is None: + elem_paths = self.simhier.GetItemElemPaths() + cids = {self.simhier.GetCollectionID(p) for p in elem_paths} + + # Warm up one heartbeat before the requested start so the value at + # start_tick is fully reconstructed (same lookback Unpack() uses). + iterator = BlobIterator(self.dtype_inspector, self.simhier) + handler = DataExtractionHandler(self.simhier, snapshot_cids=cids) + clock_ids = self._clock_ids_for_elem_paths(elem_paths) + iterator.Iterate(handler, [int(start_tick), int(end_tick)], lookback=True, clock_ids=clock_ids) + + unpacked = {} + for elem_path in elem_paths: + values_by_tick = handler.GetValuesByTick(elem_path) + clock_id = self._clock_id_for_elem_path(elem_path) + time_vals, data_vals = [], [] + for tick in sorted(values_by_tick.keys()): + if start_tick <= tick <= end_tick and self._clock_collected_at_tick(clock_id, tick): + time_vals.append(tick) + data_vals.append(values_by_tick[tick]) + unpacked[elem_path] = {'TimeVals': time_vals, 'DataVals': data_vals} + + return unpacked def GetAllTimeVals(self): return copy.deepcopy(self._time_vals) - -class PODReplayer: - NUM_BYTES_MAP = { - 'char': 1, - 'int8_t': 1, - 'int16_t': 2, - 'int32_t': 4, - 'int64_t': 8, - 'uint8_t': 1, - 'uint16_t': 2, - 'uint32_t': 4, - 'uint64_t': 8, - 'float_t': 4, - 'double_t': 8, - 'bool': 4 - } - - FORMAT_CODES_MAP = { - 'char': 'c', - 'int8_t':'b', - 'int16_t':'h', - 'int32_t':'i', - 'int64_t':'q', - 'uint8_t':'B', - 'uint16_t':'H', - 'uint32_t':'I', - 'uint64_t':'Q', - 'float_t':'f', - 'double_t':'d', - 'bool':'I' - } - - class Actions(IntEnum): - WRITE = 0 - CARRY = 1 - - def __init__(self, dtype): - self.num_bytes = PODReplayer.NUM_BYTES_MAP[dtype] - self.format_code = PODReplayer.FORMAT_CODES_MAP[dtype] - self.Reset() - - def Reset(self): - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # For collectables that are <16 bytes, we always write them directly - # without the Action enum. - if self.num_bytes < 16 or not is_auto_collected: - val = struct.unpack(self.format_code, data_blob[:self.num_bytes])[0] - self.values_by_tick[tick] = val - if is_dev_debug: - print ('[simdb verbose] {}<16, {}: {}'.format(self.num_bytes, GetDataTypeStr(self.format_code), val)) - return self.num_bytes - - # Actions are at the top of the blob as a uint8_t. - action = self.Actions(struct.unpack('B', data_blob[:1])[0]) - - if action == self.Actions.WRITE: - if is_dev_debug: - print ('[simdb verbose] WRITE {} bytes'.format(self.num_bytes)) - self.values_by_tick[tick] = struct.unpack(self.format_code, data_blob[1:1+self.num_bytes])[0] - return 1 + self.num_bytes - else: - if is_dev_debug: - print ('[simdb verbose] CARRY') - self.values_by_tick[tick] = self.values_by_tick[self.__GetLastTick(tick)] - return 1 - - def __GetLastTick(self, tick): - prev_ticks = set(self.values_by_tick.keys()) - for prev_tick in range(tick-1, -1, -1): - if prev_tick in prev_ticks: - return prev_tick - - return None - -class PODDeserializer: - NUM_BYTES_MAP = { - 'char': 1, - 'int8_t': 1, - 'int16_t': 2, - 'int32_t': 4, - 'int64_t': 8, - 'uint8_t': 1, - 'uint16_t': 2, - 'uint32_t': 4, - 'uint64_t': 8, - 'float_t': 4, - 'double_t': 8, - 'bool': 4 - } - - FORMAT_CODES_MAP = { - 'char': 'c', - 'int8_t':'b', - 'int16_t':'h', - 'int32_t':'i', - 'int64_t':'q', - 'uint8_t':'B', - 'uint16_t':'H', - 'uint32_t':'I', - 'uint64_t':'Q', - 'float_t':'f', - 'double_t':'d', - 'bool':'I' - } - - def __init__(self, dtype): - self.num_bytes = PODDeserializer.NUM_BYTES_MAP[dtype] - self.format_code = PODDeserializer.FORMAT_CODES_MAP[dtype] - - def Deserialize(self, data_blob): - return struct.unpack(self.format_code, data_blob[:self.num_bytes])[0] - -class StringReplayer: - class Actions(IntEnum): - WRITE = 0 - CARRY = 1 - - def __init__(self, strings_by_int): - self.strings_by_int = strings_by_int - self.Reset() - - def Reset(self): - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # TODO cnyce - fill this class out e.g. manually collected Collectable, - # CARRY, etc. - string_id = struct.unpack('I', data_blob[:4])[0] - self.values_by_tick[tick] = self.strings_by_int[string_id] - return 4 - -class StringDeserializer: - def __init__(self, strings_by_int): - self.strings_by_int = strings_by_int - self.format_code = 'I' - - def Deserialize(self, data_blob): - string_id = struct.unpack('I', data_blob[:4])[0] - return self.strings_by_int[string_id] - -class EnumReplayer: - class Actions(IntEnum): - WRITE = 0 - CARRY = 1 - - def __init__(self, dtype, enums_by_name): - self.enum_handler = enums_by_name[dtype] - self.num_bytes = self.enum_handler.GetNumBytes() - self.format_code = self.enum_handler.GetFormatCode() - self.Reset() - - def Reset(self): - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # For collectables that are <16 bytes, we always write them directly - # without the Action enum. - if self.num_bytes < 16 or not is_auto_collected: - enum_int = struct.unpack(self.format_code, data_blob[:self.num_bytes])[0] - self.values_by_tick[tick] = self.enum_handler.Convert(enum_int) - if is_dev_debug: - print ('[simdb verbose] {}<16, {}: {}'.format(self.num_bytes, GetDataTypeStr(self.format_code), enum_int)) - return self.num_bytes - - # Actions are at the top of the blob as a uint8_t. - action = self.Actions(struct.unpack('B', data_blob[:1])[0]) - - if action == self.Actions.WRITE: - enum_int = struct.unpack(self.format_code, data_blob[1:1+self.num_bytes])[0] - self.values_by_tick[tick] = self.enum_handler.Convert(enum_int) - if is_dev_debug: - print ('[simdb verbose] WRITE {} bytes'.format(self.num_bytes)) - return 1 + self.num_bytes - else: - self.values_by_tick[tick] = self.values_by_tick[self.__GetLastTick(tick)] - if is_dev_debug: - print ('[simdb verbose] CARRY') - return 1 - - def __GetLastTick(self, tick): - prev_ticks = set(self.values_by_tick.keys()) - for prev_tick in range(tick-1, -1, -1): - if prev_tick in prev_ticks: - return prev_tick - - return None - -class EnumDeserializer: - def __init__(self, enum_handler): - self.enum_handler = enum_handler - self.format_code = self.enum_handler.GetFormatCode() - - def Deserialize(self, data_blob): - num_bytes = self.enum_handler.GetNumBytes() - enum_int = struct.unpack(self.format_code, data_blob[:num_bytes])[0] - return self.enum_handler.Convert(enum_int) - -class StructDeserializer: - FORMAT_CODES_MAP = { - 'char_t':'c', - 'int8_t':'b', - 'int16_t':'h', - 'int32_t':'i', - 'int64_t':'q', - 'uint8_t':'B', - 'uint16_t':'H', - 'uint32_t':'I', - 'uint64_t':'Q', - 'float_t':'f', - 'double_t':'d', - } - - NUM_BYTES_MAP = { - 'c':1, - 'b':1, - 'h':2, - 'i':4, - 'q':8, - 'B':1, - 'H':2, - 'I':4, - 'Q':8, - 'f':4, - 'd':8 - } - - def __init__(self, data_retriever, struct_name, strings_by_int, enums_by_name): - self.data_retriever = data_retriever - self.struct_name = struct_name - self.strings_by_int = strings_by_int - self.enums_by_name = enums_by_name - self.field_deserializers = [] - self.field_format_codes = [] - self.all_field_names = [] - - def AddField(self, field_name, field_deserializer, format_code): - self.field_deserializers.append(field_deserializer) - self.field_format_codes.append(format_code) - self.all_field_names.append(field_name) - - def GetAllFieldNames(self): - return copy.deepcopy(self.all_field_names) - - def GetVisibleFieldNames(self): - visible_field_names = self.data_retriever.GetCurrentViewSettings().get(self.struct_name, {}).get('displayed_columns', []) - return copy.deepcopy(visible_field_names) - - def Deserialize(self, data_blob): - visible_field_names = self.data_retriever.GetCurrentViewSettings().get(self.struct_name, {}).get('displayed_columns', []) - assert len(visible_field_names) > 0 - - res = {} - tiny_start = 0 - for i, deserializer in enumerate(self.field_deserializers): - code = deserializer.format_code - nbytes = self.NUM_BYTES_MAP[code] - tiny_blob = data_blob[tiny_start:tiny_start+nbytes] - tiny_start += nbytes - - field_name = self.all_field_names[i] - if field_name not in visible_field_names: - continue - - field_deserializer = self.field_deserializers[i] - res[field_name] = field_deserializer.Deserialize(tiny_blob) - - return res - -class ScalarStructReplayer: - class Actions(IntEnum): - WRITE = 0 - CARRY = 1 - - def __init__(self, struct_name, struct_num_bytes): - self.struct_name = struct_name - self.struct_num_bytes = struct_num_bytes - self.Reset() - - def Reset(self): - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # For collectables that are <16 bytes, we always write them directly - # without the Action enum. - if self.struct_num_bytes < 16 or not is_auto_collected: - self.values_by_tick[tick] = data_blob[:self.struct_num_bytes] - if is_dev_debug: - print ('[simdb verbose] {}<16, {}: {}'.format(self.struct_num_bytes, self.struct_name, self.values_by_tick[tick])) - return self.struct_num_bytes - - # Actions are at the top of the blob as a uint8_t. - action = self.Actions(struct.unpack('B', data_blob[:1])[0]) - - if action == self.Actions.WRITE: - if is_dev_debug: - print ('[simdb verbose] WRITE {} bytes'.format(self.struct_num_bytes)) - self.values_by_tick[tick] = data_blob[1:1+self.struct_num_bytes] - return 1 + self.struct_num_bytes - else: - if is_dev_debug: - print ('[simdb verbose] CARRY') - self.values_by_tick[tick] = self.values_by_tick[self.__GetLastTick(tick)] - return 1 - - def __GetLastTick(self, tick): - prev_ticks = set(self.values_by_tick.keys()) - for prev_tick in range(tick-1, -1, -1): - if prev_tick in prev_ticks: - return prev_tick - - return None - -class ContigIterableReplayer: - class Actions(IntEnum): - ARRIVE = 0 - DEPART = 1 - BOOKENDS = 2 - CHANGE = 3 - CARRY = 4 - FULL = 5 - - def __init__(self, struct_num_bytes, capacity): - self.struct_num_bytes = struct_num_bytes - self.capacity = capacity - self.Reset() - - self.num_bytes_to_advance = { - self.Actions.ARRIVE: 1 + self.struct_num_bytes, - self.Actions.DEPART: 1, - self.Actions.BOOKENDS: 1 + self.struct_num_bytes, - self.Actions.CHANGE: 3 + self.struct_num_bytes, - self.Actions.CARRY: 1 - } - - def Reset(self): - self.values = None - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # TODO cnyce - manually collected containers - assert is_auto_collected - - # Actions are at the top of the blob as a uint8_t. - action = self.Actions(struct.unpack('B', data_blob[:1])[0]) - - if action == self.Actions.FULL: - # The number of structs is given here as a uint16_t - size = struct.unpack('H', data_blob[1:3])[0] - if is_dev_debug: - print ('[simdb verbose] FULL with {} elements'.format(size)) - self.values = [] - for i in range(size): - struct_blob = data_blob[3+i*self.struct_num_bytes:3+(i+1)*self.struct_num_bytes] - self.values.append(struct_blob) - - self.values_by_tick[tick] = copy.deepcopy(self.values) - return 3 + size*self.struct_num_bytes - - if action == self.Actions.ARRIVE: - if is_dev_debug: - print ('[simdb verbose] ARRIVE {} bytes'.format(self.struct_num_bytes)) - if self.values is not None: - # Exactly one struct arrived and gets appended to the list (back). - struct_blob = data_blob[1:1+self.struct_num_bytes] - self.values.append(struct_blob) - self.values_by_tick[tick] = copy.deepcopy(self.values) - return self.num_bytes_to_advance[action] - - elif action == self.Actions.DEPART: - if is_dev_debug: - print ('[simdb verbose] DEPART') - if self.values is not None: - # Exactly one struct left and gets removed from the list (front). - self.values.pop(0) - self.values_by_tick[tick] = copy.deepcopy(self.values) - return self.num_bytes_to_advance[action] - - elif action == self.Actions.CHANGE: - # The changed bin index is written to the blob as a uint16_t, followed - # by the changed struct's bytes. - bin_idx = struct.unpack('H', data_blob[1:3])[0] - if is_dev_debug: - print ('[simdb verbose] CHANGE index {}, {} bytes'.format(bin_idx, self.struct_num_bytes)) - if self.values is not None: - struct_blob = data_blob[3:3+self.struct_num_bytes] - self.values[bin_idx] = struct_blob - self.values_by_tick[tick] = copy.deepcopy(self.values) - return self.num_bytes_to_advance[action] - - elif action == self.Actions.BOOKENDS: - if is_dev_debug: - print ('[simdb verbose] BOOKENDS, appended {} bytes'.format(self.struct_num_bytes)) - if self.values is not None: - # This means that exactly one struct came arrived (append) and one struct - # departed (pop front). - struct_blob = data_blob[1:1+self.struct_num_bytes] - self.values.append(struct_blob) - self.values.pop(0) - self.values_by_tick[tick] = copy.deepcopy(self.values) - return self.num_bytes_to_advance[action] - - elif action == self.Actions.CARRY: - if is_dev_debug: - print ('[simdb verbose] CARRY') - if self.values is not None: - last_tick = self.__GetLastTick(tick) - self.values_by_tick[tick] = copy.deepcopy(self.values_by_tick[last_tick]) - return self.num_bytes_to_advance[action] - - assert False, 'Unreachable' - - def __GetLastTick(self, tick): - prev_ticks = set(self.values_by_tick.keys()) - for prev_tick in range(tick-1, -1, -1): - if prev_tick in prev_ticks: - return prev_tick - - return None - -class SparseIterableReplayer: - def __init__(self, struct_num_bytes, capacity): - self.struct_num_bytes = struct_num_bytes - self.capacity = capacity - self.Reset() - - def Reset(self): - self.values = [None]*self.capacity - self.values_by_tick = {} - - def Replay(self, tick, data_blob, is_auto_collected, is_dev_debug): - # TODO cnyce - manually collected containers - assert is_auto_collected - - # The top of the blob always has the number of structs as a uint16_t. - size = struct.unpack('H', data_blob[:2])[0] - if is_dev_debug: - print ('[simdb verbose] num valid: {}'.format(size)) - - for i in range(size): - # Each entry is preceeded by a uint16_t bin index. - bin_idx = struct.unpack('H', data_blob[2+i*2:2+(i+1)*2])[0] - struct_blob = data_blob[2+size*2+i*self.struct_num_bytes:2+size*2+(i+1)*self.struct_num_bytes] - - assert bin_idx >= 0 and bin_idx < self.capacity and bin_idx < len(self.values) - self.values[bin_idx] = struct_blob - - if is_dev_debug: - print ('[simdb verbose] bin {}, {} bytes'.format(bin_idx, self.struct_num_bytes)) - - self.values_by_tick[tick] = copy.deepcopy(self.values) - return 2 + size*2 + size*self.struct_num_bytes - -class EnumDef: - def __init__(self, name, int_type): - self._name = name - self.strings_by_int = {} - - if int_type == 'int8_t': - self._format = 'b' - elif int_type == 'int16_t': - self._format = 'h' - elif int_type == 'int32_t': - self._format = 'i' - elif int_type == 'int64_t': - self._format = 'q' - elif int_type == 'uint8_t': - self._format = 'B' - elif int_type == 'uint16_t': - self._format = 'H' - elif int_type == 'uint32_t': - self._format = 'I' - elif int_type == 'uint64_t': - self._format = 'Q' - else: - raise ValueError('Invalid enum integer type: ' + int_type) - - def AddEntry(self, enum_string, enum_blob): - int_val = struct.unpack(self._format, enum_blob)[0] - self.strings_by_int[int_val] = enum_string - - def GetNumBytes(self): - return struct.calcsize(self._format) - - def GetFormatCode(self): - return self._format - - def Convert(self, val): - return self.strings_by_int[val] - -def GetDataTypeStr(format_code): - if format_code == 'c': - return 'char' - elif format_code == 'b': - return 'int8_t' - elif format_code == 'h': - return 'int16_t' - elif format_code == 'i': - return 'int32_t' - elif format_code == 'q': - return 'int64_t' - elif format_code == 'B': - return 'uint8_t' - elif format_code == 'H': - return 'uint16_t' - elif format_code == 'I': - return 'uint32_t' - elif format_code == 'Q': - return 'uint64_t' - elif format_code == 'f': - return 'float_t' - elif format_code == 'd': - return 'double_t' - else: - raise ValueError('Invalid format code: ' + format_code) diff --git a/python/argos/viewer/model/dirty_reasons.py b/python/argos/viewer/model/dirty_reasons.py new file mode 100644 index 00000000..bdc3d058 --- /dev/null +++ b/python/argos/viewer/model/dirty_reasons.py @@ -0,0 +1,36 @@ +import enum + +class DirtyReasons(enum.Enum): + WidgetDropped = 1 + WidgetSplit = 2 + CanvasExploded = 3 + TabAdded = 4 + TabRenamed = 5 + TabDeleted = 6 + QueueUtilizDispQueueChanged = 7 + TimeseriesPlotSettingsChanged = 8 + QueueTableDispColsChanged = 9 + QueueTableAutoColorizeChanged = 10 + SchedulingLinesWidgetChanged = 11 + SashPositionChanged = 12 + WidgetRemoved = 13 + TrackedPacketChanged = 14 + SummaryViewsWidgetChanged = 15 + +DIRTY_REASONS = { + DirtyReasons.WidgetDropped: 'A widget was dropped onto the widget canvas', + DirtyReasons.WidgetSplit: 'A widget was split horizontally or vertically', + DirtyReasons.CanvasExploded: 'Widget canvas was exploded', + DirtyReasons.TabAdded: 'A new tab was added', + DirtyReasons.TabRenamed: 'A tab was renamed', + DirtyReasons.TabDeleted: 'A tab was deleted', + DirtyReasons.QueueUtilizDispQueueChanged: 'The displayed queues in a Queue Utilization widget were changed', + DirtyReasons.TimeseriesPlotSettingsChanged: 'Settings were changed for a timeseries plot', + DirtyReasons.QueueTableDispColsChanged: 'Displayed columns were changed for a Queue Table widget', + DirtyReasons.QueueTableAutoColorizeChanged: 'Auto-colorize column was changed for a Queue Table widget', + DirtyReasons.SchedulingLinesWidgetChanged: 'Displayed queues were changed for a Scheduling Lines widget', + DirtyReasons.SashPositionChanged: 'Widget canvas splitter window sash position changed', + DirtyReasons.WidgetRemoved: 'A widget was removed from the inspector canvas', + DirtyReasons.TrackedPacketChanged: 'Changes were made to tracked packet(s)', + DirtyReasons.SummaryViewsWidgetChanged: 'Displayed collectables were changed for a Summary Views widget' +} diff --git a/python/argos/viewer/model/dtype_inspector.py b/python/argos/viewer/model/dtype_inspector.py new file mode 100644 index 00000000..50d0cb5b --- /dev/null +++ b/python/argos/viewer/model/dtype_inspector.py @@ -0,0 +1,275 @@ +import sqlite3 +from typing import Dict, Iterator, List, Optional +from viewer.model.tiny_strings import TinyStrings +from viewer.model.data_deserializers import SimpleDeserializer + +class DataTypeInspector: + class DataTypeNode: + __slots__ = ( + "node_id", + "schema_id", + "parent_id", + "kind", + "name", + "description", + "type_name", + "enum_backing", + "special_formatter", + "effective_color_key", + "children", + "enum_members", + ) + + def __init__( + self, + node_id, + schema_id, + parent_id, + kind, + name, + description, + type_name, + enum_backing, + special_formatter, + effective_color_key="", + ): + # type: (Optional[int], int, Optional[int], str, str, str, str, str, str, str) -> None + self.node_id = node_id + self.schema_id = schema_id + self.parent_id = parent_id + self.kind = kind + self.name = name + self.description = description + self.type_name = type_name + self.enum_backing = enum_backing + self.special_formatter = special_formatter + self.effective_color_key = effective_color_key + self.children = [] # type: List[DataTypeInspector.DataTypeNode] + # Populated for Kind == "enum": member name -> numeric value (parsed from DB string) + self.enum_members = {} # type: Dict[str, int] + + def __repr__(self): + return ( + f"DataTypeNode(id={self.node_id!r}, kind={self.kind!r}, " + f"name={self.name!r}, type_name={self.type_name!r}, n_children={len(self.children)})" + ) + + def __init__(self, db_file): + self._conn = sqlite3.connect(db_file) + self._schemas = {} + self._effective_color_keys_by_schema = {} + self._effective_color_keys_by_root_type = {} + self._nodes_by_id = {} + self._top_by_schema = {} + self._root_views = [] + self._deserializers_by_typename = {} + self._tiny_strings = TinyStrings(db_file) + self._load() + + @property + def connection(self): + # type: () -> sqlite3.Connection + return self._conn + + def close(self): + # type: () -> None + self._conn.close() + + def _load(self): + # type: () -> None + cur = self._conn.cursor() + cur.execute("SELECT CID,TypeName FROM CollectableTreeNodes") + self._dtypes_by_cid = {cid:type_name for cid,type_name in cur.fetchall()} + + cur.execute("SELECT Id, RootTypeName FROM DataTypeSchemas ORDER BY Id") + for sid, root_name in cur.fetchall(): + sid = int(sid) + root_name = str(root_name) + self._schemas[sid] = root_name + self._effective_color_keys_by_schema[sid] = "" + self._effective_color_keys_by_root_type[root_name] = "" + + cur.execute("SELECT DISTINCT(SchemaId) FROM DataTypeNodes WHERE Name='DID'") + for sid in cur.fetchall(): + sid = sid[0] + root_name = self._schemas[sid] + self._effective_color_keys_by_schema[sid] = "DID" + self._effective_color_keys_by_root_type[root_name] = "DID" + + cur.execute( + """ + SELECT Id, SchemaId, Name, TypeName, FormatStr + FROM DataTypeNodes + ORDER BY Id + """ + ) + raw_rows = cur.fetchall() + + enum_defns = {} + enum_backings = {} + + cur.execute( + """ + SELECT ce.EnumName, ce.EnumIntTypeName, em.MemberName, em.MemberValueStr + FROM CollectedEnums ce + JOIN EnumMembers em ON em.EnumID = ce.Id + """ + ) + for enum_name, int_type_name, member_name, member_value_str in cur.fetchall(): + if enum_name not in enum_defns: + enum_defns[enum_name] = {} + enum_defns[enum_name][member_name] = int(member_value_str) + enum_backings[enum_name] = int_type_name + + self._nodes_by_id.clear() + self._top_by_schema = {sid: [] for sid in self._schemas} + + for row in raw_rows: + nid, sid, name, type_name, special_formatter = row + enum_back = None + if type_name in SimpleDeserializer.CONVERTERS: + kind = "pod" + elif type_name == "string": + kind = "pod" + elif type_name != "dynamic": + kind = "enum" + if type_name not in enum_defns: + # All this means is that we never ended up collecting + # anything that uses this enum. All of the enum int:str + # mappings are figured out only when first seen during + # collection. + enum_defns[type_name] = {} + else: + enum_back = enum_backings.get(type_name) + + node = DataTypeInspector.DataTypeNode( + nid, + sid, + None, + kind, + name, + None, + type_name, + enum_back, + special_formatter, + "") + + self._nodes_by_id[node.node_id] = node + if kind == "enum": + node.enum_members = enum_defns[type_name] + + for node in self._nodes_by_id.values(): + pid = node.parent_id + if pid == 0 or pid is None: + self._top_by_schema.setdefault(node.schema_id, []).append(node) + else: + parent = self._nodes_by_id.get(int(pid)) + if parent is not None: + parent.children.append(node) + else: + raise ValueError( + f"DataTypeNodes.Id={node.node_id} references missing ParentId={pid}" + ) + + for node in self._nodes_by_id.values(): + node.children.sort(key=lambda n: n.node_id or 0) + + for tops in self._top_by_schema.values(): + tops.sort(key=lambda n: n.node_id or 0) + + self._root_views = [] + for sid in sorted(self._schemas.keys()): + root_name = self._schemas[sid] + view = DataTypeInspector.DataTypeNode( + None, + sid, + None, + "schema", + "", + "", + root_name, + "", + "", + self._effective_color_keys_by_schema.get(sid, ""), + ) + view.children = list(self._top_by_schema.get(sid, [])) + self._root_views.append(view) + + def _iter_nodes(self): + # type: () -> Iterator[DataTypeInspector.DataTypeNode] + yield from self._nodes_by_id.values() + + def GetRoots(self): + # type: () -> List[DataTypeInspector.DataTypeNode] + """One synthetic node per schema row (``DataTypeSchemas``), with top-level fields as ``children``.""" + return list(self._root_views) + + def GetRootDefn(self, dtype_name): + # type: (str) -> Optional[DataTypeInspector.DataTypeNode] + """Return the synthetic schema root whose ``type_name`` matches ``DataTypeSchemas.RootTypeName``.""" + for view in self._root_views: + if view.type_name == dtype_name: + return view + return None + + def GetStructDefn(self, struct_name): + # type: (str) -> Optional[DataTypeInspector.DataTypeNode] + """First ``Kind == 'struct'`` node whose ``TypeName`` matches *struct_name*.""" + for node in self._iter_nodes(): + if node.kind == "struct" and node.type_name == struct_name: + return node + + return self.GetRootDefn(struct_name) + + def GetEnumMap(self, enum_name): + # type: (str) -> Optional[Dict[str, int]] + """Member map for ``Kind == 'enum'`` where ``TypeName`` or field ``Name`` matches *enum_name*.""" + for node in self._iter_nodes(): + if node.kind != "enum": + continue + if node.type_name == enum_name or node.name == enum_name: + return dict(node.enum_members) + return None + + def GetEnumBackingKind(self, enum_name): + # type: (str) -> Optional[Dict[str, int]] + """Member map for ``Kind == 'enum'`` where ``TypeName`` or field ``Name`` matches *enum_name*.""" + for node in self._iter_nodes(): + if node.kind != "enum": + continue + if node.type_name == enum_name or node.name == enum_name: + return node.enum_backing + return None + + def GetSimpleTypeSpecialFormatter(self, type_name): + # type: (str) -> str + """First ``Kind == 'pod'`` node whose ``TypeName`` matches *type_name* with non-empty ``FormatStr``.""" + for node in self._iter_nodes(): + if node.kind != "pod": + continue + if node.type_name != type_name: + continue + if node.special_formatter: + return node.special_formatter + return "" + + def GetDataTypeForCollectionID(self, cid): + return self._dtypes_by_cid.get(cid) + + def GetEffectiveColorKey(self, dtype_name): + # type: (str) -> str + """Return DataTypeSchemas.EffectiveColorKey for a root type, or "".""" + return self._effective_color_keys_by_root_type.get(dtype_name, "") + + def GetDeserializer(self, dtype_name, expect_exists=True): + if dtype_name in self._deserializers_by_typename: + return self._deserializers_by_typename[dtype_name] + + from viewer.model.data_deserializers import CreateDeserializer + deserializer = CreateDeserializer(self, dtype_name, self._tiny_strings) + + if deserializer: + self._deserializers_by_typename[dtype_name] = deserializer + return deserializer + + return None diff --git a/python/argos/viewer/model/frame.py b/python/argos/viewer/model/frame.py index 134beb67..2f986b61 100644 --- a/python/argos/viewer/model/frame.py +++ b/python/argos/viewer/model/frame.py @@ -1,33 +1,30 @@ import wx, sqlite3, os +from functools import partial from viewer.gui.widgets.playback_bar import PlaybackBar -from viewer.gui.explorer import DataExplorer from viewer.gui.inspector import DataInspector from viewer.gui.widgets.widget_renderer import WidgetRenderer from viewer.gui.widgets.widget_creator import WidgetCreator +from viewer.gui.canvas_grid import CanvasGrid, WidgetContainer from viewer.model.data_retriever import DataRetriever +from viewer.model.dtype_inspector import DataTypeInspector from viewer.model.simhier import SimHierarchy from viewer.gui.widgets.splitter_window import DirtySplitterWindow +from viewer.gui.view_settings import DirtyReasons class ArgosFrame(wx.Frame): - def __init__(self, db_path, view_settings, dev_debug): + def __init__(self, db_path, view_settings): super().__init__(None, title=db_path) self.db = sqlite3.connect(db_path) self.view_settings = view_settings - self.dev_debug = dev_debug - self.simhier = SimHierarchy(self.db) + self.dtype_inspector = DataTypeInspector(db_path) + self.simhier = SimHierarchy(self.db, self.dtype_inspector) self.widget_renderer = WidgetRenderer(self) self.widget_creator = WidgetCreator(self) - self.data_retriever = DataRetriever(self, self.db, self.simhier) - - self.frame_splitter = DirtySplitterWindow(self, self, style=wx.SP_LIVE_UPDATE) - self.explorer = DataExplorer(self.frame_splitter, self) - self.inspector = DataInspector(self.frame_splitter, self) + self.data_retriever = DataRetriever(self, db_path, self.simhier, self.dtype_inspector) + self.inspector = DataInspector(self, self) self.playback_bar = PlaybackBar(self) - self.frame_splitter.SplitVertically(self.explorer, self.inspector, sashPosition=300) - self.frame_splitter.SetMinimumPaneSize(300) - self.menu_bar = wx.MenuBar() file_menu = wx.Menu() @@ -47,16 +44,13 @@ def __init__(self, db_path, view_settings, dev_debug): # Layout sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(self.frame_splitter, 1, wx.EXPAND) + sizer.Add(self.inspector, 1, wx.EXPAND) sizer.Add(self.playback_bar, 0, wx.EXPAND) self.SetSizer(sizer) self.Layout() self.Maximize() def PostLoad(self, view_file): - self.widget_creator.BindToWidgetSource(self.explorer.navtree) - self.widget_creator.BindToWidgetSource(self.explorer.watchlist) - self.widget_creator.BindToWidgetSource(self.explorer.tools) self.view_settings.PostLoad(self, view_file) def CreateResourceBitmap(self, filename, size=(16, 16)): @@ -67,6 +61,46 @@ def CreateResourceBitmap(self, filename, size=(16, 16)): bitmap = bitmap.ConvertToImage().Rescale(w,h).ConvertToBitmap() return bitmap + def CreateWidgetStandardButtons(self, widget, settings_callback, settings_tooltip): + std_size = (36,22) + + settings_btn = wx.Button(widget, label=">>", size=std_size) + settings_btn.Bind(wx.EVT_BUTTON, settings_callback) + settings_btn.SetToolTip(settings_tooltip) + + def ClearWidget(evt, widget): + if isinstance(widget, WidgetContainer): + widget.DestroyAllWidgets() + self.view_settings.SetDirty(reason=DirtyReasons.WidgetRemoved) + else: + ClearWidget(evt, widget.GetParent()) + + clear_btn = wx.Button(widget, label="X", size=std_size) + clear_btn.Bind(wx.EVT_BUTTON, partial(ClearWidget, widget=widget)) + clear_btn.SetToolTip("Clear widget") + + canvas_grid = widget.GetParent() + while canvas_grid is not None and not isinstance(canvas_grid, CanvasGrid): + canvas_grid = canvas_grid.GetParent() + + assert canvas_grid is not None + splitters = [] + canvas_grid.AddQuickLinks(splitters) + + split_lr = wx.Button(widget, label="|", size=std_size) + split_lr.Bind(wx.EVT_BUTTON, splitters[0][1]) + split_lr.SetToolTip("Split left/right") + + split_tb = wx.Button(widget, label="-", size=std_size) + split_tb.Bind(wx.EVT_BUTTON, splitters[1][1]) + split_tb.SetToolTip("Split top/bottom") + + maximize_btn = wx.Button(widget, label="@", size=std_size) + maximize_btn.Bind(wx.EVT_BUTTON, splitters[2][1]) + maximize_btn.SetToolTip("Clear all widgets except this one") + + return settings_btn, clear_btn, split_lr, split_tb, maximize_btn + def __OnNewView(self, event): self.view_settings.CreateNewView() diff --git a/python/argos/viewer/model/simhier.py b/python/argos/viewer/model/simhier.py index cc6d056b..76eecde0 100644 --- a/python/argos/viewer/model/simhier.py +++ b/python/argos/viewer/model/simhier.py @@ -1,92 +1,88 @@ import copy, re +import sqlite3 class SimHierarchy: - def __init__(self, db): - child_ids_by_parent_id = {} + def __init__(self, db, dtype_inspector, show_in_ui_only=True): + full_paths = [] + metas_by_path = {} + cursor = db.cursor() - self._root_id = None - self.__RecurseBuildHierarchy(cursor, 0, child_ids_by_parent_id) - assert self._root_id is not None - - parent_ids_by_child_id = {} - for parent_id, child_ids in child_ids_by_parent_id.items(): - for child_id in child_ids: - parent_ids_by_child_id[child_id] = parent_id - - elem_names_by_id = {} - cursor.execute('SELECT Id,Name FROM ElementTreeNodes') - for id, name in cursor.fetchall(): - elem_names_by_id[id] = name - - dtype_by_elem_id = {} - self._widget_types_by_elem_id = {} - cursor.execute('SELECT ElementTreeNodeID,DataType FROM CollectableTreeNodes') - for id, dtype in cursor.fetchall(): - dtype_by_elem_id[id] = dtype + + # ShowInUI=1 + # --> This means that we only show collectables that were actually collected. + # The database has all the metadata to add everything to the hierarchy, + # but without any collected data it just looks cluttered. + cmd = "SELECT FullPath,CID,ClockID,TypeName FROM CollectableTreeNodes" + if show_in_ui_only: + cmd += " WHERE ShowInUI=1" + cursor.execute(cmd) + rows = [(r[0], r[1], r[2], r[3]) for r in cursor.fetchall()] + + for full_path, cid, clk_id, type_name in rows: + full_paths.append(full_path) + metas_by_path[full_path] = { + 'CID': cid, + 'ClkID': clk_id, + 'TypeName': type_name, + 'ArgosDefaultHiddenColumns': '' + } + + self._simhier_tree = SimHierTree() + self._simhier_tree.BuildFromList(full_paths) + for full_path, meta in metas_by_path.items(): + for meta_name, meta_value in meta.items(): + self._simhier_tree.SetMetaAtPath(full_path, meta_name, meta_value) + + def HandleLeaf(leaf): + cid = leaf.GetMeta('CID') + dtype = leaf.GetMeta('TypeName') + full_path = leaf.GetPath() + self._cids_by_elem_path[full_path] = cid if '_contig_capacity' in dtype or '_sparse_capacity' in dtype: - self._widget_types_by_elem_id[id] = 'QueueTable' - elif dtype in ('char', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'float', 'double', 'bool'): - self._widget_types_by_elem_id[id] = 'Timeseries' + self._widget_types_by_cid[cid] = 'QueueTable' + self._container_elem_paths.append(full_path) + idx = dtype.find('_capacity') + capacity = int(dtype[idx+len('_capacity'):]) + self._capacities_by_cid[cid] = capacity + if '_contig_capacity' in dtype: + self._contig_cids.add(cid) + else: + self._sparse_cids.add(cid) + elif dtype_inspector.GetStructDefn(dtype) is not None: + self._scalar_structs_elem_paths.append(leaf.GetPath()) else: - self._widget_types_by_elem_id[id] = 'ScalarStruct' - - def GetLineage(elem_id, parent_ids_by_child_id): - lineage = [] - while elem_id not in (None,0,self._root_id): - lineage.append(elem_id) - elem_id = parent_ids_by_child_id.get(elem_id) - - lineage.reverse() - return lineage - - def GetPath(elem_id, parent_ids_by_child_id, elem_names_by_id): - lineage = GetLineage(elem_id, parent_ids_by_child_id) - path = '.'.join([elem_names_by_id.get(elem_id) for elem_id in lineage]) - return path - - elem_paths_by_id = {} - for elem_id in elem_names_by_id.keys(): - path = GetPath(elem_id, parent_ids_by_child_id, elem_names_by_id) - if path: - elem_paths_by_id[elem_id] = path + self._scalar_stats_elem_paths.append(leaf.GetPath()) + self._widget_types_by_cid = {} self._scalar_stats_elem_paths = [] self._scalar_structs_elem_paths = [] self._container_elem_paths = [] - - for id, dtype in dtype_by_elem_id.items(): - if dtype in ('int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'float', 'double', 'bool'): - self._scalar_stats_elem_paths.append(elem_paths_by_id[id]) - elif '_contig_capacity' in dtype or '_sparse_capacity' in dtype: - self._container_elem_paths.append(elem_paths_by_id[id]) + self._capacities_by_cid = {} + self._sparse_cids = set() + self._contig_cids = set() + self._cids_by_elem_path = {} + self._simhier_tree.VisitLeaves(HandleLeaf) + + def HandleNode(node): + this_id = node.GetID() + child_ids = [child.GetID() for child in node.children.values()] + self._child_ids_by_parent_id[this_id] = child_ids + + if node.GetParent(): + parent_id = node.GetParent().GetID() + self._parent_ids_by_child_id[this_id] = parent_id else: - self._scalar_structs_elem_paths.append(elem_paths_by_id[id]) + self._parent_ids_by_child_id[this_id] = 0 - self._capacities_by_collectable_id = {} - self._sparse_collectable_ids = set() - self._contig_collectable_ids = set() - for id, dtype in dtype_by_elem_id.items(): - if '_contig_capacity' in dtype: - self._contig_collectable_ids.add(id) - elif '_sparse_capacity' in dtype: - self._sparse_collectable_ids.add(id) - - if '_contig_capacity' in dtype or '_sparse_capacity' in dtype: - self._capacities_by_collectable_id[id] = int(dtype.split('_capacity')[1]) + self._elem_names_by_id[this_id] = node.GetName() + self._elem_paths_by_id[this_id] = node.GetPath() - self._child_ids_by_parent_id = child_ids_by_parent_id - self._parent_ids_by_child_id = parent_ids_by_child_id - self._elem_names_by_id = elem_names_by_id - self._elem_paths_by_id = elem_paths_by_id - self._elem_ids_by_path = {v: k for k, v in elem_paths_by_id.items()} - - collectable_ids_by_elem_path = {} - for id, path in elem_paths_by_id.items(): - if id in self._widget_types_by_elem_id: - collectable_ids_by_elem_path[path] = id - - elem_paths_by_collectable_id = {v:k for k,v in collectable_ids_by_elem_path.items()} - self._collectable_id_by_elem_path = collectable_ids_by_elem_path + self._child_ids_by_parent_id = {} + self._parent_ids_by_child_id = {} + self._elem_names_by_id = {} + self._elem_paths_by_id = {} + self._simhier_tree.VisitNodes(HandleNode) + self._elem_ids_by_path = {v: k for k, v in self._elem_paths_by_id.items()} # Sanity checks to ensure that no element path contains 'root.' for _,elem_path in self._elem_paths_by_id.items(): @@ -95,13 +91,6 @@ def GetPath(elem_id, parent_ids_by_child_id, elem_names_by_id): for elem_path,_ in self._elem_ids_by_path.items(): assert elem_path.find('root.') == -1 - for elem_path,_ in collectable_ids_by_elem_path.items(): - assert elem_path.find('root.') == -1 - - for _,elem_paths in elem_paths_by_collectable_id.items(): - for elem_path in elem_paths: - assert elem_path.find('root.') == -1 - for elem_path in self._scalar_stats_elem_paths: assert elem_path.find('root.') == -1 @@ -111,11 +100,18 @@ def GetPath(elem_id, parent_ids_by_child_id, elem_names_by_id): for elem_path in self._container_elem_paths: assert elem_path.find('root.') == -1 - for elem_path,_ in self._collectable_id_by_elem_path.items(): + for elem_path,_ in self._cids_by_elem_path.items(): assert elem_path.find('root.') == -1 - def GetRootID(self): - return self._root_id + def GetTree(self): + return self._simhier_tree + + def GetMetaAtPath(self, elem_path, meta_name): + return self._simhier_tree.GetMetaAtPath(elem_path, meta_name) + + def GetMetaForCollectionID(self, elem_id, meta_name): + elem_path = self.GetElemPath(elem_id) + return self.GetMetaAtPath(elem_path, meta_name) def GetParentID(self, elem_id): return self._parent_ids_by_child_id[elem_id] @@ -130,20 +126,20 @@ def GetElemID(self, elem_path): return self._elem_ids_by_path.get(elem_path) def GetCollectionID(self, elem_path): - return self._collectable_id_by_elem_path.get(elem_path) + return self._cids_by_elem_path.get(elem_path) def GetContainerIDs(self): - return self._contig_collectable_ids | self._sparse_collectable_ids + return self._contig_cids | self._sparse_cids def GetCapacityByCollectionID(self, collectable_id): - return self._capacities_by_collectable_id.get(collectable_id) + return self._capacities_by_cid.get(collectable_id) def GetCapacityByElemPath(self, elem_path): collectable_id = self.GetCollectionID(elem_path) return self.GetCapacityByCollectionID(collectable_id) def GetSparseFlagByCollectionID(self, collectable_id): - return collectable_id in self._sparse_collectable_ids + return collectable_id in self._sparse_cids def GetSparseFlagByElemPath(self, elem_path): collectable_id = self.GetCollectionID(elem_path) @@ -152,8 +148,16 @@ def GetSparseFlagByElemPath(self, elem_path): def GetName(self, elem_id): return self._elem_names_by_id[elem_id] - def GetElemPaths(self): - return self._elem_paths_by_id.values() + def GetElemPaths(self, leaves_only=False): + if not leaves_only: + return self._elem_paths_by_id.values() + else: + paths = [] + def HandleLeaf(leaf): + paths.append(leaf.GetPath()) + + self.GetTree().VisitLeaves(HandleLeaf) + return paths def GetScalarStatsElemPaths(self): return copy.deepcopy(self._scalar_stats_elem_paths) @@ -170,21 +174,136 @@ def GetItemElemPaths(self): return elem_paths def GetWidgetType(self, elem_id): - return self._widget_types_by_elem_id.get(elem_id, '') + return self._widget_types_by_cid.get(elem_id, '') + +class SimHierTree: + class Node: + def __init__(self, name, parent=None): + self.name = name + self.parent = parent + self.children = {} + self.id = 0 # assigned later (all nodes) + self._path = None + self._meta = {} + + def GetID(self): + return self.id + + def GetName(self): + return self.name + + def GetParent(self): + return self.parent + + def GetChildren(self): + return [child for child in self.children.values()] + + def AddChild(self, name): + if name not in self.children: + self.children[name] = SimHierTree.Node(name, parent=self) + return self.children[name] + + def GetChild(self, name): + return self.children.get(name) + + def GetPath(self): + if not self._path: + parts = [] + node = self + while node and node.GetName(): + parts.append(node.GetName()) + node = node.GetParent() + + parts.reverse() + self._path = '.'.join(parts) + + return self._path + + def SetMeta(self, name, value): + self._meta[name] = value + + def GetMeta(self, name): + return self._meta[name] + + def HasMeta(self, name): + return name in self._meta + + def __repr__(self): + return f"Node(name={self.name}, id={self.id})" + + def __init__(self): + self.root = self.Node("") + self._nodes_by_path = {} + self._next_id = 1 + + def Insert(self, path_str): + parts = path_str.split(".") + curr = self.root + for part in parts: + curr = curr.AddChild(part) + + def BuildFromList(self, paths): + paths.sort() + for p in paths: + self.Insert(p) + self.__AssignDfsIDs() + self.__CacheNodesByPath() + + def GetRoot(self): + return self.root + + def GetNodeAtPath(self, path_str): + parts = path_str.split('.') + curr = self.root + for part in parts: + next = curr.GetChild(part) + if not next: + raise Exception(f'SimHierTree path does not exist: "{path_str}"') + curr = next + + return curr + + def GetMetaAtPath(self, path_str, meta_name): + node = self.GetNodeAtPath(path_str) + return node.GetMeta(meta_name) + + def SetMetaAtPath(self, path_str, meta_name, meta_value): + node = self.GetNodeAtPath(path_str) + node.SetMeta(meta_name, meta_value) + + def VisitLeaves(self, callback): + def Recurse(node, callback): + if node is not self.root and not node.children: + callback(node) + else: + for child in node.children.values(): + Recurse(child, callback) + + Recurse(self.root, callback) + + def VisitNodes(self, callback): + def Recurse(node, callback): + if node is not self.root: + callback(node) + for child in node.children.values(): + Recurse(child, callback) + + Recurse(self.root, callback) - def GetElemPathsMatchingRegex(self, elem_path_regex): - return [elem_path for elem_path in self.GetElemPaths() if re.match(elem_path_regex)] + def __AssignDfsIDs(self): + def Recurse(node): + if node is not self.root: + node.id = self._next_id + self._next_id += 1 + for child in node.children.values(): + Recurse(child) - def __RecurseBuildHierarchy(self, cursor, parent_id, child_ids_by_parent_id): - cursor.execute("SELECT Id FROM ElementTreeNodes WHERE ParentID={}".format(parent_id)) - child_rows = cursor.fetchall() + Recurse(self.root) - for row in child_rows: - if self._root_id is None and parent_id == 0: - self._root_id = row[0] - elif self._root_id is not None and parent_id == 0: - raise Exception('Multiple roots found in hierarchy') + def __CacheNodesByPath(self): + def Recurse(node, map): + map[node.GetPath()] = node + for child in node.children.values(): + Recurse(child, map) - child_id = row[0] - child_ids_by_parent_id[parent_id] = child_ids_by_parent_id.get(parent_id, []) + [child_id] - self.__RecurseBuildHierarchy(cursor, child_id, child_ids_by_parent_id) + Recurse(self.root, self._nodes_by_path) diff --git a/python/argos/viewer/model/tiny_strings.py b/python/argos/viewer/model/tiny_strings.py new file mode 100644 index 00000000..dea5b361 --- /dev/null +++ b/python/argos/viewer/model/tiny_strings.py @@ -0,0 +1,18 @@ +import sqlite3 + +class TinyStrings: + def __init__(self, db_file): + with sqlite3.connect(db_file) as conn: + cursor = conn.cursor() + cursor.execute('SELECT StringValue,StringID FROM TinyStringIDs') + + self._strings_by_id = {} + for sval, sid in cursor.fetchall(): + self._strings_by_id[sid] = sval + + def GetString(self, string_id, must_exist=False): + if string_id in self._strings_by_id: + return self._strings_by_id[string_id] + if must_exist: + raise Exception(f'String ID does not exist: {string_id}') + return None diff --git a/python/argos/viewer/model/workspace.py b/python/argos/viewer/model/workspace.py index c691761c..08d5090d 100644 --- a/python/argos/viewer/model/workspace.py +++ b/python/argos/viewer/model/workspace.py @@ -3,14 +3,14 @@ from viewer.gui.view_settings import ViewSettings class Workspace: - def __init__(self, db_path, view_file, dev_debug): + def __init__(self, db_path, view_file): self._view_settings = ViewSettings() - self._frame = ArgosFrame(db_path, self._view_settings, dev_debug) + self._frame = ArgosFrame(db_path, self._view_settings) self._frame.PostLoad(view_file) self._frame.Show() self._frame.Bind(wx.EVT_CLOSE, self.__OnCloseFrame) def __OnCloseFrame(self, event): - if self._view_settings.SaveView(on_frame_closing=True): + if self._view_settings.OnFrameClosing(): self._frame.Unbind(wx.EVT_CLOSE) self._frame.Destroy() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a5af49ed..52191f96 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1 +1,2 @@ +add_subdirectory(argos) add_subdirectory(sqlite) diff --git a/test/argos/BlobReplayer.hpp b/test/argos/BlobReplayer.hpp new file mode 100644 index 00000000..081b192f --- /dev/null +++ b/test/argos/BlobReplayer.hpp @@ -0,0 +1,500 @@ +// clang-format off + +#pragma once + +#include "simdb/Exceptions.hpp" +#include "simdb/apps/argos/Checkpoint.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/utils/Compress.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argos_test { + +using simdb::argos::Action; + +/// Minimal byte reader matching python/argos/viewer/model/data_deserializers.py ByteBuffer. +class ByteBuffer +{ +public: + explicit ByteBuffer(std::vector data) : + data_(std::move(data)) + { + } + + bool done() const { return read_idx_ >= data_.size(); } + + size_t tell() const { return read_idx_; } + + std::vector slice(size_t start, size_t end) const + { + if (end > data_.size() || start > end) + { + throw simdb::DBException("ByteBuffer slice out of range"); + } + return std::vector(data_.begin() + static_cast(start), + data_.begin() + static_cast(end)); + } + + template + T read() + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + T val{}; + readBytes(&val, sizeof(T)); + return val; + } + + void readBytes(void* out, size_t num_bytes) + { + if (read_idx_ + num_bytes > data_.size()) + { + throw simdb::DBException("ByteBuffer read past end"); + } + std::memcpy(out, data_.data() + read_idx_, num_bytes); + read_idx_ += num_bytes; + } + +private: + std::vector data_; + size_t read_idx_ = 0; +}; + +struct BlobContext +{ + uint16_t current_cid = 0; + uint64_t current_tick = 0; +}; + +class BlobHandler +{ +public: + virtual ~BlobHandler() = default; + + virtual void handleScalarClosed(BlobContext&) {} + virtual void handleScalarCarried(BlobContext&) {} + virtual void handleScalarFullDump(BlobContext&, const std::vector&) {} + + virtual void handleContigContainerClosed(BlobContext&) {} + virtual void handleContigContainerCarried(BlobContext&) {} + virtual void handleContigContainerFullDump(BlobContext&, const std::vector>&) {} + virtual void handleContigContainerSwap(BlobContext&, uint16_t, const std::vector&) {} + virtual void handleContigContainerMultiSwap(BlobContext&, const std::vector&, + const std::vector>&) {} + virtual void handleContigContainerArrival(BlobContext&, const std::vector&) {} + virtual void handleContigContainerDeparture(BlobContext&) {} + virtual void handleContigContainerBookends(BlobContext&, const std::vector&) {} + virtual void handleContigContainerMimo(BlobContext&, uint8_t /*depart_count*/, uint8_t /*arrive_count*/, + const std::vector>&) {} + + virtual void handleSparseContainerClosed(BlobContext&) {} + virtual void handleSparseContainerCarried(BlobContext&) {} + virtual void handleSparseContainerFullDump(BlobContext&, const std::map>&) {} + virtual void handleSparseContainerExchangedBin(BlobContext&, uint16_t, const std::vector&) {} + virtual void handleSparseContainerMultiSwap(BlobContext&, const std::vector&, + const std::vector>&) {} + virtual void handleSparseContainerRemovedBin(BlobContext&, uint16_t) {} + virtual void handleSparseContainerAddedBin(BlobContext&, uint16_t, const std::vector&) {} + virtual void handleSparseContainerMultiRemovedBins(BlobContext&, const std::vector&) {} + + virtual void snapshotTick(BlobContext&) {} +}; + +/// Collectable metadata loaded from CollectableTreeNodes (mirrors python SimHierarchy routing). +class CollectableRegistry +{ +public: + explicit CollectableRegistry(simdb::DatabaseManager* db_mgr) + { + auto query = db_mgr->createQuery("CollectableTreeNodes"); + int32_t cid = 0; + std::string type_name; + query->select("CID", cid); + query->select("TypeName", type_name); + + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + const auto cid_u16 = static_cast(cid); + dtype_by_cid_[cid_u16] = type_name; + if (type_name.find("_contig_") != std::string::npos || type_name.find("_sparse_") != std::string::npos) + { + container_cids_.insert(cid_u16); + if (type_name.find("_sparse_") != std::string::npos) + { + sparse_cids_.insert(cid_u16); + } + } + } + } + + bool isContainer(uint16_t cid) const { return container_cids_.count(cid) > 0; } + + bool isSparse(uint16_t cid) const { return sparse_cids_.count(cid) > 0; } + + const std::string& getTypeName(uint16_t cid) const + { + return dtype_by_cid_.at(cid); + } + +private: + std::unordered_map dtype_by_cid_; + std::unordered_set container_cids_; + std::unordered_set sparse_cids_; +}; + +struct BlobResources +{ + CollectableRegistry& registry; + BlobHandler& handler; + std::function(uint16_t cid, ByteBuffer&)> deserialize_bin; + ByteBuffer* buf = nullptr; +}; + +/// Handler step returned by HandleCID / HandleAction / HandleScalarAction (python-style chaining). +struct HandlerStep +{ + using Fn = std::function; + Fn fn; + + HandlerStep() = default; + + explicit HandlerStep(Fn step_fn) : + fn(std::move(step_fn)) + { + } + + explicit operator bool() const { return static_cast(fn); } + + HandlerStep operator()(BlobResources& resources, BlobContext& context) const + { + return fn ? fn(resources, context) : HandlerStep{}; + } +}; + +HandlerStep handleAction(BlobResources& resources, BlobContext& context); +HandlerStep handleScalarAction(BlobResources& resources, BlobContext& context, uint8_t action); +HandlerStep handleContigContainerAction(BlobResources& resources, BlobContext& context, uint8_t action); +HandlerStep handleSparseContainerAction(BlobResources& resources, BlobContext& context, uint8_t action); + +inline HandlerStep handleCID(BlobResources& resources, BlobContext& context) +{ + if (resources.buf->done()) + { + return {}; + } + + context.current_cid = resources.buf->read(); + return HandlerStep{handleAction}; +} + +inline HandlerStep handleAction(BlobResources& resources, BlobContext& context) +{ + const auto action = resources.buf->read(); + + if (!resources.registry.isContainer(context.current_cid)) + { + return HandlerStep{[action](BlobResources& res, BlobContext& ctx) { + return handleScalarAction(res, ctx, action); + }}; + } + + if (resources.registry.isSparse(context.current_cid)) + { + return HandlerStep{[action](BlobResources& res, BlobContext& ctx) { + return handleSparseContainerAction(res, ctx, action); + }}; + } + + return HandlerStep{[action](BlobResources& res, BlobContext& ctx) { + return handleContigContainerAction(res, ctx, action); + }}; +} + +inline HandlerStep handleScalarAction(BlobResources& resources, BlobContext& context, uint8_t action) +{ + switch (static_cast(action)) + { + case Action::CLOSED: + resources.handler.handleScalarClosed(context); + break; + case Action::CARRY: + resources.handler.handleScalarCarried(context); + break; + default: + { + auto deserialize = [&](ByteBuffer& byte_buf) { + return resources.deserialize_bin(context.current_cid, byte_buf); + }; + + switch (static_cast(action)) + { + case Action::FULL: + resources.handler.handleScalarFullDump(context, deserialize(*resources.buf)); + break; + default: + throw simdb::DBException("Unexpected scalar action byte: " + std::to_string(action)); + } + break; + } + } + + return HandlerStep{handleCID}; +} + +inline HandlerStep handleContigContainerAction(BlobResources& resources, BlobContext& context, uint8_t action) +{ + switch (static_cast(action)) + { + case Action::CLOSED: + resources.handler.handleContigContainerClosed(context); + break; + case Action::CARRY: + resources.handler.handleContigContainerCarried(context); + break; + case Action::FULL: + { + const auto size = resources.buf->read(); + std::vector> bins; + bins.reserve(size); + for (uint16_t i = 0; i < size; ++i) + { + (void)i; + bins.emplace_back(resources.deserialize_bin(context.current_cid, *resources.buf)); + } + + resources.handler.handleContigContainerFullDump(context, bins); + break; + } + case Action::CONTAINER_SWAP: + { + const auto bin_idx = resources.buf->read(); + auto payload = resources.deserialize_bin(context.current_cid, *resources.buf); + resources.handler.handleContigContainerSwap(context, bin_idx, payload); + break; + } + case Action::CONTAINER_MULTI_SWAP: + { + const auto count = resources.buf->read(); + std::vector indices; + std::vector> payloads; + indices.reserve(count); + payloads.reserve(count); + for (uint8_t i = 0; i < count; ++i) + { + (void)i; + indices.push_back(resources.buf->read()); + payloads.push_back(resources.deserialize_bin(context.current_cid, *resources.buf)); + } + resources.handler.handleContigContainerMultiSwap(context, indices, payloads); + break; + } + case Action::CONTIG_ARRIVE: + { + auto payload = resources.deserialize_bin(context.current_cid, *resources.buf); + resources.handler.handleContigContainerArrival(context, payload); + break; + } + case Action::CONTIG_DEPART: + resources.handler.handleContigContainerDeparture(context); + break; + case Action::CONTIG_BOOKENDS: + { + auto payload = resources.deserialize_bin(context.current_cid, *resources.buf); + resources.handler.handleContigContainerBookends(context, payload); + break; + } + case Action::CONTIG_MIMO: + { + const auto depart_count = resources.buf->read(); + const auto arrive_count = resources.buf->read(); + std::vector> arrive_payloads; + arrive_payloads.reserve(arrive_count); + for (uint8_t i = 0; i < arrive_count; ++i) + { + (void)i; + arrive_payloads.push_back(resources.deserialize_bin(context.current_cid, *resources.buf)); + } + resources.handler.handleContigContainerMimo(context, depart_count, arrive_count, arrive_payloads); + break; + } + default: + throw simdb::DBException("Unexpected contig container action byte: " + std::to_string(action)); + } + + return HandlerStep{handleCID}; +} + +inline HandlerStep handleSparseContainerAction(BlobResources& resources, BlobContext& context, uint8_t action) +{ + switch (static_cast(action)) + { + case Action::CLOSED: + resources.handler.handleSparseContainerClosed(context); + break; + case Action::CARRY: + resources.handler.handleSparseContainerCarried(context); + break; + case Action::FULL: + { + const auto size = resources.buf->read(); + std::map> bins; + for (uint16_t i = 0; i < size; ++i) + { + (void)i; + const auto bin_idx = resources.buf->read(); + bins.emplace(bin_idx, resources.deserialize_bin(context.current_cid, *resources.buf)); + } + + resources.handler.handleSparseContainerFullDump(context, bins); + break; + } + case Action::CONTAINER_SWAP: + { + const auto bin_idx = resources.buf->read(); + auto payload = resources.deserialize_bin(context.current_cid, *resources.buf); + resources.handler.handleSparseContainerExchangedBin(context, bin_idx, payload); + break; + } + case Action::CONTAINER_MULTI_SWAP: + { + const auto count = resources.buf->read(); + std::vector indices; + std::vector> payloads; + indices.reserve(count); + payloads.reserve(count); + for (uint8_t i = 0; i < count; ++i) + { + (void)i; + indices.push_back(resources.buf->read()); + payloads.push_back(resources.deserialize_bin(context.current_cid, *resources.buf)); + } + resources.handler.handleSparseContainerMultiSwap(context, indices, payloads); + break; + } + case Action::SPARSE_REMOVE: + { + const auto bin_idx = resources.buf->read(); + resources.handler.handleSparseContainerRemovedBin(context, bin_idx); + break; + } + case Action::SPARSE_ADD: + { + const auto bin_idx = resources.buf->read(); + auto payload = resources.deserialize_bin(context.current_cid, *resources.buf); + resources.handler.handleSparseContainerAddedBin(context, bin_idx, payload); + break; + } + case Action::SPARSE_MULTI_REMOVE: + { + const auto count = resources.buf->read(); + std::vector indices; + indices.reserve(count); + for (uint8_t i = 0; i < count; ++i) + { + (void)i; + indices.push_back(resources.buf->read()); + } + resources.handler.handleSparseContainerMultiRemovedBins(context, indices); + break; + } + default: + throw simdb::DBException("Unexpected sparse container action byte: " + std::to_string(action)); + } + + return HandlerStep{handleCID}; +} + +class BlobIterator +{ +public: + BlobIterator(simdb::DatabaseManager* db_mgr, CollectableRegistry& registry) : + db_mgr_(db_mgr), + registry_(registry) + { + } + + void iterate(BlobHandler& handler, + const std::function(uint16_t cid, ByteBuffer&)>& deserialize_bin) + { + auto timestamp_rows = loadTimestampRows_(); + auto blobs_by_ts_id = loadCollectionRecords_(); + + BlobResources resources{registry_, handler, deserialize_bin, nullptr}; + BlobContext context; + + for (const auto& [ts_id, tick] : timestamp_rows) + { + const auto blob_iter = blobs_by_ts_id.find(ts_id); + if (blob_iter == blobs_by_ts_id.end()) + { + continue; + } + + std::vector uncompressed; + simdb::decompressData(blob_iter->second, uncompressed); + + ByteBuffer buf(std::move(uncompressed)); + resources.buf = &buf; + context.current_tick = tick; + + HandlerStep handler_step{handleCID}; + while (handler_step) + { + handler_step = handler_step(resources, context); + } + + handler.snapshotTick(context); + } + } + +private: + std::vector> loadTimestampRows_() + { + std::vector> rows; + auto query = db_mgr_->createQuery("Timestamps"); + + int64_t ts_id = 0; + uint64_t tick = 0; + query->select("Id", ts_id); + query->select("Timestamp", tick); + + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + rows.emplace_back(ts_id, tick); + } + return rows; + } + + std::map> loadCollectionRecords_() + { + std::map> blobs; + auto query = db_mgr_->createQuery("CollectionRecords"); + + int64_t timestamp_id = 0; + std::vector records; + query->select("TimestampID", timestamp_id); + query->select("Records", records); + + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + blobs.emplace(timestamp_id, records); + } + return blobs; + } + + simdb::DatabaseManager* db_mgr_; + CollectableRegistry& registry_; +}; + +} // namespace argos_test diff --git a/test/argos/CMakeLists.txt b/test/argos/CMakeLists.txt new file mode 100644 index 00000000..e6a54f20 --- /dev/null +++ b/test/argos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Relatively quick test for simdb_regress (30 seconds) +simdb_test(ArgosSmoke main.cpp) + +# Exhaustive test for github checks (4 minutes) +add_custom_target(ArgosSmoke_10k + COMMAND $ 10000 + WORKING_DIRECTORY $ + DEPENDS ArgosSmoke + VERBATIM) diff --git a/test/argos/compare.py b/test/argos/compare.py new file mode 100644 index 00000000..6c220127 --- /dev/null +++ b/test/argos/compare.py @@ -0,0 +1,172 @@ +import os, sys, subprocess, sqlite3 +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +_ARGOS_ROOT = _REPO_ROOT / "python" / "argos" +if str(_ARGOS_ROOT) not in sys.path: + sys.path.insert(0, str(_ARGOS_ROOT)) + +# Compare two databases by deserializing collection data at every time point. +baseline_db = sys.argv[1] +test_db = sys.argv[2] + + +class _StubInspector: + def RefreshWidgetsOnAllTabs(self): + pass + + +class _StubFrame: + inspector = _StubInspector() + + +def _QueryRows(db_path, sql): + conn = sqlite3.connect(db_path) + try: + cur = conn.cursor() + cur.execute(sql) + return cur.fetchall() + finally: + conn.close() + + +def ValidateComparableSchemas(db1_path, db2_path): + db1_path = os.path.abspath(db1_path) + db2_path = os.path.abspath(db2_path) + + ctn_sql = ( + "SELECT CID, FullPath, ClockID, TypeName " + "FROM CollectableTreeNodes ORDER BY CID" + ) + rows1 = _QueryRows(db1_path, ctn_sql) + rows2 = _QueryRows(db2_path, ctn_sql) + if rows1 != rows2: + print("Error: CollectableTreeNodes differ (ignoring ShowInUI)") + print(f" baseline rows: {len(rows1)}, test rows: {len(rows2)}") + for i, (r1, r2) in enumerate(zip(rows1, rows2)): + if r1 != r2: + print(f" first differing row at index {i}: baseline={r1!r}, test={r2!r}") + break + sys.exit(1) + + schema_sql = "SELECT Id, RootTypeName FROM DataTypeSchemas ORDER BY Id" + rows1 = _QueryRows(db1_path, schema_sql) + rows2 = _QueryRows(db2_path, schema_sql) + if rows1 != rows2: + print("Error: DataTypeSchemas differ") + print(f" baseline rows: {len(rows1)}, test rows: {len(rows2)}") + for i, (r1, r2) in enumerate(zip(rows1, rows2)): + if r1 != r2: + print(f" first differing row at index {i}: baseline={r1!r}, test={r2!r}") + break + sys.exit(1) + + nodes_sql = ( + "SELECT Id, SchemaId, Name, TypeName, FormatStr " + "FROM DataTypeNodes ORDER BY Id" + ) + rows1 = _QueryRows(db1_path, nodes_sql) + rows2 = _QueryRows(db2_path, nodes_sql) + if rows1 != rows2: + print("Error: DataTypeNodes differ") + print(f" baseline rows: {len(rows1)}, test rows: {len(rows2)}") + for i, (r1, r2) in enumerate(zip(rows1, rows2)): + if r1 != r2: + print(f" first differing row at index {i}: baseline={r1!r}, test={r2!r}") + break + sys.exit(1) + + +def GetShowInUIPaths(db_path): + sql = "SELECT FullPath FROM CollectableTreeNodes WHERE ShowInUI=1" + return {row[0] for row in _QueryRows(db_path, sql)} + + +def GetComparisonTickRange(db_path): + conn = sqlite3.connect(os.path.abspath(db_path)) + try: + cur = conn.cursor() + cur.execute( + "SELECT MIN(CAST(Timestamp AS INTEGER)), MAX(CAST(Timestamp AS INTEGER)) " + "FROM Timestamps" + ) + min_tick, max_tick = cur.fetchone() + return int(min_tick), int(max_tick) + finally: + conn.close() + + +def MakeDataRetriever(db_path): + from viewer.model.dtype_inspector import DataTypeInspector + from viewer.model.simhier import SimHierarchy + from viewer.model.data_retriever import DataRetriever + + db_path = os.path.abspath(db_path) + dtype_inspector = DataTypeInspector(db_path) + simhier = SimHierarchy(dtype_inspector.connection, dtype_inspector) + retriever = DataRetriever(_StubFrame(), db_path, simhier, dtype_inspector) + return retriever + + +def SmokeTest(db_file): + return True + + +def CompareTickByTick(): + print(f'Comparing {baseline_db} and {test_db}') + ValidateComparableSchemas(baseline_db, test_db) + + min_tick1, max_tick1 = GetComparisonTickRange(baseline_db) + min_tick2, max_tick2 = GetComparisonTickRange(test_db) + start_tick = max(min_tick1, min_tick2) + end_tick = min(max_tick1, max_tick2) + + if start_tick > end_tick: + print( + f'Error: no overlapping time window ' + f'(baseline [{min_tick1}, {max_tick1}], test [{min_tick2}, {max_tick2}])' + ) + sys.exit(1) + + tick_count = end_tick - start_tick + 1 + print(f'Comparison tick range: [{start_tick}, {end_tick}] ({tick_count} ticks)') + + retriever1 = MakeDataRetriever(baseline_db) + retriever2 = MakeDataRetriever(test_db) + + elem_paths = sorted(GetShowInUIPaths(baseline_db) & GetShowInUIPaths(test_db)) + valid = set(retriever1.simhier.GetItemElemPaths()) & set(retriever2.simhier.GetItemElemPaths()) + elem_paths = [p for p in elem_paths if p in valid] + + if not elem_paths: + print('Error: no elem paths with ShowInUI=1 in both databases') + sys.exit(1) + + print(f'Comparing {len(elem_paths)} elem paths') + + tick = start_tick + while True: + all_values1 = retriever1.UnpackRange(tick, tick, elem_paths) + all_values2 = retriever2.UnpackRange(tick, tick, elem_paths) + + for elem_path in elem_paths: + values1 = all_values1[elem_path] + values2 = all_values2[elem_path] + if values1 != values2: + print(f'Error at tick {tick}, path {elem_path!r}') + print(f' baseline: {values1}') + print(f' test: {values2}') + sys.exit(1) + + if tick == end_tick: + break + tick += 1 + + print('Comparison passed.') + + +success = SmokeTest(baseline_db) and SmokeTest(test_db) +if not success: + sys.exit(1) +else: + CompareTickByTick() diff --git a/test/argos/main.cpp b/test/argos/main.cpp new file mode 100644 index 00000000..a328b620 --- /dev/null +++ b/test/argos/main.cpp @@ -0,0 +1,1448 @@ +// clang-format off + +#include "BlobReplayer.hpp" +#include "SimDBTester.hpp" +#include "simdb/apps/AppManager.hpp" +#include "simdb/apps/argos/ArgosCollector.hpp" +#include "simdb/apps/argos/StreamBuffer.hpp" +#include "simdb/sqlite/DatabaseManager.hpp" +#include "simdb/utils/Demangle.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +TEST_INIT; + +/// Call once per test function. +#define TEST_METHOD_INIT simdb::argos::EntryPoint::resetCIDs() + +std::mt19937_64& testRng() +{ + static std::mt19937_64 rng(std::random_device{}()); + return rng; +} + +void reseedTestRng(uint64_t seed) +{ + testRng().seed(seed); +} + +template +IntT randomInt(const IntT min_inclusive = std::numeric_limits::min(), + const IntT max_inclusive = std::numeric_limits::max()) +{ + std::uniform_int_distribution dist(min_inclusive, max_inclusive); + return dist(testRng()); +} + +template +double randomFloat(const FloatT min_inclusive = std::numeric_limits::min(), + const FloatT max_inclusive = std::numeric_limits::max()) +{ + std::uniform_real_distribution dist(min_inclusive, max_inclusive); + return dist(testRng()); +} + +bool randomBool() +{ + return randomInt() % 2 == 0; +} + +std::string randomString(size_t num_chars = 8) +{ + static constexpr char charset[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + constexpr size_t charset_size = sizeof(charset) - 1; // exclude null terminator + + std::string result; + result.reserve(num_chars); + + for (size_t i = 0; i < num_chars; ++i) + { + result += charset[randomInt(0, charset_size - 1)]; + } + + return result; +} + +enum class UnsignedEnum : uint32_t { __MIN__ = 0, RED = __MIN__, GREEN = 1, BLUE = 2, __MAX__, __INVALID__ = 100}; +enum class SignedEnum : int32_t { __MIN__ = -1, RED = __MIN__, GREEN = 0, BLUE = 1, __MAX__, __INVALID__ = 100}; + +static std::map unsigned_enum_map{ + {UnsignedEnum::RED, "RED"}, + {UnsignedEnum::GREEN, "GREEN"}, + {UnsignedEnum::BLUE, "BLUE"}, + {UnsignedEnum::__INVALID__, "INVALID"} +}; + +static std::map signed_enum_map{ + {SignedEnum::RED, "RED"}, + {SignedEnum::GREEN, "GREEN"}, + {SignedEnum::BLUE, "BLUE"}, + {SignedEnum::__INVALID__, "INVALID"} +}; + +std::ostream& operator<<(std::ostream& os, const UnsignedEnum e) +{ + const auto iter = unsigned_enum_map.find(e); + if (iter != unsigned_enum_map.end()) + { + return os << iter->second; + } + return os << static_cast(e); +} + +std::ostream& operator<<(std::ostream& os, const SignedEnum e) +{ + const auto iter = signed_enum_map.find(e); + if (iter != signed_enum_map.end()) + { + return os << iter->second; + } + return os << static_cast(e); +} + +template +E randomEnum() +{ + switch (randomInt(0, 2)) + { + case 0: + return E::RED; + case 1: + return E::GREEN; + default: + return E::BLUE; + } +} + +template +E readEnum(argos_test::ByteBuffer& buf) +{ + using underlying_t = std::underlying_type_t; + return static_cast(buf.read()); +} + +struct MultipleTypes +{ + int32_t i = 0; + std::string s; + bool b = false; + UnsignedEnum ue = UnsignedEnum::RED; + SignedEnum se = SignedEnum::RED; + + bool operator==(const MultipleTypes& other) const + { + return i == other.i && s == other.s && b == other.b && ue == other.ue && se == other.se; + } + + static MultipleTypes random() + { + return {randomInt(), randomString(), randomBool(), randomEnum(), randomEnum()}; + } + + void writeToBuffer(simdb::argos::StreamBuffer& buf) const + { + buf.append(i); + buf.append(s); + buf.append(b); + buf.append(ue); + buf.append(se); + } + + static MultipleTypes readFromBuffer(argos_test::ByteBuffer& buf, const std::unordered_map& strings) + { + MultipleTypes val; + val.i = buf.read(); + const auto sid = buf.read(); + const auto str_iter = strings.find(sid); + if (str_iter == strings.end()) + { + throw simdb::DBException("Unknown TinyString ID during replay: " + std::to_string(sid)); + } + val.s = str_iter->second; + val.b = static_cast(buf.read()); + val.ue = readEnum(buf); + val.se = readEnum(buf); + return val; + } +}; + +std::ostream& operator<<(std::ostream& os, const MultipleTypes& val) +{ + return os << "MultipleTypes{i=" << val.i << ",s=" << val.s << ",b=" << std::boolalpha << val.b + << ",ue=" << val.ue << ",se=" << val.se << "}"; +} + +using ScalarValue = std::variant; +using ContigValue = std::vector; +using SparseValue = std::map; +using CollectableValue = std::variant; + +constexpr size_t kContainerCapacity = 8; + +std::ostream& operator<<(std::ostream& os, const ScalarValue& val) +{ + std::visit([&os](const auto& v) { os << v; }, val); + return os; +} + +std::ostream& operator<<(std::ostream& os, const ContigValue& val) +{ + os << "["; + for (size_t i = 0; i < val.size(); ++i) + { + if (i != 0) + { + os << ","; + } + os << val[i]; + } + return os << "]"; +} + +std::ostream& operator<<(std::ostream& os, const SparseValue& val) +{ + os << "{"; + bool first = true; + for (const auto& [idx, elem] : val) + { + if (!first) + { + os << ","; + } + first = false; + os << idx << ":" << elem; + } + return os << "}"; +} + +std::ostream& operator<<(std::ostream& os, const CollectableValue& val) +{ + std::visit([&os](const auto& v) { os << v; }, val); + return os; +} + +template +std::vector getScalarBytes(const ScalarT& val, simdb::TinyStrings<>* tiny_strings) +{ + if constexpr (std::is_same_v) + { + auto sid = tiny_strings->getStringID(val); + return getScalarBytes(sid, tiny_strings); + } + else + { + static_assert(std::is_trivial_v && std::is_standard_layout_v); + std::vector bytes; + simdb::argos::StreamBuffer buf(bytes); + buf.append(val); + return bytes; + } +} + +std::vector getScalarBytes(const MultipleTypes& val, simdb::TinyStrings<>* tiny_strings) +{ + std::vector bytes; + simdb::argos::StreamBuffer buf(bytes, tiny_strings); + val.writeToBuffer(buf); + return bytes; +} + +std::vector scalarValueToBytes(const ScalarValue& val, simdb::TinyStrings<>* tiny_strings) +{ + return std::visit([&](const auto& v) { return getScalarBytes(v, tiny_strings); }, val); +} + +std::unordered_map loadStringTable(simdb::DatabaseManager* db_mgr) +{ + std::unordered_map strings; + auto query = db_mgr->createQuery("TinyStringIDs"); + + std::string string_value; + uint32_t string_id = 0; + query->select("StringValue", string_value); + query->select("StringID", string_id); + + auto results = query->getResultSet(); + while (results.getNextRecord()) + { + strings.emplace(string_id, string_value); + } + return strings; +} + +ScalarValue deserializeScalarValue(const std::string& encoded_type, + argos_test::ByteBuffer& buf, + const std::unordered_map& strings) +{ + if (encoded_type == "int") + { + return buf.read(); + } + if (encoded_type == "string") + { + const auto sid = buf.read(); + const auto str_iter = strings.find(sid); + if (str_iter == strings.end()) + { + throw simdb::DBException("Unknown TinyString ID during replay: " + std::to_string(sid)); + } + return str_iter->second; + } + if (encoded_type == "bool") + { + return static_cast(buf.read()); + } + if (encoded_type == "UnsignedEnum") + { + return readEnum(buf); + } + if (encoded_type == "SignedEnum") + { + return readEnum(buf); + } + if (encoded_type == "MultipleTypes") + { + return MultipleTypes::readFromBuffer(buf, strings); + } + throw simdb::DBException("Unsupported scalar encoded type: " + encoded_type); +} + +ScalarValue deserializeScalarPayload(const std::string& encoded_type, + const std::vector& payload, + const std::unordered_map& strings) +{ + argos_test::ByteBuffer buf(payload); + return deserializeScalarValue(encoded_type, buf, strings); +} + +void writeMultipleTypesMetadata(simdb::DatabaseManager* db_mgr) +{ + const auto struct_schema_id = db_mgr->INSERT(SQL_TABLE("DataTypeSchemas"), SQL_VALUES("MultipleTypes"))->getId(); + + auto node_inserter = db_mgr->prepareINSERT(SQL_TABLE("DataTypeNodes")); + node_inserter->createRecordWithColValues(struct_schema_id, "i", "int", ""); + node_inserter->createRecordWithColValues(struct_schema_id, "s", "string", ""); + node_inserter->createRecordWithColValues(struct_schema_id, "b", "bool", ""); + node_inserter->createRecordWithColValues(struct_schema_id, "ue", "UnsignedEnum", ""); + node_inserter->createRecordWithColValues(struct_schema_id, "se", "SignedEnum", ""); + + auto collected_enums = db_mgr->prepareINSERT(SQL_TABLE("CollectedEnums")); + auto unsigned_enum_id = + collected_enums->createRecordWithColValues("UnsignedEnum", simdb::demangle_type()); + auto signed_enum_id = collected_enums->createRecordWithColValues("SignedEnum", simdb::demangle_type()); + + auto enum_members = db_mgr->prepareINSERT(SQL_TABLE("EnumMembers")); + for (const auto& [e, name] : unsigned_enum_map) + { + auto e_str = std::to_string(static_cast(e)); + enum_members->createRecordWithColValues(unsigned_enum_id, name, e_str); + } + + for (const auto& [e, name] : signed_enum_map) + { + auto e_str = std::to_string(static_cast(e)); + enum_members->createRecordWithColValues(signed_enum_id, name, e_str); + } +} + +std::function*)> makeRandomScalarValueFn_(size_t palette_index) +{ + switch (palette_index) + { + case 0: + return [](simdb::TinyStrings<>*) { return ScalarValue{randomInt()}; }; + case 1: + return [](simdb::TinyStrings<>*) { return ScalarValue{randomString()}; }; + case 2: + return [](simdb::TinyStrings<>*) { return ScalarValue{randomBool()}; }; + case 3: + return [](simdb::TinyStrings<>*) { return ScalarValue{randomEnum()}; }; + case 4: + return [](simdb::TinyStrings<>*) { return ScalarValue{randomEnum()}; }; + case 5: + return [](simdb::TinyStrings<>*) { return ScalarValue{MultipleTypes::random()}; }; + default: + throw simdb::DBException("Invalid scalar palette index"); + } +} + +MultipleTypes deserializeBinValue(const std::vector& payload, + const std::unordered_map& strings) +{ + return std::get(deserializeScalarPayload("MultipleTypes", payload, strings)); +} + +ContigValue contigBinsToValue(const std::vector>& bins, + const std::unordered_map& strings) +{ + ContigValue value; + value.reserve(bins.size()); + for (const auto& bin : bins) + { + value.push_back(deserializeBinValue(bin, strings)); + } + return value; +} + +SparseValue sparseBinsToValue(const std::map>& bins, + const std::unordered_map& strings) +{ + SparseValue value; + for (const auto& [idx, bin] : bins) + { + value.emplace(idx, deserializeBinValue(bin, strings)); + } + return value; +} + +bool collectableHasPayload(const CollectableValue& value) +{ + if (std::holds_alternative(value)) + { + return true; + } + if (const auto* contig = std::get_if(&value)) + { + return !contig->empty(); + } + if (const auto* sparse = std::get_if(&value)) + { + return !sparse->empty(); + } + return false; +} + +ContigValue makeRandomContig() +{ + const size_t num_bins = randomInt(0, kContainerCapacity); + ContigValue value; + value.reserve(num_bins); + for (size_t i = 0; i < num_bins; ++i) + { + value.push_back(MultipleTypes::random()); + } + return value; +} + +SparseValue makeRandomSparse() +{ + const size_t num_bins = randomInt(0, kContainerCapacity); + SparseValue value; + std::set occupied; + while (occupied.size() < num_bins) + { + occupied.insert(static_cast(randomInt(0, kContainerCapacity - 1))); + } + for (const uint16_t idx : occupied) + { + value.emplace(idx, MultipleTypes::random()); + } + return value; +} + +struct LiveState +{ + bool closed = false; + bool has_data = false; + std::optional logical; + std::optional latent; +}; + +enum class CollectableKind { Scalar, Contig, Sparse }; + +struct ActiveCollectable +{ + CollectableKind kind = CollectableKind::Scalar; + std::string encoded_type; + std::string bin_element_type; + size_t palette_index = 0; + simdb::argos::EntryPoint* entry_point = nullptr; +}; + +void stageCollectable(const ActiveCollectable& active, const CollectableValue& value, simdb::TinyStrings<>* tiny_strings) +{ + switch (active.kind) + { + case CollectableKind::Scalar: + { + auto bytes = scalarValueToBytes(std::get(value), tiny_strings); + active.entry_point->setScalarValueBytes(std::move(bytes)); + break; + } + case CollectableKind::Contig: + { + const auto& contig = std::get(value); + std::vector> bins; + bins.reserve(contig.size()); + for (const auto& elem : contig) + { + bins.push_back(getScalarBytes(elem, tiny_strings)); + } + active.entry_point->setContigContainerBinBytes(std::move(bins)); + break; + } + case CollectableKind::Sparse: + { + const auto& sparse = std::get(value); + std::map> bins; + for (const auto& [idx, elem] : sparse) + { + bins.emplace(idx, getScalarBytes(elem, tiny_strings)); + } + active.entry_point->setSparseContainerBinBytes(std::move(bins)); + break; + } + } +} + +namespace { + +int randomRoll() +{ + return randomInt(0, 99); +} + +bool maybeClose(LiveState& state, simdb::argos::EntryPoint* ep) +{ + if (!state.closed && state.has_data && randomRoll() < 6) + { + state.latent = state.logical; + state.logical.reset(); + state.closed = true; + state.has_data = false; + ep->closeRecord(); + return true; + } + return false; +} + +bool maybeReopen(LiveState& state, const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) +{ + if (!state.closed) + { + return false; + } + + if (!(state.latent.has_value() && randomRoll() < 8)) + { + return true; + } + + state.closed = false; + state.logical = state.latent; + state.has_data = collectableHasPayload(*state.logical); + stageCollectable(active, *state.logical, tiny_strings); + return true; +} + +struct ActionPolicy +{ + virtual ~ActionPolicy() = default; + virtual void apply(LiveState& state, const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) = 0; +}; + +class ScalarActionPolicy final : public ActionPolicy +{ +public: + void apply(LiveState& state, const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) override + { + auto* ep = active.entry_point; + + if (state.closed) + { + (void)maybeReopen(state, active, tiny_strings); + return; + } + + if (maybeClose(state, ep)) + { + return; + } + + if (randomRoll() > 70) + { + return; + } + + const auto scalar = makeRandomScalarValueFn_(active.palette_index)(tiny_strings); + state.logical = CollectableValue{scalar}; + state.has_data = true; + stageCollectable(active, *state.logical, tiny_strings); + } +}; + +class ContigActionPolicy final : public ActionPolicy +{ +public: + void apply(LiveState& state, const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) override + { + auto* ep = active.entry_point; + + if (state.closed) + { + (void)maybeReopen(state, active, tiny_strings); + return; + } + + if (maybeClose(state, ep)) + { + return; + } + + const auto roll = randomRoll(); + if (roll > 70) + { + return; + } + + ContigValue contig; + if (state.logical && std::holds_alternative(*state.logical)) + { + contig = std::get(*state.logical); + } + + if (!state.has_data || roll < 40) + { + contig = makeRandomContig(); + } else if (roll < 75) + { + if (contig.size() > 1) + { + contig[1] = MultipleTypes::random(); + } else + { + contig = makeRandomContig(); + } + } else if (contig.size() > 1) + { + contig.erase(contig.begin() + 1); + } else + { + contig = makeRandomContig(); + } + + state.logical = CollectableValue{std::move(contig)}; + state.has_data = collectableHasPayload(*state.logical); + stageCollectable(active, *state.logical, tiny_strings); + } +}; + +class SparseActionPolicy final : public ActionPolicy +{ +public: + void apply(LiveState& state, const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) override + { + auto* ep = active.entry_point; + + if (state.closed) + { + (void)maybeReopen(state, active, tiny_strings); + return; + } + + if (maybeClose(state, ep)) + { + return; + } + + const auto roll = randomRoll(); + if (roll > 70) + { + return; + } + + SparseValue sparse; + if (state.logical && std::holds_alternative(*state.logical)) + { + sparse = std::get(*state.logical); + } + + if (!state.has_data || roll < 40) + { + sparse = makeRandomSparse(); + } else if (roll < 70) + { + for (auto& [idx, elem] : sparse) + { + (void)idx; + elem = MultipleTypes::random(); + } + } else if (sparse.size() > 1) + { + auto iter = sparse.begin(); + std::advance(iter, 1); + sparse.erase(iter); + } else + { + sparse = makeRandomSparse(); + } + + state.logical = CollectableValue{std::move(sparse)}; + state.has_data = collectableHasPayload(*state.logical); + stageCollectable(active, *state.logical, tiny_strings); + } +}; + +class SimulationEngine +{ +public: + void seedInitial(uint16_t cid, CollectableValue value) + { + auto& state = live_[cid]; + state.logical = std::move(value); + state.has_data = collectableHasPayload(*state.logical); + state.closed = false; + state.latent.reset(); + } + + void applyRandomAction(const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) + { + auto& state = live_[active.entry_point->getID()]; + switch (active.kind) + { + case CollectableKind::Scalar: + scalar_policy_.apply(state, active, tiny_strings); + break; + case CollectableKind::Contig: + contig_policy_.apply(state, active, tiny_strings); + break; + case CollectableKind::Sparse: + sparse_policy_.apply(state, active, tiny_strings); + break; + } + } + + const LiveState& at(uint16_t cid) const { return live_.at(cid); } + +private: + std::map live_; + ScalarActionPolicy scalar_policy_; + ContigActionPolicy contig_policy_; + SparseActionPolicy sparse_policy_; +}; + +} // namespace + +struct ScalarSpec +{ + std::string path; + std::string encoded_type; +}; + +struct ContainerSpec +{ + std::string path; + std::string encoded_type; + std::string bin_element_type; + size_t capacity = kContainerCapacity; + bool sparse = false; +}; + +/// Reconstructs live collectable values while replaying blobs (python DataExtractionHandler style). +class DataExtractionHandler : public argos_test::BlobHandler +{ +public: + explicit DataExtractionHandler(const std::unordered_map& encoded_type_by_cid, + const std::unordered_map& strings) : + encoded_type_by_cid_(encoded_type_by_cid), + strings_(strings) + { + } + + void handleScalarClosed(argos_test::BlobContext& context) override + { + values_by_cid_.erase(context.current_cid); + } + + void handleScalarCarried(argos_test::BlobContext&) override {} + + void handleScalarFullDump(argos_test::BlobContext& context, const std::vector& payload) override + { + values_by_cid_[context.current_cid] = decodeScalar_(context.current_cid, payload); + } + + void handleContigContainerClosed(argos_test::BlobContext& context) override + { + values_by_cid_.erase(context.current_cid); + } + + void handleContigContainerCarried(argos_test::BlobContext&) override {} + + void handleContigContainerFullDump(argos_test::BlobContext& context, + const std::vector>& bins) override + { + values_by_cid_[context.current_cid] = CollectableValue{contigBinsToValue(bins, strings_)}; + } + + void handleContigContainerSwap(argos_test::BlobContext& context, uint16_t bin_idx, + const std::vector& payload) override + { + auto& contig = contigAt_(context.current_cid); + if (bin_idx >= contig.size()) + { + throw simdb::DBException("Contig swap index out of range"); + } + contig[bin_idx] = deserializeBinValue(payload, strings_); + } + + void handleContigContainerMultiSwap(argos_test::BlobContext& context, const std::vector& indices, + const std::vector>& payloads) override + { + assert(indices.size() == payloads.size()); + for (size_t i = 0; i < indices.size(); ++i) + { + handleContigContainerSwap(context, indices[i], payloads[i]); + } + } + + void handleContigContainerArrival(argos_test::BlobContext& context, const std::vector& payload) override + { + contigAt_(context.current_cid).push_back(deserializeBinValue(payload, strings_)); + } + + void handleContigContainerDeparture(argos_test::BlobContext& context) override + { + auto& contig = contigAt_(context.current_cid); + if (contig.empty()) + { + throw simdb::DBException("Contig departure on empty container"); + } + contig.erase(contig.begin()); + } + + void handleContigContainerBookends(argos_test::BlobContext& context, const std::vector& payload) override + { + handleContigContainerDeparture(context); + handleContigContainerArrival(context, payload); + } + + void handleContigContainerMimo(argos_test::BlobContext& context, uint8_t depart_count, uint8_t arrive_count, + const std::vector>& arrive_payloads) override + { + (void)arrive_count; + for (uint8_t i = 0; i < depart_count; ++i) + { + (void)i; + handleContigContainerDeparture(context); + } + for (const auto& payload : arrive_payloads) + { + handleContigContainerArrival(context, payload); + } + } + + void handleSparseContainerClosed(argos_test::BlobContext& context) override + { + values_by_cid_.erase(context.current_cid); + } + + void handleSparseContainerCarried(argos_test::BlobContext&) override {} + + void handleSparseContainerFullDump(argos_test::BlobContext& context, + const std::map>& bins) override + { + values_by_cid_[context.current_cid] = CollectableValue{sparseBinsToValue(bins, strings_)}; + } + + void handleSparseContainerExchangedBin(argos_test::BlobContext& context, uint16_t bin_idx, + const std::vector& payload) override + { + sparseAt_(context.current_cid)[bin_idx] = deserializeBinValue(payload, strings_); + } + + void handleSparseContainerMultiSwap(argos_test::BlobContext& context, const std::vector& indices, + const std::vector>& payloads) override + { + assert(indices.size() == payloads.size()); + for (size_t i = 0; i < indices.size(); ++i) + { + handleSparseContainerExchangedBin(context, indices[i], payloads[i]); + } + } + + void handleSparseContainerRemovedBin(argos_test::BlobContext& context, uint16_t bin_idx) override + { + sparseAt_(context.current_cid).erase(bin_idx); + } + + void handleSparseContainerAddedBin(argos_test::BlobContext& context, uint16_t bin_idx, + const std::vector& payload) override + { + sparseAt_(context.current_cid)[bin_idx] = deserializeBinValue(payload, strings_); + } + + void handleSparseContainerMultiRemovedBins(argos_test::BlobContext& context, + const std::vector& indices) override + { + for (const auto bin_idx : indices) + { + handleSparseContainerRemovedBin(context, bin_idx); + } + } + + const std::unordered_map& getValuesByCid() const { return values_by_cid_; } + +private: + CollectableValue decodeScalar_(uint16_t cid, const std::vector& payload) const + { + return CollectableValue{deserializeScalarPayload(encoded_type_by_cid_.at(cid), payload, strings_)}; + } + + ContigValue& contigAt_(uint16_t cid) + { + auto iter = values_by_cid_.find(cid); + if (iter == values_by_cid_.end() || !std::holds_alternative(iter->second)) + { + throw simdb::DBException("Contig replay state missing for cid " + std::to_string(cid)); + } + return std::get(iter->second); + } + + SparseValue& sparseAt_(uint16_t cid) + { + auto iter = values_by_cid_.find(cid); + if (iter == values_by_cid_.end() || !std::holds_alternative(iter->second)) + { + throw simdb::DBException("Sparse replay state missing for cid " + std::to_string(cid)); + } + return std::get(iter->second); + } + + const std::unordered_map& encoded_type_by_cid_; + const std::unordered_map& strings_; + std::unordered_map values_by_cid_; +}; + +/// Compares replayed per-tick state against simulator truth. +class TruthValidationHandler final : public DataExtractionHandler +{ +public: + TruthValidationHandler(const std::unordered_map& encoded_type_by_cid, + const std::unordered_map& strings, + const std::map>& truth_by_tick, + const std::vector& tracked_cids) : + DataExtractionHandler(encoded_type_by_cid, strings), + truth_by_tick_(truth_by_tick), + tracked_cids_(tracked_cids) + { + } + + void snapshotTick(argos_test::BlobContext& context) override + { + const auto tick_truth_iter = truth_by_tick_.find(context.current_tick); + if (tick_truth_iter == truth_by_tick_.end()) + { + return; + } + + const auto& truth_at_tick = tick_truth_iter->second; + const auto& replayed = getValuesByCid(); + + for (const auto cid : tracked_cids_) + { + const auto truth_iter = truth_at_tick.find(cid); + if (truth_iter == truth_at_tick.end()) + { + continue; + } + + const auto replay_iter = replayed.find(cid); + if (replay_iter == replayed.end()) + { + continue; + } + + EXPECT_EQUAL(replay_iter->second, truth_iter->second); + } + } + +private: + const std::map>& truth_by_tick_; + const std::vector& tracked_cids_; +}; + +class Harness +{ +public: + Harness(const std::string& db_file, size_t heartbeat = 10) + : db_file_(db_file) + , heartbeat_(heartbeat) + { + } + + void createScalars(size_t count) + { + static constexpr struct PaletteEntry + { + const char* suffix; + const char* encoded_type; + } kPalette[] = { + {"int", "int"}, + {"string", "string"}, + {"bool", "bool"}, + {"uenum", "UnsignedEnum"}, + {"senum", "SignedEnum"}, + {"struct", "MultipleTypes"}, + }; + constexpr size_t kPaletteSize = sizeof(kPalette) / sizeof(kPalette[0]); + + scalar_specs_.clear(); + scalar_specs_.reserve(count); + + for (size_t i = 0; i < count; ++i) + { + const auto& entry = kPalette[i % kPaletteSize]; + const size_t instance = i / kPaletteSize; + + ScalarSpec spec; + spec.path = "top.scalar." + std::string(entry.suffix) + "_" + std::to_string(instance); + spec.encoded_type = entry.encoded_type; + scalar_specs_.push_back(std::move(spec)); + } + } + + void createContigs(size_t count) + { + contig_specs_.clear(); + contig_specs_.reserve(count); + + for (size_t i = 0; i < count; ++i) + { + ContainerSpec spec; + spec.path = "top.contig.exhaustive_" + std::to_string(i); + spec.bin_element_type = "MultipleTypes"; + spec.encoded_type = spec.bin_element_type + "_contig_capacity" + std::to_string(kContainerCapacity); + spec.capacity = kContainerCapacity; + spec.sparse = false; + contig_specs_.push_back(std::move(spec)); + } + } + + void createSparses(size_t count) + { + sparse_specs_.clear(); + sparse_specs_.reserve(count); + + for (size_t i = 0; i < count; ++i) + { + ContainerSpec spec; + spec.path = "top.sparse.exhaustive_" + std::to_string(i); + spec.bin_element_type = "MultipleTypes"; + spec.encoded_type = spec.bin_element_type + "_sparse_capacity" + std::to_string(kContainerCapacity); + spec.capacity = kContainerCapacity; + spec.sparse = true; + sparse_specs_.push_back(std::move(spec)); + } + } + + void runSimulation(uint64_t end_sim_time) + { + setupAppManagers_(); + createEntryPoints_(); + openPipelines_(); + simulate_(end_sim_time); + teardown_(); + writeMultipleTypesMetadata(db_mgr_); + postSimValidate_(); + } + +private: + CollectableValue makeInitialValue_(const ActiveCollectable& active, simdb::TinyStrings<>* tiny_strings) const + { + switch (active.kind) + { + case CollectableKind::Scalar: + return CollectableValue{makeRandomScalarValueFn_(active.palette_index)(tiny_strings)}; + case CollectableKind::Contig: + return CollectableValue{makeRandomContig()}; + case CollectableKind::Sparse: + return CollectableValue{makeRandomSparse()}; + } + throw simdb::DBException("Unknown collectable kind"); + } + + void recordTruth_(uint64_t tick, uint16_t cid, const LiveState& state) + { + if (state.closed || !state.logical.has_value()) + { + return; + } + truth_by_tick_[tick][cid] = *state.logical; + } + + void registerActive_(ActiveCollectable active) + { + const auto cid = active.entry_point->getID(); + tracked_cids_.push_back(cid); + encoded_type_by_cid_.emplace(cid, active.encoded_type); + bin_element_type_by_cid_.emplace(cid, active.bin_element_type); + active_.push_back(std::move(active)); + } + + void setupAppManagers_() + { + app_mgrs_.registerApp(); + auto& app_mgr = app_mgrs_.createAppManager(db_file_); + db_mgr_ = app_mgr.getDatabaseManager(); + + app_mgr.enableApp(simdb::argos::ArgosCollector::NAME); + app_mgrs_.createEnabledApps(); + + argos_collector_ = app_mgr.getApp(); + argos_collector_->setHeartbeat(heartbeat_); + argos_collector_->timestampWith(&tick_); + argos_collector_->addClock("root", 1); + } + + void createEntryPoints_() + { + active_.clear(); + tracked_cids_.clear(); + encoded_type_by_cid_.clear(); + bin_element_type_by_cid_.clear(); + + for (size_t i = 0; i < scalar_specs_.size(); ++i) + { + const auto& spec = scalar_specs_[i]; + ActiveCollectable active; + active.kind = CollectableKind::Scalar; + active.encoded_type = spec.encoded_type; + active.bin_element_type = spec.encoded_type; + active.palette_index = i % 6; + active.entry_point = argos_collector_->createScalarCollector(spec.path, "root", spec.encoded_type); + registerActive_(std::move(active)); + } + + for (const auto& spec : contig_specs_) + { + ActiveCollectable active; + active.kind = CollectableKind::Contig; + active.encoded_type = spec.encoded_type; + active.bin_element_type = spec.bin_element_type; + active.entry_point = argos_collector_->createContainerCollector(spec.path, "root", spec.encoded_type); + registerActive_(std::move(active)); + } + + for (const auto& spec : sparse_specs_) + { + ActiveCollectable active; + active.kind = CollectableKind::Sparse; + active.encoded_type = spec.encoded_type; + active.bin_element_type = spec.bin_element_type; + active.entry_point = argos_collector_->createContainerCollector(spec.path, "root", spec.encoded_type); + registerActive_(std::move(active)); + } + } + + + void openPipelines_() + { + app_mgrs_.createSchemas(); + app_mgrs_.postInit(0, nullptr); + app_mgrs_.initializePipelines(); + app_mgrs_.openPipelines(); + } + + void simulate_(uint64_t end_sim_time) + { + auto* tiny_strings = argos_collector_->getTinyStrings(); + SimulationEngine engine; + + while (++tick_ <= end_sim_time) + { + for (const auto& active : active_) + { + const auto cid = active.entry_point->getID(); + + if (tick_ == 1) + { + const auto initial = makeInitialValue_(active, tiny_strings); + engine.seedInitial(cid, initial); + stageCollectable(active, initial, tiny_strings); + } else + { + engine.applyRandomAction(active, tiny_strings); + } + + recordTruth_(tick_, cid, engine.at(cid)); + } + } + } + + void teardown_() + { + app_mgrs_.postSimLoopTeardown(); + } + + void postSimValidate_() + { + const auto strings = loadStringTable(db_mgr_); + argos_test::CollectableRegistry registry(db_mgr_); + + const auto deserialize_bin = [&](uint16_t cid, argos_test::ByteBuffer& buf) -> std::vector { + const auto start = buf.tell(); + (void)deserializeScalarValue(bin_element_type_by_cid_.at(cid), buf, strings); + return buf.slice(start, buf.tell()); + }; + + TruthValidationHandler handler(encoded_type_by_cid_, strings, truth_by_tick_, tracked_cids_); + argos_test::BlobIterator iterator(db_mgr_, registry); + iterator.iterate(handler, deserialize_bin); + } + + const std::string db_file_; + const size_t heartbeat_; + simdb::AppManagers app_mgrs_; + simdb::argos::ArgosCollector* argos_collector_ = nullptr; + simdb::DatabaseManager* db_mgr_ = nullptr; + uint64_t tick_ = 0; + std::vector scalar_specs_; + std::vector contig_specs_; + std::vector sparse_specs_; + std::vector active_; + std::vector tracked_cids_; + std::unordered_map encoded_type_by_cid_; + std::unordered_map bin_element_type_by_cid_; + std::map> truth_by_tick_; +}; + +namespace { + +constexpr uint64_t kHarnessSeed = 0xC0FFEEULL; + +std::filesystem::path findRepoRoot() +{ + auto path = std::filesystem::current_path(); + for (int depth = 0; depth < 12; ++depth) + { + if (std::filesystem::exists(path / "test" / "argos" / "compare.py")) + { + return path; + } + if (!path.has_parent_path() || path == path.parent_path()) + { + break; + } + path = path.parent_path(); + } + throw simdb::DBException("Could not locate repo root (test/argos/compare.py)"); +} + +std::string pythonArgosCmd(const std::filesystem::path& repo_root, const std::string& inner_cmd) +{ + std::ostringstream cmd; + cmd << "cd " << (repo_root / "test" / "argos").string() + << " && PYTHONPATH=" << (repo_root / "python" / "argos").string() << " python3 " << inner_cmd; + return cmd.str(); +} + +bool compareAvailable(const std::filesystem::path& repo_root) +{ + return std::system(pythonArgosCmd(repo_root, "-c \"from viewer.model.data_retriever import DataRetriever\"").c_str()) == 0; +} + +void runCompare(const std::filesystem::path& repo_root, const std::filesystem::path& baseline_db, + const std::filesystem::path& test_db) +{ + const auto baseline = std::filesystem::absolute(baseline_db).string(); + const auto test = std::filesystem::absolute(test_db).string(); + const auto cmd = pythonArgosCmd(repo_root, "compare.py " + baseline + " " + test); + + const int rc = std::system(cmd.c_str()); + EXPECT_EQUAL(rc, 0); + if (rc != 0) + { + std::ostringstream msg; + msg << "compare.py failed (exit=" << rc << ") comparing " << baseline_db.filename().string() << " vs " + << test_db.filename().string() << "\nCommand: " << cmd; + throw simdb::DBException(msg.str()); + } +} + +void runHarness(const std::filesystem::path& db_path, size_t heartbeat, uint64_t end_sim_time) +{ + TEST_METHOD_INIT; + reseedTestRng(kHarnessSeed); + std::filesystem::remove(db_path); + + Harness harness(db_path.string(), heartbeat); + harness.createScalars(10); + harness.createContigs(10); + harness.createSparses(10); + harness.runSimulation(end_sim_time); +} + +class LifecycleReplayHandler final : public DataExtractionHandler +{ +public: + LifecycleReplayHandler(uint16_t cid, const std::unordered_map& encoded_type_by_cid, + const std::unordered_map& strings) : + DataExtractionHandler(encoded_type_by_cid, strings), + cid_(cid) + { + } + + void snapshotTick(argos_test::BlobContext& context) override + { + const auto iter = getValuesByCid().find(cid_); + if (iter == getValuesByCid().end()) + { + snapshots_[context.current_tick] = std::nullopt; + return; + } + + snapshots_[context.current_tick] = std::get(std::get(iter->second)); + } + + std::optional valueAt(uint64_t tick) const + { + const auto iter = snapshots_.find(tick); + if (iter == snapshots_.end()) + { + return std::nullopt; + } + return iter->second; + } + +private: + uint16_t cid_; + std::map> snapshots_; +}; + +void testCloseCollectLifecycle() +{ + TEST_METHOD_INIT; + + const auto db_path = std::filesystem::current_path() / "lifecycle.db"; + std::filesystem::remove(db_path); + + simdb::AppManagers app_mgrs; + app_mgrs.registerApp(); + auto& app_mgr = app_mgrs.createAppManager(db_path.string()); + auto* db_mgr = app_mgr.getDatabaseManager(); + app_mgr.enableApp(simdb::argos::ArgosCollector::NAME); + app_mgrs.createEnabledApps(); + + auto* collector = app_mgr.getApp(); + collector->setHeartbeat(10); + uint64_t tick = 0; + collector->timestampWith(&tick); + collector->addClock("root", 1); + + auto* ep = collector->createScalarCollector("top.lifecycle", "root", "int"); + const auto cid = ep->getID(); + auto* tiny_strings = collector->getTinyStrings(); + + app_mgrs.createSchemas(); + app_mgrs.postInit(0, nullptr); + app_mgrs.initializePipelines(); + app_mgrs.openPipelines(); + + ActiveCollectable active; + active.kind = CollectableKind::Scalar; + active.encoded_type = "int"; + active.bin_element_type = "int"; + active.entry_point = ep; + + tick = 100; + stageCollectable(active, CollectableValue{static_cast(tick)}, tiny_strings); + tick = 150; + stageCollectable(active, CollectableValue{static_cast(tick)}, tiny_strings); + tick = 200; + ep->closeRecord(); + tick = 250; + stageCollectable(active, CollectableValue{static_cast(tick)}, tiny_strings); + tick = 300; + ep->closeRecord(); + tick = 350; + + app_mgrs.postSimLoopTeardown(); + + const auto strings = loadStringTable(db_mgr); + argos_test::CollectableRegistry registry(db_mgr); + const std::unordered_map encoded_type_by_cid{{cid, "int"}}; + + const auto deserialize_bin = [&](uint16_t replay_cid, argos_test::ByteBuffer& buf) -> std::vector { + const auto start = buf.tell(); + (void)deserializeScalarValue("int", buf, strings); + (void)replay_cid; + return buf.slice(start, buf.tell()); + }; + + LifecycleReplayHandler handler(cid, encoded_type_by_cid, strings); + argos_test::BlobIterator iterator(db_mgr, registry); + iterator.iterate(handler, deserialize_bin); + + const auto expectAbsent = [&](uint64_t at_tick) { + const auto value = handler.valueAt(at_tick); + EXPECT_EQUAL(value.has_value(), false); + }; + + const auto expectPresent = [&](uint64_t at_tick, int32_t expected) { + const auto value = handler.valueAt(at_tick); + EXPECT_EQUAL(value.has_value(), true); + EXPECT_EQUAL(value.value(), expected); + }; + + expectPresent(100, 100); + expectPresent(150, 150); + expectAbsent(200); + expectPresent(250, 250); + expectAbsent(300); + expectAbsent(350); +} + +} // namespace + +constexpr uint64_t DEFAULT_END_SIM_TIME = 1000; + +void testEverything(uint64_t end_sim_time) +{ + const auto repo_root = findRepoRoot(); + const auto db_dir = std::filesystem::current_path(); + const auto hb1_db = db_dir / "hb1.db"; + const auto hb3_db = db_dir / "hb3.db"; + const auto hb10_db = db_dir / "hb10.db"; + + runHarness(hb1_db, 1, end_sim_time); + runHarness(hb3_db, 3, end_sim_time); + runHarness(hb10_db, 10, end_sim_time); + + if (!compareAvailable(repo_root)) + { + throw simdb::DBException( + "python3 could not import viewer.model.data_retriever; " + "ensure python3 is on PATH and run from the SimDB repo"); + } + + runCompare(repo_root, hb1_db, hb3_db); + runCompare(repo_root, hb1_db, hb10_db); +} + +int main(int argc, char** argv) +{ + TEST_METHOD_INIT; + + uint64_t end_sim_time = DEFAULT_END_SIM_TIME; + if (argc > 1) + { + end_sim_time = static_cast(std::stoull(argv[1])); + } + + testEverything(end_sim_time); + testCloseCollectLifecycle(); + + REPORT_ERROR; + return ERROR_CODE; +} diff --git a/test/sqlite/Query/main.cpp b/test/sqlite/Query/main.cpp index 8ace62dc..514cba3c 100644 --- a/test/sqlite/Query/main.cpp +++ b/test/sqlite/Query/main.cpp @@ -5,6 +5,8 @@ #include "TestSchema.hpp" #include "simdb/sqlite/DatabaseManager.hpp" +#include + TEST_INIT; /// This test covers basic SELECT functionality for SimDB. @@ -812,6 +814,177 @@ int main() EXPECT_EQUAL(customer_first_name, "Jane"); EXPECT_FALSE(result_set.getNextRecord()); + // Test std::optional SELECT overloads for NULL/unset column values. + { + simdb::Schema optional_schema; + auto& optional_tbl = optional_schema.addTable("OptionalTypes"); + optional_tbl.addColumn("SomeInt32", dt::int32_t); + optional_tbl.addColumn("SomeUInt32", dt::uint32_t); + optional_tbl.addColumn("SomeInt64", dt::int64_t); + optional_tbl.addColumn("SomeUInt64", dt::uint64_t); + optional_tbl.addColumn("SomeDouble", dt::double_t); + db_mgr.appendSchema(optional_schema); + + db_mgr.INSERT(SQL_TABLE("OptionalTypes"), SQL_COLUMNS("SomeInt32"), SQL_VALUES((int32_t)42)); + + db_mgr.INSERT( + SQL_TABLE("OptionalTypes"), + SQL_COLUMNS("SomeInt32", "SomeUInt32", "SomeInt64", "SomeUInt64", "SomeDouble"), + SQL_VALUES((int32_t)100, (uint32_t)200, (int64_t)300, (uint64_t)400, TEST_DOUBLE_PI)); + + db_mgr.INSERT(SQL_TABLE("OptionalTypes"), SQL_COLUMNS("SomeUInt64"), SQL_VALUES((uint64_t)UINT64_MAX)); + + auto optional_query = db_mgr.createQuery("OptionalTypes"); + + std::optional opt_i32; + std::optional opt_u32; + std::optional opt_i64; + std::optional opt_u64; + std::optional opt_dbl; + + optional_query->select("SomeInt32", opt_i32); + optional_query->select("SomeUInt32", opt_u32); + optional_query->select("SomeInt64", opt_i64); + optional_query->select("SomeUInt64", opt_u64); + optional_query->select("SomeDouble", opt_dbl); + optional_query->orderBy("Id", simdb::QueryOrder::ASC); + + EXPECT_EQUAL(optional_query->count(), 3); + { + auto result_set = optional_query->getResultSet(); + + // Row 1: only SomeInt32 is set. + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_TRUE(opt_i32.has_value()); + EXPECT_EQUAL(opt_i32.value(), 42); + EXPECT_FALSE(opt_u32.has_value()); + EXPECT_FALSE(opt_i64.has_value()); + EXPECT_FALSE(opt_u64.has_value()); + EXPECT_FALSE(opt_dbl.has_value()); + + // Row 2: all columns set. + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_TRUE(opt_i32.has_value()); + EXPECT_EQUAL(opt_i32.value(), 100); + EXPECT_TRUE(opt_u32.has_value()); + EXPECT_EQUAL(opt_u32.value(), 200u); + EXPECT_TRUE(opt_i64.has_value()); + EXPECT_EQUAL(opt_i64.value(), 300); + EXPECT_TRUE(opt_u64.has_value()); + EXPECT_EQUAL(opt_u64.value(), 400u); + EXPECT_TRUE(opt_dbl.has_value()); + const double actual_dbl = opt_dbl.value(); + EXPECT_WITHIN_EPSILON(actual_dbl, TEST_DOUBLE_PI); + + // Row 3: only SomeUInt64 is set (NULL columns must not throw). + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_FALSE(opt_i32.has_value()); + EXPECT_FALSE(opt_u32.has_value()); + EXPECT_FALSE(opt_i64.has_value()); + EXPECT_TRUE(opt_u64.has_value()); + EXPECT_EQUAL(opt_u64.value(), UINT64_MAX); + EXPECT_FALSE(opt_dbl.has_value()); + + EXPECT_FALSE(result_set.getNextRecord()); + + // Verify reset() reuses optional writers via clone(). + result_set.reset(); + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_TRUE(opt_i32.has_value()); + EXPECT_EQUAL(opt_i32.value(), 42); + EXPECT_FALSE(opt_u64.has_value()); + } + + // Non-optional SELECT throws on NULL columns instead of coercing to zero. + { + auto plain_query = db_mgr.createQuery("OptionalTypes"); + uint32_t u32 = 999; + plain_query->select("SomeUInt32", u32); + plain_query->addConstraintForInt("SomeInt32", simdb::Constraints::EQUAL, 42); + + auto result_set = plain_query->getResultSet(); + EXPECT_THROW(result_set.getNextRecord()); + } + { + auto plain_query = db_mgr.createQuery("OptionalTypes"); + uint64_t u64 = 999; + plain_query->select("SomeUInt64", u64); + plain_query->addConstraintForInt("SomeInt32", simdb::Constraints::EQUAL, 42); + + auto result_set = plain_query->getResultSet(); + EXPECT_THROW(result_set.getNextRecord()); + } + { + auto plain_query = db_mgr.createQuery("OptionalTypes"); + uint32_t u32 = 0; + plain_query->select("SomeUInt32", u32); + plain_query->addConstraintForInt("SomeInt32", simdb::Constraints::EQUAL, 100); + + auto result_set = plain_query->getResultSet(); + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_EQUAL(u32, 200u); + EXPECT_FALSE(result_set.getNextRecord()); + } + } + + // Test std::optional to distinguish SQL NULL from "". + { + simdb::Schema optional_string_schema; + optional_string_schema.addTable("OptionalStrings") + .addColumn("Label", dt::int32_t) + .addColumn("SomeString", dt::string_t); + db_mgr.appendSchema(optional_string_schema); + + db_mgr.INSERT(SQL_TABLE("OptionalStrings"), SQL_COLUMNS("Label"), SQL_VALUES((int32_t)1)); + db_mgr.INSERT(SQL_TABLE("OptionalStrings"), SQL_COLUMNS("Label", "SomeString"), SQL_VALUES((int32_t)2, "")); + db_mgr.INSERT(SQL_TABLE("OptionalStrings"), SQL_COLUMNS("Label", "SomeString"), SQL_VALUES((int32_t)3, "hello")); + + int32_t label = 0; + + // NULL columns are written as "" for plain std::string. + { + auto plain_query = db_mgr.createQuery("OptionalStrings"); + std::string str; + plain_query->select("Label", label); + plain_query->select("SomeString", str); + plain_query->addConstraintForInt("Label", simdb::Constraints::EQUAL, 1); + + auto result_set = plain_query->getResultSet(); + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_EQUAL(label, 1); + EXPECT_EQUAL(str, ""); + EXPECT_FALSE(result_set.getNextRecord()); + } + + // std::optional distinguishes SQL NULL from "". + { + auto opt_query = db_mgr.createQuery("OptionalStrings"); + std::optional opt_str; + opt_query->select("Label", label); + opt_query->select("SomeString", opt_str); + opt_query->orderBy("Label", simdb::QueryOrder::ASC); + + EXPECT_EQUAL(opt_query->count(), 3); + auto result_set = opt_query->getResultSet(); + + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_EQUAL(label, 1); + EXPECT_FALSE(opt_str.has_value()); + + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_EQUAL(label, 2); + EXPECT_TRUE(opt_str.has_value()); + EXPECT_EQUAL(opt_str.value(), ""); + + EXPECT_TRUE(result_set.getNextRecord()); + EXPECT_EQUAL(label, 3); + EXPECT_TRUE(opt_str.has_value()); + EXPECT_EQUAL(opt_str.value(), "hello"); + + EXPECT_FALSE(result_set.getNextRecord()); + } + } + REPORT_ERROR; return ERROR_CODE; }